From 8c8e9bd530b5232414dce3f8a1fb4791e3ea9bcf Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Tue, 5 Apr 2016 21:55:44 -0700 Subject: [PATCH 01/27] Untested autoimage HTTPS server/client cert support by copying into the image. --- server/modules/autoimage.py | 41 ++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/server/modules/autoimage.py b/server/modules/autoimage.py index 16f3547..d37686d 100755 --- a/server/modules/autoimage.py +++ b/server/modules/autoimage.py @@ -28,6 +28,18 @@ parser.add_argument( dest='device', action='store', required=True) +parser.add_argument( + '--https-ca-cert', + dest='https_ca_cert', + action='store') +parser.add_argument( + '--https-client-cert', + dest='https_client_cert', + action='store') +parser.add_argument( + '--https-client-key', + dest='https_client_key', + action='store') parser.add_argument( '--persistent-percent', dest='persistent_percent', @@ -64,6 +76,32 @@ def main(): FLAGS.ca_cert, os.path.join(FLAGS.chroot_path, 'autoimage', 'config', 'ca.cert.pem')) + image_flags = [] + + if FLAGS.https_ca_cert: + https_ca_cert_path = os.path.join('autoimage', 'config', 'https-ca.cert.pem') + shutil.copyfile( + FLAGS.https_ca_cert, + os.path.join(FLAGS.chroot_path, https_ca_cert_path)) + image_flags.extend([ + '--https-ca-cert', os.path.join('/', https_ca_cert_path), + ]) + + if FLAGS.https_client_cert and FLAGS.https_client_key: + https_client_cert_path = os.path.join('autoimage', 'config', 'https-client.cert.pem') + shutil.copyfile( + FLAGS.https_client_cert, + os.path.join(FLAGS.chroot_path, https_client_cert_path)) + https_client_key_path = os.path.join('autoimage', 'config', 'https-client.key.pem') + shutil.copyfile( + FLAGS.https_client_key, + os.path.join(FLAGS.chroot_path, https_client_key_path)) + os.chmod(os.path.join(FLAGS.chroot_path, https_client_key_path), 0o400) + image_flags.extend([ + '--https-client-cert', os.path.join('/', https_client_cert_path), + '--https-client-key', os.path.join('/', https_client_key_path), + ]) + parsed = parse.urlparse(FLAGS.base_url) init = os.path.join(FLAGS.chroot_path, 'etc', 'init', 'autoimage.conf') @@ -79,7 +117,7 @@ script chvt 7 /autoimage/client/wait_for_service.py --host=%(host)s --service=%(service)s /dev/tty7 2>&1 chvt 7 - /autoimage/imager/image.py --device=%(device)s --persistent-percent=%(persistent_percent)d --ca-cert=/autoimage/config/ca.cert.pem --base-url=%(base_url)s /dev/tty7 2>&1 + /autoimage/imager/image.py --device=%(device)s --persistent-percent=%(persistent_percent)d --ca-cert=/autoimage/config/ca.cert.pem --base-url=%(base_url)s %(image_flags)s /dev/tty7 2>&1 chvt 7 echo >/dev/tty7 @@ -95,6 +133,7 @@ end script 'device': FLAGS.device, 'persistent_percent': FLAGS.persistent_percent, 'base_url': FLAGS.base_url, + 'image_flags': ' '.join(image_flags), }) From 5e7f9f03515d6e46e5aa1f94cfb6166b8124d428 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Wed, 6 Apr 2016 23:37:40 +0000 Subject: [PATCH 02/27] Required module --- server/modules/autoimage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/autoimage.py b/server/modules/autoimage.py index d37686d..96dac7a 100755 --- a/server/modules/autoimage.py +++ b/server/modules/autoimage.py @@ -63,7 +63,7 @@ def main(): 'apt-get', 'install', '--assume-yes', - 'git', 'grub-pc', 'python3-openssl') + 'git', 'grub-pc', 'python3-openssl', 'python3-requests') ExecChroot( 'git', From 427b69497836db5df19e1ae50e90d139f4a897c0 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Wed, 6 Apr 2016 23:38:44 +0000 Subject: [PATCH 03/27] Web UI for generating systemid tags. --- www/code-39.woff | Bin 0 -> 2536 bytes www/systemid.html | 103 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 www/code-39.woff create mode 100644 www/systemid.html diff --git a/www/code-39.woff b/www/code-39.woff new file mode 100644 index 0000000000000000000000000000000000000000..a9daeb1815f1ed3ecc35020bf03cdc2f8a93df19 GIT binary patch literal 2536 zcmY+GdpuMBAIHx&wiz3_<+h@taxam~N&ielK`#c`c$N9Wo=W}_Vb3X6S=ft|$*#Qs$EMl_&MtIev z{;P%c|1U1?diwzYVhg&5LGTe#A?XtPV>sH9g%&)TPa<;px)GsF1B zho-45_%5xl^400RAroJb`@BLalWlK|NO@~x9WF<4kcTx}{lI|1?;6MQzhQo?OPKZ38Tjj_;BTfq``B)<&^5bU zBXgN7&8Mu&e-Y1GU3dOmaY+@iw#0E%WeuKKaO6}QM@O*HvG2d;k=!icsit#rO=-^u z7ovQ5*BX}aTNgcLXhk#2dlfX@u*29d*y!r3SzW9v&L8!XHk|QXJ-z%Fc3GUTR-3|R zElqPxBJD6yIdp}mx->YKdu!hQQZ4D_=gu<2gE@#JTOWCdC`?*;+t^&!Tr!i-$K&>i zu#~0(f4Df?dV4rZe8(xV-LeR2hAbKL@%$GL#pQMz)~_LU&5?yG0Rq4PB!B|*NA%Ah zKr)UsaN8xa&r-Ji?j-!5_fN&-JCcrQXNK;Po(nV@Jq^>WvTuM)08{X*AEC;a&&dNa zmDXFx^5zMjsvL*cCaYc(d#+-h$b7yw_4Cfc`DG|QcPyZOyb)6Wxmm^-vi;Bq zS1O%Lj|Y>)e0%HV0-{21jU-}KqR0X@@hJca>9#iJ8ggTj$Vp}Y%x@{T@q1lgP|$U1 zJ*r(#rPR~v-3wI0iwb1soWB|$z1hQwd1l$nQ&e>46$D`j9JJO<%w)8Zh?|sg9r7^J zBwBPYK36PwL!;)5MnhBWx~9P3>*9?Y@aEu@chJ5u&2^tQyidkj3l-Tfc}ltXev`dy zhWisZ%6safPf&Jzjx$Uln>I)oT`9?EEMtM4N-(Q|_>bGY3W)f&_VTFq$(%xPkU@QuZ zp=#ba=D>)wq2=wSwt^sJIg>A+Ie$;9!LWq=aO5!`=6OadwkgGs#@!?7~$Jk32tS z?)1wv!YQ(Nx$HR9tmla{9Jx$Z@iQ9p?K(GTcs@$ej?y<89x*5%P3D<*)OURBNq1Ob zpI9>LFGo1XmMwalbtlBxKd5C7#(NH6d)F^mq}ICLtI+IzU__EqP3YSDxn=#t@{Us_y$Tomn;jweRbUdanBSu~SE12!|rlQa>VX7hV@hZ5D`T#y+BL4W^J-0VIBVM=?I z@;5((IbsDbV?hEWaX$fN*evJ*JqFwOJqL)xLIB{85$rw!3EE{J%Tq zh06?vRbKo)n5&3rC;&KULw|zz&%NeyFI=|5TnjplH^D`iOmD=ke6#OO?AYd9MVG3T zsJ7IWd^iPZq3!Gz&IlGyfDM5{PD09|B2XV_B@7PpgH?+lMNWx46-9~SMW4YX;DPXZ z1O`DuJV#0)gOMCDX|YhTmnd0OIEpJSFCHP@BB3A=Dba>jLPw(8FiIFIrd?7692)|n z3+{@d!ioMW0NfRXk0U%kz~^baX&%+*Dw`5u7S?wG$wZxy7{K}#nWoDmYgjp)caB2Gwk=Axbb{7GQ#M{~V47A2YsKw=tT&BS^hsE!|-Zg(QU5bX$pjO6v67DZZGRExJt?$HUXN?!R7-S6TPD$l2tC(LpC zn@}@v$CJf#Lr^V;aUC3K-OTa2#8*Bpc9Ra+pBdT2;g|v6vh?xoJ+Rp7mD=L&hwj-9 z*2Np|J~&;%`>!>g)&EhJHlO-mj%ClG;WOXd?@SLlc2zfoqe}_| zv2^UQgoZ#WOZEGan!F^Qd!_)s+1YncQ(?20bj^J=r#0<pTS6Q~4x(}AzidXR7 zz@Kf(x#WFWbnsm+K55^MJt{>R!~#r2xoN3=vNht6Ik8i5@^};9zxCdXgu8AwPg>*P z-4OQGY?x+6oZx02?6@4-Z!)2zWBAG?f$X*cIn7C)+|!eNuhhF8hXZ{b@0N@x$DntQ zoKEQ~cl%1cMp+zv(0U^u!E$z-x{HlS9SZj9lo5s;0GQAVzn2PE zKB>^0juZ$sGm0;<8u?7+W33r3B>whwd?X<(h(aWnAZux^rPzFyzu)MHIj)g|Zl!25 db93` + + SystemID + + + + + +
+
+
+
+
+ +
+
+
+
+
+ +
+
+
+
+
+ + + + From 234d1859764da5f4d3af397453494dc288b4a519 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Thu, 7 Apr 2016 00:26:14 +0000 Subject: [PATCH 04/27] Auto-resizing, and larger. --- www/systemid.html | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/www/systemid.html b/www/systemid.html index cba19fd..de0a4bc 100644 --- a/www/systemid.html +++ b/www/systemid.html @@ -12,8 +12,6 @@ } .barcode { - width: 400px; - height: 50px; font-family: 'code-39'; font-size: 24px; } @@ -38,10 +36,8 @@ } .sysid { - width: 400px; - height: 120px; font-family: 'droid-sans-mono'; - font-size: 96px; + font-size: 140px; text-align: center; text-transform: uppercase; } @@ -51,6 +47,7 @@ } .block { + width: 600px; margin: 20px; } @@ -89,6 +86,7 @@ document.addEventListener('DOMContentLoaded', (e) => { break; } barcodes.push(barcode); + barcode.innerText = '**'; } sysid.addEventListener('input', (e) => { let barcodeText = '*' + sysid.innerText.toUpperCase() + '*'; From ed9e49a0db35e98a5989d947144c699ee9cfa5b0 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Thu, 7 Apr 2016 00:47:35 +0000 Subject: [PATCH 05/27] Change some cert filenames. --- server/modules/autoimage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/modules/autoimage.py b/server/modules/autoimage.py index 96dac7a..79fbfaa 100755 --- a/server/modules/autoimage.py +++ b/server/modules/autoimage.py @@ -79,7 +79,7 @@ def main(): image_flags = [] if FLAGS.https_ca_cert: - https_ca_cert_path = os.path.join('autoimage', 'config', 'https-ca.cert.pem') + https_ca_cert_path = os.path.join('autoimage', 'config', 'ca.https.cert.pem') shutil.copyfile( FLAGS.https_ca_cert, os.path.join(FLAGS.chroot_path, https_ca_cert_path)) @@ -88,11 +88,11 @@ def main(): ]) if FLAGS.https_client_cert and FLAGS.https_client_key: - https_client_cert_path = os.path.join('autoimage', 'config', 'https-client.cert.pem') + https_client_cert_path = os.path.join('autoimage', 'config', 'client.https.cert.pem') shutil.copyfile( FLAGS.https_client_cert, os.path.join(FLAGS.chroot_path, https_client_cert_path)) - https_client_key_path = os.path.join('autoimage', 'config', 'https-client.key.pem') + https_client_key_path = os.path.join('autoimage', 'config', 'client.https.key.pem') shutil.copyfile( FLAGS.https_client_key, os.path.join(FLAGS.chroot_path, https_client_key_path)) From e8b5cdea294c352c06a014d1be1e422f2d191435 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Thu, 7 Apr 2016 01:00:40 +0000 Subject: [PATCH 06/27] init style cleanup --- server/modules/autoimage.py | 19 +++++++++---------- server/modules/persistent.py | 4 ++++ server/modules/systemid.py | 4 ++++ 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/server/modules/autoimage.py b/server/modules/autoimage.py index 79fbfaa..a91cd70 100755 --- a/server/modules/autoimage.py +++ b/server/modules/autoimage.py @@ -109,23 +109,22 @@ def main(): fh.write(""" description "AutoImage" -start on stopped rc RUNLEVEL=[2345] - -stop on runlevel [!2345] +start on runlevel [2345] script + exec /dev/tty7 2>&1 chvt 7 - /autoimage/client/wait_for_service.py --host=%(host)s --service=%(service)s /dev/tty7 2>&1 + /autoimage/client/wait_for_service.py --host=%(host)s --service=%(service)s chvt 7 - /autoimage/imager/image.py --device=%(device)s --persistent-percent=%(persistent_percent)d --ca-cert=/autoimage/config/ca.cert.pem --base-url=%(base_url)s %(image_flags)s /dev/tty7 2>&1 + /autoimage/imager/image.py --device=%(device)s --persistent-percent=%(persistent_percent)d --ca-cert=/autoimage/config/ca.cert.pem --base-url=%(base_url)s %(image_flags)s chvt 7 - echo >/dev/tty7 - echo "==================" >/dev/tty7 - echo "autoimage complete" >/dev/tty7 - echo "==================" >/dev/tty7 + echo + echo "==================" + echo "autoimage complete" + echo "==================" - /autoimage/client/alert.py --type=happy /dev/tty7 + /autoimage/client/alert.py --type=happy end script """ % { 'host': parsed.hostname, diff --git a/server/modules/persistent.py b/server/modules/persistent.py index 1b30e91..1448507 100755 --- a/server/modules/persistent.py +++ b/server/modules/persistent.py @@ -22,9 +22,13 @@ def main(): description "Mount /persistent" start on filesystem +task + +emits persistent-ready script mount LABEL=PERSISTENT /persistent + initctl emit --no-wait persistent-ready end script """) diff --git a/server/modules/systemid.py b/server/modules/systemid.py index d496d3e..8159c54 100755 --- a/server/modules/systemid.py +++ b/server/modules/systemid.py @@ -22,6 +22,9 @@ def main(): description "Mount /systemid" start on filesystem +task + +emits systemid-ready script mount LABEL=SYSTEMID /systemid @@ -29,6 +32,7 @@ script echo ${SYSTEMID} > /etc/hostname hostname --file /etc/hostname grep ${SYSTEMID} /etc/hosts >/dev/null || echo "127.0.2.1 ${SYSTEMID}" >> /etc/hosts + initctl emit --no-wait systemid-ready end script """) From eebd3906fd1fee7f86b368b743c18b13a68577e8 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Thu, 7 Apr 2016 01:37:53 +0000 Subject: [PATCH 07/27] Add certclient module for use with autoimage --- server/modules/certclient.py | 134 +++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100755 server/modules/certclient.py diff --git a/server/modules/certclient.py b/server/modules/certclient.py new file mode 100755 index 0000000..a0e1838 --- /dev/null +++ b/server/modules/certclient.py @@ -0,0 +1,134 @@ +#!/usr/bin/python3 + +import argparse +import os +import shutil +import subprocess +from urllib import parse + + +parser = argparse.ArgumentParser(description='iconograph autoimage') +parser.add_argument( + '--chroot-path', + dest='chroot_path', + action='store', + required=True) +parser.add_argument( + '--ca-cert', + dest='ca_cert', + action='store', + required=True) +parser.add_argument( + '--client-cert', + dest='client_cert', + action='store', + required=True) +parser.add_argument( + '--client-key', + dest='client_key', + action='store', + required=True) +parser.add_argument( + '--subject', + dest='subject', + action='store', + required=True) +parser.add_argument( + '--tag', + dest='tag', + action='store', + required=True) +parser.add_argument( + '--server', + dest='server', + action='store', + required=True) +FLAGS = parser.parse_args() + + +def Exec(*args, **kwargs): + print('+', args) + subprocess.check_call(args, **kwargs) + + +def ExecChroot(*args, **kwargs): + Exec('chroot', FLAGS.chroot_path, *args, **kwargs) + + +def main(): + ExecChroot( + 'apt-get', + 'install', + '--assume-yes', + 'git', 'python3-requests', 'openssl') + + ExecChroot( + 'git', + 'clone', + 'https://github.com/robot-tools/iconograph.git', + 'certclient-icon') + + ExecChroot( + 'git', + 'clone', + 'https://github.com/robot-tools/certserver.git', + 'certserver') + + os.mkdir(os.path.join(FLAGS.chroot_path, 'certclient-icon', 'config')) + + ca_cert_path = os.path.join('certclient-icon', 'config', 'ca.%s.certserver.cert.pem' % FLAGS.tag) + shutil.copyfile( + FLAGS.ca_cert, + os.path.join(FLAGS.chroot_path, ca_cert_path)) + + client_cert_path = os.path.join('certclient-icon', 'config', 'client.%s.certserver.cert.pem' % FLAGS.tag) + shutil.copyfile( + FLAGS.client_cert, + os.path.join(FLAGS.chroot_path, client_cert_path)) + client_key_path = os.path.join('certclient-icon', 'config', 'client.%s.certserver.key.pem' % FLAGS.tag) + shutil.copyfile( + FLAGS.client_key, + os.path.join(FLAGS.chroot_path, client_key_path)) + os.chmod(os.path.join(FLAGS.chroot_path, client_key_path), 0o400) + + parsed = parse.urlparse(FLAGS.server) + + init = os.path.join(FLAGS.chroot_path, 'etc', 'init', 'certclient.%s.conf' % FLAGS.tag) + with open(init, 'w') as fh: + fh.write(""" +description "CertClient %(tag)s" + +start on systemid-ready + +script + exec /dev/tty8 2>&1 + chvt 8 + KEY="/systemid/$(hostname).%(tag)s.key.pem" + CERT="/systemid/$(hostname).%(tag)s.cert.pem" + SUBJECT="$(echo '%(subject)s' | sed s/SYSTEMID/$(hostname)/g)" + openssl ecparam -name secp384r1 -genkey | openssl ec -out "${KEY}" + chmod 0400 "${KEY}" + chvt 8 + + /certclient-icon/client/wait_for_service.py --host=%(host)s --service=%(service)s + chvt 8 + openssl req -new -key "${KEY}" -subj "${SUBJECT}" | /certserver/certclient.py --ca-cert=/certclient-icon/config/ca.%(tag)s.certserver.cert.pem --client-cert=/certclient-icon/config/client.%(tag)s.certserver.cert.pem --client-key=/certclient-icon/config/client.%(tag)s.certserver.key.pem --server=%(server)s > "${CERT}" + chmod 0444 "${CERT}" + chvt 8 + + echo + echo "==================" + echo "certclient %(tag)s complete" + echo "==================" +end script +""" % { + 'host': parsed.hostname, + 'service': parsed.port or parsed.scheme, + 'subject': FLAGS.subject, + 'tag': FLAGS.tag, + 'server': FLAGS.server, + }) + + +if __name__ == '__main__': + main() From 8eb18f1ccfcdccf984ac99c44c5944ff79d6a262 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Wed, 6 Apr 2016 22:38:57 -0700 Subject: [PATCH 08/27] Simplify directory tree --- client/.gitignore | 1 - client/fetch_and_update.sh | 13 +++++++------ server/modules/autoimage.py | 28 +++++++++++++++------------- server/modules/certclient.py | 34 ++++++++++++++++++---------------- server/modules/iconograph.py | 21 ++++++++++++--------- 5 files changed, 52 insertions(+), 45 deletions(-) delete mode 100644 client/.gitignore diff --git a/client/.gitignore b/client/.gitignore deleted file mode 100644 index 2cdfa72..0000000 --- a/client/.gitignore +++ /dev/null @@ -1 +0,0 @@ -flags diff --git a/client/fetch_and_update.sh b/client/fetch_and_update.sh index 0011327..b248fd4 100755 --- a/client/fetch_and_update.sh +++ b/client/fetch_and_update.sh @@ -1,15 +1,16 @@ -#!/bin/sh +#!/bin/bash set -ex BASE=$(dirname $0) -IMAGES=/isodevice/iconograph +IMAGES="/isodevice/iconograph" mkdir -p "${IMAGES}" -BOOT=/isodevice +BOOT="/isodevice" -FLAGS=$(cat ${BASE}/flags) +FLAGS="$(cat /icon/config/fetcher.flags)" +CA_CERT="/icon/config/ca.image.cert.pem" -${BASE}/fetcher.py --image-dir="${IMAGES}" --ca-cert=${BASE}/../config/ca.cert.pem ${FLAGS} -${BASE}/update_grub.py --image-dir="${IMAGES}" --boot-dir="${BOOT}" > ${BOOT}/grub/grub.cfg.tmp && mv ${BOOT}/grub/grub.cfg.tmp ${BOOT}/grub/grub.cfg +"${BASE}/fetcher.py" --image-dir="${IMAGES}" --ca-cert="${CA_CERT}" ${FLAGS} +"${BASE}/update_grub.py" --image-dir="${IMAGES}" --boot-dir="${BOOT}" > "${BOOT}/grub/grub.cfg.tmp" && mv "${BOOT}/grub/grub.cfg.tmp" "${BOOT}/grub/grub.cfg" diff --git a/server/modules/autoimage.py b/server/modules/autoimage.py index a91cd70..cb24ec0 100755 --- a/server/modules/autoimage.py +++ b/server/modules/autoimage.py @@ -65,21 +65,23 @@ def main(): '--assume-yes', 'git', 'grub-pc', 'python3-openssl', 'python3-requests') - ExecChroot( - 'git', - 'clone', - 'https://github.com/robot-tools/iconograph.git', - 'autoimage') + os.makedirs(os.path.join(FLAGS.chroot_path, 'icon', 'config'), exist_ok=True) + + if not os.path.exists(os.path.join(FLAGS.chroot_path, 'icon', 'iconograph')): + ExecChroot( + 'git', + 'clone', + 'https://github.com/robot-tools/iconograph.git', + 'icon/iconograph') - os.mkdir(os.path.join(FLAGS.chroot_path, 'autoimage', 'config')) shutil.copyfile( FLAGS.ca_cert, - os.path.join(FLAGS.chroot_path, 'autoimage', 'config', 'ca.cert.pem')) + os.path.join(FLAGS.chroot_path, 'icon', 'config', 'ca.image.cert.pem')) image_flags = [] if FLAGS.https_ca_cert: - https_ca_cert_path = os.path.join('autoimage', 'config', 'ca.https.cert.pem') + https_ca_cert_path = os.path.join('icon', 'config', 'ca.https.cert.pem') shutil.copyfile( FLAGS.https_ca_cert, os.path.join(FLAGS.chroot_path, https_ca_cert_path)) @@ -88,11 +90,11 @@ def main(): ]) if FLAGS.https_client_cert and FLAGS.https_client_key: - https_client_cert_path = os.path.join('autoimage', 'config', 'client.https.cert.pem') + https_client_cert_path = os.path.join('icon', 'config', 'client.https.cert.pem') shutil.copyfile( FLAGS.https_client_cert, os.path.join(FLAGS.chroot_path, https_client_cert_path)) - https_client_key_path = os.path.join('autoimage', 'config', 'client.https.key.pem') + https_client_key_path = os.path.join('icon', 'config', 'client.https.key.pem') shutil.copyfile( FLAGS.https_client_key, os.path.join(FLAGS.chroot_path, https_client_key_path)) @@ -114,9 +116,9 @@ start on runlevel [2345] script exec /dev/tty7 2>&1 chvt 7 - /autoimage/client/wait_for_service.py --host=%(host)s --service=%(service)s + /icon/iconograph/client/wait_for_service.py --host=%(host)s --service=%(service)s chvt 7 - /autoimage/imager/image.py --device=%(device)s --persistent-percent=%(persistent_percent)d --ca-cert=/autoimage/config/ca.cert.pem --base-url=%(base_url)s %(image_flags)s + /icon/iconograph/imager/image.py --device=%(device)s --persistent-percent=%(persistent_percent)d --ca-cert=/icon/config/ca.image.cert.pem --base-url=%(base_url)s %(image_flags)s chvt 7 echo @@ -124,7 +126,7 @@ script echo "autoimage complete" echo "==================" - /autoimage/client/alert.py --type=happy + /icon/iconograph/client/alert.py --type=happy end script """ % { 'host': parsed.hostname, diff --git a/server/modules/certclient.py b/server/modules/certclient.py index a0e1838..1e7a0a8 100755 --- a/server/modules/certclient.py +++ b/server/modules/certclient.py @@ -62,30 +62,32 @@ def main(): '--assume-yes', 'git', 'python3-requests', 'openssl') - ExecChroot( - 'git', - 'clone', - 'https://github.com/robot-tools/iconograph.git', - 'certclient-icon') + os.makedirs(os.path.join(FLAGS.chroot_path, 'icon', 'config'), exist_ok=True) - ExecChroot( - 'git', - 'clone', - 'https://github.com/robot-tools/certserver.git', - 'certserver') + if not os.path.exists(os.path.join(FLAGS.chroot_path, 'icon', 'iconograph')): + ExecChroot( + 'git', + 'clone', + 'https://github.com/robot-tools/iconograph.git', + 'icon/iconograph') - os.mkdir(os.path.join(FLAGS.chroot_path, 'certclient-icon', 'config')) + if not os.path.exists(os.path.join(FLAGS.chroot_path, 'icon', 'certserver')): + ExecChroot( + 'git', + 'clone', + 'https://github.com/robot-tools/certserver.git', + 'icon/certserver') - ca_cert_path = os.path.join('certclient-icon', 'config', 'ca.%s.certserver.cert.pem' % FLAGS.tag) + ca_cert_path = os.path.join('icon', 'config', 'ca.%s.certserver.cert.pem' % FLAGS.tag) shutil.copyfile( FLAGS.ca_cert, os.path.join(FLAGS.chroot_path, ca_cert_path)) - client_cert_path = os.path.join('certclient-icon', 'config', 'client.%s.certserver.cert.pem' % FLAGS.tag) + client_cert_path = os.path.join('icon', 'config', 'client.%s.certserver.cert.pem' % FLAGS.tag) shutil.copyfile( FLAGS.client_cert, os.path.join(FLAGS.chroot_path, client_cert_path)) - client_key_path = os.path.join('certclient-icon', 'config', 'client.%s.certserver.key.pem' % FLAGS.tag) + client_key_path = os.path.join('icon', 'config', 'client.%s.certserver.key.pem' % FLAGS.tag) shutil.copyfile( FLAGS.client_key, os.path.join(FLAGS.chroot_path, client_key_path)) @@ -110,9 +112,9 @@ script chmod 0400 "${KEY}" chvt 8 - /certclient-icon/client/wait_for_service.py --host=%(host)s --service=%(service)s + /icon/iconograph/client/wait_for_service.py --host=%(host)s --service=%(service)s chvt 8 - openssl req -new -key "${KEY}" -subj "${SUBJECT}" | /certserver/certclient.py --ca-cert=/certclient-icon/config/ca.%(tag)s.certserver.cert.pem --client-cert=/certclient-icon/config/client.%(tag)s.certserver.cert.pem --client-key=/certclient-icon/config/client.%(tag)s.certserver.key.pem --server=%(server)s > "${CERT}" + openssl req -new -key "${KEY}" -subj "${SUBJECT}" | /icon/certserver/certclient.py --ca-cert=/icon/config/ca.%(tag)s.certserver.cert.pem --client-cert=/icon/config/client.%(tag)s.certserver.cert.pem --client-key=/icon/config/client.%(tag)s.certserver.key.pem --server=%(server)s > "${CERT}" chmod 0444 "${CERT}" chvt 8 diff --git a/server/modules/iconograph.py b/server/modules/iconograph.py index 345e5e8..673c347 100755 --- a/server/modules/iconograph.py +++ b/server/modules/iconograph.py @@ -47,17 +47,20 @@ def main(): '--assume-yes', 'daemontools-run', 'git', 'python3-openssl') - ExecChroot( - 'git', - 'clone', - 'https://github.com/robot-tools/iconograph.git') + os.makedirs(os.path.join(FLAGS.chroot_path, 'icon', 'config'), exist_ok=True) + + if not os.path.exists(os.path.join(FLAGS.chroot_path, 'icon', 'iconograph')): + ExecChroot( + 'git', + 'clone', + 'https://github.com/robot-tools/iconograph.git', + 'icon/iconograph') - os.mkdir(os.path.join(FLAGS.chroot_path, 'iconograph', 'config')) shutil.copyfile( FLAGS.ca_cert, - os.path.join(FLAGS.chroot_path, 'iconograph', 'config', 'ca.cert.pem')) + os.path.join(FLAGS.chroot_path, 'icon', 'config', 'ca.image.cert.pem')) - path = os.path.join(FLAGS.chroot_path, 'iconograph', 'client', 'flags') + path = os.path.join(FLAGS.chroot_path, 'icon', 'config', 'fetcher.flags') with open(path, 'w') as fh: fh.write('--base-url=%(base_url)s --max-images=%(max_images)d\n' % { 'base_url': FLAGS.base_url, @@ -65,8 +68,8 @@ def main(): }) os.symlink( - '/iconograph/client', - os.path.join(FLAGS.chroot_path, 'etc', 'service', 'iconograph')) + '/icon/iconograph/client', + os.path.join(FLAGS.chroot_path, 'etc', 'service', 'iconograph-client')) if __name__ == '__main__': From 1e1fff2e1cf31cc5a26bc7f7f037898167e432fe Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Wed, 6 Apr 2016 22:40:43 -0700 Subject: [PATCH 09/27] Only fetch certs if they don't exist at the target --- server/modules/certclient.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/server/modules/certclient.py b/server/modules/certclient.py index 1e7a0a8..a8c4fdf 100755 --- a/server/modules/certclient.py +++ b/server/modules/certclient.py @@ -105,17 +105,25 @@ start on systemid-ready script exec /dev/tty8 2>&1 chvt 8 + KEY="/systemid/$(hostname).%(tag)s.key.pem" CERT="/systemid/$(hostname).%(tag)s.cert.pem" SUBJECT="$(echo '%(subject)s' | sed s/SYSTEMID/$(hostname)/g)" - openssl ecparam -name secp384r1 -genkey | openssl ec -out "${KEY}" - chmod 0400 "${KEY}" - chvt 8 + if test ! -e "${KEY}"; then + openssl ecparam -name secp384r1 -genkey | openssl ec -out "${KEY}" + chmod 0400 "${KEY}" + fi + + chvt 8 /icon/iconograph/client/wait_for_service.py --host=%(host)s --service=%(service)s chvt 8 - openssl req -new -key "${KEY}" -subj "${SUBJECT}" | /icon/certserver/certclient.py --ca-cert=/icon/config/ca.%(tag)s.certserver.cert.pem --client-cert=/icon/config/client.%(tag)s.certserver.cert.pem --client-key=/icon/config/client.%(tag)s.certserver.key.pem --server=%(server)s > "${CERT}" - chmod 0444 "${CERT}" + + if test ! -e "${CERT}"; then + openssl req -new -key "${KEY}" -subj "${SUBJECT}" | /icon/certserver/certclient.py --ca-cert=/icon/config/ca.%(tag)s.certserver.cert.pem --client-cert=/icon/config/client.%(tag)s.certserver.cert.pem --client-key=/icon/config/client.%(tag)s.certserver.key.pem --server=%(server)s > "${CERT}" + chmod 0444 "${CERT}" + fi + chvt 8 echo From 14f2be7e10228aebae68812b8c200d8c054257bc Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Wed, 6 Apr 2016 22:52:48 -0700 Subject: [PATCH 10/27] Support certs in fetcher --- client/fetch_and_update.sh | 13 ++++++++++++- server/modules/iconograph.py | 10 ++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/client/fetch_and_update.sh b/client/fetch_and_update.sh index b248fd4..c866ba4 100755 --- a/client/fetch_and_update.sh +++ b/client/fetch_and_update.sh @@ -12,5 +12,16 @@ BOOT="/isodevice" FLAGS="$(cat /icon/config/fetcher.flags)" CA_CERT="/icon/config/ca.image.cert.pem" -"${BASE}/fetcher.py" --image-dir="${IMAGES}" --ca-cert="${CA_CERT}" ${FLAGS} +HTTPS_CLIENT_KEY="/systemid/$(hostname).www.key.pem" +HTTPS_CLIENT_CERT="/systemid/$(hostname).www.cert.pem" +HTTPS_CA_CERT="/icon/config/ca.www.cert.pem" + +if test -e "${HTTPS_CLIENT_KEY}" -a -e "${HTTPS_CLIENT_CERT}"; then + HTTPS_CLIENT_FLAGS="--https-client-cert=${HTTPS_CLIENT_CERT} --https-client-key=${HTTPS_CLIENT_KEY}" +fi +if test -e "${HTTPS_CA_CERT}"; then + HTTPS_CA_FLAGS="--https-ca-cert=${HTTPS_CA_CERT}" +fi + +"${BASE}/fetcher.py" --image-dir="${IMAGES}" --ca-cert="${CA_CERT}" ${FLAGS} ${HTTPS_CLIENT_FLAGS} ${HTTPS_CA_FLAGS} "${BASE}/update_grub.py" --image-dir="${IMAGES}" --boot-dir="${BOOT}" > "${BOOT}/grub/grub.cfg.tmp" && mv "${BOOT}/grub/grub.cfg.tmp" "${BOOT}/grub/grub.cfg" diff --git a/server/modules/iconograph.py b/server/modules/iconograph.py index 673c347..2776279 100755 --- a/server/modules/iconograph.py +++ b/server/modules/iconograph.py @@ -22,6 +22,10 @@ parser.add_argument( dest='chroot_path', action='store', required=True) +parser.add_argument( + '--https-ca-cert', + dest='https_ca_cert', + action='store') parser.add_argument( '--max-images', dest='max_images', @@ -60,6 +64,12 @@ def main(): FLAGS.ca_cert, os.path.join(FLAGS.chroot_path, 'icon', 'config', 'ca.image.cert.pem')) + if FLAGS.https_ca_cert: + shutil.copyfile( + FLAGS.https_ca_cert, + os.path.join(FLAGS.chroot_path, 'icon', 'config', 'ca.www.cert.pem')) + + path = os.path.join(FLAGS.chroot_path, 'icon', 'config', 'fetcher.flags') with open(path, 'w') as fh: fh.write('--base-url=%(base_url)s --max-images=%(max_images)d\n' % { From a6e4ddb1a04e59a8fb7008cb8890a3bd59a90174 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Wed, 6 Apr 2016 22:59:57 -0700 Subject: [PATCH 11/27] Document https flags, systemid. --- README.md | 63 ++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index c4e6df4..9da9adc 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,33 @@ flags to build_image.py as long as the modules are compatible with each other. Stock modules: +### autoimage.py + +Build an image that will partition, mkfs, and install an image from a different +URL onto a target system. Used to create install USB drives, PXE boot, etc. +Use the build_image.py flag: + +```bash +--module="server/modules/autoimage.py --base-url=http://yourhost/ --ca-cert=/path/to/signing/cert.pem --device=/dev/sdx --persistent-percent=50" +``` + +`--device` specifies the device to partition and install to on the target +system. + +Optional flags: + +`--persistent-percent`, if non-zero, specifies the percent of the target +device to allocate to a LABEL=PERSISTENT filesystem. If the inner image uses +persistent.py, this filesystem will be automatically mounted. + +`--https-ca-cert` specifies a local path to a PEM-encoded certificate to +validate the HTTPS image server cert against. This differs from `--ca-cert`, +which is used to validate the manifest.json signature. + +`--https-client-cert` and `--https-client-key` are used together to specify +local paths to a PEM-encoded certificate and key pair that will be provided +to the server over HTTPS. This can be used to limit image availability. + ### iconograph.py Install icon inside the image. This allows the image to auto-update over HTTP. @@ -95,6 +122,10 @@ Use the build_image.py flag: Optional flags: +`--https-ca-cert` specifies a local path to a PEM-encoded certificate to +validate the HTTPS image server cert against. This differs from `--ca-cert`, +which is used to validate the manifest.json signature. + `--max-images` sets the number of recent images to keep. Older images are deleted. Defaults to 5. 0 means unlimited. @@ -108,23 +139,21 @@ Use the build_image.py flag: --module="server/modules/persistent.py" ``` -### autoimage.py +See [imager/image.py](imager/image.py)'s or +[server/module/autoimage.py](autoimage.py)'s `--persistent-percent` flag to +create this partition. -Build an image that will partition, mkfs, and install an image from a different -URL onto a target system. Used to create install USB drives, PXE boot, etc. -Use the build_image.py flag: +### systemid.py + +Mount a /systemid partition from a filesystem with LABEL=SYSTEMID. This is +intended to a be separate device (possibly a USB flash drive, SD card, etc.) +which contains data that persists across re-images and identifies the system, +including system-specific keys and certificates. ```bash ---module="server/modules/autoimage.py --base-url=http://yourhost/ --ca-cert=/path/to/signing/cert.pem --device=/dev/sdx --persistent-percent=50" +--module="server/modules/systemid.py" ``` -`--device` specifies the device to partition and install to on the target -system. - -`--persistent-percent`, if non-zero, specifies the percent of the target -device to allocate to a LABEL=PERSISTENT filesystem. If the inner image uses -persistent.py, this filesystem will be automatically mounted. - ## Module API Modules are passed the following long-style arguments: @@ -184,3 +213,13 @@ or manually write them to a drive. To do so: # Needs sudo to partition and mkfs devices sudo imager/image.py --base-url=http://yourhost/ --ca-cert=/path/to/signing/cert.pem --device=/dev/sdx --persistent-percent=50 ``` + +Optional flags: + +`--https-ca-cert` specifies a local path to a PEM-encoded certificate to +validate the HTTPS image server cert against. This differs from `--ca-cert`, +which is used to validate the manifest.json signature. + +`--https-client-cert` and `--https-client-key` are used together to specify +local paths to a PEM-encoded certificate and key pair that will be provided +to the server over HTTPS. This can be used to limit image availability. From 004a58a4839882dfaeb6fff26a7e377aad28abec Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Wed, 6 Apr 2016 23:06:43 -0700 Subject: [PATCH 12/27] Document certclient.py --- README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/README.md b/README.md index 9da9adc..9e4d881 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,28 @@ which is used to validate the manifest.json signature. local paths to a PEM-encoded certificate and key pair that will be provided to the server over HTTPS. This can be used to limit image availability. +### certclient.py + +Use a local master key/cert pair to authenticate to a +[https://github.com/robot-tools/certserver](certserver) instance and retrieve +a system-specific key. Mainly intended to be used with autoimage.py and +systemid.py. + +Use the build_image.py flag: + +```bash +--module="server/modules/certclient.py --server=https://certserver/ --ca-cert=/path/to/server/cert.pem --client-cert=/path/to/client/cert.pem --client-key=/path/to/client/key.pem --tag=www --subject='/C=US/ST=California/O=XXXX/OU=XXXX Test/CN=HOSTNAME'" +``` + +The new key and cert are saved to /systemid + +`--tag` specifies a value added to the filename, so certclient.py can be +used more than once with different servers (e.g. once for an HTTPS client +key/cert pair, and once for an EAP-TLS key/cert pair). + +`--subject` specifics the subject string passed to openssl. `HOSTNAME` is +replaced with the system hostname, possibly as set by systemid.py + ### iconograph.py Install icon inside the image. This allows the image to auto-update over HTTP. @@ -150,6 +172,9 @@ intended to a be separate device (possibly a USB flash drive, SD card, etc.) which contains data that persists across re-images and identifies the system, including system-specific keys and certificates. +It also sets the hostname to the value found in the systemid config on the +device. + ```bash --module="server/modules/systemid.py" ``` From 63b0e393f7f8eafb5fe3b045d2fb276f176b3268 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Wed, 6 Apr 2016 23:07:50 -0700 Subject: [PATCH 13/27] Fix link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9e4d881..77e4deb 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ to the server over HTTPS. This can be used to limit image availability. ### certclient.py Use a local master key/cert pair to authenticate to a -[https://github.com/robot-tools/certserver](certserver) instance and retrieve +[certserver](https://github.com/robot-tools/certserver) instance and retrieve a system-specific key. Mainly intended to be used with autoimage.py and systemid.py. From 22e072c19f2f76c2087f6f3aa6db7c92a3d84804 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Thu, 7 Apr 2016 20:29:48 +0000 Subject: [PATCH 14/27] Ignore zero-sized files --- server/modules/certclient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/modules/certclient.py b/server/modules/certclient.py index a8c4fdf..20505eb 100755 --- a/server/modules/certclient.py +++ b/server/modules/certclient.py @@ -110,7 +110,7 @@ script CERT="/systemid/$(hostname).%(tag)s.cert.pem" SUBJECT="$(echo '%(subject)s' | sed s/SYSTEMID/$(hostname)/g)" - if test ! -e "${KEY}"; then + if test ! -s "${KEY}"; then openssl ecparam -name secp384r1 -genkey | openssl ec -out "${KEY}" chmod 0400 "${KEY}" fi @@ -119,7 +119,7 @@ script /icon/iconograph/client/wait_for_service.py --host=%(host)s --service=%(service)s chvt 8 - if test ! -e "${CERT}"; then + if test ! -s "${CERT}"; then openssl req -new -key "${KEY}" -subj "${SUBJECT}" | /icon/certserver/certclient.py --ca-cert=/icon/config/ca.%(tag)s.certserver.cert.pem --client-cert=/icon/config/client.%(tag)s.certserver.cert.pem --client-key=/icon/config/client.%(tag)s.certserver.key.pem --server=%(server)s > "${CERT}" chmod 0444 "${CERT}" fi From d8852ac1bb0610ee6135262770947a0d1fa0116c Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Thu, 7 Apr 2016 20:39:50 +0000 Subject: [PATCH 15/27] Correct docs --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 77e4deb..68afc2c 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ systemid.py. Use the build_image.py flag: ```bash ---module="server/modules/certclient.py --server=https://certserver/ --ca-cert=/path/to/server/cert.pem --client-cert=/path/to/client/cert.pem --client-key=/path/to/client/key.pem --tag=www --subject='/C=US/ST=California/O=XXXX/OU=XXXX Test/CN=HOSTNAME'" +--module="server/modules/certclient.py --server=https://certserver/ --ca-cert=/path/to/server/cert.pem --client-cert=/path/to/client/cert.pem --client-key=/path/to/client/key.pem --tag=www --subject='/C=US/ST=California/O=XXXX/OU=XXXX Test/CN=SYSTEMID'" ``` The new key and cert are saved to /systemid @@ -130,7 +130,7 @@ The new key and cert are saved to /systemid used more than once with different servers (e.g. once for an HTTPS client key/cert pair, and once for an EAP-TLS key/cert pair). -`--subject` specifics the subject string passed to openssl. `HOSTNAME` is +`--subject` specifics the subject string passed to openssl. `SYSTEMID` is replaced with the system hostname, possibly as set by systemid.py ### iconograph.py From eb2a56d8e2f0b76fb5534e10ff574499b658dce5 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Thu, 7 Apr 2016 20:42:20 +0000 Subject: [PATCH 16/27] Rename certs --- server/modules/autoimage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/modules/autoimage.py b/server/modules/autoimage.py index cb24ec0..8c29f92 100755 --- a/server/modules/autoimage.py +++ b/server/modules/autoimage.py @@ -81,7 +81,7 @@ def main(): image_flags = [] if FLAGS.https_ca_cert: - https_ca_cert_path = os.path.join('icon', 'config', 'ca.https.cert.pem') + https_ca_cert_path = os.path.join('icon', 'config', 'ca.www.cert.pem') shutil.copyfile( FLAGS.https_ca_cert, os.path.join(FLAGS.chroot_path, https_ca_cert_path)) @@ -90,11 +90,11 @@ def main(): ]) if FLAGS.https_client_cert and FLAGS.https_client_key: - https_client_cert_path = os.path.join('icon', 'config', 'client.https.cert.pem') + https_client_cert_path = os.path.join('icon', 'config', 'client.www.cert.pem') shutil.copyfile( FLAGS.https_client_cert, os.path.join(FLAGS.chroot_path, https_client_cert_path)) - https_client_key_path = os.path.join('icon', 'config', 'client.https.key.pem') + https_client_key_path = os.path.join('icon', 'config', 'client.www.key.pem') shutil.copyfile( FLAGS.https_client_key, os.path.join(FLAGS.chroot_path, https_client_key_path)) From 79961af5c26b3bca3278417065c10e4f82aed1fc Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Thu, 7 Apr 2016 20:48:32 +0000 Subject: [PATCH 17/27] Move off tty7, which is used by upstart --- server/modules/autoimage.py | 8 ++++---- server/modules/certclient.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/server/modules/autoimage.py b/server/modules/autoimage.py index 8c29f92..8d931a5 100755 --- a/server/modules/autoimage.py +++ b/server/modules/autoimage.py @@ -114,12 +114,12 @@ description "AutoImage" start on runlevel [2345] script - exec /dev/tty7 2>&1 - chvt 7 + exec /dev/tty8 2>&1 + chvt 8 /icon/iconograph/client/wait_for_service.py --host=%(host)s --service=%(service)s - chvt 7 + chvt 8 /icon/iconograph/imager/image.py --device=%(device)s --persistent-percent=%(persistent_percent)d --ca-cert=/icon/config/ca.image.cert.pem --base-url=%(base_url)s %(image_flags)s - chvt 7 + chvt 8 echo echo "==================" diff --git a/server/modules/certclient.py b/server/modules/certclient.py index 20505eb..7d8b4f1 100755 --- a/server/modules/certclient.py +++ b/server/modules/certclient.py @@ -103,8 +103,8 @@ description "CertClient %(tag)s" start on systemid-ready script - exec /dev/tty8 2>&1 - chvt 8 + exec /dev/tty9 2>&1 + chvt 9 KEY="/systemid/$(hostname).%(tag)s.key.pem" CERT="/systemid/$(hostname).%(tag)s.cert.pem" @@ -115,16 +115,16 @@ script chmod 0400 "${KEY}" fi - chvt 8 + chvt 9 /icon/iconograph/client/wait_for_service.py --host=%(host)s --service=%(service)s - chvt 8 + chvt 9 if test ! -s "${CERT}"; then openssl req -new -key "${KEY}" -subj "${SUBJECT}" | /icon/certserver/certclient.py --ca-cert=/icon/config/ca.%(tag)s.certserver.cert.pem --client-cert=/icon/config/client.%(tag)s.certserver.cert.pem --client-key=/icon/config/client.%(tag)s.certserver.key.pem --server=%(server)s > "${CERT}" chmod 0444 "${CERT}" fi - chvt 8 + chvt 9 echo echo "==================" From 22a27eb0bd1e3dbaf92971c00d2d03b86e5c61f4 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Thu, 7 Apr 2016 20:56:01 +0000 Subject: [PATCH 18/27] Icon updates need python3-requests --- server/modules/iconograph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/iconograph.py b/server/modules/iconograph.py index 2776279..2e50656 100755 --- a/server/modules/iconograph.py +++ b/server/modules/iconograph.py @@ -49,7 +49,7 @@ def main(): 'apt-get', 'install', '--assume-yes', - 'daemontools-run', 'git', 'python3-openssl') + 'daemontools-run', 'git', 'python3-openssl', 'python3-requests') os.makedirs(os.path.join(FLAGS.chroot_path, 'icon', 'config'), exist_ok=True) From 3b8b1cc11ec426f3f8022fe84024d3c39428336b Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Fri, 8 Apr 2016 22:51:37 +0000 Subject: [PATCH 19/27] Mount /proc in the chroot, and provide more shell flags. --- server/build_image.py | 47 ++++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/server/build_image.py b/server/build_image.py index 1563a4f..8aba608 100755 --- a/server/build_image.py +++ b/server/build_image.py @@ -35,8 +35,13 @@ parser.add_argument( action='store', required=True) parser.add_argument( - '--shell', - dest='shell', + '--post-module-shell', + dest='post_module_shell', + action='store_true', + default=False) +parser.add_argument( + '--pre-module-shell', + dest='pre_module_shell', action='store_true', default=False) parser.add_argument( @@ -100,6 +105,27 @@ class ImageBuilder(object): def _ExecChroot(self, chroot_path, *args, **kwargs): self._Exec('chroot', chroot_path, *args, **kwargs) + def _CreateRoot(self, timestamp): + root = tempfile.mkdtemp() + self._rmtree.append(root) + self._Exec( + 'mount', + '--types', 'tmpfs', + 'none', + root) + self._umount.append(root) + return root + + def _MountProc(self, root): + path = os.path.join(root, 'proc') + self._Exec( + 'mount', + '--types', 'proc', + 'none', + path) + self._umount.append(path) + return path + def _Debootstrap(self, root): path = os.path.join(root, 'chroot') os.mkdir(path) @@ -241,30 +267,23 @@ class ImageBuilder(object): return dest_iso def _BuildImage(self): - root = tempfile.mkdtemp() - self._rmtree.append(root) - timestamp = int(time.time()) - + root = self._CreateRoot(timestamp) print('Building image in:', root) - self._Exec( - 'mount', - '--types', 'tmpfs', - 'none', - root) - self._umount.append(root) - chroot_path = self._Debootstrap(root) union_path = self._CreateUnion(root) + self._MountProc(chroot_path) self._FixSourcesList(chroot_path) self._AddDiversions(chroot_path) self._InstallPackages(chroot_path) self._WriteVersion(chroot_path, timestamp) + if FLAGS.pre_module_shell: + self._Exec('bash', cwd=root) self._RunModules(chroot_path) self._CleanPackages(chroot_path) self._RemoveDiversions(chroot_path) - if FLAGS.shell: + if FLAGS.post_module_shell: self._Exec('bash', cwd=root) self._Squash(chroot_path, union_path) self._CopyISOFiles(union_path) From fd656ab7f1feb2bb771a368980e7c3dc75fb394c Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Fri, 8 Apr 2016 23:33:42 +0000 Subject: [PATCH 20/27] Unmount /proc before squashing --- server/build_image.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/server/build_image.py b/server/build_image.py index 8aba608..7639703 100755 --- a/server/build_image.py +++ b/server/build_image.py @@ -116,8 +116,8 @@ class ImageBuilder(object): self._umount.append(root) return root - def _MountProc(self, root): - path = os.path.join(root, 'proc') + def _MountProc(self, chroot_path): + path = os.path.join(chroot_path, 'proc') self._Exec( 'mount', '--types', 'proc', @@ -245,6 +245,12 @@ class ImageBuilder(object): source) os.unlink(os.path.join(chroot_path, 'usr', 'sbin', 'policy-rc.d')) + def _Unmount(self, path): + self._Exec( + 'umount', + path) + self._umount.remove(path) + def _Squash(self, chroot_path, union_path): self._Exec( 'mksquashfs', @@ -273,7 +279,7 @@ class ImageBuilder(object): chroot_path = self._Debootstrap(root) union_path = self._CreateUnion(root) - self._MountProc(chroot_path) + proc_path = self._MountProc(chroot_path) self._FixSourcesList(chroot_path) self._AddDiversions(chroot_path) self._InstallPackages(chroot_path) @@ -285,6 +291,7 @@ class ImageBuilder(object): self._RemoveDiversions(chroot_path) if FLAGS.post_module_shell: self._Exec('bash', cwd=root) + self._Unmount(proc_path) self._Squash(chroot_path, union_path) self._CopyISOFiles(union_path) iso_path = self._CreateISO(union_path, timestamp) From ddabe391c890fc590f880854012cb43febe533a7 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Fri, 8 Apr 2016 23:41:48 +0000 Subject: [PATCH 21/27] Boot to text. --- server/iso_files/grub.cfg | 2 +- server/iso_files/loopback.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/iso_files/grub.cfg b/server/iso_files/grub.cfg index 95e87de..ac32bf2 100644 --- a/server/iso_files/grub.cfg +++ b/server/iso_files/grub.cfg @@ -2,6 +2,6 @@ set timeout=5 terminal_output console menuentry "Ubuntu Server Live Image" { - linux /casper/vmlinuz.efi boot=casper root=LABEL=ISOIMAGE + linux /casper/vmlinuz.efi boot=casper root=LABEL=ISOIMAGE nomodeset initrd /casper/initrd.lz } diff --git a/server/iso_files/loopback.cfg b/server/iso_files/loopback.cfg index 03c6b46..98bf4ea 100644 --- a/server/iso_files/loopback.cfg +++ b/server/iso_files/loopback.cfg @@ -2,6 +2,6 @@ set timeout=5 terminal_output console menuentry "Ubuntu Server Live Image" { - linux /casper/vmlinuz.efi boot=casper iso-scan/filename=${iso_path} + linux /casper/vmlinuz.efi boot=casper iso-scan/filename=${iso_path} nomodeset initrd /casper/initrd.lz } From 0b2e6e6341422cfb4238334f40e0c7bde835fc84 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Sat, 9 Apr 2016 00:06:49 +0000 Subject: [PATCH 22/27] Re-use old hashes to speed up build_manifest --- server/build_manifest.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/server/build_manifest.py b/server/build_manifest.py index e79330c..fef112d 100755 --- a/server/build_manifest.py +++ b/server/build_manifest.py @@ -43,14 +43,14 @@ class ManifestBuilder(object): self._image_dir = image_dir self._old_manifest = old_manifest - def _Rollouts(self): + def _OldImages(self): if not self._old_manifest: return {} try: with open(self._old_manifest, 'r') as fh: parsed = json.load(fh) return dict( - (image['timestamp'], image['rollout_‱']) + (image['timestamp'], image) for image in parsed['images']) except FileNotFoundError: return {} @@ -60,7 +60,7 @@ class ManifestBuilder(object): 'timestamp': int(time.time()), 'images': [], } - rollouts = self._Rollouts() + old_images = self._OldImages() for filename in os.listdir(self._image_dir): match = self._FILE_REGEX.match(filename) if not match: @@ -68,8 +68,15 @@ class ManifestBuilder(object): timestamp = int(match.group('timestamp')) image = { 'timestamp': timestamp, - 'rollout_‱': rollouts.get(timestamp, FLAGS.default_rollout), + 'rollout_‱': FLAGS.default_rollout, } + ret['images'].append(image) + + old_image = old_images.get(timestamp) + if old_image: + image.update(old_image) + continue + with open(os.path.join(self._image_dir, filename), 'rb') as fh: hash_obj = hashlib.sha256() while True: @@ -78,7 +85,7 @@ class ManifestBuilder(object): break hash_obj.update(data) image['hash'] = hash_obj.hexdigest() - ret['images'].append(image) + ret['images'].sort(key=lambda x: x['timestamp'], reverse=True) return ret From 9fb8b2bc70a34c9f91c652eb5582b3d894a069d6 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Fri, 8 Apr 2016 21:47:49 -0700 Subject: [PATCH 23/27] Add utility to fetch and unpack tarballs securely from a server. --- util/fetch_archive.py | 70 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100755 util/fetch_archive.py diff --git a/util/fetch_archive.py b/util/fetch_archive.py new file mode 100755 index 0000000..7e4f2c8 --- /dev/null +++ b/util/fetch_archive.py @@ -0,0 +1,70 @@ +#!/usr/bin/python3 + +import argparse +import os +import requests +import shutil +import subprocess + + +parser = argparse.ArgumentParser(description='iconograph fetch_archive') +parser.add_argument( + '--dest-dir', + dest='dest_dir', + action='store', + default='.') +parser.add_argument( + '--https-ca-cert', + dest='https_ca_cert', + action='store') +parser.add_argument( + '--https-client-cert', + dest='https_client_cert', + action='store') +parser.add_argument( + '--https-client-key', + dest='https_client_key', + action='store') +parser.add_argument( + '--url', + dest='url', + action='store', + required=True) +FLAGS = parser.parse_args() + + +class ArchiveFetcher(object): + + _BUF_SIZE = 2 ** 16 + + def __init__(self, https_ca_cert, https_client_cert, https_client_key): + self._session = requests.Session() + if https_ca_cert: + self._session.verify = https_ca_cert + if https_client_cert and https_client_key: + self._session.cert = (https_client_cert, https_client_key) + + def Fetch(self, url, dest_dir='.'): + resp = self._session.get(url, stream=True) + + tar = subprocess.Popen( + ['tar', '--extract', '--verbose'], + stdin=subprocess.PIPE, + cwd=dest_dir) + for data in resp.iter_content(self._BUF_SIZE): + tar.stdin.write(data) + tar.wait() + + +def main(): + fetcher = ArchiveFetcher( + FLAGS.https_ca_cert, + FLAGS.https_client_cert, + FLAGS.https_client_key) + fetcher.Fetch( + FLAGS.url, + FLAGS.dest_dir) + + +if __name__ == '__main__': + main() From 5a55f3bb279119d0e72ca0964fdf88df2cf93f3e Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Tue, 12 Apr 2016 05:00:04 +0000 Subject: [PATCH 24/27] Move openssh-server to its own module. Move the key onto the systemid device. --- server/build_image.py | 1 - server/modules/openssh.py | 43 +++++++++++++++++++++++++++++++++++++++ systemid/image.py | 9 ++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100755 server/modules/openssh.py diff --git a/server/build_image.py b/server/build_image.py index 7639703..89e8b91 100755 --- a/server/build_image.py +++ b/server/build_image.py @@ -60,7 +60,6 @@ class ImageBuilder(object): 'iputils-ping', 'linux-firmware', 'linux-firmware-nonfree', - 'openssh-server', 'ubuntu-minimal', 'ubuntu-standard', 'user-setup', diff --git a/server/modules/openssh.py b/server/modules/openssh.py new file mode 100755 index 0000000..2643cd9 --- /dev/null +++ b/server/modules/openssh.py @@ -0,0 +1,43 @@ +#!/usr/bin/python3 + +import argparse +import glob +import os +import subprocess + + +parser = argparse.ArgumentParser(description='iconograph openssh') +parser.add_argument( + '--chroot-path', + dest='chroot_path', + action='store', + required=True) +FLAGS = parser.parse_args() + + +def Exec(*args, **kwargs): + print('+', args) + subprocess.check_call(args, **kwargs) + + +def ExecChroot(*args, **kwargs): + Exec('chroot', FLAGS.chroot_path, *args, **kwargs) + + +def main(): + ExecChroot( + 'apt-get', + 'install', + '--assume-yes', + 'openssh-server') + + for path in glob.glob(os.path.join(FLAGS.chroot_path, 'etc', 'ssh', 'ssh_host_*')): + os.unlink(path) + + os.symlink( + '/systemid/ssh_host_ed25519_key', + os.path.join(FLAGS.chroot_path, 'etc', 'ssh', 'ssh_host_ed25519_key')) + + +if __name__ == '__main__': + main() diff --git a/systemid/image.py b/systemid/image.py index e331ee2..5360523 100755 --- a/systemid/image.py +++ b/systemid/image.py @@ -120,6 +120,14 @@ SYSTEMID=%(system_id)s 'system_id': new_id, }) + def _GenerateSSHKey(self, root): + self._Exec( + 'ssh-keygen', + '-f', os.path.join(root, 'ssh_host_ed25519_key'), + '-N', '', + '-t', 'ed25519', + ) + def _Image(self): self._PartitionAndMkFS() root = self._Mount() @@ -135,6 +143,7 @@ New ID: \033[91m%s\033[00m ============== """ % new_id) + self._GenerateSSHKey(root) def Image(self): self._umount = [] From 45b03b1bc38ac2f536a136e285fcf87857673e7b Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Tue, 12 Apr 2016 19:47:43 +0000 Subject: [PATCH 25/27] Drop the verbose flag. --- util/fetch_archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/fetch_archive.py b/util/fetch_archive.py index 7e4f2c8..78f9b96 100755 --- a/util/fetch_archive.py +++ b/util/fetch_archive.py @@ -48,7 +48,7 @@ class ArchiveFetcher(object): resp = self._session.get(url, stream=True) tar = subprocess.Popen( - ['tar', '--extract', '--verbose'], + ['tar', '--extract'], stdin=subprocess.PIPE, cwd=dest_dir) for data in resp.iter_content(self._BUF_SIZE): From 5de9a8d2c62f08c67799826df604de4faae717e3 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Fri, 22 Apr 2016 21:11:58 +0000 Subject: [PATCH 26/27] Support update_grub flags. --- client/fetch_and_update.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/client/fetch_and_update.sh b/client/fetch_and_update.sh index c866ba4..ef7b7a7 100755 --- a/client/fetch_and_update.sh +++ b/client/fetch_and_update.sh @@ -9,7 +9,10 @@ mkdir -p "${IMAGES}" BOOT="/isodevice" -FLAGS="$(cat /icon/config/fetcher.flags)" +FETCHER_FLAGS="$(cat /icon/config/fetcher.flags)" +if -f /icon/config/update_grub.flags; then + UPDATE_GRUB_FLAGS="$(cat /icon/config/update_grub.flags)" +fi CA_CERT="/icon/config/ca.image.cert.pem" HTTPS_CLIENT_KEY="/systemid/$(hostname).www.key.pem" @@ -23,5 +26,5 @@ if test -e "${HTTPS_CA_CERT}"; then HTTPS_CA_FLAGS="--https-ca-cert=${HTTPS_CA_CERT}" fi -"${BASE}/fetcher.py" --image-dir="${IMAGES}" --ca-cert="${CA_CERT}" ${FLAGS} ${HTTPS_CLIENT_FLAGS} ${HTTPS_CA_FLAGS} -"${BASE}/update_grub.py" --image-dir="${IMAGES}" --boot-dir="${BOOT}" > "${BOOT}/grub/grub.cfg.tmp" && mv "${BOOT}/grub/grub.cfg.tmp" "${BOOT}/grub/grub.cfg" +"${BASE}/fetcher.py" --image-dir="${IMAGES}" --ca-cert="${CA_CERT}" ${FETCHER_FLAGS} ${HTTPS_CLIENT_FLAGS} ${HTTPS_CA_FLAGS} +"${BASE}/update_grub.py" --image-dir="${IMAGES}" --boot-dir="${BOOT}" ${UPDATE_GRUB_FLAGS} > "${BOOT}/grub/grub.cfg.tmp" && mv "${BOOT}/grub/grub.cfg.tmp" "${BOOT}/grub/grub.cfg" From 5952adfd55a06b725edd41b7a54b58ac81cb30ea Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Fri, 22 Apr 2016 21:19:29 +0000 Subject: [PATCH 27/27] Add kernel arg passing support --- server/build_image.py | 18 +++++++++++++----- server/iso_files/grub.cfg | 2 +- server/iso_files/loopback.cfg | 2 +- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/server/build_image.py b/server/build_image.py index 89e8b91..a79db6c 100755 --- a/server/build_image.py +++ b/server/build_image.py @@ -25,6 +25,10 @@ parser.add_argument( dest='image_dir', action='store', required=True) +parser.add_argument( + '--kernel-arg', + dest='kernel_args', + action='append') parser.add_argument( '--module', dest='modules', @@ -87,13 +91,14 @@ class ImageBuilder(object): 'loopback.cfg': 'boot/grub/loopback.cfg', } - def __init__(self, source_iso, image_dir, archive, arch, release, modules): + def __init__(self, source_iso, image_dir, archive, arch, release, modules, kernel_args): self._source_iso = source_iso self._image_dir = image_dir self._archive = archive self._arch = arch self._release = release self._modules = modules or [] + self._kernel_args = kernel_args or [] self._ico_server_path = os.path.dirname(sys.argv[0]) @@ -259,9 +264,11 @@ class ImageBuilder(object): def _CopyISOFiles(self, union_path): for source, dest in self._ISO_COPIES.items(): - shutil.copyfile( - os.path.join(self._ico_server_path, 'iso_files', source), - os.path.join(union_path, dest)) + source_path = os.path.join(self._ico_server_path, 'iso_files', source) + dest_path = os.path.join(union_path, dest) + with open(source_path, 'r') as source_fh, open(dest_path, 'w') as dest_fh: + for line in source_fh: + dest_fh.write(line.replace('$KERNEL_ARGS', ' '.join(self._kernel_args))) def _CreateISO(self, union_path, timestamp): dest_iso = os.path.join(self._image_dir, '%d.iso' % timestamp) @@ -322,7 +329,8 @@ def main(): FLAGS.archive, FLAGS.arch, FLAGS.release, - FLAGS.modules) + FLAGS.modules, + FLAGS.kernel_args) builder.BuildImage() diff --git a/server/iso_files/grub.cfg b/server/iso_files/grub.cfg index ac32bf2..2cc51d4 100644 --- a/server/iso_files/grub.cfg +++ b/server/iso_files/grub.cfg @@ -2,6 +2,6 @@ set timeout=5 terminal_output console menuentry "Ubuntu Server Live Image" { - linux /casper/vmlinuz.efi boot=casper root=LABEL=ISOIMAGE nomodeset + linux /casper/vmlinuz.efi boot=casper root=LABEL=ISOIMAGE nomodeset $KERNEL_ARGS initrd /casper/initrd.lz } diff --git a/server/iso_files/loopback.cfg b/server/iso_files/loopback.cfg index 98bf4ea..ef37d44 100644 --- a/server/iso_files/loopback.cfg +++ b/server/iso_files/loopback.cfg @@ -2,6 +2,6 @@ set timeout=5 terminal_output console menuentry "Ubuntu Server Live Image" { - linux /casper/vmlinuz.efi boot=casper iso-scan/filename=${iso_path} nomodeset + linux /casper/vmlinuz.efi boot=casper iso-scan/filename=${iso_path} nomodeset $KERNEL_ARGS initrd /casper/initrd.lz }