From 66f2a7f0bd561c11302e9998caed0b34a4cbf4bb Mon Sep 17 00:00:00 2001 From: Evan Broder Date: Wed, 25 Nov 2009 23:43:12 -0500 Subject: [PATCH 01/16] First stab at the remctl script to handle new build queue submission for the Invirtibuilder. svn path=/trunk/packages/invirt-dev/; revision=2563 --- invirt-submit-build | 52 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100755 invirt-submit-build diff --git a/invirt-submit-build b/invirt-submit-build new file mode 100755 index 0000000..caa949f --- /dev/null +++ b/invirt-submit-build @@ -0,0 +1,52 @@ +#!/usr/bin/python + +"""Validate and add a new item to the Invirt build queue. + +This script, intended to be invoked by remctl, first validates the +build submitted parameters, and then adds a new item to the +Invirtibuilder build queue, triggering the Invirtibuilder to start the +build. + +The expected arguments are + + pocket package commit + +This script will also automatically extract the Kerberos principal +used to submit the job, and include that in the queue file for records +keeping. +""" + + +import datetime +import os +import sys +import tempfile +import uuid + +import invirt.builder as b + + +def main(): + pocket, package, commit = sys.argv[1:4] + principal = os.environ['REMOTE_USER'] + request_time = datetime.datetime.utcnow() + q_path = os.path.join(b._QUEUE_DIR, + '%s_%s' % (request_time.strftime('%Y%m%d%H%M%S'), + uuid.uuid4())) + + try: + validateBuild(pocket, package, commit) + except b.InvalidBuild, e: + print >>sys.stderr, "E: %s" % e + sys.exit(1) + + # To keep from triggering the Invirtibuilder before we've actually + # written the file out, first write the queue entry to a temporary + # file, and then move it into the queue directory. + q = tempfile.NamedTemporaryFile(delete=False) + print >>q, "%s %s %s %s" % (pocket, package, commit, principal) + os.rename(q.name, q_path) + + +if __name__ == '__main__': + main() -- 1.7.9.5 From 8ed2c5ca63dfcfeedc7f980d4944000216b989e6 Mon Sep 17 00:00:00 2001 From: Evan Broder Date: Wed, 25 Nov 2009 23:43:15 -0500 Subject: [PATCH 02/16] Update invirt-dev.dirs to create directories used by the Invirtbuilder. Also punt the directories it was creating before - there's no need to create directories that are being used as destinations in a .install file. svn path=/trunk/packages/invirt-dev/; revision=2564 --- debian/invirt-dev.dirs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/invirt-dev.dirs b/debian/invirt-dev.dirs index 996edf4..a17355d 100644 --- a/debian/invirt-dev.dirs +++ b/debian/invirt-dev.dirs @@ -1,3 +1,3 @@ -usr/bin -usr/sbin -srv/repository/conf +var/lib/invirt-dev/queue +var/log/invirt/builds +usr/share/invirt-dev/build.d -- 1.7.9.5 From c81a9160bc47b724cba617285da6fec429734a21 Mon Sep 17 00:00:00 2001 From: Evan Broder Date: Wed, 25 Nov 2009 23:43:18 -0500 Subject: [PATCH 03/16] Add a script for generating the remctl configuration to trigger the Invirtibuilder. svn path=/trunk/packages/invirt-dev/; revision=2565 --- debian/invirt-dev.init | 1 + invirt-build-conf | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100755 invirt-build-conf diff --git a/debian/invirt-dev.init b/debian/invirt-dev.init index 60ceebd..160aabd 100755 --- a/debian/invirt-dev.init +++ b/debian/invirt-dev.init @@ -23,6 +23,7 @@ case "$1" in log_begin_msg "Reloading config for $PACKAGE" gen_files reprepro-env export + invirt-build-conf log_end_msg $? ;; stop) diff --git a/invirt-build-conf b/invirt-build-conf new file mode 100755 index 0000000..f74b05d --- /dev/null +++ b/invirt-build-conf @@ -0,0 +1,39 @@ +#!/usr/bin/python + +"""Re-generate the remctl configuration for build submissions. + +This script generates the remctl ACL and configuration for each build +pocket defined in the configuration. +""" + + +import os +import tempfile + +from invirt.authz import mech as authz +from invirt.config import structs as config + + +def main(): + # Python could really use a file-like object that gets written to + # a temporary path and moved to its final resting place on + # .close(). Oh well. + conf = tempfile.NamedTemporaryFile(delete=False) + build_handler = '/usr/sbin/invirt-submit-build' + + for pocket in config.git.pockets: + acl = authz.expandAdmin(getattr(config.git.pockets, pocket).acl, None) + + acl_fd = tempfile.NamedTemporaryFile(delete=False) + print >>acl_fd, '\n'.join(acl) + + acl_path = os.path.join('/etc/remctl/acl/build-%s' % pocket) + + os.rename(acl_fd.name, acl_path) + print >>conf, 'build %s %s %s' % (pocket, build_handler, acl_path) + + os.rename(conf, '/etc/remctl/conf.d/build') + + +if __name__ == '__main__': + main() -- 1.7.9.5 From f8aea528d7813a1e6ea648510483ca6d371f75ba Mon Sep 17 00:00:00 2001 From: Evan Broder Date: Wed, 25 Nov 2009 23:43:21 -0500 Subject: [PATCH 04/16] Until we switch to storing krb5 principals in the database, we need to convert the AFS-style principals in the database to krb5 principals. Hopefully this code can be torn out one of these days in the not-so-distant future. svn path=/trunk/packages/invirt-dev/; revision=2566 --- invirt-build-conf | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/invirt-build-conf b/invirt-build-conf index f74b05d..4c5039f 100755 --- a/invirt-build-conf +++ b/invirt-build-conf @@ -14,6 +14,17 @@ from invirt.authz import mech as authz from invirt.config import structs as config +def userToPrinc(user): + """Convert an AFS principal to a Kerberos v5 principal.""" + if '@' in user: + (princ, realm) = user.split('@') + else: + princ = user + realm = config.kerberos.realm + + return princ.replace('.', '/') + '@' + realm + + def main(): # Python could really use a file-like object that gets written to # a temporary path and moved to its final resting place on @@ -25,7 +36,7 @@ def main(): acl = authz.expandAdmin(getattr(config.git.pockets, pocket).acl, None) acl_fd = tempfile.NamedTemporaryFile(delete=False) - print >>acl_fd, '\n'.join(acl) + print >>acl_fd, '\n'.join(user(a) for a in acl) acl_path = os.path.join('/etc/remctl/acl/build-%s' % pocket) -- 1.7.9.5 From 643e51926cb33b020b1a5b2726239e96a754b65f Mon Sep 17 00:00:00 2001 From: Evan Broder Date: Thu, 26 Nov 2009 09:45:03 -0500 Subject: [PATCH 05/16] Replace NamedTemporaryFile with mkstemp in invirt-build-conf and invirt-submit-build. NamedTemporaryFile lacks the delete kwarg in Python 2.5, meaning that files are always deleted when the fd is closed. svn path=/trunk/packages/invirt-dev/; revision=2567 --- invirt-build-conf | 10 ++++++---- invirt-submit-build | 5 +++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/invirt-build-conf b/invirt-build-conf index 4c5039f..b7c3bdd 100755 --- a/invirt-build-conf +++ b/invirt-build-conf @@ -29,21 +29,23 @@ def main(): # Python could really use a file-like object that gets written to # a temporary path and moved to its final resting place on # .close(). Oh well. - conf = tempfile.NamedTemporaryFile(delete=False) + conf_fd, conf_name = tempfile.mkstemp() + conf = os.fdopen(conf_fd) build_handler = '/usr/sbin/invirt-submit-build' for pocket in config.git.pockets: acl = authz.expandAdmin(getattr(config.git.pockets, pocket).acl, None) - acl_fd = tempfile.NamedTemporaryFile(delete=False) + acl_fd, acl_name = tempfile.mkstemp() + acl_fd = os.fdopen(acl_fd) print >>acl_fd, '\n'.join(user(a) for a in acl) acl_path = os.path.join('/etc/remctl/acl/build-%s' % pocket) - os.rename(acl_fd.name, acl_path) + os.rename(acl_name, acl_path) print >>conf, 'build %s %s %s' % (pocket, build_handler, acl_path) - os.rename(conf, '/etc/remctl/conf.d/build') + os.rename(conf_name, '/etc/remctl/conf.d/build') if __name__ == '__main__': diff --git a/invirt-submit-build b/invirt-submit-build index caa949f..713f3da 100755 --- a/invirt-submit-build +++ b/invirt-submit-build @@ -43,9 +43,10 @@ def main(): # To keep from triggering the Invirtibuilder before we've actually # written the file out, first write the queue entry to a temporary # file, and then move it into the queue directory. - q = tempfile.NamedTemporaryFile(delete=False) + q_fd, q_name = tempfile.mkstemp() + q = os.fdopen(q_fd) print >>q, "%s %s %s %s" % (pocket, package, commit, principal) - os.rename(q.name, q_path) + os.rename(q_name, q_path) if __name__ == '__main__': -- 1.7.9.5 From 884c3218e348c1d9acff1934bd0852644338e141 Mon Sep 17 00:00:00 2001 From: Evan Broder Date: Thu, 26 Nov 2009 09:46:28 -0500 Subject: [PATCH 06/16] Fix typo in invirt-build-conf. svn path=/trunk/packages/invirt-dev/; revision=2568 --- invirt-build-conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/invirt-build-conf b/invirt-build-conf index b7c3bdd..7ebd542 100755 --- a/invirt-build-conf +++ b/invirt-build-conf @@ -38,7 +38,7 @@ def main(): acl_fd, acl_name = tempfile.mkstemp() acl_fd = os.fdopen(acl_fd) - print >>acl_fd, '\n'.join(user(a) for a in acl) + print >>acl_fd, '\n'.join(userToPrinc(a) for a in acl) acl_path = os.path.join('/etc/remctl/acl/build-%s' % pocket) -- 1.7.9.5 From a78d67017e7c0aee98261751bfb4eb1815fa2bb1 Mon Sep 17 00:00:00 2001 From: Evan Broder Date: Thu, 26 Nov 2009 10:15:42 -0500 Subject: [PATCH 07/16] Pass a mode to os.fdopen in invirt-build-conf and invirt-submit-build. By default, os.fdopen opens files in read-only mode. svn path=/trunk/packages/invirt-dev/; revision=2569 --- invirt-build-conf | 4 ++-- invirt-submit-build | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/invirt-build-conf b/invirt-build-conf index 7ebd542..b5a8d3d 100755 --- a/invirt-build-conf +++ b/invirt-build-conf @@ -30,14 +30,14 @@ def main(): # a temporary path and moved to its final resting place on # .close(). Oh well. conf_fd, conf_name = tempfile.mkstemp() - conf = os.fdopen(conf_fd) + conf = os.fdopen(conf_fd, 'r+') build_handler = '/usr/sbin/invirt-submit-build' for pocket in config.git.pockets: acl = authz.expandAdmin(getattr(config.git.pockets, pocket).acl, None) acl_fd, acl_name = tempfile.mkstemp() - acl_fd = os.fdopen(acl_fd) + acl_fd = os.fdopen(acl_fd, 'r+') print >>acl_fd, '\n'.join(userToPrinc(a) for a in acl) acl_path = os.path.join('/etc/remctl/acl/build-%s' % pocket) diff --git a/invirt-submit-build b/invirt-submit-build index 713f3da..7115bf6 100755 --- a/invirt-submit-build +++ b/invirt-submit-build @@ -44,7 +44,7 @@ def main(): # written the file out, first write the queue entry to a temporary # file, and then move it into the queue directory. q_fd, q_name = tempfile.mkstemp() - q = os.fdopen(q_fd) + q = os.fdopen(q_fd, 'r+') print >>q, "%s %s %s %s" % (pocket, package, commit, principal) os.rename(q_name, q_path) -- 1.7.9.5 From 30aa08f9ba6f3b1f0867348f59a4777636d4bc79 Mon Sep 17 00:00:00 2001 From: Evan Broder Date: Thu, 26 Nov 2009 10:18:14 -0500 Subject: [PATCH 08/16] Add remctl-server as a dependency for invirt-dev, for the build scripts. svn path=/trunk/packages/invirt-dev/; revision=2570 --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 4b32573..b0fd8be 100644 --- a/debian/control +++ b/debian/control @@ -7,6 +7,6 @@ Standards-Version: 3.7.2 Package: invirt-dev Architecture: all -Depends: ${shlibs:Depends}, ${misc:Depends}, dpkg-dev-el, emacs21, reprepro, apache2, postfix, screen, dh-make, fakeroot, quilt, patchutils, config-package-dev, pbuilder, equivs, invirt-base, invirt-database +Depends: ${shlibs:Depends}, ${misc:Depends}, dpkg-dev-el, emacs21, reprepro, apache2, postfix, screen, dh-make, fakeroot, quilt, patchutils, config-package-dev, pbuilder, equivs, invirt-base, invirt-database, remctl-server Description: Invirt build and apt server This packages the build scripts and apt-repository configuration for Invirt. -- 1.7.9.5 From e693fbc9c639083e3003b8de0aff71501afddee2 Mon Sep 17 00:00:00 2001 From: Evan Broder Date: Sat, 5 Dec 2009 15:36:21 -0500 Subject: [PATCH 09/16] Stuff all of our repos into an /invirt subdirectory, so that we have room for future expansion. svn path=/trunk/packages/invirt-dev/; revision=2577 --- invirtibuilder | 2 +- python/invirt/builder.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/invirtibuilder b/invirtibuilder index 0a19484..61efa9f 100755 --- a/invirtibuilder +++ b/invirtibuilder @@ -182,7 +182,7 @@ def updateSuperrepo(pocket, package, commit, principal): Note that there's no locking issue here, because we disallow all pushes to the superrepo. """ - superrepo = os.path.join(b._REPO_DIR, 'packages.git') + superrepo = os.path.join(b._REPO_DIR, 'invirt/packages.git') branch = b.pocketToGit(pocket) tree = c.captureOutput(['git', 'ls-tree', branch], cwd=superrepo) diff --git a/python/invirt/builder.py b/python/invirt/builder.py index 27e21a6..b8e9c0b 100644 --- a/python/invirt/builder.py +++ b/python/invirt/builder.py @@ -26,7 +26,7 @@ class InvalidBuild(ValueError): def getRepo(package): """Return the path to the git repo for a given package.""" - return os.path.join(_REPO_DIR, 'packages', '%s.git' % package) + return os.path.join(_REPO_DIR, 'invirt/packages', '%s.git' % package) def pocketToGit(pocket): -- 1.7.9.5 From 25d2b37398f10c1767e518843907b5c2adb81b57 Mon Sep 17 00:00:00 2001 From: Evan Broder Date: Sat, 5 Dec 2009 16:12:48 -0500 Subject: [PATCH 10/16] Update the git user's .k5login in invirt-build-conf. svn path=/trunk/packages/invirt-dev/; revision=2579 --- invirt-build-conf | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/invirt-build-conf b/invirt-build-conf index b5a8d3d..637d396 100755 --- a/invirt-build-conf +++ b/invirt-build-conf @@ -3,7 +3,8 @@ """Re-generate the remctl configuration for build submissions. This script generates the remctl ACL and configuration for each build -pocket defined in the configuration. +pocket defined in the configuration. It also updates the .k5login for +the git user that developers can push through. """ @@ -26,6 +27,8 @@ def userToPrinc(user): def main(): + all_devs = set() + # Python could really use a file-like object that gets written to # a temporary path and moved to its final resting place on # .close(). Oh well. @@ -40,6 +43,8 @@ def main(): acl_fd = os.fdopen(acl_fd, 'r+') print >>acl_fd, '\n'.join(userToPrinc(a) for a in acl) + all_devs.update(set(userToPrinc(a) for a in acl)) + acl_path = os.path.join('/etc/remctl/acl/build-%s' % pocket) os.rename(acl_name, acl_path) @@ -47,6 +52,10 @@ def main(): os.rename(conf_name, '/etc/remctl/conf.d/build') + k5login_fd, k5login_name = tempfile.mkstemp() + k5login = os.fdopen(k5login_fd, 'r+') + print >>k5login, '\n'.join(all_devs) + if __name__ == '__main__': main() -- 1.7.9.5 From abc36100a14abd2187ec4396bcc49e8313ce3fb3 Mon Sep 17 00:00:00 2001 From: Evan Broder Date: Sat, 5 Dec 2009 16:12:49 -0500 Subject: [PATCH 11/16] Actually create a git user. svn path=/trunk/packages/invirt-dev/; revision=2580 --- debian/invirt-dev.postinst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/debian/invirt-dev.postinst b/debian/invirt-dev.postinst index b873ea2..3574e88 100755 --- a/debian/invirt-dev.postinst +++ b/debian/invirt-dev.postinst @@ -11,6 +11,10 @@ case "$1" in if ! getent group repo >/dev/null 2>&1; then addgroup --system repo fi + + if ! getent passwd git >/dev/null 2>&1; then + adduser --system --home /srv/git --shell /usr/bin/git-shell git + fi cat >>/etc/sudoers < Date: Sat, 5 Dec 2009 16:12:50 -0500 Subject: [PATCH 12/16] Update the Invirt git configuration at install-time. svn path=/trunk/packages/invirt-dev/; revision=2581 --- debian/invirt-dev.postinst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/debian/invirt-dev.postinst b/debian/invirt-dev.postinst index 3574e88..55bff25 100755 --- a/debian/invirt-dev.postinst +++ b/debian/invirt-dev.postinst @@ -15,6 +15,8 @@ case "$1" in if ! getent passwd git >/dev/null 2>&1; then adduser --system --home /srv/git --shell /usr/bin/git-shell git fi + + invirt-build-conf cat >>/etc/sudoers < Date: Sat, 5 Dec 2009 16:12:52 -0500 Subject: [PATCH 13/16] Actually install the invirt-build-conf script. svn path=/trunk/packages/invirt-dev/; revision=2582 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cdacc32..4a4323c 100755 --- a/setup.py +++ b/setup.py @@ -21,5 +21,5 @@ setup( py_modules = ['invirt.builder'], package_dir = {'': 'python'}, - scripts = ['invirtibuilder'] + scripts = ['invirtibuilder', 'invirt-build-conf'] ) -- 1.7.9.5 From ab38b956a35fade013259fc795894589175b3d95 Mon Sep 17 00:00:00 2001 From: Evan Broder Date: Sat, 5 Dec 2009 16:12:52 -0500 Subject: [PATCH 14/16] Automatically regenerate the Invirt git configuration daily. svn path=/trunk/packages/invirt-dev/; revision=2583 --- debian/invirt-dev.cron.daily | 3 +++ 1 file changed, 3 insertions(+) create mode 100755 debian/invirt-dev.cron.daily diff --git a/debian/invirt-dev.cron.daily b/debian/invirt-dev.cron.daily new file mode 100755 index 0000000..1b41bd9 --- /dev/null +++ b/debian/invirt-dev.cron.daily @@ -0,0 +1,3 @@ +#!/bin/sh + +[ -x /usr/bin/invirt-build-conf ] && /usr/bin/invirt-build-conf -- 1.7.9.5 From 921681d96bf39509c3a4354fe0e3d0331aa8d27b Mon Sep 17 00:00:00 2001 From: Evan Broder Date: Sat, 5 Dec 2009 16:29:46 -0500 Subject: [PATCH 15/16] When generating the git user's k5login, actually move it into place. svn path=/trunk/packages/invirt-dev/; revision=2584 --- invirt-build-conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/invirt-build-conf b/invirt-build-conf index 637d396..13ce3c4 100755 --- a/invirt-build-conf +++ b/invirt-build-conf @@ -12,6 +12,7 @@ import os import tempfile from invirt.authz import mech as authz +from invirt import builder from invirt.config import structs as config @@ -56,6 +57,8 @@ def main(): k5login = os.fdopen(k5login_fd, 'r+') print >>k5login, '\n'.join(all_devs) + os.rename(k5login_name, os.path.join(builder._REPO_DIR, '.k5login')) + if __name__ == '__main__': main() -- 1.7.9.5 From 06f7bbac2e822ec98fbca4cb6d0e4ae38d297136 Mon Sep 17 00:00:00 2001 From: Evan Broder Date: Sat, 5 Dec 2009 16:53:40 -0500 Subject: [PATCH 16/16] Add a debian/pyversions file to invirt-dev. Our code doesn't work on Python 2.4 svn path=/trunk/packages/invirt-dev/; revision=2585 --- debian/pyversions | 1 + 1 file changed, 1 insertion(+) create mode 100644 debian/pyversions diff --git a/debian/pyversions b/debian/pyversions new file mode 100644 index 0000000..b3dc41e --- /dev/null +++ b/debian/pyversions @@ -0,0 +1 @@ +2.5- -- 1.7.9.5