From 1c9ac2563d5afefb47da45c352f4bdd25acf8635 Mon Sep 17 00:00:00 2001 From: Greg Brockman Date: Mon, 5 Jul 2010 02:01:45 -0400 Subject: [PATCH] Make invirtibuilder work svn path=/trunk/packages/invirt-dev/; revision=3028 --- README | 3 ++ debian/changelog | 7 ++++ debian/control | 2 +- invirt-submit-build | 9 +++++- invirtibuilder | 80 ++++++++++++++++++++++++++++------------------ python/invirt/builder.py | 22 ++++++++++--- 6 files changed, 86 insertions(+), 37 deletions(-) diff --git a/README b/README index 3339738..3978406 100644 --- a/README +++ b/README @@ -5,3 +5,6 @@ Installation: The public key should then be in the apt keyring of systems using the packages. + + - Packages should be located at + /srv/git/invirt/packages/${packagename}.git diff --git a/debian/changelog b/debian/changelog index 0cf4983..3fc0322 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +invirt-dev (0.1.1) unstable; urgency=low + + * Added missing imports + * Other minor functional changes + + -- Greg Brockman Mon, 05 Jul 2010 01:43:45 -0400 + invirt-dev (0.1.0) unstable; urgency=low * Switch to using git instead of svn. diff --git a/debian/control b/debian/control index 384b314..2f138d6 100644 --- a/debian/control +++ b/debian/control @@ -7,6 +7,6 @@ Standards-Version: 3.7.2 Package: invirt-dev Architecture: all -Depends: ${shlibs:Depends}, ${misc:Depends}, dpkg-dev-el, emacs21, reprepro, apache2, postfix, screen, dh-make, fakeroot, quilt, patchutils, config-package-dev, pbuilder, equivs, invirt-base, invirt-database, remctl-server, update-inetd, openbsd-inetd | inet-superserver, python-pyinotify +Depends: ${shlibs:Depends}, ${misc:Depends}, dpkg-dev-el, emacs21, reprepro, apache2, postfix, screen, dh-make, fakeroot, quilt, patchutils, config-package-dev, sbuild, equivs, invirt-base, invirt-database, remctl-server, update-inetd, openbsd-inetd | inet-superserver, python-pyinotify, python-debian, git-core (>= 1.6.4) Description: Invirt build and apt server This packages the build scripts and apt-repository configuration for Invirt. diff --git a/invirt-submit-build b/invirt-submit-build index 42446b0..8175865 100755 --- a/invirt-submit-build +++ b/invirt-submit-build @@ -18,6 +18,7 @@ keeping. import datetime +import optparse import os import sys import tempfile @@ -27,7 +28,12 @@ import invirt.builder as b def main(): - pocket, package, commit = sys.argv[1:4] + parser = optparse.OptionParser('Usage: %prog pocket package commit') + opts, args = parser.parse_args() + if len(args) != 3: + parser.print_help() + return 1 + pocket, package, commit = args principal = os.environ['REMOTE_USER'] request_time = datetime.datetime.utcnow() q_path = os.path.join(b._QUEUE_DIR, @@ -46,6 +52,7 @@ def main(): q_fd, q_name = tempfile.mkstemp() q = os.fdopen(q_fd, 'r+') print >>q, "%s %s %s %s" % (pocket, package, commit, principal) + q.close() os.rename(q_name, q_path) diff --git a/invirtibuilder b/invirtibuilder index f5c3821..d2a571c 100755 --- a/invirtibuilder +++ b/invirtibuilder @@ -29,15 +29,22 @@ principal is the Kerberos principal that requested the build. from __future__ import with_statement import contextlib +import glob import os import re import shutil import subprocess +import tempfile +import traceback import pyinotify +from debian_bundle import deb822 + import invirt.builder as b +import invirt.common as c from invirt import database +from invirt.config import structs as config DISTRIBUTION = 'hardy' @@ -51,7 +58,7 @@ def getControl(package, ref): acts roughly like a dict. """ return deb822.Deb822.iter_paragraphs( - getGitFile(package, ref, 'debian/control').split('\n')) + b.getGitFile(package, ref, 'debian/control').split('\n')) def getBinaries(package, ref): @@ -71,7 +78,7 @@ def getArches(package, ref): def getDscName(package, ref): """Return the .dsc file that will be generated for this package.""" - v = getVersion(package, ref) + v = b.getVersion(package, ref) if v.debian_version: v_str = '%s-%s' % (v.upstream_version, v.debian_version) @@ -95,13 +102,13 @@ def sanitizeVersion(version): return v.replace('~', '.') -def aptCopy(packages, dst_pocket, src_pocket): +def aptCopy(package, commit, dst_pocket, src_pocket): """Copy a package from one pocket to another.""" binaries = getBinaries(package, commit) - cpatureOutput(['reprepro-env', 'copy', - b.pocketToApt(dst_pocket), - b.pocketToApt(src_pocket), - package] + binaries) + c.captureOutput(['reprepro-env', 'copy', + b.pocketToApt(dst_pocket), + b.pocketToApt(src_pocket), + package] + binaries) def sbuild(package, ref, arch, workdir, arch_all=False): @@ -122,7 +129,7 @@ def sbuildAll(package, ref, workdir): sbuild(package, ref, 'i386', workdir) -def tagSubmodule(pocket, package, ref, principal): +def tagSubmodule(pocket, package, principal, version, env): """Tag a new version of a submodule. If this pocket does not allow_backtracking, then this will create @@ -138,12 +145,7 @@ def tagSubmodule(pocket, package, ref, principal): legitimacy of my reasoning. """ if not config.build.pockets[pocket].get('allow_backtracking', False): - env = dict(os.environ) branch = b.pocketToGit(pocket) - version = b.getVersion(package, ref) - - env['GIT_COMMITTER_NAME'] = config.build.tagger.name - env['GIT_COMMITTER_EMAIL'] = config.build.tagger.email tag_msg = ('Tag %s of %s\n\n' 'Requested by %s' % (version.full_version, package, @@ -159,7 +161,7 @@ def updateSubmoduleBranch(pocket, package, ref): """Update the appropriately named branch in the submodule.""" branch = b.pocketToGit(pocket) c.captureOutput( - ['git', 'update-ref', 'refs/heads/%s' % branch, ref]) + ['git', 'update-ref', 'refs/heads/%s' % branch, ref], cwd=b.getRepo(package)) def uploadBuild(pocket, workdir): @@ -167,13 +169,13 @@ def uploadBuild(pocket, workdir): apt = b.pocketToApt(pocket) for changes in glob.glob(os.path.join(workdir, '*.changes')): c.captureOutput(['reprepro-env', - 'include', '--ignore=wrongdistribution', + 'include', apt, changes]) -def updateSuperproject(pocket, package, commit, principal): +def updateSuperproject(pocket, package, commit, principal, version, env): """Update the superproject. This will create a new commit on the branch for the given pocket @@ -185,32 +187,35 @@ def updateSuperproject(pocket, package, commit, principal): superproject = os.path.join(b._REPO_DIR, 'invirt/packages.git') branch = b.pocketToGit(pocket) tree = c.captureOutput(['git', 'ls-tree', branch], - cwd=superproject) + cwd=superproject).strip() new_tree = re.compile( r'^(160000 commit )[0-9a-f]*(\t%s)$' % package, re.M).sub( - r'\1%s\2' % commit, + r'\g<1>%s\g<2>' % commit, tree) - new_tree_id = c.captureOutput(['git', 'mktree'], - cwd=superproject, - stdin_str=new_tree) + new_tree_id = c.captureOutput(['git', 'mktree', '--missing'], + cwd=superproject, + stdin_str=new_tree).strip() commit_msg = ('Update %s to version %s\n\n' 'Requested by %s' % (package, version.full_version, principal)) new_commit = c.captureOutput( - ['git', 'commit-tree', new_tree_hash, '-p', branch], + ['git', 'commit-tree', new_tree_id, '-p', branch], cwd=superproject, env=env, - stdin_str=commit_msg) + stdin_str=commit_msg).strip() c.captureOutput( ['git', 'update-ref', 'refs/heads/%s' % branch, new_commit], cwd=superproject) +def makeReadable(workdir): + os.chmod(workdir, 0755) + @contextlib.contextmanager def packageWorkdir(package, commit): """Checkout the package in a temporary working directory. @@ -227,7 +232,7 @@ def packageWorkdir(package, commit): p_archive = subprocess.Popen( ['git', 'archive', '--remote=file://%s' % b.getRepo(package), - '--prefix=%s' % package, + '--prefix=%s/' % package, commit, ], stdout=subprocess.PIPE, @@ -277,13 +282,17 @@ def build(): db.commit = commit db.principal = principal database.session.save_or_update(db) - database.commit() + database.session.commit() - database.begin() + database.session.begin() try: db.failed_stage = 'validating job' - src = validateBuild(pocket, package, commit) + src = b.validateBuild(pocket, package, commit) + # Don't expand the commit in the DB until we're sure the user + # isn't trying to be tricky. + db.commit = commit = c.captureOutput(['git', 'rev-parse', commit], + cwd=b.getRepo(package)).strip() db.version = str(b.getVersion(package, commit)) @@ -294,7 +303,7 @@ def build(): # raised an exception) if src != True: db.failed_stage = 'copying package from another pocket' - aptCopy(packages, pocket, src) + aptCopy(package, commit, pocket, src) # If we can't copy the package from somewhere, but # validateBuild didn't raise an exception, then we need to # do the build ourselves @@ -319,18 +328,27 @@ def build(): db.failed_stage = 'building binary packages' sbuildAll(package, commit, workdir) finally: - logdir = os.path.join(b._LOG_DIR, db.build_id) + logdir = os.path.join(b._LOG_DIR, str(db.build_id)) if not os.path.exists(logdir): os.makedirs(logdir) for log in glob.glob(os.path.join(workdir, '*.build')): os.copy2(log, logdir) + + db.failed_stage = 'processing metadata' + env = dict(os.environ) + env['GIT_COMMITTER_NAME'] = config.build.tagger.name + env['GIT_COMMITTER_EMAIL'] = config.build.tagger.email + version = b.getVersion(package, commit) + db.failed_stage = 'tagging submodule' - tagSubmodule(pocket, package, commit, principal) + tagSubmodule(pocket, package, principal, version, env) db.failed_stage = 'updating submodule branches' updateSubmoduleBranch(pocket, package, commit) db.failed_stage = 'updating superproject' - updateSuperproject(pocket, package, commit, principal) + updateSuperproject(pocket, package, commit, principal, version, env) + db.failed_stage = 'relaxing permissions on workdir' + makeReadable(workdir) db.failed_stage = 'uploading packages to apt repo' uploadBuild(pocket, workdir) diff --git a/python/invirt/builder.py b/python/invirt/builder.py index c1730df..f2cb421 100644 --- a/python/invirt/builder.py +++ b/python/invirt/builder.py @@ -28,6 +28,19 @@ def getRepo(package): """Return the path to the git repo for a given package.""" return os.path.join(_REPO_DIR, 'invirt/packages', '%s.git' % package) +def ensureValidRepo(package): + """Perform some basic sanity checks that the requested repo is in a + subdirectory of _REPO_DIR/invirt/packages. This prevents weirdness + such as submitting a package like '../prod/...git'. Also ensures that + the repo exists.""" + # TODO: this might be easier just to regex + repo = os.path.abspath(getRepo(package)) + parent_dir = os.path.dirname(repo) + prefix = os.path.join(_REPO_DIR, 'invirt/packages') + if not parent_dir.startswith(prefix): + raise InvalidBuild('Invalid package name %s' % package) + elif not os.path.exists(repo): + raise InvalidBuild('Nonexisting package %s' % package) def pocketToGit(pocket): """Map a pocket in the configuration to a git branch.""" @@ -42,7 +55,7 @@ def pocketToApt(pocket): def getGitFile(package, ref, path): """Return the contents of a path from a git ref in a package.""" return c.captureOutput(['git', 'cat-file', 'blob', '%s:%s' % (ref, path)], - cwd=getRepo(package)) + cwd=getRepo(package)) def getChangelog(package, ref): @@ -85,6 +98,7 @@ def validateBuild(pocket, package, commit): another pocket, then this function returns that pocket. Otherwise, it returns True. """ + ensureValidRepo(package) package_repo = getRepo(package) new_version = getVersion(package, commit) @@ -96,18 +110,18 @@ def validateBuild(pocket, package, commit): b = pocketToGit(p) current_commit = c.captureOutput(['git', 'rev-parse', b], - cwd=package_repo) + cwd=package_repo).strip() current_version = getVersion(package, b) if current_version == new_version: if current_commit == commit: ret = p else: - raise InvalidBuild('Version %s of %s already available in ' + raise InvalidBuild('Version %s of %s already available is in ' 'pocket %s from commit %s' % (new_version, package, p, current_commit)) - if config.build.pockets[pocket].get('allow_backtracking', False): + if not config.build.pockets[pocket].get('allow_backtracking', False): branch = pocketToGit(pocket) current_version = getVersion(package, branch) if new_version <= current_version: -- 1.7.9.5