X-Git-Url: http://xvm.mit.edu/gitweb/invirt/packages/invirt-dev.git/blobdiff_plain/10277b30403ade535cdd169a3517561e56f53311..refs/heads/prod:/invirtibuilder diff --git a/invirtibuilder b/invirtibuilder index 32744ae..eac57f3 100755 --- a/invirtibuilder +++ b/invirtibuilder @@ -29,19 +29,41 @@ principal is the Kerberos principal that requested the build. from __future__ import with_statement import contextlib +import glob import os import re import shutil import subprocess +import tempfile +import traceback import pyinotify +from debian_bundle import deb822 + import invirt.builder as b +import invirt.common as c from invirt import database - - -DISTRIBUTION = 'hardy' - +from invirt.config import structs as config + + +logfile = None + +def logAndRun(cmd, *args, **kwargs): + # Always grab stdout, even if the caller doesn't need it. + # TODO: don't slurp it all into memory in that case. + if 'stdout' in kwargs and kwargs['stdout'] is None: + del kwargs['stdout'] + kwargs['stderr'] = logfile + logfile.write('---> Ran %s\n' % (cmd, )) + if 'stdin_str' in kwargs: + logfile.write('STDIN:\n') + logfile.write(kwargs['stdin_str']) + logfile.write('STDERR:\n') + output = c.captureOutput(cmd, *args, **kwargs) + logfile.write('STDOUT:\n') + logfile.write(output) + return output def getControl(package, ref): """Get the parsed debian/control file for a given package. @@ -51,7 +73,7 @@ def getControl(package, ref): acts roughly like a dict. """ return deb822.Deb822.iter_paragraphs( - getGitFile(package, ref, 'debian/control').split('\n')) + b.getGitFile(package, ref, 'debian/control').split('\n')) def getBinaries(package, ref): @@ -71,7 +93,7 @@ def getArches(package, ref): def getDscName(package, ref): """Return the .dsc file that will be generated for this package.""" - v = getVersion(package, ref) + v = b.getVersion(package, ref) if v.debian_version: v_str = '%s-%s' % (v.upstream_version, v.debian_version) @@ -86,43 +108,53 @@ def sanitizeVersion(version): """Sanitize a Debian package version for use as a git tag. This function strips the epoch from the version number and - replaces any tildes with periods.""" - if v.debian_version: + replaces any tildes with underscores.""" + if version.debian_version: v = '%s-%s' % (version.upstream_version, version.debian_version) else: v = version.upstream_version - return v.replace('~', '.') + return v.replace('~', '_') -def aptCopy(packages, dst_pocket, src_pocket): +def aptCopy(package, commit, dst_pocket, src_pocket): """Copy a package from one pocket to another.""" binaries = getBinaries(package, commit) - cpatureOutput(['reprepro-env', 'copy', - b.pocketToApt(dst_pocket), - b.pocketToApt(src_pocket), - package] + binaries) - - -def sbuild(package, ref, arch, workdir, arch_all=False): - """Build a package for a particular architecture.""" - args = ['sbuild', '-d', DISTRIBUTION, '--arch', arch] + logAndRun(['reprepro-env', 'copy', + b.pocketToApt(dst_pocket), + b.pocketToApt(src_pocket), + package] + binaries) + + +def sbuild(package, ref, distro, arch, workdir, arch_all=False): + """Build a package for a particular architecture and distro.""" + # We append a suffix like ~ubuntu8.04 to differentiate the same + # version built for multiple distros + nmutag = b.distroToSuffix(distro) + env = os.environ.copy() + env['NMUTAG'] = nmutag + + # Run sbuild with a hack in place to append arbitrary versions + args = ['perl', '-I/usr/share/invirt-dev', '-MSbuildHack', + '/usr/bin/sbuild', + '--binNMU=171717', '--make-binNMU=Build with sbuild', + '-v', '-d', distro, '--arch', arch] if arch_all: args.append('-A') args.append(getDscName(package, ref)) - c.captureOutput(args, cwd=workdir, stdout=None) + logAndRun(args, cwd=workdir, env=env) -def sbuildAll(package, ref, workdir): +def sbuildAll(package, ref, distro, workdir): """Build a package for all architectures it supports.""" arches = getArches(package, ref) if 'all' in arches or 'any' in arches or 'amd64' in arches: - sbuild(package, ref, 'amd64', workdir, arch_all=True) + sbuild(package, ref, distro, 'amd64', workdir, arch_all=True) if 'any' in arches or 'i386' in arches: - sbuild(package, ref, 'i386', workdir) + sbuild(package, ref, distro, 'i386', workdir) -def tagSubmodule(pocket, package, ref, principal): +def tagSubmodule(pocket, package, commit, principal, version, env): """Tag a new version of a submodule. If this pocket does not allow_backtracking, then this will create @@ -134,46 +166,51 @@ def tagSubmodule(pocket, package, ref, principal): hook. Because we reject pushes to tags in the update hook, no push can ever take out a lock on any tags. - I'm sure that long description gives you great confidence in teh + I'm sure that long description gives you great confidence in the legitimacy of my reasoning. """ - if config.build.pockets[pocket].get('allow_backtracking', False): - env = dict(os.environ) + if not config.build.pockets[pocket].get('allow_backtracking', False): branch = b.pocketToGit(pocket) - version = b.getVersion(package, ref) - - env['GIT_COMMITTER_NAME'] = config.build.tagger.name - env['GIT_COMMITTER_EMAIL'] = config.build.tagger.email tag_msg = ('Tag %s of %s\n\n' 'Requested by %s' % (version.full_version, package, principal)) - c.captureOutput( - ['git', 'tag', '-m', tag_msg, commit], - stdout=None, - env=env) + logAndRun( + ['git', 'tag', '-m', tag_msg, '--', sanitizeVersion(version), + commit], + env=env, + cwd=b.getRepo(package)) -def updateSubmoduleBranch(pocket, package, ref): +def updateSubmoduleBranch(pocket, package, commit): """Update the appropriately named branch in the submodule.""" branch = b.pocketToGit(pocket) - c.captureOutput( - ['git', 'update-ref', 'refs/heads/%s' % branch, ref]) + logAndRun( + ['git', 'update-ref', 'refs/heads/%s' % branch, commit], cwd=b.getRepo(package)) def uploadBuild(pocket, workdir): """Upload all build products in the work directory.""" + force = config.build.pockets[pocket].get('allow_backtracking', False) apt = b.pocketToApt(pocket) for changes in glob.glob(os.path.join(workdir, '*.changes')): - c.captureOutput(['reprepro-env', - 'include', - '--ignore=wrongdistribution', - apt, - changes]) - - -def updateSuperproject(pocket, package, commit, principal): + upload = ['reprepro-env', '--ignore=wrongdistribution', + 'include', apt, changes] + try: + logAndRun(upload) + except subprocess.CalledProcessError, e: + if not force: + raise + changelog = deb822.Changes(open(changes).read()) + packages = set(changelog['Binary'].split()) + packages.add(changelog['Source']) + for package in packages: + logAndRun(['reprepro-env', 'remove', apt, package]) + logAndRun(upload) + + +def updateSuperproject(pocket, package, commit, principal, version, env): """Update the superproject. This will create a new commit on the branch for the given pocket @@ -183,36 +220,85 @@ def updateSuperproject(pocket, package, commit, principal): pushes to the superproject. """ superproject = os.path.join(b._REPO_DIR, 'invirt/packages.git') - branch = b.pocketToGit(pocket) - tree = c.captureOutput(['git', 'ls-tree', branch], - cwd=superproject) - new_tree = re.compile( - r'^(160000 commit )[0-9a-f]*(\t%s)$' % package, re.M).sub( - r'\1%s\2' % commit, - tree) + branch = b.pocketToGit(pocket) - new_tree_id = c.captureOutput(['git', 'mktree'], + if not b.pocketExists(pocket, superproject): + gitmodules = "\n" + gitmodules_hash = logAndRun(['git', 'hash-object', '-w', '--stdin'], + cwd=superproject, + stdin_str=gitmodules).strip() + tree_items = {'.gitmodules': "100644 blob "+gitmodules_hash} + new_tree = "\n".join("%s\t%s" % (v, k) for (k, v) in tree_items.iteritems()) + new_tree_id = logAndRun(['git', 'mktree', '--missing'], cwd=superproject, - stdin_str=new_tree) - - commit_msg = ('Update %s to version %s\n\n' - 'Requested by %s' % (package, - version.full_version, - principal)) - new_commit = c.captureOutput( - ['git', 'commit-tree', new_tree_hash, '-p', branch], + stdin_str=new_tree).strip() + env2 = dict(os.environ) + env2['GIT_AUTHOR_NAME'] = config.build.tagger.name + env2['GIT_AUTHOR_EMAIL'] = config.build.tagger.email + env2['GIT_COMMITTER_NAME'] = config.build.tagger.name + env2['GIT_COMMITTER_EMAIL'] = config.build.tagger.email + new_commit = logAndRun(['git', 'commit-tree', new_tree_id], + cwd=superproject, + env=env2, + stdin_str="Create new pocket").strip() + logAndRun(['git', 'update-ref', 'refs/heads/%s' % branch, new_commit], + cwd=superproject) + + tree = logAndRun(['git', 'ls-tree', branch], + cwd=superproject).strip() + + tree_items = dict((k, v) for (v, k) in (x.split("\t") for x in tree.split("\n"))) + + created = not (package in tree_items) + + tree_items[package] = "160000 commit "+commit + + # If "created" is true, we need to check if the package is + # mentioned in .gitmodules, and add it if not. + if created: + gitmodules = logAndRun(['git', 'cat-file', 'blob', '%s:.gitmodules' % (branch)], + cwd=superproject) + if ('[submodule "%s"]' % (package)) not in gitmodules.split("\n"): + gitmodules += """[submodule "%s"] +\tpath = %s +\turl = ../packages/%s.git +""" % (package, package, package) + gitmodules_hash = logAndRun(['git', 'hash-object', '-w', '--stdin'], + cwd=superproject, + stdin_str=gitmodules).strip() + tree_items['.gitmodules'] = "100644 blob "+gitmodules_hash + + new_tree = "\n".join("%s\t%s" % (v, k) for (k, v) in tree_items.iteritems()) + + new_tree_id = logAndRun(['git', 'mktree', '--missing'], + cwd=superproject, + stdin_str=new_tree).strip() + + if created: + commit_msg = 'Add %s at version %s' + else: + commit_msg = 'Update %s to version %s' + commit_msg = ((commit_msg + '\n\n' + 'Requested by %s') % (package, + version.full_version, + principal)) + new_commit = logAndRun( + ['git', 'commit-tree', new_tree_id, '-p', branch], cwd=superproject, env=env, - stdin_str=commit_msg) + stdin_str=commit_msg).strip() - c.captureOutput( + logAndRun( ['git', 'update-ref', 'refs/heads/%s' % branch, new_commit], cwd=superproject) +def makeReadable(workdir): + os.chmod(workdir, 0755) + @contextlib.contextmanager -def packageWorkdir(package): +def packageWorkdir(package, commit, build_id): """Checkout the package in a temporary working directory. This context manager returns that working directory. The requested @@ -222,12 +308,12 @@ def packageWorkdir(package): When the context wrapped with this context manager is exited, the working directory is automatically deleted. """ - workdir = tempfile.mkdtemp() + workdir = tempfile.mkdtemp(prefix=("b%d-" % build_id)) try: p_archive = subprocess.Popen( - ['git', 'archive', - '--remote=file://%s' % b.getRepo(package), - '--prefix=%s' % package, + ['git', '--git-dir=%s' % (b.getRepo(package),), + 'archive', + '--prefix=%s/' % package, commit, ], stdout=subprocess.PIPE, @@ -244,22 +330,14 @@ def packageWorkdir(package): finally: shutil.rmtree(workdir) - -def reportBuild(build): - """Run hooks to report the results of a build attempt.""" - - c.captureOutput(['run-parts', - '--arg=%s' % build.build_id, - '--', - b._HOOKS_DIR]) - - def build(): """Deal with items in the build queue. When triggered, iterate over build queue items one at a time, until there are no more pending build jobs. """ + global logfile + while True: stage = 'processing incoming job' queue = os.listdir(b._QUEUE_DIR) @@ -277,15 +355,33 @@ def build(): db.commit = commit db.principal = principal database.session.save_or_update(db) - database.commit() + database.session.commit() + + database.session.begin() - database.begin() + logdir = os.path.join(b._LOG_DIR, str(db.build_id)) + if not os.path.exists(logdir): + os.makedirs(logdir) try: db.failed_stage = 'validating job' - src = validateBuild(pocket, package, commit) + # Don't expand the commit in the DB until we're sure the user + # isn't trying to be tricky. + b.ensureValidPackage(package) + + logfile = open(os.path.join(logdir, '%s.log' % db.package), 'w') - db.version = str(b.getVersion(package, commit)) + db.commit = commit = b.canonicalize_commit(package, commit) + src = b.validateBuild(pocket, package, commit) + version = b.getVersion(package, commit) + db.version = str(version) + b.runHook('pre-build', [str(db.build_id)]) + + env = dict(os.environ) + env['GIT_COMMITTER_NAME'] = config.build.tagger.name + env['GIT_COMMITTER_EMAIL'] = config.build.tagger.email + env['GIT_AUTHOR_NAME'] = principal.split('@')[0] + env['GIT_AUTHOR_EMAIL'] = principal # If validateBuild returns something other than True, then # it means we should copy from that pocket to our pocket. @@ -293,14 +389,22 @@ def build(): # (If the validation failed, validateBuild would have # raised an exception) if src != True: + # TODO: cut out this code duplication + db.failed_stage = 'tagging submodule before copying package' + tagSubmodule(pocket, package, commit, principal, version, env) + db.failed_stage = 'updating submodule branches before copying package' + updateSubmoduleBranch(pocket, package, commit) + db.failed_stage = 'updating superproject before copying package' + updateSuperproject(pocket, package, commit, principal, version, env) db.failed_stage = 'copying package from another pocket' - aptCopy(packages, pocket, src) + aptCopy(package, commit, pocket, src) + # If we can't copy the package from somewhere, but # validateBuild didn't raise an exception, then we need to # do the build ourselves else: db.failed_stage = 'checking out package source' - with packageWorkdir(package) as workdir: + with packageWorkdir(package, commit, db.build_id) as workdir: db.failed_stage = 'preparing source package' packagedir = os.path.join(workdir, package) @@ -311,52 +415,52 @@ def build(): # If we were, we could use debuild and get nice # environment scrubbing. Since we're not, debuild # complains about not having an orig.tar.gz - c.captureOutput(['dpkg-buildpackage', '-us', '-uc', '-S'], - cwd=packagedir, - stdout=None) - - try: - db.failed_stage = 'building binary packages' - sbuildAll(package, commit, workdir) - finally: - logdir = os.path.join(b._LOG_DIR, db.build_id) - if not os.path.exists(logdir): - os.makedirs(logdir) - - for log in glob.glob(os.path.join(workdir, '*.build')): - os.copy2(log, logdir) + logAndRun(['schroot', '-c', + '%s-amd64-sbuild' % (b.pocketToDistro(pocket),), + '--', 'dpkg-buildpackage', '-us', '-uc', '-S'], + cwd=packagedir) + + db.failed_stage = 'building binary packages' + sbuildAll(package, commit, b.pocketToDistro(pocket), workdir) db.failed_stage = 'tagging submodule' - tagSubmodule(pocket, package, commit, principal) + tagSubmodule(pocket, package, commit, principal, version, env) db.failed_stage = 'updating submodule branches' updateSubmoduleBranch(pocket, package, commit) db.failed_stage = 'updating superproject' - updateSuperproject(pocket, package, commit, principal) + updateSuperproject(pocket, package, commit, principal, version, env) + db.failed_stage = 'relaxing permissions on workdir' + makeReadable(workdir) db.failed_stage = 'uploading packages to apt repo' uploadBuild(pocket, workdir) db.failed_stage = 'cleaning up' - - # Finally, now that everything is done, remove the - # build queue item - os.unlink(os.path.join(b._QUEUE_DIR, build)) except: db.traceback = traceback.format_exc() else: db.succeeded = True db.failed_stage = None finally: + if logfile is not None: + logfile.close() + database.session.save_or_update(db) database.session.commit() - reportBuild(db) + # Finally, now that everything is done, remove the + # build queue item + os.unlink(os.path.join(b._QUEUE_DIR, build)) + if db.succeeded: + b.runHook('post-build', [str(db.build_id)]) + else: + b.runHook('failed-build', [str(db.build_id)]) class Invirtibuilder(pyinotify.ProcessEvent): """Process inotify triggers to build new packages.""" - def process_IN_CREATE(self, event): - """Handle a created file or directory. + def process_default(self, event): + """Handle an inotify event. - When an IN_CREATE event comes in, trigger the builder. + When an inotify event comes in, trigger the builder. """ build() @@ -369,7 +473,8 @@ def main(): invirtibuilder = Invirtibuilder() notifier = pyinotify.Notifier(watch_manager, invirtibuilder) watch_manager.add_watch(b._QUEUE_DIR, - pyinotify.EventsCodes.ALL_FLAGS['IN_CREATE']) + pyinotify.EventsCodes.ALL_FLAGS['IN_CREATE'] | + pyinotify.EventsCodes.ALL_FLAGS['IN_MOVED_TO']) # Before inotifying, run any pending builds; otherwise we won't # get notified for them.