X-Git-Url: http://xvm.mit.edu/gitweb/invirt/packages/invirt-dev.git/blobdiff_plain/705b811db5dd577929aa161b2fb666ff3f4d3abe..811f7dd06268328f86a506881485cbbcfc0a16e8:/invirtibuilder diff --git a/invirtibuilder b/invirtibuilder index 9198549..6f3d917 100755 --- a/invirtibuilder +++ b/invirtibuilder @@ -48,7 +48,20 @@ from invirt.config import structs as config DISTRIBUTION = 'hardy' - +logfile = None + +def logAndRun(cmd, *args, **kwargs): + # Always grab stdout, even if the caller doesn't need it. + # TODO: don't slurp it all into memory in that case. + if 'stdout' in kwargs and kwargs['stdout'] is None: + del kwargs['stdout'] + kwargs['stderr'] = logfile + logfile.write('---> Ran %s\n' % (cmd, )) + logfile.write('STDERR:\n') + output = c.captureOutput(cmd, *args, **kwargs) + logfile.write('STDOUT:\n') + logfile.write(output) + return output def getControl(package, ref): """Get the parsed debian/control file for a given package. @@ -93,22 +106,22 @@ def sanitizeVersion(version): """Sanitize a Debian package version for use as a git tag. This function strips the epoch from the version number and - replaces any tildes with periods.""" - if v.debian_version: + replaces any tildes with underscores.""" + if version.debian_version: v = '%s-%s' % (version.upstream_version, version.debian_version) else: v = version.upstream_version - return v.replace('~', '.') + return v.replace('~', '_') def aptCopy(package, commit, dst_pocket, src_pocket): """Copy a package from one pocket to another.""" binaries = getBinaries(package, commit) - c.captureOutput(['reprepro-env', 'copy', - b.pocketToApt(dst_pocket), - b.pocketToApt(src_pocket), - package] + binaries) + logAndRun(['reprepro-env', 'copy', + b.pocketToApt(dst_pocket), + b.pocketToApt(src_pocket), + package] + binaries) def sbuild(package, ref, arch, workdir, arch_all=False): @@ -117,7 +130,7 @@ def sbuild(package, ref, arch, workdir, arch_all=False): if arch_all: args.append('-A') args.append(getDscName(package, ref)) - c.captureOutput(args, cwd=workdir) + logAndRun(args, cwd=workdir) def sbuildAll(package, ref, workdir): @@ -151,8 +164,9 @@ def tagSubmodule(pocket, package, commit, principal, version, env): package, principal)) - c.captureOutput( - ['git', 'tag', '-m', tag_msg, commit], + logAndRun( + ['git', 'tag', '-m', tag_msg, '--', sanitizeVersion(version), + commit], env=env, cwd=b.getRepo(package)) @@ -160,7 +174,7 @@ def tagSubmodule(pocket, package, commit, principal, version, env): def updateSubmoduleBranch(pocket, package, commit): """Update the appropriately named branch in the submodule.""" branch = b.pocketToGit(pocket) - c.captureOutput( + logAndRun( ['git', 'update-ref', 'refs/heads/%s' % branch, commit], cwd=b.getRepo(package)) @@ -172,13 +186,13 @@ def uploadBuild(pocket, workdir): upload = ['reprepro-env', '--ignore=wrongdistribution', 'include', apt, changes] try: - c.captureOutput(upload) + logAndRun(upload) except subprocess.CalledProcessError, e: if not force: raise package = deb822.Changes(open(changes).read())['Binary'] - c.captureOutput(['reprepro-env', 'remove', apt, package]) - c.captureOutput(upload) + logAndRun(['reprepro-env', 'remove', apt, package]) + logAndRun(upload) def updateSuperproject(pocket, package, commit, principal, version, env): @@ -192,29 +206,29 @@ def updateSuperproject(pocket, package, commit, principal, version, env): """ superproject = os.path.join(b._REPO_DIR, 'invirt/packages.git') branch = b.pocketToGit(pocket) - tree = c.captureOutput(['git', 'ls-tree', branch], - cwd=superproject).strip() + tree = logAndRun(['git', 'ls-tree', branch], + cwd=superproject).strip() new_tree = re.compile( r'^(160000 commit )[0-9a-f]*(\t%s)$' % package, re.M).sub( r'\g<1>%s\g<2>' % commit, tree) - new_tree_id = c.captureOutput(['git', 'mktree', '--missing'], - cwd=superproject, - stdin_str=new_tree).strip() + new_tree_id = logAndRun(['git', 'mktree', '--missing'], + cwd=superproject, + stdin_str=new_tree).strip() commit_msg = ('Update %s to version %s\n\n' 'Requested by %s' % (package, version.full_version, principal)) - new_commit = c.captureOutput( + new_commit = logAndRun( ['git', 'commit-tree', new_tree_id, '-p', branch], cwd=superproject, env=env, stdin_str=commit_msg).strip() - c.captureOutput( + logAndRun( ['git', 'update-ref', 'refs/heads/%s' % branch, new_commit], cwd=superproject) @@ -261,6 +275,8 @@ def build(): When triggered, iterate over build queue items one at a time, until there are no more pending build jobs. """ + global logfile + while True: stage = 'processing incoming job' queue = os.listdir(b._QUEUE_DIR) @@ -282,11 +298,18 @@ def build(): database.session.begin() + logdir = os.path.join(b._LOG_DIR, str(db.build_id)) + if not os.path.exists(logdir): + os.makedirs(logdir) + try: db.failed_stage = 'validating job' # Don't expand the commit in the DB until we're sure the user # isn't trying to be tricky. b.ensureValidPackage(package) + + logfile = open(os.path.join(logdir, '%s.log' % db.package), 'w') + db.commit = commit = b.canonicalize_commit(package, commit) src = b.validateBuild(pocket, package, commit) version = b.getVersion(package, commit) @@ -330,21 +353,11 @@ def build(): # If we were, we could use debuild and get nice # environment scrubbing. Since we're not, debuild # complains about not having an orig.tar.gz - c.captureOutput(['dpkg-buildpackage', '-us', '-uc', '-S'], - cwd=packagedir, - stdout=None) - - try: - db.failed_stage = 'building binary packages' - sbuildAll(package, commit, workdir) - finally: - logdir = os.path.join(b._LOG_DIR, str(db.build_id)) - if not os.path.exists(logdir): - os.makedirs(logdir) - - for log in glob.glob(os.path.join(workdir, 'build-*.log')): - os.copy(log, logdir) + logAndRun(['dpkg-buildpackage', '-us', '-uc', '-S'], + cwd=packagedir) + db.failed_stage = 'building binary packages' + sbuildAll(package, commit, workdir) db.failed_stage = 'tagging submodule' tagSubmodule(pocket, package, commit, principal, version, env) db.failed_stage = 'updating submodule branches' @@ -363,6 +376,9 @@ def build(): db.succeeded = True db.failed_stage = None finally: + if logfile is not None: + logfile.close() + database.session.save_or_update(db) database.session.commit()