From: Evan Broder Date: Sun, 22 Nov 2009 21:07:20 +0000 (-0500) Subject: Pull functions that are needed for the git remctl scripts out of the X-Git-Tag: 0.1.5~81 X-Git-Url: http://xvm.mit.edu/gitweb/invirt/packages/invirt-dev.git/commitdiff_plain/9679ebb058b3be6f4aa3df45506380b535d9b1f6 Pull functions that are needed for the git remctl scripts out of the invirtibuilder and into a common module. svn path=/trunk/packages/invirt-dev/; revision=2543 --- diff --git a/invirtibuilder b/invirtibuilder index 716d7d7..f5cdfb4 100755 --- a/invirtibuilder +++ b/invirtibuilder @@ -34,82 +34,13 @@ import subprocess import pyinotify -from invirt.config import structs as config +import invirt.builder as b from invirt import database -_QUEUE_DIR = '/var/lib/invirt-dev/queue' -_REPO_DIR = '/srv/git' -_LOG_DIR = '/var/log/invirt/builds' -_HOOKS_DIR = '/usr/share/invirt-dev/build.d' - - DISTRIBUTION = 'hardy' -class InvalidBuild(ValueError): - pass - - -def captureOutput(popen_args, stdin_str=None, *args, **kwargs): - """Capture stdout from a command. - - This method will proxy the arguments to subprocess.Popen. It - returns the output from the command if the call succeeded and - raises an exception if the process returns a non-0 value. - - This is intended to be a variant on the subprocess.check_call - function that also allows you access to the output from the - command. - """ - if 'stdin' not in kwargs: - kwargs['stdin'] = subprocess.PIPE - if 'stdout' not in kwargs: - kwargs['stdout'] = subprocess.PIPE - if 'stderr' not in kwargs: - kwargs['stderr'] = subprocess.STDOUT - p = subprocess.Popen(popen_args, *args, **kwargs) - out, _ = p.communicate(stdin_str) - if p.returncode: - raise subprocess.CalledProcessError(p.returncode, popen_args, out) - return out - - -def getRepo(package): - """Return the path to the git repo for a given package.""" - return os.path.join(_REPO_DIR, 'packages', '%s.git' % package) - - -def pocketToGit(pocket): - """Map a pocket in the configuration to a git branch.""" - return config.git.pockets[pocket].get('git', pocket) - - -def pocketToApt(pocket): - """Map a pocket in the configuration to an apt repo pocket.""" - return config.git.pockets[pocket].get('apt', pocket) - - -def getGitFile(package, ref, path): - """Return the contents of a path from a git ref in a package.""" - return captureOutput(['git', 'cat-file', 'blob', '%s:%s' % (ref, path)], - cwd=getRepo(package)) - - -def getChangelog(package, ref): - """Get a changelog object for a given ref in a given package. - - This returns a debian_bundle.changelog.Changelog object for a - given ref of a given package. - """ - return changelog.Changelog(getGitFile(package, ref, 'debian/changelog')) - - -def getVersion(package, ref): - """Get the version of a given package at a particular ref.""" - return getChangelog(package, ref).get_version() - - def getControl(package, ref): """Get the parsed debian/control file for a given package. @@ -145,68 +76,6 @@ def getDscName(package, ref): version.debian_version) -def validateBuild(pocket, package, commit): - """Given the parameters of a new build, validate that build. - - The checks this function performs vary based on whether or not the - pocket is configured with allow_backtracking. - - A build of a pocket without allow_backtracking set must be a - fast-forward of the previous revision, and the most recent version - in the changelog most be strictly greater than the version - currently in the repository. - - In all cases, this revision of the package can only have the same - version number as any other revision currently in the apt - repository if they have the same commit ID. - - If it's unspecified, it is assumed that pocket do not - allow_backtracking. - - If this build request fails validation, this function will raise a - InvalidBuild exception, with information about why the validation - failed. - - If this build request can be satisfied by copying the package from - another pocket, then this function returns that pocket. Otherwise, - it returns True. - """ - package_repo = getRepo(package) - new_version = getVersion(package, commit) - - for p in config.git.pockets: - if p == pocket: - continue - - b = pocketToGit(p) - current_commit = captureOutput(['git', 'rev-parse', b], - cwd=package_repo) - current_version = getVersion(package, b) - - if current_version == new_version: - if current_commit == commit: - return p - else: - raise InvalidBuild('Version %s of %s already available in ' - 'pocket %s from commit %s' % - (new_version, package, p, current_commit)) - - if config.git.pockets[pocket].get('allow_backtracking', False): - branch = pocketToGit(pocket) - current_version = getVersion(package, branch) - if new_version <= current_version: - raise InvalidBuild('New version %s of %s is not newer than ' - 'version %s currently in pocket %s' % - (new_version, package, current_version, pocket)) - - # Almost by definition, A is a fast-forward of B if B..A is - # empty - if not captureOutput(['git', 'rev-list', '%s..%s' % (commit, branch)]): - raise InvalidBuild('New commit %s of %s is not a fast-forward of' - 'commit currently in pocket %s' % - (commit, package, pocket)) - - def sanitizeVersion(version): """Sanitize a Debian package version for use as a git tag. @@ -220,14 +89,14 @@ def sanitizeVersion(version): def aptCopy(packages, dst_pocket, src_pocket): """Copy a package from one pocket to another.""" binaries = [] - for line in getGitFile(package, commit, 'debian/control').split('\n'): + for line in b.getGitFile(package, commit, 'debian/control').split('\n'): m = re.match('Package: (.*)$') if m: binaries.append(m.group(1)) cpatureOutput(['reprepro-env', 'copy', - pocketToApt(dst_pocket), - pocketToApt(src_pocket), + b.pocketToApt(dst_pocket), + b.pocketToApt(src_pocket), package] + binaries) @@ -237,7 +106,7 @@ def sbuild(package, ref, arch, workdir, arch_all=False): if arch_all: args.append('-A') args.append(getDscName(package, ref)) - captureOutput(args, cwd=workdir, stdout=None) + c.captureOutput(args, cwd=workdir, stdout=None) def sbuildAll(package, ref, workdir): @@ -266,8 +135,8 @@ def tagSubmodule(pocket, package, ref, principal): """ if config.git.pockets[pocket].get('allow_backtracking', False): env = dict(os.environ) - branch = pocketToGit(pocket) - version = getVersion(package, ref) + branch = b.pocketToGit(pocket) + version = b.getVersion(package, ref) env['GIT_COMMITTER_NAME'] = config.git.tagger.name env['GIT_COMMITTER_EMAIL'] = config.git.tagger.email @@ -276,7 +145,7 @@ def tagSubmodule(pocket, package, ref, principal): package, principal)) - captureOutput( + c.captureOutput( ['git', 'tag', '-m', tag_msg, commit], stdout=None, env=env) @@ -284,16 +153,16 @@ def tagSubmodule(pocket, package, ref, principal): def updateSubmoduleBranch(pocket, package, ref): """Update the appropriately named branch in the submodule.""" - branch = pocketToGit(pocket) - captureOutput( + branch = b.pocketToGit(pocket) + c.captureOutput( ['git', 'update-ref', 'refs/heads/%s' % branch, ref]) def uploadBuild(pocket, workdir): """Upload all build products in the work directory.""" - apt = pocketToApt(pocket) + apt = b.pocketToApt(pocket) for changes in glob.glob(os.path.join(workdir, '*.changes')): - captureOutput(['reprepro-env', + c.captureOutput(['reprepro-env', 'include', '--ignore=wrongdistribution', apt, @@ -309,9 +178,9 @@ def updateSuperrepo(pocket, package, commit, principal): Note that there's no locking issue here, because we disallow all pushes to the superrepo. """ - superrepo = os.path.join(_REPO_DIR, 'packages.git') - branch = pocketToGit(pocket) - tree = captureOutput(['git', 'ls-tree', branch], + superrepo = os.path.join(b._REPO_DIR, 'packages.git') + branch = b.pocketToGit(pocket) + tree = c.captureOutput(['git', 'ls-tree', branch], cwd=superrepo) new_tree = re.compile( @@ -319,7 +188,7 @@ def updateSuperrepo(pocket, package, commit, principal): r'\1%s\2' % commit, tree) - new_tree_id = captureOutput(['git', 'mktree'], + new_tree_id = c.captureOutput(['git', 'mktree'], cwd=superrepo, stdin_str=new_tree) @@ -327,13 +196,13 @@ def updateSuperrepo(pocket, package, commit, principal): 'Requested by %s' % (package, version.full_version, principal)) - new_commit = captureOutput( + new_commit = c.captureOutput( ['git', 'commit-tree', new_tree_hash, '-p', branch], cwd=superrepo, env=env, stdin_str=commit_msg) - captureOutput( + c.captureOutput( ['git', 'update-ref', 'refs/heads/%s' % branch, new_commit], cwd=superrepo) @@ -353,7 +222,7 @@ def packageWorkdir(package): try: p_archive = subprocess.Popen( ['git', 'archive', - '--remote=file://%s' % getRepo(package), + '--remote=file://%s' % b.getRepo(package), '--prefix=%s' % package, commit, ], @@ -375,10 +244,10 @@ def packageWorkdir(package): def reportBuild(build): """Run hooks to report the results of a build attempt.""" - captureOutput(['run-parts', + c.captureOutput(['run-parts', '--arg=%s' % build.build_id, '--', - _HOOKS_DIR]) + b._HOOKS_DIR]) def build(): @@ -389,12 +258,12 @@ def build(): """ while True: stage = 'processing incoming job' - queue = os.listdir(_QUEUE_DIR) + queue = os.listdir(b._QUEUE_DIR) if not queue: break build = min(queue) - job = open(os.path.join(_QUEUE_DIR, build)).read().strip() + job = open(os.path.join(b._QUEUE_DIR, build)).read().strip() pocket, package, commit, principal = job.split() database.session.begin() @@ -412,7 +281,7 @@ def build(): db.failed_stage = 'validating job' src = validateBuild(pocket, package, commit) - db.version = str(getVersion(package, commit)) + db.version = str(b.getVersion(package, commit)) # If validateBuild returns something other than True, then # it means we should copy from that pocket to our pocket. @@ -438,7 +307,7 @@ def build(): # If we were, we could use debuild and get nice # environment scrubbing. Since we're not, debuild # complains about not having an orig.tar.gz - captureOutput(['dpkg-buildpackage', '-us', '-uc', '-S'], + c.captureOutput(['dpkg-buildpackage', '-us', '-uc', '-S'], cwd=packagedir, stdout=None) @@ -446,7 +315,7 @@ def build(): db.failed_stage = 'building binary packages' sbuildAll(package, commit, workdir) finally: - logdir = os.path.join(_LOG_DIR, db.build_id) + logdir = os.path.join(b._LOG_DIR, db.build_id) if not os.path.exists(logdir): os.makedirs(logdir) @@ -465,7 +334,7 @@ def build(): # Finally, now that everything is done, remove the # build queue item - os.unlink(os.path.join(_QUEUE_DIR, build)) + os.unlink(os.path.join(b._QUEUE_DIR, build)) except: db.traceback = traceback.format_exc() else: @@ -495,7 +364,7 @@ def main(): watch_manager = pyinotify.WatchManager() invirtibuilder = Invirtibuilder() notifier = pyinotify.Notifier(watch_manager, invirtibuilder) - watch_manager.add_watch(_QUEUE_DIR, + watch_manager.add_watch(b._QUEUE_DIR, pyinotify.EventsCodes.ALL_FLAGS['IN_CREATE']) # Before inotifying, run any pending builds; otherwise we won't diff --git a/python/invirt/builder.py b/python/invirt/builder.py new file mode 100644 index 0000000..27e21a6 --- /dev/null +++ b/python/invirt/builder.py @@ -0,0 +1,123 @@ +"""Invirt build utilities. + +This module contains utility functions used by both the invirtibuilder +and the remctl submission scripts that insert items into its queue. +""" + + +import os + +from debian_bundle import changelog +from debian_bundle import deb822 + +import invirt.common as c +from invirt.config import structs as config + + +_QUEUE_DIR = '/var/lib/invirt-dev/queue' +_REPO_DIR = '/srv/git' +_LOG_DIR = '/var/log/invirt/builds' +_HOOKS_DIR = '/usr/share/invirt-dev/build.d' + + +class InvalidBuild(ValueError): + pass + + +def getRepo(package): + """Return the path to the git repo for a given package.""" + return os.path.join(_REPO_DIR, 'packages', '%s.git' % package) + + +def pocketToGit(pocket): + """Map a pocket in the configuration to a git branch.""" + return config.git.pockets[pocket].get('git', pocket) + + +def pocketToApt(pocket): + """Map a pocket in the configuration to an apt repo pocket.""" + return config.git.pockets[pocket].get('apt', pocket) + + +def getGitFile(package, ref, path): + """Return the contents of a path from a git ref in a package.""" + return c.captureOutput(['git', 'cat-file', 'blob', '%s:%s' % (ref, path)], + cwd=getRepo(package)) + + +def getChangelog(package, ref): + """Get a changelog object for a given ref in a given package. + + This returns a debian_bundle.changelog.Changelog object for a + given ref of a given package. + """ + return changelog.Changelog(getGitFile(package, ref, 'debian/changelog')) + + +def getVersion(package, ref): + """Get the version of a given package at a particular ref.""" + return getChangelog(package, ref).get_version() + + +def validateBuild(pocket, package, commit): + """Given the parameters of a new build, validate that build. + + The checks this function performs vary based on whether or not the + pocket is configured with allow_backtracking. + + A build of a pocket without allow_backtracking set must be a + fast-forward of the previous revision, and the most recent version + in the changelog most be strictly greater than the version + currently in the repository. + + In all cases, this revision of the package can only have the same + version number as any other revision currently in the apt + repository if they have the same commit ID. + + If it's unspecified, it is assumed that pocket do not + allow_backtracking. + + If this build request fails validation, this function will raise a + InvalidBuild exception, with information about why the validation + failed. + + If this build request can be satisfied by copying the package from + another pocket, then this function returns that pocket. Otherwise, + it returns True. + """ + package_repo = getRepo(package) + new_version = getVersion(package, commit) + + for p in config.git.pockets: + if p == pocket: + continue + + b = pocketToGit(p) + current_commit = c.captureOutput(['git', 'rev-parse', b], + cwd=package_repo) + current_version = getVersion(package, b) + + if current_version == new_version: + if current_commit == commit: + return p + else: + raise InvalidBuild('Version %s of %s already available in ' + 'pocket %s from commit %s' % + (new_version, package, p, current_commit)) + + if config.git.pockets[pocket].get('allow_backtracking', False): + branch = pocketToGit(pocket) + current_version = getVersion(package, branch) + if new_version <= current_version: + raise InvalidBuild('New version %s of %s is not newer than ' + 'version %s currently in pocket %s' % + (new_version, package, current_version, pocket)) + + # Almost by definition, A is a fast-forward of B if B..A is + # empty + if not c.captureOutput(['git', 'rev-list', '%s..%s' % (commit, branch)]): + raise InvalidBuild('New commit %s of %s is not a fast-forward of' + 'commit currently in pocket %s' % + (commit, package, pocket)) + +