--- /dev/null
+#!/usr/bin/python
+
+"""Process the Invirt build queue.
+
+The Invirtibuilder handles package builds and uploads. On demand, it
+attempts to build a particular package.
+
+If the build succeeds, the new version of the package is uploaded to
+the apt repository, tagged in its git repository, and the Invirt
+superrepo is updated to point at the new version.
+
+If the build fails, the Invirtibuilder sends mail with the build log.
+
+The build queue is tracked via files in /var/lib/invirt-dev/queue. In
+order to maintain ordering, all filenames in that directory are the
+timestamp of their creation time.
+
+Each queue file contains a file of the form
+
+ pocket package hash principal
+
+where pocket is one of the pockets globally configured in
+git.pockets. For instance, the pockets in XVM are "prod" and "dev".
+
+principal is the Kerberos principal that requested the build.
+"""
+
+
+import contextlib
+import os
+import re
+import shutil
+import subprocess
+
+import pyinotify
+
+from invirt.config import structs as config
+from invirt import database
+
+
+_QUEUE_DIR = '/var/lib/invirt-dev/queue'
+_REPO_DIR = '/srv/git'
+_LOG_DIR = '/var/log/invirt/builds'
+_HOOKS_DIR = '/usr/share/invirt-dev/build.d'
+
+
+DISTRIBUTION = 'hardy'
+
+
+class InvalidBuild(ValueError):
+ pass
+
+
+def captureOutput(popen_args, stdin_str=None, *args, **kwargs):
+ """Capture stdout from a command.
+
+ This method will proxy the arguments to subprocess.Popen. It
+ returns the output from the command if the call succeeded and
+ raises an exception if the process returns a non-0 value.
+
+ This is intended to be a variant on the subprocess.check_call
+ function that also allows you access to the output from the
+ command.
+ """
+ if 'stdin' not in kwargs:
+ kwargs['stdin'] = subprocess.PIPE
+ if 'stdout' not in kwargs:
+ kwargs['stdout'] = subprocess.PIPE
+ if 'stderr' not in kwargs:
+ kwargs['stderr'] = subprocess.STDOUT
+ p = subprocess.Popen(popen_args, *args, **kwargs)
+ out, _ = p.communicate(stdin_str)
+ if p.returncode:
+ raise subprocess.CalledProcessError(p.returncode, popen_args, out)
+ return out
+
+
+def getRepo(package):
+ """Return the path to the git repo for a given package."""
+ return os.path.join(_REPO_DIR, 'packages', '%s.git' % package)
+
+
+def pocketToGit(pocket):
+ """Map a pocket in the configuration to a git branch."""
+ return config.git.pockets[pocket].get('git', pocket)
+
+
+def pocketToApt(pocket):
+ """Map a pocket in the configuration to an apt repo pocket."""
+ return config.git.pockets[pocket].get('apt', pocket)
+
+
+def getGitFile(package, ref, path):
+ """Return the contents of a path from a git ref in a package."""
+ return captureOutput(['git', 'cat-file', 'blob', '%s:%s' % (ref, path)],
+ cwd=getRepo(package))
+
+
+def getChangelog(package, ref):
+ """Get a changelog object for a given ref in a given package.
+
+ This returns a debian_bundle.changelog.Changelog object for a
+ given ref of a given package.
+ """
+ return changelog.Changelog(getGitFile(package, ref, 'debian/changelog'))
+
+
+def getVersion(package, ref):
+ """Get the version of a given package at a particular ref."""
+ return getChangelog(package, ref).get_version()
+
+
+def getControl(package, ref):
+ """Get the parsed debian/control file for a given package.
+
+ This returns a list of debian_bundle.deb822.Deb822 objects, one
+ for each section of the debian/control file. Each Deb822 object
+ acts roughly like a dict.
+ """
+ return deb822.Deb822.iter_paragraphs(
+ getGitFile(package, ref, 'debian/control').split('\n'))
+
+
+def getBinaries(package, ref):
+ """Get a list of binary packages in a package at a given ref."""
+ return [p['Package'] for p in getControl(package, ref)
+ if 'Package' in p]
+
+
+def getArches(package, ref):
+ """Get the set of all architectures in any binary package."""
+ arches = set()
+ for section in getControl(package, ref):
+ if 'Architecture' in section:
+ arches.update(section['Architecture'].split())
+ return arches
+
+
+def getDscName(package, ref):
+ """Return the .dsc file that will be generated for this package."""
+ v = getVersion(package, ref)
+ return '%s_%s-%s.dsc' % (
+ package,
+ version.upstream_version,
+ version.debian_version)
+
+
+def validateBuild(pocket, package, commit):
+ """Given the parameters of a new build, validate that build.
+
+ The checks this function performs vary based on whether or not the
+ pocket is configured with allow_backtracking.
+
+ A build of a pocket without allow_backtracking set must be a
+ fast-forward of the previous revision, and the most recent version
+ in the changelog most be strictly greater than the version
+ currently in the repository.
+
+ In all cases, this revision of the package can only have the same
+ version number as any other revision currently in the apt
+ repository if they have the same commit ID.
+
+ If it's unspecified, it is assumed that pocket do not
+ allow_backtracking.
+
+ If this build request fails validation, this function will raise a
+ InvalidBuild exception, with information about why the validation
+ failed.
+
+ If this build request can be satisfied by copying the package from
+ another pocket, then this function returns that pocket. Otherwise,
+ it returns True.
+ """
+ package_repo = getRepo(package)
+ new_version = getVersion(package, commit)
+
+ for p in config.git.pockets:
+ if p == pocket:
+ continue
+
+ b = pocketToGit(p)
+ current_commit = captureOutput(['git', 'rev-parse', b],
+ cwd=package_repo)
+ current_version = getVersion(package, b)
+
+ if current_version == new_version:
+ if current_commit == commit:
+ return p
+ else:
+ raise InvalidBuild('Version %s of %s already available in '
+ 'pocket %s from commit %s' %
+ (new_version, package, p, current_commit))
+
+ if config.git.pockets[pocket].get('allow_backtracking', False):
+ branch = pocketToGit(pocket)
+ current_version = getVersion(package, branch)
+ if new_version <= current_version:
+ raise InvalidBuild('New version %s of %s is not newer than '
+ 'version %s currently in pocket %s' %
+ (new_version, package, current_version, pocket))
+
+ # Almost by definition, A is a fast-forward of B if B..A is
+ # empty
+ if not captureOutput(['git', 'rev-list', '%s..%s' % (commit, branch)]):
+ raise InvalidBuild('New commit %s of %s is not a fast-forward of'
+ 'commit currently in pocket %s' %
+ (commit, package, pocket))
+
+
+def sanitizeVersion(version):
+ """Sanitize a Debian package version for use as a git tag.
+
+ This function strips the epoch from the version number and
+ replaces any tildes with periods."""
+ v = '%s-%s' % (version.upstream_version,
+ version.debian_version)
+ return v.replace('~', '.')
+
+
+def aptCopy(packages, dst_pocket, src_pocket):
+ """Copy a package from one pocket to another."""
+ binaries = []
+ for line in getGitFile(package, commit, 'debian/control').split('\n'):
+ m = re.match('Package: (.*)$')
+ if m:
+ binaries.append(m.group(1))
+
+ cpatureOutput(['reprepro-env', 'copy',
+ pocketToApt(dst_pocket),
+ pocketToApt(src_pocket),
+ package] + binaries)
+
+
+def sbuild(package, ref, arch, workdir, arch_all=False):
+ """Build a package for a particular architecture."""
+ args = ['sbuild', '-d', DISTRIBUTION, '--arch', arch]
+ if arch_all:
+ args.append('-A')
+ args.append(getDscName(package, ref))
+ captureOutput(args, cwd=workdir, stdout=None)
+
+
+def sbuildAll(package, ref, workdir):
+ """Build a package for all architectures it supports."""
+ arches = getArches(package, ref)
+ if 'all' in arches or 'any' in arches or 'amd64' in arches:
+ sbuild(package, ref, 'amd64', workdir, arch_all=True)
+ if 'any' in arches or 'i386' in arches:
+ sbuild(package, ref, 'i386', workdir)
+
+
+def tagSubmodule(pocket, package, ref, principal):
+ """Tag a new version of a submodule.
+
+ If this pocket does not allow_backtracking, then this will create
+ a new tag of the version at ref.
+
+ This function doesn't need to care about lock
+ contention. git-receive-pack updates one ref at a time, and only
+ takes out a lock for that ref after it's passed the update
+ hook. Because we reject pushes to tags in the update hook, no push
+ can ever take out a lock on any tags.
+
+ I'm sure that long description gives you great confidence in teh
+ legitimacy of my reasoning.
+ """
+ if config.git.pockets[pocket].get('allow_backtracking', False):
+ env = dict(os.environ)
+ branch = pocketToGit(pocket)
+ version = getVersion(package, ref)
+
+ env['GIT_COMMITTER_NAME'] = config.git.tagger.name
+ env['GIT_COMMITTER_EMAIL'] = config.git.tagger.email
+ tag_msg = ('Tag %s of %s\n\n'
+ 'Requested by %s' % (version.full_version,
+ package,
+ principal))
+
+ captureOutput(
+ ['git', 'tag', '-m', tag_msg, commit],
+ stdout=None,
+ env=env)
+
+
+def updateSubmoduleBranch(pocket, package, ref):
+ """Update the appropriately named branch in the submodule."""
+ branch = pocketToGit(pocket)
+ captureOutput(
+ ['git', 'update-ref', 'refs/heads/%s' % branch, ref])
+
+
+def uploadBuild(pocket, workdir):
+ """Upload all build products in the work directory."""
+ apt = pocketToApt(pocket)
+ for changes in glob.glob(os.path.join(workdir, '*.changes')):
+ captureOutput(['reprepro-env',
+ 'include',
+ '--ignore=wrongdistribution',
+ apt,
+ changes])
+
+
+def updateSuperrepo(pocket, package, commit, principal):
+ """Update the superrepo.
+
+ This will create a new commit on the branch for the given pocket
+ that sets the commit for the package submodule to commit.
+
+ Note that there's no locking issue here, because we disallow all
+ pushes to the superrepo.
+ """
+ superrepo = os.path.join(_REPO_DIR, 'packages.git')
+ branch = pocketToGit(pocket)
+ tree = captureOutput(['git', 'ls-tree', branch],
+ cwd=superrepo)
+
+ new_tree = re.compile(
+ r'^(160000 commit )[0-9a-f]*(\t%s)$' % package, re.M).sub(
+ r'\1%s\2' % commit,
+ tree)
+
+ new_tree_id = captureOutput(['git', 'mktree'],
+ cwd=superrepo,
+ stdin_str=new_tree)
+
+ commit_msg = ('Update %s to version %s\n\n'
+ 'Requested by %s' % (package,
+ version.full_version,
+ principal))
+ new_commit = captureOutput(
+ ['git', 'commit-tree', new_tree_hash, '-p', branch],
+ cwd=superrepo,
+ env=env,
+ stdin_str=commit_msg)
+
+ captureOutput(
+ ['git', 'update-ref', 'refs/heads/%s' % branch, new_commit],
+ cwd=superrepo)
+
+
+@contextlib.contextmanager
+def packageWorkdir(package):
+ """Checkout the package in a temporary working directory.
+
+ This context manager returns that working directory. The requested
+ package is checked out into a subdirectory of the working
+ directory with the same name as the package.
+
+ When the context wrapped with this context manager is exited, the
+ working directory is automatically deleted.
+ """
+ workdir = tempfile.mkdtemp()
+ try:
+ p_archive = subprocess.Popen(
+ ['git', 'archive',
+ '--remote=file://%s' % getRepo(package),
+ '--prefix=%s' % package,
+ commit,
+ ],
+ stdout=subprocess.PIPE,
+ )
+ p_tar = subprocess.Popen(
+ ['tar', '-x'],
+ stdin=p_archive.stdout,
+ cwd=workdir,
+ )
+ p_archive.wait()
+ p_tar.wait()
+
+ yield workdir
+ finally:
+ shutil.rmtree(workdir)
+
+
+def reportBuild(build):
+ """Run hooks to report the results of a build attempt."""
+
+ captureOutput(['run-parts',
+ '--arg=%s' % build.build_id,
+ '--',
+ _HOOKS_DIR])
+
+
+def build():
+ """Deal with items in the build queue.
+
+ When triggered, iterate over build queue items one at a time,
+ until there are no more pending build jobs.
+ """
+ while True:
+ stage = 'processing incoming job'
+ queue = os.listdir(_QUEUE_DIR)
+ if not queue:
+ break
+
+ build = min(queue)
+ job = open(os.path.join(_QUEUE_DIR, build)).read().strip()
+ pocket, package, commit, principal = job.split()
+
+ database.session.begin()
+ db = database.Build()
+ db.package = package
+ db.pocket = pocket
+ db.commit = commit
+ db.principal = principal
+ database.session.save_or_update(db)
+ database.commit()
+
+ database.begin()
+
+ try:
+ db.failed_stage = 'validating job'
+ src = validateBuild(pocket, package, commit)
+
+ db.version = str(getVersion(package, commit))
+
+ # If validateBuild returns something other than True, then
+ # it means we should copy from that pocket to our pocket.
+ #
+ # (If the validation failed, validateBuild would have
+ # raised an exception)
+ if src != True:
+ db.failed_stage = 'copying package from another pocket'
+ aptCopy(packages, pocket, src)
+ # If we can't copy the package from somewhere, but
+ # validateBuild didn't raise an exception, then we need to
+ # do the build ourselves
+ else:
+ db.failed_stage = 'checking out package source'
+ with packageWorkdir(package) as workdir:
+ db.failed_stage = 'preparing source package'
+ packagedir = os.path.join(workdir, package)
+
+ # We should be more clever about dealing with
+ # things like non-Debian-native packages than we
+ # are.
+ #
+ # If we were, we could use debuild and get nice
+ # environment scrubbing. Since we're not, debuild
+ # complains about not having an orig.tar.gz
+ captureOutput(['dpkg-buildpackage', '-us', '-uc', '-S'],
+ cwd=packagedir,
+ stdout=None)
+
+ try:
+ db.failed_stage = 'building binary packages'
+ sbuildAll(package, commit, workdir)
+ finally:
+ logdir = os.path.join(_LOG_DIR, db.build_id)
+ if not os.path.exists(logdir):
+ os.makedirs(logdir)
+
+ for log in glob.glob(os.path.join(workdir, '*.build')):
+ os.copy2(log, logdir)
+ db.failed_stage = 'tagging submodule'
+ tagSubmodule(pocket, package, commit, principal)
+ db.failed_stage = 'updating submodule branches'
+ updateSubmoduleBranch(pocket, package, commit)
+ db.failed_stage = 'updating superrepo'
+ updateSuperrepo(pocket, package, commit, principal)
+ db.failed_stage = 'uploading packages to apt repo'
+ uploadBuild(pocket, workdir)
+
+ db.failed_stage = 'cleaning up'
+
+ # Finally, now that everything is done, remove the
+ # build queue item
+ os.unlink(os.path.join(_QUEUE_DIR, build))
+ except:
+ db.traceback = traceback.format_exc()
+ else:
+ db.succeeded = True
+ db.failed_stage = None
+ finally:
+ database.session.save_or_update(db)
+ database.session.commit()
+
+ reportBuild(db)
+
+
+class Invirtibuilder(pyinotify.ProcessEvent):
+ """Process inotify triggers to build new packages."""
+ def process_IN_CREATE(self, event):
+ """Handle a created file or directory.
+
+ When an IN_CREATE event comes in, trigger the builder.
+ """
+ build()
+
+
+def main():
+ """Initialize the inotifications and start the main loop."""
+ database.connect()
+
+ watch_manager = pyinotify.WatchManager()
+ invirtibuilder = Invirtibuilder()
+ notifier = pyinotify.Notifier(watch_manager, invirtibuilder)
+ watch_manager.add_watch(_QUEUE_DIR,
+ pyinotify.EventsCodes.ALL_FLAGS['IN_CREATE'])
+
+ # Before inotifying, run any pending builds; otherwise we won't
+ # get notified for them.
+ build()
+
+ while True:
+ notifier.process_events()
+ if notifier.check_events():
+ notifier.read_events()
+
+
+if __name__ == '__main__':
+ main()
projects / invirt/packages/invirt-dev.git / commitdiff
