Use the expanded commit name internally
[invirt/packages/invirt-dev.git] / invirtibuilder
index 716d7d7..24861f5 100755 (executable)
@@ -7,7 +7,7 @@ attempts to build a particular package.
 
 If the build succeeds, the new version of the package is uploaded to
 the apt repository, tagged in its git repository, and the Invirt
-superrepo is updated to point at the new version.
+superproject is updated to point at the new version.
 
 If the build fails, the Invirtibuilder sends mail with the build log.
 
@@ -20,96 +20,36 @@ Each queue file contains a file of the form
     pocket package hash principal
 
 where pocket is one of the pockets globally configured in
-git.pockets. For instance, the pockets in XVM are "prod" and "dev".
+build.pockets. For instance, the pockets in XVM are "prod" and "dev".
 
 principal is the Kerberos principal that requested the build.
 """
 
 
+from __future__ import with_statement
+
 import contextlib
+import glob
 import os
 import re
 import shutil
 import subprocess
+import tempfile
+import traceback
 
 import pyinotify
 
-from invirt.config import structs as config
-from invirt import database
-
+from debian_bundle import deb822
 
-_QUEUE_DIR = '/var/lib/invirt-dev/queue'
-_REPO_DIR = '/srv/git'
-_LOG_DIR = '/var/log/invirt/builds'
-_HOOKS_DIR = '/usr/share/invirt-dev/build.d'
+import invirt.builder as b
+import invirt.common as c
+from invirt import database
+from invirt.config import structs as config
 
 
 DISTRIBUTION = 'hardy'
 
 
-class InvalidBuild(ValueError):
-    pass
-
-
-def captureOutput(popen_args, stdin_str=None, *args, **kwargs):
-    """Capture stdout from a command.
-
-    This method will proxy the arguments to subprocess.Popen. It
-    returns the output from the command if the call succeeded and
-    raises an exception if the process returns a non-0 value.
-
-    This is intended to be a variant on the subprocess.check_call
-    function that also allows you access to the output from the
-    command.
-    """
-    if 'stdin' not in kwargs:
-        kwargs['stdin'] = subprocess.PIPE
-    if 'stdout' not in kwargs:
-        kwargs['stdout'] = subprocess.PIPE
-    if 'stderr' not in kwargs:
-        kwargs['stderr'] = subprocess.STDOUT
-    p = subprocess.Popen(popen_args, *args, **kwargs)
-    out, _ = p.communicate(stdin_str)
-    if p.returncode:
-        raise subprocess.CalledProcessError(p.returncode, popen_args, out)
-    return out
-
-
-def getRepo(package):
-    """Return the path to the git repo for a given package."""
-    return os.path.join(_REPO_DIR, 'packages', '%s.git' % package)
-
-
-def pocketToGit(pocket):
-    """Map a pocket in the configuration to a git branch."""
-    return config.git.pockets[pocket].get('git', pocket)
-
-
-def pocketToApt(pocket):
-    """Map a pocket in the configuration to an apt repo pocket."""
-    return config.git.pockets[pocket].get('apt', pocket)
-
-
-def getGitFile(package, ref, path):
-    """Return the contents of a path from a git ref in a package."""
-    return captureOutput(['git', 'cat-file', 'blob', '%s:%s' % (ref, path)],
-                         cwd=getRepo(package))
-
-
-def getChangelog(package, ref):
-    """Get a changelog object for a given ref in a given package.
-
-    This returns a debian_bundle.changelog.Changelog object for a
-    given ref of a given package.
-    """
-    return changelog.Changelog(getGitFile(package, ref, 'debian/changelog'))
-
-
-def getVersion(package, ref):
-    """Get the version of a given package at a particular ref."""
-    return getChangelog(package, ref).get_version()
-
-
 def getControl(package, ref):
     """Get the parsed debian/control file for a given package.
 
@@ -118,7 +58,7 @@ def getControl(package, ref):
     acts roughly like a dict.
     """
     return deb822.Deb822.iter_paragraphs(
-        getGitFile(package, ref, 'debian/control').split('\n'))
+        b.getGitFile(package, ref, 'debian/control').split('\n'))
 
 
 def getBinaries(package, ref):
@@ -138,73 +78,15 @@ def getArches(package, ref):
 
 def getDscName(package, ref):
     """Return the .dsc file that will be generated for this package."""
-    v = getVersion(package, ref)
-    return '%s_%s-%s.dsc' % (
+    v = b.getVersion(package, ref)
+    if v.debian_version:
+        v_str = '%s-%s' % (v.upstream_version,
+                           v.debian_version)
+    else:
+        v_str = v.upstream_version
+    return '%s_%s.dsc' % (
         package,
-        version.upstream_version,
-        version.debian_version)
-
-
-def validateBuild(pocket, package, commit):
-    """Given the parameters of a new build, validate that build.
-
-    The checks this function performs vary based on whether or not the
-    pocket is configured with allow_backtracking.
-
-    A build of a pocket without allow_backtracking set must be a
-    fast-forward of the previous revision, and the most recent version
-    in the changelog most be strictly greater than the version
-    currently in the repository.
-
-    In all cases, this revision of the package can only have the same
-    version number as any other revision currently in the apt
-    repository if they have the same commit ID.
-
-    If it's unspecified, it is assumed that pocket do not
-    allow_backtracking.
-
-    If this build request fails validation, this function will raise a
-    InvalidBuild exception, with information about why the validation
-    failed.
-
-    If this build request can be satisfied by copying the package from
-    another pocket, then this function returns that pocket. Otherwise,
-    it returns True.
-    """
-    package_repo = getRepo(package)
-    new_version = getVersion(package, commit)
-
-    for p in config.git.pockets:
-        if p == pocket:
-            continue
-
-        b = pocketToGit(p)
-        current_commit = captureOutput(['git', 'rev-parse', b],
-                                       cwd=package_repo)
-        current_version = getVersion(package, b)
-
-        if current_version == new_version:
-            if current_commit == commit:
-                return p
-            else:
-                raise InvalidBuild('Version %s of %s already available in '
-                                   'pocket %s from commit %s' %
-                                   (new_version, package, p, current_commit))
-
-    if config.git.pockets[pocket].get('allow_backtracking', False):
-        branch = pocketToGit(pocket)
-        current_version = getVersion(package, branch)
-        if new_version <= current_version:
-            raise InvalidBuild('New version %s of %s is not newer than '
-                               'version %s currently in pocket %s' %
-                               (new_version, package, current_version, pocket))
-
-        # Almost by definition, A is a fast-forward of B if B..A is
-        # empty
-        if not captureOutput(['git', 'rev-list', '%s..%s' % (commit, branch)]):
-            raise InvalidBuild('New commit %s of %s is not a fast-forward of'
-                               'commit currently in pocket %s' %
-                               (commit, package, pocket))
+        v_str)
 
 
 def sanitizeVersion(version):
@@ -212,32 +94,30 @@ def sanitizeVersion(version):
 
     This function strips the epoch from the version number and
     replaces any tildes with periods."""
-    v = '%s-%s' % (version.upstream_version,
-                   version.debian_version)
+    if v.debian_version:
+        v = '%s-%s' % (version.upstream_version,
+                       version.debian_version)
+    else:
+        v = version.upstream_version
     return v.replace('~', '.')
 
 
-def aptCopy(packages, dst_pocket, src_pocket):
+def aptCopy(package, commit, dst_pocket, src_pocket):
     """Copy a package from one pocket to another."""
-    binaries = []
-    for line in getGitFile(package, commit, 'debian/control').split('\n'):
-        m = re.match('Package: (.*)$')
-        if m:
-            binaries.append(m.group(1))
-
-    cpatureOutput(['reprepro-env', 'copy',
-                   pocketToApt(dst_pocket),
-                   pocketToApt(src_pocket),
-                   package] + binaries)
+    binaries = getBinaries(package, commit)
+    c.captureOutput(['reprepro-env', 'copy',
+                     b.pocketToApt(dst_pocket),
+                     b.pocketToApt(src_pocket),
+                     package] + binaries)
 
 
 def sbuild(package, ref, arch, workdir, arch_all=False):
     """Build a package for a particular architecture."""
-    args = ['sbuild', '-d', DISTRIBUTION, '--arch', arch]
+    args = ['sbuild', '-v', '-d', DISTRIBUTION, '--arch', arch]
     if arch_all:
         args.append('-A')
     args.append(getDscName(package, ref))
-    captureOutput(args, cwd=workdir, stdout=None)
+    c.captureOutput(args, cwd=workdir)
 
 
 def sbuildAll(package, ref, workdir):
@@ -249,7 +129,7 @@ def sbuildAll(package, ref, workdir):
         sbuild(package, ref, 'i386', workdir)
 
 
-def tagSubmodule(pocket, package, ref, principal):
+def tagSubmodule(pocket, package, principal, version, env):
     """Tag a new version of a submodule.
 
     If this pocket does not allow_backtracking, then this will create
@@ -261,22 +141,17 @@ def tagSubmodule(pocket, package, ref, principal):
     hook. Because we reject pushes to tags in the update hook, no push
     can ever take out a lock on any tags.
 
-    I'm sure that long description gives you great confidence in teh
+    I'm sure that long description gives you great confidence in the
     legitimacy of my reasoning.
     """
-    if config.git.pockets[pocket].get('allow_backtracking', False):
-        env = dict(os.environ)
-        branch = pocketToGit(pocket)
-        version = getVersion(package, ref)
-
-        env['GIT_COMMITTER_NAME'] = config.git.tagger.name
-        env['GIT_COMMITTER_EMAIL'] = config.git.tagger.email
+    if not config.build.pockets[pocket].get('allow_backtracking', False):
+        branch = b.pocketToGit(pocket)
         tag_msg = ('Tag %s of %s\n\n'
                    'Requested by %s' % (version.full_version,
                                         package,
                                         principal))
 
-        captureOutput(
+        c.captureOutput(
             ['git', 'tag', '-m', tag_msg, commit],
             stdout=None,
             env=env)
@@ -284,62 +159,65 @@ def tagSubmodule(pocket, package, ref, principal):
 
 def updateSubmoduleBranch(pocket, package, ref):
     """Update the appropriately named branch in the submodule."""
-    branch = pocketToGit(pocket)
-    captureOutput(
-        ['git', 'update-ref', 'refs/heads/%s' % branch, ref])
+    branch = b.pocketToGit(pocket)
+    c.captureOutput(
+        ['git', 'update-ref', 'refs/heads/%s' % branch, ref], cwd=b.getRepo(package))
 
 
 def uploadBuild(pocket, workdir):
     """Upload all build products in the work directory."""
-    apt = pocketToApt(pocket)
+    apt = b.pocketToApt(pocket)
     for changes in glob.glob(os.path.join(workdir, '*.changes')):
-        captureOutput(['reprepro-env',
-                       'include',
+        c.captureOutput(['reprepro-env',
                        '--ignore=wrongdistribution',
+                       'include',
                        apt,
                        changes])
 
 
-def updateSuperrepo(pocket, package, commit, principal):
-    """Update the superrepo.
+def updateSuperproject(pocket, package, commit, principal, version, env):
+    """Update the superproject.
 
     This will create a new commit on the branch for the given pocket
     that sets the commit for the package submodule to commit.
 
     Note that there's no locking issue here, because we disallow all
-    pushes to the superrepo.
+    pushes to the superproject.
     """
-    superrepo = os.path.join(_REPO_DIR, 'packages.git')
-    branch = pocketToGit(pocket)
-    tree = captureOutput(['git', 'ls-tree', branch],
-                         cwd=superrepo)
+    superproject = os.path.join(b._REPO_DIR, 'invirt/packages.git')
+    branch = b.pocketToGit(pocket)
+    tree = c.captureOutput(['git', 'ls-tree', branch],
+                           cwd=superproject).strip()
 
     new_tree = re.compile(
         r'^(160000 commit )[0-9a-f]*(\t%s)$' % package, re.M).sub(
-        r'\1%s\2' % commit,
+        r'\g<1>%s\g<2>' % commit,
         tree)
 
-    new_tree_id = captureOutput(['git', 'mktree'],
-                                cwd=superrepo,
-                                stdin_str=new_tree)
+    new_tree_id = c.captureOutput(['git', 'mktree', '--missing'],
+                                  cwd=superproject,
+                                  stdin_str=new_tree).strip()
 
     commit_msg = ('Update %s to version %s\n\n'
                   'Requested by %s' % (package,
                                        version.full_version,
                                        principal))
-    new_commit = captureOutput(
-        ['git', 'commit-tree', new_tree_hash, '-p', branch],
-        cwd=superrepo,
+    new_commit = c.captureOutput(
+        ['git', 'commit-tree', new_tree_id, '-p', branch],
+        cwd=superproject,
         env=env,
-        stdin_str=commit_msg)
+        stdin_str=commit_msg).strip()
 
-    captureOutput(
+    c.captureOutput(
         ['git', 'update-ref', 'refs/heads/%s' % branch, new_commit],
-        cwd=superrepo)
+        cwd=superproject)
 
 
+def makeReadable(workdir):
+    os.chmod(workdir, 0755)
+
 @contextlib.contextmanager
-def packageWorkdir(package):
+def packageWorkdir(package, commit):
     """Checkout the package in a temporary working directory.
 
     This context manager returns that working directory. The requested
@@ -353,8 +231,8 @@ def packageWorkdir(package):
     try:
         p_archive = subprocess.Popen(
             ['git', 'archive',
-             '--remote=file://%s' % getRepo(package),
-             '--prefix=%s' % package,
+             '--remote=file://%s' % b.getRepo(package),
+             '--prefix=%s/' % package,
              commit,
              ],
             stdout=subprocess.PIPE,
@@ -375,10 +253,10 @@ def packageWorkdir(package):
 def reportBuild(build):
     """Run hooks to report the results of a build attempt."""
 
-    captureOutput(['run-parts',
+    c.captureOutput(['run-parts',
                    '--arg=%s' % build.build_id,
                    '--',
-                   _HOOKS_DIR])
+                   b._HOOKS_DIR])
 
 
 def build():
@@ -389,12 +267,12 @@ def build():
     """
     while True:
         stage = 'processing incoming job'
-        queue = os.listdir(_QUEUE_DIR)
+        queue = os.listdir(b._QUEUE_DIR)
         if not queue:
             break
 
         build = min(queue)
-        job = open(os.path.join(_QUEUE_DIR, build)).read().strip()
+        job = open(os.path.join(b._QUEUE_DIR, build)).read().strip()
         pocket, package, commit, principal = job.split()
 
         database.session.begin()
@@ -404,15 +282,19 @@ def build():
         db.commit = commit
         db.principal = principal
         database.session.save_or_update(db)
-        database.commit()
+        database.session.commit()
 
-        database.begin()
+        database.session.begin()
 
         try:
             db.failed_stage = 'validating job'
-            src = validateBuild(pocket, package, commit)
+            # Don't expand the commit in the DB until we're sure the user
+            # isn't trying to be tricky.
+            b.ensureValidPackage(package)
+            db.commit = commit = b.canonicalize_commit(package, commit)
+            src = b.validateBuild(pocket, package, commit)
 
-            db.version = str(getVersion(package, commit))
+            db.version = str(b.getVersion(package, commit))
 
             # If validateBuild returns something other than True, then
             # it means we should copy from that pocket to our pocket.
@@ -421,13 +303,13 @@ def build():
             # raised an exception)
             if src != True:
                 db.failed_stage = 'copying package from another pocket'
-                aptCopy(packages, pocket, src)
+                aptCopy(package, commit, pocket, src)
             # If we can't copy the package from somewhere, but
             # validateBuild didn't raise an exception, then we need to
             # do the build ourselves
             else:
                 db.failed_stage = 'checking out package source'
-                with packageWorkdir(package) as workdir:
+                with packageWorkdir(package, commit) as workdir:
                     db.failed_stage = 'preparing source package'
                     packagedir = os.path.join(workdir, package)
 
@@ -438,7 +320,7 @@ def build():
                     # If we were, we could use debuild and get nice
                     # environment scrubbing. Since we're not, debuild
                     # complains about not having an orig.tar.gz
-                    captureOutput(['dpkg-buildpackage', '-us', '-uc', '-S'],
+                    c.captureOutput(['dpkg-buildpackage', '-us', '-uc', '-S'],
                                   cwd=packagedir,
                                   stdout=None)
 
@@ -446,26 +328,31 @@ def build():
                         db.failed_stage = 'building binary packages'
                         sbuildAll(package, commit, workdir)
                     finally:
-                        logdir = os.path.join(_LOG_DIR, db.build_id)
+                        logdir = os.path.join(b._LOG_DIR, str(db.build_id))
                         if not os.path.exists(logdir):
                             os.makedirs(logdir)
 
-                        for log in glob.glob(os.path.join(workdir, '*.build')):
-                            os.copy2(log, logdir)
+                        for log in glob.glob(os.path.join(workdir, 'build-*.log')):
+                            os.copy(log, logdir)
+
+                    db.failed_stage = 'processing metadata'
+                    env = dict(os.environ)
+                    env['GIT_COMMITTER_NAME'] = config.build.tagger.name
+                    env['GIT_COMMITTER_EMAIL'] = config.build.tagger.email
+                    version = b.getVersion(package, commit)
+
                     db.failed_stage = 'tagging submodule'
-                    tagSubmodule(pocket, package, commit, principal)
+                    tagSubmodule(pocket, package, principal, version, env)
                     db.failed_stage = 'updating submodule branches'
                     updateSubmoduleBranch(pocket, package, commit)
-                    db.failed_stage = 'updating superrepo'
-                    updateSuperrepo(pocket, package, commit, principal)
+                    db.failed_stage = 'updating superproject'
+                    updateSuperproject(pocket, package, commit, principal, version, env)
+                    db.failed_stage = 'relaxing permissions on workdir'
+                    makeReadable(workdir)
                     db.failed_stage = 'uploading packages to apt repo'
                     uploadBuild(pocket, workdir)
 
                     db.failed_stage = 'cleaning up'
-
-                # Finally, now that everything is done, remove the
-                # build queue item
-                os.unlink(os.path.join(_QUEUE_DIR, build))
         except:
             db.traceback = traceback.format_exc()
         else:
@@ -475,15 +362,19 @@ def build():
             database.session.save_or_update(db)
             database.session.commit()
 
+            # Finally, now that everything is done, remove the
+            # build queue item
+            os.unlink(os.path.join(b._QUEUE_DIR, build))
+
             reportBuild(db)
 
 
 class Invirtibuilder(pyinotify.ProcessEvent):
     """Process inotify triggers to build new packages."""
-    def process_IN_CREATE(self, event):
-        """Handle a created file or directory.
+    def process_default(self, event):
+        """Handle an inotify event.
 
-        When an IN_CREATE event comes in, trigger the builder.
+        When an inotify event comes in, trigger the builder.
         """
         build()
 
@@ -495,8 +386,9 @@ def main():
     watch_manager = pyinotify.WatchManager()
     invirtibuilder = Invirtibuilder()
     notifier = pyinotify.Notifier(watch_manager, invirtibuilder)
-    watch_manager.add_watch(_QUEUE_DIR,
-                            pyinotify.EventsCodes.ALL_FLAGS['IN_CREATE'])
+    watch_manager.add_watch(b._QUEUE_DIR,
+                            pyinotify.EventsCodes.ALL_FLAGS['IN_CREATE'] |
+                            pyinotify.EventsCodes.ALL_FLAGS['IN_MOVED_TO'])
 
     # Before inotifying, run any pending builds; otherwise we won't
     # get notified for them.