Log and capture sbuild results

[invirt/packages/invirt-dev.git] / invirtibuilder

diff --git a/invirtibuilder b/invirtibuilder
index f5cdfb4..aa70b53 100755 (executable)
--- a/invirtibuilder
+++ b/invirtibuilder
@@ -7,7 +7,7 @@ attempts to build a particular package.
 
 If the build succeeds, the new version of the package is uploaded to
 the apt repository, tagged in its git repository, and the Invirt
 
 If the build succeeds, the new version of the package is uploaded to
 the apt repository, tagged in its git repository, and the Invirt

-superrepo is updated to point at the new version.
+superproject is updated to point at the new version.

 
 If the build fails, the Invirtibuilder sends mail with the build log.
 
 
 If the build fails, the Invirtibuilder sends mail with the build log.
 

@@ -20,22 +20,31 @@ Each queue file contains a file of the form
     pocket package hash principal
 
 where pocket is one of the pockets globally configured in
     pocket package hash principal
 
 where pocket is one of the pockets globally configured in

-git.pockets. For instance, the pockets in XVM are "prod" and "dev".
+build.pockets. For instance, the pockets in XVM are "prod" and "dev".

 
 principal is the Kerberos principal that requested the build.
 """
 
 
 
 principal is the Kerberos principal that requested the build.
 """
 
 

+from __future__ import with_statement
+

 import contextlib
 import contextlib

+import glob

 import os
 import re
 import shutil
 import subprocess
 import os
 import re
 import shutil
 import subprocess

+import tempfile
+import traceback

 
 import pyinotify
 
 
 import pyinotify
 

+from debian_bundle import deb822
+

 import invirt.builder as b
 import invirt.builder as b

+import invirt.common as c

 from invirt import database
 from invirt import database

+from invirt.config import structs as config

 
 
 DISTRIBUTION = 'hardy'
 
 
 DISTRIBUTION = 'hardy'

@@ -49,7 +58,7 @@ def getControl(package, ref):
     acts roughly like a dict.
     """
     return deb822.Deb822.iter_paragraphs(
     acts roughly like a dict.
     """
     return deb822.Deb822.iter_paragraphs(

-        getGitFile(package, ref, 'debian/control').split('\n'))
+        b.getGitFile(package, ref, 'debian/control').split('\n'))

 
 
 def getBinaries(package, ref):
 
 
 def getBinaries(package, ref):

@@ -69,11 +78,15 @@ def getArches(package, ref):
 
 def getDscName(package, ref):
     """Return the .dsc file that will be generated for this package."""
 
 def getDscName(package, ref):
     """Return the .dsc file that will be generated for this package."""

-    v = getVersion(package, ref)
-    return '%s_%s-%s.dsc' % (
+    v = b.getVersion(package, ref)
+    if v.debian_version:
+        v_str = '%s-%s' % (v.upstream_version,
+                           v.debian_version)
+    else:
+        v_str = v.upstream_version
+    return '%s_%s.dsc' % (

         package,
         package,

-        version.upstream_version,
-        version.debian_version)
+        v_str)

 
 
 def sanitizeVersion(version):
 
 
 def sanitizeVersion(version):

@@ -81,32 +94,30 @@ def sanitizeVersion(version):
 
     This function strips the epoch from the version number and
     replaces any tildes with periods."""
 
     This function strips the epoch from the version number and
     replaces any tildes with periods."""

-    v = '%s-%s' % (version.upstream_version,
-                   version.debian_version)
+    if v.debian_version:
+        v = '%s-%s' % (version.upstream_version,
+                       version.debian_version)
+    else:
+        v = version.upstream_version

     return v.replace('~', '.')
 
 
     return v.replace('~', '.')
 
 

-def aptCopy(packages, dst_pocket, src_pocket):
+def aptCopy(package, commit, dst_pocket, src_pocket):

     """Copy a package from one pocket to another."""
     """Copy a package from one pocket to another."""

-    binaries = []
-    for line in b.getGitFile(package, commit, 'debian/control').split('\n'):
-        m = re.match('Package: (.*)$')
-        if m:
-            binaries.append(m.group(1))
-
-    cpatureOutput(['reprepro-env', 'copy',
-                   b.pocketToApt(dst_pocket),
-                   b.pocketToApt(src_pocket),
-                   package] + binaries)
+    binaries = getBinaries(package, commit)
+    c.captureOutput(['reprepro-env', 'copy',
+                     b.pocketToApt(dst_pocket),
+                     b.pocketToApt(src_pocket),
+                     package] + binaries)

 
 
 def sbuild(package, ref, arch, workdir, arch_all=False):
     """Build a package for a particular architecture."""
 
 
 def sbuild(package, ref, arch, workdir, arch_all=False):
     """Build a package for a particular architecture."""

-    args = ['sbuild', '-d', DISTRIBUTION, '--arch', arch]
+    args = ['sbuild', '-v', '-d', DISTRIBUTION, '--arch', arch]

     if arch_all:
         args.append('-A')
     args.append(getDscName(package, ref))
     if arch_all:
         args.append('-A')
     args.append(getDscName(package, ref))

-    c.captureOutput(args, cwd=workdir, stdout=None)
+    c.captureOutput(args, cwd=workdir)

 
 
 def sbuildAll(package, ref, workdir):
 
 
 def sbuildAll(package, ref, workdir):

@@ -118,7 +129,7 @@ def sbuildAll(package, ref, workdir):
         sbuild(package, ref, 'i386', workdir)
 
 
         sbuild(package, ref, 'i386', workdir)
 
 

-def tagSubmodule(pocket, package, ref, principal):
+def tagSubmodule(pocket, package, principal, version, env):

     """Tag a new version of a submodule.
 
     If this pocket does not allow_backtracking, then this will create
     """Tag a new version of a submodule.
 
     If this pocket does not allow_backtracking, then this will create

@@ -130,16 +141,11 @@ def tagSubmodule(pocket, package, ref, principal):
     hook. Because we reject pushes to tags in the update hook, no push
     can ever take out a lock on any tags.
 
     hook. Because we reject pushes to tags in the update hook, no push
     can ever take out a lock on any tags.
 

-    I'm sure that long description gives you great confidence in teh
+    I'm sure that long description gives you great confidence in the

     legitimacy of my reasoning.
     """
     legitimacy of my reasoning.
     """

-    if config.git.pockets[pocket].get('allow_backtracking', False):
-        env = dict(os.environ)
+    if not config.build.pockets[pocket].get('allow_backtracking', False):

         branch = b.pocketToGit(pocket)
         branch = b.pocketToGit(pocket)

-        version = b.getVersion(package, ref)
-
-        env['GIT_COMMITTER_NAME'] = config.git.tagger.name
-        env['GIT_COMMITTER_EMAIL'] = config.git.tagger.email

         tag_msg = ('Tag %s of %s\n\n'
                    'Requested by %s' % (version.full_version,
                                         package,
         tag_msg = ('Tag %s of %s\n\n'
                    'Requested by %s' % (version.full_version,
                                         package,

@@ -155,7 +161,7 @@ def updateSubmoduleBranch(pocket, package, ref):
     """Update the appropriately named branch in the submodule."""
     branch = b.pocketToGit(pocket)
     c.captureOutput(
     """Update the appropriately named branch in the submodule."""
     branch = b.pocketToGit(pocket)
     c.captureOutput(

-        ['git', 'update-ref', 'refs/heads/%s' % branch, ref])
+        ['git', 'update-ref', 'refs/heads/%s' % branch, ref], cwd=b.getRepo(package))

 
 
 def uploadBuild(pocket, workdir):
 
 
 def uploadBuild(pocket, workdir):

@@ -163,52 +169,55 @@ def uploadBuild(pocket, workdir):
     apt = b.pocketToApt(pocket)
     for changes in glob.glob(os.path.join(workdir, '*.changes')):
         c.captureOutput(['reprepro-env',
     apt = b.pocketToApt(pocket)
     for changes in glob.glob(os.path.join(workdir, '*.changes')):
         c.captureOutput(['reprepro-env',

-                       'include',

                        '--ignore=wrongdistribution',
                        '--ignore=wrongdistribution',

+                       'include',

                        apt,
                        changes])
 
 
                        apt,
                        changes])
 
 

-def updateSuperrepo(pocket, package, commit, principal):
-    """Update the superrepo.
+def updateSuperproject(pocket, package, commit, principal, version, env):
+    """Update the superproject.

 
     This will create a new commit on the branch for the given pocket
     that sets the commit for the package submodule to commit.
 
     Note that there's no locking issue here, because we disallow all
 
     This will create a new commit on the branch for the given pocket
     that sets the commit for the package submodule to commit.
 
     Note that there's no locking issue here, because we disallow all

-    pushes to the superrepo.
+    pushes to the superproject.

     """
     """

-    superrepo = os.path.join(b._REPO_DIR, 'packages.git')
+    superproject = os.path.join(b._REPO_DIR, 'invirt/packages.git')

     branch = b.pocketToGit(pocket)
     tree = c.captureOutput(['git', 'ls-tree', branch],
     branch = b.pocketToGit(pocket)
     tree = c.captureOutput(['git', 'ls-tree', branch],

-                         cwd=superrepo)
+                           cwd=superproject).strip()

 
     new_tree = re.compile(
         r'^(160000 commit )[0-9a-f]*(\t%s)$' % package, re.M).sub(
 
     new_tree = re.compile(
         r'^(160000 commit )[0-9a-f]*(\t%s)$' % package, re.M).sub(

-        r'\1%s\2' % commit,
+        r'\g<1>%s\g<2>' % commit,

         tree)
 
         tree)
 

-    new_tree_id = c.captureOutput(['git', 'mktree'],
-                                cwd=superrepo,
-                                stdin_str=new_tree)
+    new_tree_id = c.captureOutput(['git', 'mktree', '--missing'],
+                                  cwd=superproject,
+                                  stdin_str=new_tree).strip()

 
     commit_msg = ('Update %s to version %s\n\n'
                   'Requested by %s' % (package,
                                        version.full_version,
                                        principal))
     new_commit = c.captureOutput(
 
     commit_msg = ('Update %s to version %s\n\n'
                   'Requested by %s' % (package,
                                        version.full_version,
                                        principal))
     new_commit = c.captureOutput(

-        ['git', 'commit-tree', new_tree_hash, '-p', branch],
-        cwd=superrepo,
+        ['git', 'commit-tree', new_tree_id, '-p', branch],
+        cwd=superproject,

         env=env,
         env=env,

-        stdin_str=commit_msg)
+        stdin_str=commit_msg).strip()

 
     c.captureOutput(
         ['git', 'update-ref', 'refs/heads/%s' % branch, new_commit],
 
     c.captureOutput(
         ['git', 'update-ref', 'refs/heads/%s' % branch, new_commit],

-        cwd=superrepo)
+        cwd=superproject)

 
 
 
 

+def makeReadable(workdir):
+    os.chmod(workdir, 0755)
+

 @contextlib.contextmanager
 @contextlib.contextmanager

-def packageWorkdir(package):
+def packageWorkdir(package, commit):

     """Checkout the package in a temporary working directory.
 
     This context manager returns that working directory. The requested
     """Checkout the package in a temporary working directory.
 
     This context manager returns that working directory. The requested

@@ -223,7 +232,7 @@ def packageWorkdir(package):
         p_archive = subprocess.Popen(
             ['git', 'archive',
              '--remote=file://%s' % b.getRepo(package),
         p_archive = subprocess.Popen(
             ['git', 'archive',
              '--remote=file://%s' % b.getRepo(package),

-             '--prefix=%s' % package,
+             '--prefix=%s/' % package,

              commit,
              ],
             stdout=subprocess.PIPE,
              commit,
              ],
             stdout=subprocess.PIPE,

@@ -273,13 +282,17 @@ def build():
         db.commit = commit
         db.principal = principal
         database.session.save_or_update(db)
         db.commit = commit
         db.principal = principal
         database.session.save_or_update(db)

-        database.commit()
+        database.session.commit()

 
 

-        database.begin()
+        database.session.begin()

 
         try:
             db.failed_stage = 'validating job'
 
         try:
             db.failed_stage = 'validating job'

-            src = validateBuild(pocket, package, commit)
+            src = b.validateBuild(pocket, package, commit)
+            # Don't expand the commit in the DB until we're sure the user
+            # isn't trying to be tricky.
+            db.commit = commit = c.captureOutput(['git', 'rev-parse', commit],
+                                                 cwd=b.getRepo(package)).strip()

 
             db.version = str(b.getVersion(package, commit))
 
 
             db.version = str(b.getVersion(package, commit))
 

@@ -290,13 +303,13 @@ def build():
             # raised an exception)
             if src != True:
                 db.failed_stage = 'copying package from another pocket'
             # raised an exception)
             if src != True:
                 db.failed_stage = 'copying package from another pocket'

-                aptCopy(packages, pocket, src)
+                aptCopy(package, commit, pocket, src)

             # If we can't copy the package from somewhere, but
             # validateBuild didn't raise an exception, then we need to
             # do the build ourselves
             else:
                 db.failed_stage = 'checking out package source'
             # If we can't copy the package from somewhere, but
             # validateBuild didn't raise an exception, then we need to
             # do the build ourselves
             else:
                 db.failed_stage = 'checking out package source'

-                with packageWorkdir(package) as workdir:
+                with packageWorkdir(package, commit) as workdir:

                     db.failed_stage = 'preparing source package'
                     packagedir = os.path.join(workdir, package)
 
                     db.failed_stage = 'preparing source package'
                     packagedir = os.path.join(workdir, package)
 

@@ -315,26 +328,31 @@ def build():
                         db.failed_stage = 'building binary packages'
                         sbuildAll(package, commit, workdir)
                     finally:
                         db.failed_stage = 'building binary packages'
                         sbuildAll(package, commit, workdir)
                     finally:

-                        logdir = os.path.join(b._LOG_DIR, db.build_id)
+                        logdir = os.path.join(b._LOG_DIR, str(db.build_id))

                         if not os.path.exists(logdir):
                             os.makedirs(logdir)
 
                         if not os.path.exists(logdir):
                             os.makedirs(logdir)
 

-                        for log in glob.glob(os.path.join(workdir, '*.build')):
-                            os.copy2(log, logdir)
+                        for log in glob.glob(os.path.join(workdir, 'build-*.log')):
+                            os.copy(log, logdir)
+
+                    db.failed_stage = 'processing metadata'
+                    env = dict(os.environ)
+                    env['GIT_COMMITTER_NAME'] = config.build.tagger.name
+                    env['GIT_COMMITTER_EMAIL'] = config.build.tagger.email
+                    version = b.getVersion(package, commit)
+

                     db.failed_stage = 'tagging submodule'
                     db.failed_stage = 'tagging submodule'

-                    tagSubmodule(pocket, package, commit, principal)
+                    tagSubmodule(pocket, package, principal, version, env)

                     db.failed_stage = 'updating submodule branches'
                     updateSubmoduleBranch(pocket, package, commit)
                     db.failed_stage = 'updating submodule branches'
                     updateSubmoduleBranch(pocket, package, commit)

-                    db.failed_stage = 'updating superrepo'
-                    updateSuperrepo(pocket, package, commit, principal)
+                    db.failed_stage = 'updating superproject'
+                    updateSuperproject(pocket, package, commit, principal, version, env)
+                    db.failed_stage = 'relaxing permissions on workdir'
+                    makeReadable(workdir)

                     db.failed_stage = 'uploading packages to apt repo'
                     uploadBuild(pocket, workdir)
 
                     db.failed_stage = 'cleaning up'
                     db.failed_stage = 'uploading packages to apt repo'
                     uploadBuild(pocket, workdir)
 
                     db.failed_stage = 'cleaning up'

-
-                # Finally, now that everything is done, remove the
-                # build queue item
-                os.unlink(os.path.join(b._QUEUE_DIR, build))

         except:
             db.traceback = traceback.format_exc()
         else:
         except:
             db.traceback = traceback.format_exc()
         else:

@@ -344,15 +362,19 @@ def build():
             database.session.save_or_update(db)
             database.session.commit()
 
             database.session.save_or_update(db)
             database.session.commit()
 

+            # Finally, now that everything is done, remove the
+            # build queue item
+            os.unlink(os.path.join(b._QUEUE_DIR, build))
+

             reportBuild(db)
 
 
 class Invirtibuilder(pyinotify.ProcessEvent):
     """Process inotify triggers to build new packages."""
             reportBuild(db)
 
 
 class Invirtibuilder(pyinotify.ProcessEvent):
     """Process inotify triggers to build new packages."""

-    def process_IN_CREATE(self, event):
-        """Handle a created file or directory.
+    def process_default(self, event):
+        """Handle an inotify event.

 
 

-        When an IN_CREATE event comes in, trigger the builder.
+        When an inotify event comes in, trigger the builder.

         """
         build()
 
         """
         build()
 

@@ -365,7 +387,8 @@ def main():
     invirtibuilder = Invirtibuilder()
     notifier = pyinotify.Notifier(watch_manager, invirtibuilder)
     watch_manager.add_watch(b._QUEUE_DIR,
     invirtibuilder = Invirtibuilder()
     notifier = pyinotify.Notifier(watch_manager, invirtibuilder)
     watch_manager.add_watch(b._QUEUE_DIR,

-                            pyinotify.EventsCodes.ALL_FLAGS['IN_CREATE'])
+                            pyinotify.EventsCodes.ALL_FLAGS['IN_CREATE'] |
+                            pyinotify.EventsCodes.ALL_FLAGS['IN_MOVED_TO'])

 
     # Before inotifying, run any pending builds; otherwise we won't
     # get notified for them.
 
     # Before inotifying, run any pending builds; otherwise we won't
     # get notified for them.