Fixed version

[invirt/packages/invirt-dev.git] / invirtibuilder

diff --git a/invirtibuilder b/invirtibuilder
index e1c3d80..bf3b499 100755 (executable)
--- a/invirtibuilder
+++ b/invirtibuilder
@@ -47,8 +47,23 @@ from invirt import database
 from invirt.config import structs as config
 
 
 from invirt.config import structs as config
 
 

-DISTRIBUTION = 'hardy'
-
+logfile = None
+
+def logAndRun(cmd, *args, **kwargs):
+    # Always grab stdout, even if the caller doesn't need it.
+    # TODO: don't slurp it all into memory in that case.
+    if 'stdout' in kwargs and kwargs['stdout'] is None:
+        del kwargs['stdout']
+    kwargs['stderr'] = logfile
+    logfile.write('---> Ran %s\n' % (cmd, ))
+    if 'stdin_str' in kwargs:
+        logfile.write('STDIN:\n')
+        logfile.write(kwargs['stdin_str'])
+    logfile.write('STDERR:\n')
+    output = c.captureOutput(cmd, *args, **kwargs)
+    logfile.write('STDOUT:\n')
+    logfile.write(output)
+    return output

 
 def getControl(package, ref):
     """Get the parsed debian/control file for a given package.
 
 def getControl(package, ref):
     """Get the parsed debian/control file for a given package.

@@ -93,40 +108,50 @@ def sanitizeVersion(version):
     """Sanitize a Debian package version for use as a git tag.
 
     This function strips the epoch from the version number and
     """Sanitize a Debian package version for use as a git tag.
 
     This function strips the epoch from the version number and

-    replaces any tildes with periods."""
-    if v.debian_version:
+    replaces any tildes with underscores."""
+    if version.debian_version:

         v = '%s-%s' % (version.upstream_version,
                        version.debian_version)
     else:
         v = version.upstream_version
         v = '%s-%s' % (version.upstream_version,
                        version.debian_version)
     else:
         v = version.upstream_version

-    return v.replace('~', '.')
+    return v.replace('~', '_')

 
 
 def aptCopy(package, commit, dst_pocket, src_pocket):
     """Copy a package from one pocket to another."""
     binaries = getBinaries(package, commit)
 
 
 def aptCopy(package, commit, dst_pocket, src_pocket):
     """Copy a package from one pocket to another."""
     binaries = getBinaries(package, commit)

-    c.captureOutput(['reprepro-env', 'copy',
-                     b.pocketToApt(dst_pocket),
-                     b.pocketToApt(src_pocket),
-                     package] + binaries)
-
-
-def sbuild(package, ref, arch, workdir, arch_all=False):
-    """Build a package for a particular architecture."""
-    args = ['sbuild', '-v', '-d', DISTRIBUTION, '--arch', arch]
+    logAndRun(['reprepro-env', 'copy',
+               b.pocketToApt(dst_pocket),
+               b.pocketToApt(src_pocket),
+               package] + binaries)
+
+
+def sbuild(package, ref, distro, arch, workdir, arch_all=False):
+    """Build a package for a particular architecture and distro."""
+    # We append a suffix like ~ubuntu8.04 to differentiate the same
+    # version built for multiple distros
+    nmutag = b.distroToSuffix(distro)
+    env = os.environ.copy()
+    env['NMUTAG'] = nmutag
+
+    # Run sbuild with a hack in place to append arbitrary versions
+    args = ['perl', '-I/usr/share/invirt-dev', '-MSbuildHack',
+            '/usr/bin/sbuild',
+            '--binNMU=171717', '--make-binNMU=Build with sbuild',
+            '-v', '-d', distro, '--arch', arch]

     if arch_all:
         args.append('-A')
     args.append(getDscName(package, ref))
     if arch_all:
         args.append('-A')
     args.append(getDscName(package, ref))

-    c.captureOutput(args, cwd=workdir)
+    logAndRun(args, cwd=workdir, env=env)

 
 
 
 

-def sbuildAll(package, ref, workdir):
+def sbuildAll(package, ref, distro, workdir):

     """Build a package for all architectures it supports."""
     arches = getArches(package, ref)
     if 'all' in arches or 'any' in arches or 'amd64' in arches:
     """Build a package for all architectures it supports."""
     arches = getArches(package, ref)
     if 'all' in arches or 'any' in arches or 'amd64' in arches:

-        sbuild(package, ref, 'amd64', workdir, arch_all=True)
+        sbuild(package, ref, distro, 'amd64', workdir, arch_all=True)

     if 'any' in arches or 'i386' in arches:
     if 'any' in arches or 'i386' in arches:

-        sbuild(package, ref, 'i386', workdir)
+        sbuild(package, ref, distro, 'i386', workdir)

 
 
 def tagSubmodule(pocket, package, commit, principal, version, env):
 
 
 def tagSubmodule(pocket, package, commit, principal, version, env):

@@ -151,8 +176,9 @@ def tagSubmodule(pocket, package, commit, principal, version, env):
                                         package,
                                         principal))
 
                                         package,
                                         principal))
 

-        c.captureOutput(
-            ['git', 'tag', '-m', tag_msg, commit],
+        logAndRun(
+            ['git', 'tag', '-m', tag_msg, '--', sanitizeVersion(version),
+             commit],

             env=env,
             cwd=b.getRepo(package))
 
             env=env,
             cwd=b.getRepo(package))
 

@@ -160,7 +186,7 @@ def tagSubmodule(pocket, package, commit, principal, version, env):
 def updateSubmoduleBranch(pocket, package, commit):
     """Update the appropriately named branch in the submodule."""
     branch = b.pocketToGit(pocket)
 def updateSubmoduleBranch(pocket, package, commit):
     """Update the appropriately named branch in the submodule."""
     branch = b.pocketToGit(pocket)

-    c.captureOutput(
+    logAndRun(

         ['git', 'update-ref', 'refs/heads/%s' % branch, commit], cwd=b.getRepo(package))
 
 
         ['git', 'update-ref', 'refs/heads/%s' % branch, commit], cwd=b.getRepo(package))
 
 

@@ -172,13 +198,16 @@ def uploadBuild(pocket, workdir):
         upload = ['reprepro-env', '--ignore=wrongdistribution',
                   'include', apt, changes]
         try:
         upload = ['reprepro-env', '--ignore=wrongdistribution',
                   'include', apt, changes]
         try:

-            c.captureOutput(upload)
+            logAndRun(upload)

         except subprocess.CalledProcessError, e:
             if not force:
                 raise
         except subprocess.CalledProcessError, e:
             if not force:
                 raise

-            package = deb822.Changes(open(changes).read())['Binary']
-            c.captureOutput(['reprepro-env', 'remove', apt, package])
-            c.captureOutput(upload)
+            changelog = deb822.Changes(open(changes).read())
+            packages = set(changelog['Binary'].split())
+            packages.add(changelog['Source'])
+            for package in packages:
+                logAndRun(['reprepro-env', 'remove', apt, package])
+            logAndRun(upload)

 
 
 def updateSuperproject(pocket, package, commit, principal, version, env):
 
 
 def updateSuperproject(pocket, package, commit, principal, version, env):

@@ -192,29 +221,51 @@ def updateSuperproject(pocket, package, commit, principal, version, env):
     """
     superproject = os.path.join(b._REPO_DIR, 'invirt/packages.git')
     branch = b.pocketToGit(pocket)
     """
     superproject = os.path.join(b._REPO_DIR, 'invirt/packages.git')
     branch = b.pocketToGit(pocket)

-    tree = c.captureOutput(['git', 'ls-tree', branch],
-                           cwd=superproject).strip()
-
-    new_tree = re.compile(
-        r'^(160000 commit )[0-9a-f]*(\t%s)$' % package, re.M).sub(
-        r'\g<1>%s\g<2>' % commit,
-        tree)
-
-    new_tree_id = c.captureOutput(['git', 'mktree', '--missing'],
-                                  cwd=superproject,
-                                  stdin_str=new_tree).strip()
-
-    commit_msg = ('Update %s to version %s\n\n'
-                  'Requested by %s' % (package,
-                                       version.full_version,
-                                       principal))
-    new_commit = c.captureOutput(
+    tree = logAndRun(['git', 'ls-tree', branch],
+                     cwd=superproject).strip()
+
+    tree_items = dict((k, v) for (v, k) in (x.split("\t") for x in tree.split("\n")))
+
+    created = not (package in tree_items)
+
+    tree_items[package] = "160000 commit "+commit
+
+    # If "created" is true, we need to check if the package is
+    # mentioned in .gitmodules, and add it if not.
+    if created:
+        gitmodules = logAndRun(['git', 'cat-file', 'blob', '%s:.gitmodules' % (branch)],
+                               cwd=superproject)
+        if ('[submodule "%s"]' % (package)) not in gitmodules.split("\n"):
+            gitmodules += """[submodule "%s"]
+\tpath = %s
+\turl = ../packages/%s.git
+""" % (package, package, package)
+            gitmodules_hash = logAndRun(['git', 'hash-object', '-w', '--stdin'],
+                                        cwd=superproject,
+                                        stdin_str=gitmodules).strip()
+            tree_items[package] = "100644 blob "+gitmodules_hash
+
+    new_tree = "\n".join("%s\t%s" % (v, k) for (k, v) in tree_items.iteritems())
+
+    new_tree_id = logAndRun(['git', 'mktree', '--missing'],
+                            cwd=superproject,
+                            stdin_str=new_tree).strip()
+
+    if created:
+        commit_msg = 'Add %s at version %s'
+    else:
+        commit_msg = 'Update %s to version %s'
+    commit_msg = ((commit_msg + '\n\n'
+                   'Requested by %s') % (package,
+                                         version.full_version,
+                                         principal))
+    new_commit = logAndRun(

         ['git', 'commit-tree', new_tree_id, '-p', branch],
         cwd=superproject,
         env=env,
         stdin_str=commit_msg).strip()
 
         ['git', 'commit-tree', new_tree_id, '-p', branch],
         cwd=superproject,
         env=env,
         stdin_str=commit_msg).strip()
 

-    c.captureOutput(
+    logAndRun(

         ['git', 'update-ref', 'refs/heads/%s' % branch, new_commit],
         cwd=superproject)
 
         ['git', 'update-ref', 'refs/heads/%s' % branch, new_commit],
         cwd=superproject)
 

@@ -223,7 +274,7 @@ def makeReadable(workdir):
     os.chmod(workdir, 0755)
 
 @contextlib.contextmanager
     os.chmod(workdir, 0755)
 
 @contextlib.contextmanager

-def packageWorkdir(package, commit):
+def packageWorkdir(package, commit, build_id):

     """Checkout the package in a temporary working directory.
 
     This context manager returns that working directory. The requested
     """Checkout the package in a temporary working directory.
 
     This context manager returns that working directory. The requested

@@ -233,7 +284,7 @@ def packageWorkdir(package, commit):
     When the context wrapped with this context manager is exited, the
     working directory is automatically deleted.
     """
     When the context wrapped with this context manager is exited, the
     working directory is automatically deleted.
     """

-    workdir = tempfile.mkdtemp()
+    workdir = tempfile.mkdtemp(prefix=("b%d-" % build_id))

     try:
         p_archive = subprocess.Popen(
             ['git', 'archive',
     try:
         p_archive = subprocess.Popen(
             ['git', 'archive',

@@ -261,6 +312,8 @@ def build():
     When triggered, iterate over build queue items one at a time,
     until there are no more pending build jobs.
     """
     When triggered, iterate over build queue items one at a time,
     until there are no more pending build jobs.
     """

+    global logfile
+

     while True:
         stage = 'processing incoming job'
         queue = os.listdir(b._QUEUE_DIR)
     while True:
         stage = 'processing incoming job'
         queue = os.listdir(b._QUEUE_DIR)

@@ -282,17 +335,27 @@ def build():
 
         database.session.begin()
 
 
         database.session.begin()
 

+        logdir = os.path.join(b._LOG_DIR, str(db.build_id))
+        if not os.path.exists(logdir):
+            os.makedirs(logdir)
+

         try:
             db.failed_stage = 'validating job'
             # Don't expand the commit in the DB until we're sure the user
             # isn't trying to be tricky.
             b.ensureValidPackage(package)
         try:
             db.failed_stage = 'validating job'
             # Don't expand the commit in the DB until we're sure the user
             # isn't trying to be tricky.
             b.ensureValidPackage(package)

+
+            logfile = open(os.path.join(logdir, '%s.log' % db.package), 'w')
+

             db.commit = commit = b.canonicalize_commit(package, commit)
             src = b.validateBuild(pocket, package, commit)
             db.commit = commit = b.canonicalize_commit(package, commit)
             src = b.validateBuild(pocket, package, commit)

+            version = b.getVersion(package, commit)
+            db.version = str(version)
+            b.runHook('pre-build', [str(db.build_id)])

 
 

-            db.version = str(b.getVersion(package, commit))
-            b.runHook('pre-build', [str(db.build_id), db.pocket, db.package,
-                                    db.commit, db.principal, db.version, str(db.inserted_at)])
+            env = dict(os.environ)
+            env['GIT_COMMITTER_NAME'] = config.build.tagger.name
+            env['GIT_COMMITTER_EMAIL'] = config.build.tagger.email

 
             # If validateBuild returns something other than True, then
             # it means we should copy from that pocket to our pocket.
 
             # If validateBuild returns something other than True, then
             # it means we should copy from that pocket to our pocket.

@@ -300,14 +363,22 @@ def build():
             # (If the validation failed, validateBuild would have
             # raised an exception)
             if src != True:
             # (If the validation failed, validateBuild would have
             # raised an exception)
             if src != True:

+                # TODO: cut out this code duplication
+                db.failed_stage = 'tagging submodule before copying package'
+                tagSubmodule(pocket, package, commit, principal, version, env)
+                db.failed_stage = 'updating submodule branches before copying package'
+                updateSubmoduleBranch(pocket, package, commit)
+                db.failed_stage = 'updating superproject before copying package'
+                updateSuperproject(pocket, package, commit, principal, version, env)

                 db.failed_stage = 'copying package from another pocket'
                 aptCopy(package, commit, pocket, src)
                 db.failed_stage = 'copying package from another pocket'
                 aptCopy(package, commit, pocket, src)

+                

             # If we can't copy the package from somewhere, but
             # validateBuild didn't raise an exception, then we need to
             # do the build ourselves
             else:
                 db.failed_stage = 'checking out package source'
             # If we can't copy the package from somewhere, but
             # validateBuild didn't raise an exception, then we need to
             # do the build ourselves
             else:
                 db.failed_stage = 'checking out package source'

-                with packageWorkdir(package, commit) as workdir:
+                with packageWorkdir(package, commit, db.build_id) as workdir:

                     db.failed_stage = 'preparing source package'
                     packagedir = os.path.join(workdir, package)
 
                     db.failed_stage = 'preparing source package'
                     packagedir = os.path.join(workdir, package)
 

@@ -318,29 +389,13 @@ def build():
                     # If we were, we could use debuild and get nice
                     # environment scrubbing. Since we're not, debuild
                     # complains about not having an orig.tar.gz
                     # If we were, we could use debuild and get nice
                     # environment scrubbing. Since we're not, debuild
                     # complains about not having an orig.tar.gz

-                    c.captureOutput(['dpkg-buildpackage', '-us', '-uc', '-S'],
-                                  cwd=packagedir,
-                                  stdout=None)
-
-                    try:
-                        db.failed_stage = 'building binary packages'
-                        sbuildAll(package, commit, workdir)
-                    finally:
-                        logdir = os.path.join(b._LOG_DIR, str(db.build_id))
-                        if not os.path.exists(logdir):
-                            os.makedirs(logdir)
-
-                        for log in glob.glob(os.path.join(workdir, 'build-*.log')):
-                            os.copy(log, logdir)
-
-                    db.failed_stage = 'processing metadata'
-                    env = dict(os.environ)
-                    env['GIT_COMMITTER_NAME'] = config.build.tagger.name
-                    env['GIT_COMMITTER_EMAIL'] = config.build.tagger.email
-                    version = b.getVersion(package, commit)
+                    logAndRun(['dpkg-buildpackage', '-us', '-uc', '-S'],
+                              cwd=packagedir)

 
 

+                    db.failed_stage = 'building binary packages'
+                    sbuildAll(package, commit, b.pocketToDistro(pocket), workdir)

                     db.failed_stage = 'tagging submodule'
                     db.failed_stage = 'tagging submodule'

-                    tagSubmodule(pocket, package, principal, version, env)
+                    tagSubmodule(pocket, package, commit, principal, version, env)

                     db.failed_stage = 'updating submodule branches'
                     updateSubmoduleBranch(pocket, package, commit)
                     db.failed_stage = 'updating superproject'
                     db.failed_stage = 'updating submodule branches'
                     updateSubmoduleBranch(pocket, package, commit)
                     db.failed_stage = 'updating superproject'

@@ -357,6 +412,9 @@ def build():
             db.succeeded = True
             db.failed_stage = None
         finally:
             db.succeeded = True
             db.failed_stage = None
         finally:

+            if logfile is not None:
+                logfile.close()
+

             database.session.save_or_update(db)
             database.session.commit()
 
             database.session.save_or_update(db)
             database.session.commit()