Add submodules to the superrepo if a package has never been built

[invirt/packages/invirt-dev.git] / invirtibuilder

diff --git a/invirtibuilder b/invirtibuilder
index 9198549..3df3539 100755 (executable)
--- a/invirtibuilder
+++ b/invirtibuilder
@@ -47,8 +47,20 @@ from invirt import database
 from invirt.config import structs as config
 
 
 from invirt.config import structs as config
 
 

-DISTRIBUTION = 'hardy'
-
+logfile = None
+
+def logAndRun(cmd, *args, **kwargs):
+    # Always grab stdout, even if the caller doesn't need it.
+    # TODO: don't slurp it all into memory in that case.
+    if 'stdout' in kwargs and kwargs['stdout'] is None:
+        del kwargs['stdout']
+    kwargs['stderr'] = logfile
+    logfile.write('---> Ran %s\n' % (cmd, ))
+    logfile.write('STDERR:\n')
+    output = c.captureOutput(cmd, *args, **kwargs)
+    logfile.write('STDOUT:\n')
+    logfile.write(output)
+    return output

 
 def getControl(package, ref):
     """Get the parsed debian/control file for a given package.
 
 def getControl(package, ref):
     """Get the parsed debian/control file for a given package.

@@ -93,40 +105,50 @@ def sanitizeVersion(version):
     """Sanitize a Debian package version for use as a git tag.
 
     This function strips the epoch from the version number and
     """Sanitize a Debian package version for use as a git tag.
 
     This function strips the epoch from the version number and

-    replaces any tildes with periods."""
-    if v.debian_version:
+    replaces any tildes with underscores."""
+    if version.debian_version:

         v = '%s-%s' % (version.upstream_version,
                        version.debian_version)
     else:
         v = version.upstream_version
         v = '%s-%s' % (version.upstream_version,
                        version.debian_version)
     else:
         v = version.upstream_version

-    return v.replace('~', '.')
+    return v.replace('~', '_')

 
 
 def aptCopy(package, commit, dst_pocket, src_pocket):
     """Copy a package from one pocket to another."""
     binaries = getBinaries(package, commit)
 
 
 def aptCopy(package, commit, dst_pocket, src_pocket):
     """Copy a package from one pocket to another."""
     binaries = getBinaries(package, commit)

-    c.captureOutput(['reprepro-env', 'copy',
-                     b.pocketToApt(dst_pocket),
-                     b.pocketToApt(src_pocket),
-                     package] + binaries)
-
-
-def sbuild(package, ref, arch, workdir, arch_all=False):
-    """Build a package for a particular architecture."""
-    args = ['sbuild', '-v', '-d', DISTRIBUTION, '--arch', arch]
+    logAndRun(['reprepro-env', 'copy',
+               b.pocketToApt(dst_pocket),
+               b.pocketToApt(src_pocket),
+               package] + binaries)
+
+
+def sbuild(package, ref, distro, arch, workdir, arch_all=False):
+    """Build a package for a particular architecture and distro."""
+    # We append a suffix like ~ubuntu8.04 to differentiate the same
+    # version built for multiple distros
+    nmutag = b.distroToSuffix(distro)
+    env = os.environ.copy()
+    env['NMUTAG'] = nmutag
+
+    # Run sbuild with a hack in place to append arbitrary versions
+    args = ['perl', '-I/usr/share/invirt-dev',
+            '/usr/bin/sbuild',
+            '--binNMU=171717', '--make-binNMU=Build with sbuild',
+            '-v', '-d', distro, '--arch', arch]

     if arch_all:
         args.append('-A')
     args.append(getDscName(package, ref))
     if arch_all:
         args.append('-A')
     args.append(getDscName(package, ref))

-    c.captureOutput(args, cwd=workdir)
+    logAndRun(args, cwd=workdir, env=env)

 
 
 
 

-def sbuildAll(package, ref, workdir):
+def sbuildAll(package, ref, distro, workdir):

     """Build a package for all architectures it supports."""
     arches = getArches(package, ref)
     if 'all' in arches or 'any' in arches or 'amd64' in arches:
     """Build a package for all architectures it supports."""
     arches = getArches(package, ref)
     if 'all' in arches or 'any' in arches or 'amd64' in arches:

-        sbuild(package, ref, 'amd64', workdir, arch_all=True)
+        sbuild(package, ref, distro, 'amd64', workdir, arch_all=True)

     if 'any' in arches or 'i386' in arches:
     if 'any' in arches or 'i386' in arches:

-        sbuild(package, ref, 'i386', workdir)
+        sbuild(package, ref, distro, 'i386', workdir)

 
 
 def tagSubmodule(pocket, package, commit, principal, version, env):
 
 
 def tagSubmodule(pocket, package, commit, principal, version, env):

@@ -151,8 +173,9 @@ def tagSubmodule(pocket, package, commit, principal, version, env):
                                         package,
                                         principal))
 
                                         package,
                                         principal))
 

-        c.captureOutput(
-            ['git', 'tag', '-m', tag_msg, commit],
+        logAndRun(
+            ['git', 'tag', '-m', tag_msg, '--', sanitizeVersion(version),
+             commit],

             env=env,
             cwd=b.getRepo(package))
 
             env=env,
             cwd=b.getRepo(package))
 

@@ -160,7 +183,7 @@ def tagSubmodule(pocket, package, commit, principal, version, env):
 def updateSubmoduleBranch(pocket, package, commit):
     """Update the appropriately named branch in the submodule."""
     branch = b.pocketToGit(pocket)
 def updateSubmoduleBranch(pocket, package, commit):
     """Update the appropriately named branch in the submodule."""
     branch = b.pocketToGit(pocket)

-    c.captureOutput(
+    logAndRun(

         ['git', 'update-ref', 'refs/heads/%s' % branch, commit], cwd=b.getRepo(package))
 
 
         ['git', 'update-ref', 'refs/heads/%s' % branch, commit], cwd=b.getRepo(package))
 
 

@@ -172,13 +195,13 @@ def uploadBuild(pocket, workdir):
         upload = ['reprepro-env', '--ignore=wrongdistribution',
                   'include', apt, changes]
         try:
         upload = ['reprepro-env', '--ignore=wrongdistribution',
                   'include', apt, changes]
         try:

-            c.captureOutput(upload)
+            logAndRun(upload)

         except subprocess.CalledProcessError, e:
             if not force:
                 raise
             package = deb822.Changes(open(changes).read())['Binary']
         except subprocess.CalledProcessError, e:
             if not force:
                 raise
             package = deb822.Changes(open(changes).read())['Binary']

-            c.captureOutput(['reprepro-env', 'remove', apt, package])
-            c.captureOutput(upload)
+            logAndRun(['reprepro-env', 'remove', apt, package])
+            logAndRun(upload)

 
 
 def updateSuperproject(pocket, package, commit, principal, version, env):
 
 
 def updateSuperproject(pocket, package, commit, principal, version, env):

@@ -192,29 +215,50 @@ def updateSuperproject(pocket, package, commit, principal, version, env):
     """
     superproject = os.path.join(b._REPO_DIR, 'invirt/packages.git')
     branch = b.pocketToGit(pocket)
     """
     superproject = os.path.join(b._REPO_DIR, 'invirt/packages.git')
     branch = b.pocketToGit(pocket)

-    tree = c.captureOutput(['git', 'ls-tree', branch],
-                           cwd=superproject).strip()
+    tree = logAndRun(['git', 'ls-tree', branch],
+                     cwd=superproject).strip()
+
+    tree_items = dict(k, v for v, k in (x.split("\t" for x in tree.split("\n"))))
+
+    created = not (package in tree_items)
+
+    tree_items[package] = "160000 commit "+commit
+
+    # If "created" is true, we need to check if the package is
+    # mentioned in .gitmodules, and add it if not.
+    if created:
+        gitmodules = c.captureOutput(['git', 'cat-file', 'blob', '%s:.gitmodules' % (branch)],
+                                     cwd=superproject)
+        if ('[submodule "%s"]' % (package)) not in gitmodules.split("\n"):
+            gitmodules += """[submodule "%s"]
+\tpath = %s
+\turl = ../packages/%s.git
+""" % (package, package, package)
+            gitmodules_hash = c.captureOutput(['git', 'hash-object', '-w', '--stdin'],
+                                              cwd=superproject).strip()
+            tree_items[package] = "100644 blob "+gitmodules_hash

 
 

-    new_tree = re.compile(
-        r'^(160000 commit )[0-9a-f]*(\t%s)$' % package, re.M).sub(
-        r'\g<1>%s\g<2>' % commit,
-        tree)
+    new_tree = "\n".join("%s\t%s" % (v, k) for (k, v) in tree_items.iteritems())

 
 

-    new_tree_id = c.captureOutput(['git', 'mktree', '--missing'],
-                                  cwd=superproject,
-                                  stdin_str=new_tree).strip()
+    new_tree_id = logAndRun(['git', 'mktree', '--missing'],
+                            cwd=superproject,
+                            stdin_str=new_tree).strip()

 
 

-    commit_msg = ('Update %s to version %s\n\n'
+    if created:
+        commit_msg = 'Add %s at version %s'
+    else:
+        commit_msg = 'Update %s to version %s'
+    commit_msg = (commit_msg + '\n\n'

                   'Requested by %s' % (package,
                                        version.full_version,
                                        principal))
                   'Requested by %s' % (package,
                                        version.full_version,
                                        principal))

-    new_commit = c.captureOutput(
+    new_commit = logAndRun(

         ['git', 'commit-tree', new_tree_id, '-p', branch],
         cwd=superproject,
         env=env,
         stdin_str=commit_msg).strip()
 
         ['git', 'commit-tree', new_tree_id, '-p', branch],
         cwd=superproject,
         env=env,
         stdin_str=commit_msg).strip()
 

-    c.captureOutput(
+    logAndRun(

         ['git', 'update-ref', 'refs/heads/%s' % branch, new_commit],
         cwd=superproject)
 
         ['git', 'update-ref', 'refs/heads/%s' % branch, new_commit],
         cwd=superproject)
 

@@ -261,6 +305,8 @@ def build():
     When triggered, iterate over build queue items one at a time,
     until there are no more pending build jobs.
     """
     When triggered, iterate over build queue items one at a time,
     until there are no more pending build jobs.
     """

+    global logfile
+

     while True:
         stage = 'processing incoming job'
         queue = os.listdir(b._QUEUE_DIR)
     while True:
         stage = 'processing incoming job'
         queue = os.listdir(b._QUEUE_DIR)

@@ -282,17 +328,23 @@ def build():
 
         database.session.begin()
 
 
         database.session.begin()
 

+        logdir = os.path.join(b._LOG_DIR, str(db.build_id))
+        if not os.path.exists(logdir):
+            os.makedirs(logdir)
+

         try:
             db.failed_stage = 'validating job'
             # Don't expand the commit in the DB until we're sure the user
             # isn't trying to be tricky.
             b.ensureValidPackage(package)
         try:
             db.failed_stage = 'validating job'
             # Don't expand the commit in the DB until we're sure the user
             # isn't trying to be tricky.
             b.ensureValidPackage(package)

+
+            logfile = open(os.path.join(logdir, '%s.log' % db.package), 'w')
+

             db.commit = commit = b.canonicalize_commit(package, commit)
             src = b.validateBuild(pocket, package, commit)
             version = b.getVersion(package, commit)
             db.version = str(version)
             db.commit = commit = b.canonicalize_commit(package, commit)
             src = b.validateBuild(pocket, package, commit)
             version = b.getVersion(package, commit)
             db.version = str(version)

-            b.runHook('pre-build', [str(db.build_id), db.pocket, db.package,
-                                    db.commit, db.principal, db.version, str(db.inserted_at)])
+            b.runHook('pre-build', [str(db.build_id)])

 
             env = dict(os.environ)
             env['GIT_COMMITTER_NAME'] = config.build.tagger.name
 
             env = dict(os.environ)
             env['GIT_COMMITTER_NAME'] = config.build.tagger.name

@@ -330,21 +382,11 @@ def build():
                     # If we were, we could use debuild and get nice
                     # environment scrubbing. Since we're not, debuild
                     # complains about not having an orig.tar.gz
                     # If we were, we could use debuild and get nice
                     # environment scrubbing. Since we're not, debuild
                     # complains about not having an orig.tar.gz

-                    c.captureOutput(['dpkg-buildpackage', '-us', '-uc', '-S'],
-                                  cwd=packagedir,
-                                  stdout=None)
-
-                    try:
-                        db.failed_stage = 'building binary packages'
-                        sbuildAll(package, commit, workdir)
-                    finally:
-                        logdir = os.path.join(b._LOG_DIR, str(db.build_id))
-                        if not os.path.exists(logdir):
-                            os.makedirs(logdir)
-
-                        for log in glob.glob(os.path.join(workdir, 'build-*.log')):
-                            os.copy(log, logdir)
+                    logAndRun(['dpkg-buildpackage', '-us', '-uc', '-S'],
+                              cwd=packagedir)

 
 

+                    db.failed_stage = 'building binary packages'
+                    sbuildAll(package, commit, b.pocketToDistro(pocket), workdir)

                     db.failed_stage = 'tagging submodule'
                     tagSubmodule(pocket, package, commit, principal, version, env)
                     db.failed_stage = 'updating submodule branches'
                     db.failed_stage = 'tagging submodule'
                     tagSubmodule(pocket, package, commit, principal, version, env)
                     db.failed_stage = 'updating submodule branches'

@@ -363,6 +405,9 @@ def build():
             db.succeeded = True
             db.failed_stage = None
         finally:
             db.succeeded = True
             db.failed_stage = None
         finally:

+            if logfile is not None:
+                logfile.close()
+

             database.session.save_or_update(db)
             database.session.commit()
 
             database.session.save_or_update(db)
             database.session.commit()