Put the build number in the temporary directory builds happen in

[invirt/packages/invirt-dev.git] / invirtibuilder

diff --git a/invirtibuilder b/invirtibuilder
index 2f28103..9260b1e 100755 (executable)
--- a/invirtibuilder
+++ b/invirtibuilder
@@ -47,7 +47,6 @@ from invirt import database
 from invirt.config import structs as config
 
 
 from invirt.config import structs as config
 
 

-DISTRIBUTION = 'hardy'

 logfile = None
 
 def logAndRun(cmd, *args, **kwargs):
 logfile = None
 
 def logAndRun(cmd, *args, **kwargs):

@@ -124,22 +123,32 @@ def aptCopy(package, commit, dst_pocket, src_pocket):
                package] + binaries)
 
 
                package] + binaries)
 
 

-def sbuild(package, ref, arch, workdir, arch_all=False):
-    """Build a package for a particular architecture."""
-    args = ['sbuild', '-v', '-d', DISTRIBUTION, '--arch', arch]
+def sbuild(package, ref, distro, arch, workdir, arch_all=False):
+    """Build a package for a particular architecture and distro."""
+    # We append a suffix like ~ubuntu8.04 to differentiate the same
+    # version built for multiple distros
+    nmutag = b.distroToSuffix(distro)
+    env = os.environ.copy()
+    env['NMUTAG'] = nmutag
+
+    # Run sbuild with a hack in place to append arbitrary versions
+    args = ['perl', '-I/usr/share/invirt-dev', '-MSbuildHack',
+            '/usr/bin/sbuild',
+            '--binNMU=171717', '--make-binNMU=Build with sbuild',
+            '-v', '-d', distro, '--arch', arch]

     if arch_all:
         args.append('-A')
     args.append(getDscName(package, ref))
     if arch_all:
         args.append('-A')
     args.append(getDscName(package, ref))

-    logAndRun(args, cwd=workdir)
+    logAndRun(args, cwd=workdir, env=env)

 
 
 
 

-def sbuildAll(package, ref, workdir):
+def sbuildAll(package, ref, distro, workdir):

     """Build a package for all architectures it supports."""
     arches = getArches(package, ref)
     if 'all' in arches or 'any' in arches or 'amd64' in arches:
     """Build a package for all architectures it supports."""
     arches = getArches(package, ref)
     if 'all' in arches or 'any' in arches or 'amd64' in arches:

-        sbuild(package, ref, 'amd64', workdir, arch_all=True)
+        sbuild(package, ref, distro, 'amd64', workdir, arch_all=True)

     if 'any' in arches or 'i386' in arches:
     if 'any' in arches or 'i386' in arches:

-        sbuild(package, ref, 'i386', workdir)
+        sbuild(package, ref, distro, 'i386', workdir)

 
 
 def tagSubmodule(pocket, package, commit, principal, version, env):
 
 
 def tagSubmodule(pocket, package, commit, principal, version, env):

@@ -209,19 +218,40 @@ def updateSuperproject(pocket, package, commit, principal, version, env):
     tree = logAndRun(['git', 'ls-tree', branch],
                      cwd=superproject).strip()
 
     tree = logAndRun(['git', 'ls-tree', branch],
                      cwd=superproject).strip()
 

-    new_tree = re.compile(
-        r'^(160000 commit )[0-9a-f]*(\t%s)$' % package, re.M).sub(
-        r'\g<1>%s\g<2>' % commit,
-        tree)
+    tree_items = dict((k, v) for (v, k) in (x.split("\t") for x in tree.split("\n")))
+
+    created = not (package in tree_items)
+
+    tree_items[package] = "160000 commit "+commit
+
+    # If "created" is true, we need to check if the package is
+    # mentioned in .gitmodules, and add it if not.
+    if created:
+        gitmodules = logAndRun(['git', 'cat-file', 'blob', '%s:.gitmodules' % (branch)],
+                               cwd=superproject)
+        if ('[submodule "%s"]' % (package)) not in gitmodules.split("\n"):
+            gitmodules += """[submodule "%s"]
+\tpath = %s
+\turl = ../packages/%s.git
+""" % (package, package, package)
+            gitmodules_hash = logAndRun(['git', 'hash-object', '-w', '--stdin'],
+                                        cwd=superproject).strip()
+            tree_items[package] = "100644 blob "+gitmodules_hash
+
+    new_tree = "\n".join("%s\t%s" % (v, k) for (k, v) in tree_items.iteritems())

 
     new_tree_id = logAndRun(['git', 'mktree', '--missing'],
                             cwd=superproject,
                             stdin_str=new_tree).strip()
 
 
     new_tree_id = logAndRun(['git', 'mktree', '--missing'],
                             cwd=superproject,
                             stdin_str=new_tree).strip()
 

-    commit_msg = ('Update %s to version %s\n\n'
-                  'Requested by %s' % (package,
-                                       version.full_version,
-                                       principal))
+    if created:
+        commit_msg = 'Add %s at version %s'
+    else:
+        commit_msg = 'Update %s to version %s'
+    commit_msg = ((commit_msg + '\n\n'
+                   'Requested by %s') % (package,
+                                         version.full_version,
+                                         principal))

     new_commit = logAndRun(
         ['git', 'commit-tree', new_tree_id, '-p', branch],
         cwd=superproject,
     new_commit = logAndRun(
         ['git', 'commit-tree', new_tree_id, '-p', branch],
         cwd=superproject,

@@ -237,7 +267,7 @@ def makeReadable(workdir):
     os.chmod(workdir, 0755)
 
 @contextlib.contextmanager
     os.chmod(workdir, 0755)
 
 @contextlib.contextmanager

-def packageWorkdir(package, commit):
+def packageWorkdir(package, commit, build_id):

     """Checkout the package in a temporary working directory.
 
     This context manager returns that working directory. The requested
     """Checkout the package in a temporary working directory.
 
     This context manager returns that working directory. The requested

@@ -247,7 +277,7 @@ def packageWorkdir(package, commit):
     When the context wrapped with this context manager is exited, the
     working directory is automatically deleted.
     """
     When the context wrapped with this context manager is exited, the
     working directory is automatically deleted.
     """

-    workdir = tempfile.mkdtemp()
+    workdir = tempfile.mkdtemp(prefix=("b%d-" % build_id))

     try:
         p_archive = subprocess.Popen(
             ['git', 'archive',
     try:
         p_archive = subprocess.Popen(
             ['git', 'archive',

@@ -301,19 +331,20 @@ def build():
         logdir = os.path.join(b._LOG_DIR, str(db.build_id))
         if not os.path.exists(logdir):
             os.makedirs(logdir)
         logdir = os.path.join(b._LOG_DIR, str(db.build_id))
         if not os.path.exists(logdir):
             os.makedirs(logdir)

-        logfile = open(os.path.join(logdir, 'build.log'), 'w')

 
         try:
             db.failed_stage = 'validating job'
             # Don't expand the commit in the DB until we're sure the user
             # isn't trying to be tricky.
             b.ensureValidPackage(package)
 
         try:
             db.failed_stage = 'validating job'
             # Don't expand the commit in the DB until we're sure the user
             # isn't trying to be tricky.
             b.ensureValidPackage(package)

+
+            logfile = open(os.path.join(logdir, '%s.log' % db.package), 'w')
+

             db.commit = commit = b.canonicalize_commit(package, commit)
             src = b.validateBuild(pocket, package, commit)
             version = b.getVersion(package, commit)
             db.version = str(version)
             db.commit = commit = b.canonicalize_commit(package, commit)
             src = b.validateBuild(pocket, package, commit)
             version = b.getVersion(package, commit)
             db.version = str(version)

-            b.runHook('pre-build', [str(db.build_id), db.pocket, db.package,
-                                    db.commit, db.principal, db.version, str(db.inserted_at)])
+            b.runHook('pre-build', [str(db.build_id)])

 
             env = dict(os.environ)
             env['GIT_COMMITTER_NAME'] = config.build.tagger.name
 
             env = dict(os.environ)
             env['GIT_COMMITTER_NAME'] = config.build.tagger.name

@@ -340,7 +371,7 @@ def build():
             # do the build ourselves
             else:
                 db.failed_stage = 'checking out package source'
             # do the build ourselves
             else:
                 db.failed_stage = 'checking out package source'

-                with packageWorkdir(package, commit) as workdir:
+                with packageWorkdir(package, commit, db.build_id) as workdir:

                     db.failed_stage = 'preparing source package'
                     packagedir = os.path.join(workdir, package)
 
                     db.failed_stage = 'preparing source package'
                     packagedir = os.path.join(workdir, package)
 

@@ -355,7 +386,7 @@ def build():
                               cwd=packagedir)
 
                     db.failed_stage = 'building binary packages'
                               cwd=packagedir)
 
                     db.failed_stage = 'building binary packages'

-                    sbuildAll(package, commit, workdir)
+                    sbuildAll(package, commit, b.pocketToDistro(pocket), workdir)

                     db.failed_stage = 'tagging submodule'
                     tagSubmodule(pocket, package, commit, principal, version, env)
                     db.failed_stage = 'updating submodule branches'
                     db.failed_stage = 'tagging submodule'
                     tagSubmodule(pocket, package, commit, principal, version, env)
                     db.failed_stage = 'updating submodule branches'

@@ -374,7 +405,8 @@ def build():
             db.succeeded = True
             db.failed_stage = None
         finally:
             db.succeeded = True
             db.failed_stage = None
         finally:

-            logfile.close()
+            if logfile is not None:
+                logfile.close()

 
             database.session.save_or_update(db)
             database.session.commit()
 
             database.session.save_or_update(db)
             database.session.commit()