Fixes to invirtibuilder for case of missing package in super-repo, missing

[invirt/packages/invirt-dev.git] / invirtibuilder

diff --git a/invirtibuilder b/invirtibuilder
index 527695a..eac57f3 100755 (executable)
--- a/invirtibuilder
+++ b/invirtibuilder
@@ -56,6 +56,9 @@ def logAndRun(cmd, *args, **kwargs):
         del kwargs['stdout']
     kwargs['stderr'] = logfile
     logfile.write('---> Ran %s\n' % (cmd, ))
         del kwargs['stdout']
     kwargs['stderr'] = logfile
     logfile.write('---> Ran %s\n' % (cmd, ))

+    if 'stdin_str' in kwargs:
+        logfile.write('STDIN:\n')
+        logfile.write(kwargs['stdin_str'])

     logfile.write('STDERR:\n')
     output = c.captureOutput(cmd, *args, **kwargs)
     logfile.write('STDOUT:\n')
     logfile.write('STDERR:\n')
     output = c.captureOutput(cmd, *args, **kwargs)
     logfile.write('STDOUT:\n')

@@ -199,8 +202,11 @@ def uploadBuild(pocket, workdir):
         except subprocess.CalledProcessError, e:
             if not force:
                 raise
         except subprocess.CalledProcessError, e:
             if not force:
                 raise

-            package = deb822.Changes(open(changes).read())['Binary']
-            logAndRun(['reprepro-env', 'remove', apt, package])
+            changelog = deb822.Changes(open(changes).read())
+            packages = set(changelog['Binary'].split())
+            packages.add(changelog['Source'])
+            for package in packages:
+                logAndRun(['reprepro-env', 'remove', apt, package])

             logAndRun(upload)
 
 
             logAndRun(upload)
 
 

@@ -214,11 +220,35 @@ def updateSuperproject(pocket, package, commit, principal, version, env):
     pushes to the superproject.
     """
     superproject = os.path.join(b._REPO_DIR, 'invirt/packages.git')
     pushes to the superproject.
     """
     superproject = os.path.join(b._REPO_DIR, 'invirt/packages.git')

+

     branch = b.pocketToGit(pocket)
     branch = b.pocketToGit(pocket)

+
+    if not b.pocketExists(pocket, superproject):
+        gitmodules = "\n"
+        gitmodules_hash = logAndRun(['git', 'hash-object', '-w', '--stdin'],
+                                    cwd=superproject,
+                                    stdin_str=gitmodules).strip()
+        tree_items = {'.gitmodules': "100644 blob "+gitmodules_hash}
+        new_tree = "\n".join("%s\t%s" % (v, k) for (k, v) in tree_items.iteritems())
+        new_tree_id = logAndRun(['git', 'mktree', '--missing'],
+                                cwd=superproject,
+                                stdin_str=new_tree).strip()
+        env2 = dict(os.environ)
+        env2['GIT_AUTHOR_NAME'] = config.build.tagger.name
+        env2['GIT_AUTHOR_EMAIL'] = config.build.tagger.email
+        env2['GIT_COMMITTER_NAME'] = config.build.tagger.name
+        env2['GIT_COMMITTER_EMAIL'] = config.build.tagger.email
+        new_commit = logAndRun(['git', 'commit-tree', new_tree_id],
+                               cwd=superproject,
+                               env=env2,
+                               stdin_str="Create new pocket").strip()
+        logAndRun(['git', 'update-ref', 'refs/heads/%s' % branch, new_commit],
+                  cwd=superproject)
+

     tree = logAndRun(['git', 'ls-tree', branch],
                      cwd=superproject).strip()
 
     tree = logAndRun(['git', 'ls-tree', branch],
                      cwd=superproject).strip()
 

-    tree_items = dict(k, v for v, k in (x.split("\t" for x in tree.split("\n"))))
+    tree_items = dict((k, v) for (v, k) in (x.split("\t") for x in tree.split("\n")))

 
     created = not (package in tree_items)
 
 
     created = not (package in tree_items)
 

@@ -227,16 +257,17 @@ def updateSuperproject(pocket, package, commit, principal, version, env):
     # If "created" is true, we need to check if the package is
     # mentioned in .gitmodules, and add it if not.
     if created:
     # If "created" is true, we need to check if the package is
     # mentioned in .gitmodules, and add it if not.
     if created:

-        gitmodules = c.captureOutput(['git', 'cat-file', 'blob', '%s:.gitmodules' % (branch)],
-                                     cwd=superproject)
+        gitmodules = logAndRun(['git', 'cat-file', 'blob', '%s:.gitmodules' % (branch)],
+                               cwd=superproject)

         if ('[submodule "%s"]' % (package)) not in gitmodules.split("\n"):
             gitmodules += """[submodule "%s"]
 \tpath = %s
 \turl = ../packages/%s.git
 """ % (package, package, package)
         if ('[submodule "%s"]' % (package)) not in gitmodules.split("\n"):
             gitmodules += """[submodule "%s"]
 \tpath = %s
 \turl = ../packages/%s.git
 """ % (package, package, package)

-            gitmodules_hash = c.captureOutput(['git', 'hash-object', '-w', '--stdin'],
-                                              cwd=superproject).strip()
-            tree_items[package] = "100644 blob "+gitmodules_hash
+            gitmodules_hash = logAndRun(['git', 'hash-object', '-w', '--stdin'],
+                                        cwd=superproject,
+                                        stdin_str=gitmodules).strip()
+            tree_items['.gitmodules'] = "100644 blob "+gitmodules_hash

 
     new_tree = "\n".join("%s\t%s" % (v, k) for (k, v) in tree_items.iteritems())
 
 
     new_tree = "\n".join("%s\t%s" % (v, k) for (k, v) in tree_items.iteritems())
 

@@ -248,10 +279,10 @@ def updateSuperproject(pocket, package, commit, principal, version, env):
         commit_msg = 'Add %s at version %s'
     else:
         commit_msg = 'Update %s to version %s'
         commit_msg = 'Add %s at version %s'
     else:
         commit_msg = 'Update %s to version %s'

-    commit_msg = (commit_msg + '\n\n'
-                  'Requested by %s' % (package,
-                                       version.full_version,
-                                       principal))
+    commit_msg = ((commit_msg + '\n\n'
+                   'Requested by %s') % (package,
+                                         version.full_version,
+                                         principal))

     new_commit = logAndRun(
         ['git', 'commit-tree', new_tree_id, '-p', branch],
         cwd=superproject,
     new_commit = logAndRun(
         ['git', 'commit-tree', new_tree_id, '-p', branch],
         cwd=superproject,

@@ -267,7 +298,7 @@ def makeReadable(workdir):
     os.chmod(workdir, 0755)
 
 @contextlib.contextmanager
     os.chmod(workdir, 0755)
 
 @contextlib.contextmanager

-def packageWorkdir(package, commit):
+def packageWorkdir(package, commit, build_id):

     """Checkout the package in a temporary working directory.
 
     This context manager returns that working directory. The requested
     """Checkout the package in a temporary working directory.
 
     This context manager returns that working directory. The requested

@@ -277,11 +308,11 @@ def packageWorkdir(package, commit):
     When the context wrapped with this context manager is exited, the
     working directory is automatically deleted.
     """
     When the context wrapped with this context manager is exited, the
     working directory is automatically deleted.
     """

-    workdir = tempfile.mkdtemp()
+    workdir = tempfile.mkdtemp(prefix=("b%d-" % build_id))

     try:
         p_archive = subprocess.Popen(
     try:
         p_archive = subprocess.Popen(

-            ['git', 'archive',
-             '--remote=file://%s' % b.getRepo(package),
+            ['git', '--git-dir=%s' % (b.getRepo(package),),
+             'archive',

              '--prefix=%s/' % package,
              commit,
              ],
              '--prefix=%s/' % package,
              commit,
              ],

@@ -349,6 +380,8 @@ def build():
             env = dict(os.environ)
             env['GIT_COMMITTER_NAME'] = config.build.tagger.name
             env['GIT_COMMITTER_EMAIL'] = config.build.tagger.email
             env = dict(os.environ)
             env['GIT_COMMITTER_NAME'] = config.build.tagger.name
             env['GIT_COMMITTER_EMAIL'] = config.build.tagger.email

+            env['GIT_AUTHOR_NAME'] = principal.split('@')[0]
+            env['GIT_AUTHOR_EMAIL'] = principal

 
             # If validateBuild returns something other than True, then
             # it means we should copy from that pocket to our pocket.
 
             # If validateBuild returns something other than True, then
             # it means we should copy from that pocket to our pocket.

@@ -371,7 +404,7 @@ def build():
             # do the build ourselves
             else:
                 db.failed_stage = 'checking out package source'
             # do the build ourselves
             else:
                 db.failed_stage = 'checking out package source'

-                with packageWorkdir(package, commit) as workdir:
+                with packageWorkdir(package, commit, db.build_id) as workdir:

                     db.failed_stage = 'preparing source package'
                     packagedir = os.path.join(workdir, package)
 
                     db.failed_stage = 'preparing source package'
                     packagedir = os.path.join(workdir, package)
 

@@ -382,7 +415,9 @@ def build():
                     # If we were, we could use debuild and get nice
                     # environment scrubbing. Since we're not, debuild
                     # complains about not having an orig.tar.gz
                     # If we were, we could use debuild and get nice
                     # environment scrubbing. Since we're not, debuild
                     # complains about not having an orig.tar.gz

-                    logAndRun(['dpkg-buildpackage', '-us', '-uc', '-S'],
+                    logAndRun(['schroot', '-c', 
+                               '%s-amd64-sbuild' % (b.pocketToDistro(pocket),), 
+                               '--', 'dpkg-buildpackage', '-us', '-uc', '-S'],

                               cwd=packagedir)
 
                     db.failed_stage = 'building binary packages'
                               cwd=packagedir)
 
                     db.failed_stage = 'building binary packages'