Make invirtibuilder work

svn path=/trunk/packages/invirt-dev/; revision=3028

diff --git a/README b/README
index 3339738..3978406 100644 (file)
--- a/README
+++ b/README
@@ -5,3 +5,6 @@ Installation:
 
    The public key should then be in the apt keyring of systems using
    the packages.
 
    The public key should then be in the apt keyring of systems using
    the packages.

+
+ - Packages should be located at
+   /srv/git/invirt/packages/${packagename}.git

diff --git a/debian/changelog b/debian/changelog
index 0cf4983..3fc0322 100644 (file)
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+invirt-dev (0.1.1) unstable; urgency=low
+
+  * Added missing imports
+  * Other minor functional changes
+
+ -- Greg Brockman <gdb@mit.edu>  Mon, 05 Jul 2010 01:43:45 -0400
+

 invirt-dev (0.1.0) unstable; urgency=low
 
   * Switch to using git instead of svn.
 invirt-dev (0.1.0) unstable; urgency=low
 
   * Switch to using git instead of svn.

diff --git a/debian/control b/debian/control
index 384b314..2f138d6 100644 (file)
--- a/debian/control
+++ b/debian/control
@@ -7,6 +7,6 @@ Standards-Version: 3.7.2
 
 Package: invirt-dev
 Architecture: all
 
 Package: invirt-dev
 Architecture: all

-Depends: ${shlibs:Depends}, ${misc:Depends}, dpkg-dev-el, emacs21, reprepro, apache2, postfix, screen, dh-make, fakeroot, quilt, patchutils, config-package-dev, pbuilder, equivs, invirt-base, invirt-database, remctl-server, update-inetd, openbsd-inetd | inet-superserver, python-pyinotify
+Depends: ${shlibs:Depends}, ${misc:Depends}, dpkg-dev-el, emacs21, reprepro, apache2, postfix, screen, dh-make, fakeroot, quilt, patchutils, config-package-dev, sbuild, equivs, invirt-base, invirt-database, remctl-server, update-inetd, openbsd-inetd | inet-superserver, python-pyinotify, python-debian, git-core (>= 1.6.4)

 Description: Invirt build and apt server
  This packages the build scripts and apt-repository configuration for Invirt.
 Description: Invirt build and apt server
  This packages the build scripts and apt-repository configuration for Invirt.

diff --git a/invirt-submit-build b/invirt-submit-build
index 42446b0..8175865 100755 (executable)
--- a/invirt-submit-build
+++ b/invirt-submit-build
@@ -18,6 +18,7 @@ keeping.
 
 
 import datetime
 
 
 import datetime

+import optparse

 import os
 import sys
 import tempfile
 import os
 import sys
 import tempfile

@@ -27,7 +28,12 @@ import invirt.builder as b
 
 
 def main():
 
 
 def main():

-    pocket, package, commit = sys.argv[1:4]
+    parser = optparse.OptionParser('Usage: %prog pocket package commit')
+    opts, args = parser.parse_args()
+    if len(args) != 3:
+        parser.print_help()
+        return 1
+    pocket, package, commit = args

     principal = os.environ['REMOTE_USER']
     request_time = datetime.datetime.utcnow()
     q_path = os.path.join(b._QUEUE_DIR,
     principal = os.environ['REMOTE_USER']
     request_time = datetime.datetime.utcnow()
     q_path = os.path.join(b._QUEUE_DIR,

@@ -46,6 +52,7 @@ def main():
     q_fd, q_name = tempfile.mkstemp()
     q = os.fdopen(q_fd, 'r+')
     print >>q, "%s %s %s %s" % (pocket, package, commit, principal)
     q_fd, q_name = tempfile.mkstemp()
     q = os.fdopen(q_fd, 'r+')
     print >>q, "%s %s %s %s" % (pocket, package, commit, principal)

+    q.close()

     os.rename(q_name, q_path)
 
 
     os.rename(q_name, q_path)
 
 

diff --git a/invirtibuilder b/invirtibuilder
index f5c3821..d2a571c 100755 (executable)
--- a/invirtibuilder
+++ b/invirtibuilder
@@ -29,15 +29,22 @@ principal is the Kerberos principal that requested the build.
 from __future__ import with_statement
 
 import contextlib
 from __future__ import with_statement
 
 import contextlib

+import glob

 import os
 import re
 import shutil
 import subprocess
 import os
 import re
 import shutil
 import subprocess

+import tempfile
+import traceback

 
 import pyinotify
 
 
 import pyinotify
 

+from debian_bundle import deb822
+

 import invirt.builder as b
 import invirt.builder as b

+import invirt.common as c

 from invirt import database
 from invirt import database

+from invirt.config import structs as config

 
 
 DISTRIBUTION = 'hardy'
 
 
 DISTRIBUTION = 'hardy'

@@ -51,7 +58,7 @@ def getControl(package, ref):
     acts roughly like a dict.
     """
     return deb822.Deb822.iter_paragraphs(
     acts roughly like a dict.
     """
     return deb822.Deb822.iter_paragraphs(

-        getGitFile(package, ref, 'debian/control').split('\n'))
+        b.getGitFile(package, ref, 'debian/control').split('\n'))

 
 
 def getBinaries(package, ref):
 
 
 def getBinaries(package, ref):

@@ -71,7 +78,7 @@ def getArches(package, ref):
 
 def getDscName(package, ref):
     """Return the .dsc file that will be generated for this package."""
 
 def getDscName(package, ref):
     """Return the .dsc file that will be generated for this package."""

-    v = getVersion(package, ref)
+    v = b.getVersion(package, ref)

     if v.debian_version:
         v_str = '%s-%s' % (v.upstream_version,
                            v.debian_version)
     if v.debian_version:
         v_str = '%s-%s' % (v.upstream_version,
                            v.debian_version)

@@ -95,13 +102,13 @@ def sanitizeVersion(version):
     return v.replace('~', '.')
 
 
     return v.replace('~', '.')
 
 

-def aptCopy(packages, dst_pocket, src_pocket):
+def aptCopy(package, commit, dst_pocket, src_pocket):

     """Copy a package from one pocket to another."""
     binaries = getBinaries(package, commit)
     """Copy a package from one pocket to another."""
     binaries = getBinaries(package, commit)

-    cpatureOutput(['reprepro-env', 'copy',
-                   b.pocketToApt(dst_pocket),
-                   b.pocketToApt(src_pocket),
-                   package] + binaries)
+    c.captureOutput(['reprepro-env', 'copy',
+                     b.pocketToApt(dst_pocket),
+                     b.pocketToApt(src_pocket),
+                     package] + binaries)

 
 
 def sbuild(package, ref, arch, workdir, arch_all=False):
 
 
 def sbuild(package, ref, arch, workdir, arch_all=False):

@@ -122,7 +129,7 @@ def sbuildAll(package, ref, workdir):
         sbuild(package, ref, 'i386', workdir)
 
 
         sbuild(package, ref, 'i386', workdir)
 
 

-def tagSubmodule(pocket, package, ref, principal):
+def tagSubmodule(pocket, package, principal, version, env):

     """Tag a new version of a submodule.
 
     If this pocket does not allow_backtracking, then this will create
     """Tag a new version of a submodule.
 
     If this pocket does not allow_backtracking, then this will create

@@ -138,12 +145,7 @@ def tagSubmodule(pocket, package, ref, principal):
     legitimacy of my reasoning.
     """
     if not config.build.pockets[pocket].get('allow_backtracking', False):
     legitimacy of my reasoning.
     """
     if not config.build.pockets[pocket].get('allow_backtracking', False):

-        env = dict(os.environ)

         branch = b.pocketToGit(pocket)
         branch = b.pocketToGit(pocket)

-        version = b.getVersion(package, ref)
-
-        env['GIT_COMMITTER_NAME'] = config.build.tagger.name
-        env['GIT_COMMITTER_EMAIL'] = config.build.tagger.email

         tag_msg = ('Tag %s of %s\n\n'
                    'Requested by %s' % (version.full_version,
                                         package,
         tag_msg = ('Tag %s of %s\n\n'
                    'Requested by %s' % (version.full_version,
                                         package,

@@ -159,7 +161,7 @@ def updateSubmoduleBranch(pocket, package, ref):
     """Update the appropriately named branch in the submodule."""
     branch = b.pocketToGit(pocket)
     c.captureOutput(
     """Update the appropriately named branch in the submodule."""
     branch = b.pocketToGit(pocket)
     c.captureOutput(

-        ['git', 'update-ref', 'refs/heads/%s' % branch, ref])
+        ['git', 'update-ref', 'refs/heads/%s' % branch, ref], cwd=b.getRepo(package))

 
 
 def uploadBuild(pocket, workdir):
 
 
 def uploadBuild(pocket, workdir):

@@ -167,13 +169,13 @@ def uploadBuild(pocket, workdir):
     apt = b.pocketToApt(pocket)
     for changes in glob.glob(os.path.join(workdir, '*.changes')):
         c.captureOutput(['reprepro-env',
     apt = b.pocketToApt(pocket)
     for changes in glob.glob(os.path.join(workdir, '*.changes')):
         c.captureOutput(['reprepro-env',

-                       'include',

                        '--ignore=wrongdistribution',
                        '--ignore=wrongdistribution',

+                       'include',

                        apt,
                        changes])
 
 
                        apt,
                        changes])
 
 

-def updateSuperproject(pocket, package, commit, principal):
+def updateSuperproject(pocket, package, commit, principal, version, env):

     """Update the superproject.
 
     This will create a new commit on the branch for the given pocket
     """Update the superproject.
 
     This will create a new commit on the branch for the given pocket

@@ -185,32 +187,35 @@ def updateSuperproject(pocket, package, commit, principal):
     superproject = os.path.join(b._REPO_DIR, 'invirt/packages.git')
     branch = b.pocketToGit(pocket)
     tree = c.captureOutput(['git', 'ls-tree', branch],
     superproject = os.path.join(b._REPO_DIR, 'invirt/packages.git')
     branch = b.pocketToGit(pocket)
     tree = c.captureOutput(['git', 'ls-tree', branch],

-                         cwd=superproject)
+                           cwd=superproject).strip()

 
     new_tree = re.compile(
         r'^(160000 commit )[0-9a-f]*(\t%s)$' % package, re.M).sub(
 
     new_tree = re.compile(
         r'^(160000 commit )[0-9a-f]*(\t%s)$' % package, re.M).sub(

-        r'\1%s\2' % commit,
+        r'\g<1>%s\g<2>' % commit,

         tree)
 
         tree)
 

-    new_tree_id = c.captureOutput(['git', 'mktree'],
-                                cwd=superproject,
-                                stdin_str=new_tree)
+    new_tree_id = c.captureOutput(['git', 'mktree', '--missing'],
+                                  cwd=superproject,
+                                  stdin_str=new_tree).strip()

 
     commit_msg = ('Update %s to version %s\n\n'
                   'Requested by %s' % (package,
                                        version.full_version,
                                        principal))
     new_commit = c.captureOutput(
 
     commit_msg = ('Update %s to version %s\n\n'
                   'Requested by %s' % (package,
                                        version.full_version,
                                        principal))
     new_commit = c.captureOutput(

-        ['git', 'commit-tree', new_tree_hash, '-p', branch],
+        ['git', 'commit-tree', new_tree_id, '-p', branch],

         cwd=superproject,
         env=env,
         cwd=superproject,
         env=env,

-        stdin_str=commit_msg)
+        stdin_str=commit_msg).strip()

 
     c.captureOutput(
         ['git', 'update-ref', 'refs/heads/%s' % branch, new_commit],
         cwd=superproject)
 
 
 
     c.captureOutput(
         ['git', 'update-ref', 'refs/heads/%s' % branch, new_commit],
         cwd=superproject)
 
 

+def makeReadable(workdir):
+    os.chmod(workdir, 0755)
+

 @contextlib.contextmanager
 def packageWorkdir(package, commit):
     """Checkout the package in a temporary working directory.
 @contextlib.contextmanager
 def packageWorkdir(package, commit):
     """Checkout the package in a temporary working directory.

@@ -227,7 +232,7 @@ def packageWorkdir(package, commit):
         p_archive = subprocess.Popen(
             ['git', 'archive',
              '--remote=file://%s' % b.getRepo(package),
         p_archive = subprocess.Popen(
             ['git', 'archive',
              '--remote=file://%s' % b.getRepo(package),

-             '--prefix=%s' % package,
+             '--prefix=%s/' % package,

              commit,
              ],
             stdout=subprocess.PIPE,
              commit,
              ],
             stdout=subprocess.PIPE,

@@ -277,13 +282,17 @@ def build():
         db.commit = commit
         db.principal = principal
         database.session.save_or_update(db)
         db.commit = commit
         db.principal = principal
         database.session.save_or_update(db)

-        database.commit()
+        database.session.commit()

 
 

-        database.begin()
+        database.session.begin()

 
         try:
             db.failed_stage = 'validating job'
 
         try:
             db.failed_stage = 'validating job'

-            src = validateBuild(pocket, package, commit)
+            src = b.validateBuild(pocket, package, commit)
+            # Don't expand the commit in the DB until we're sure the user
+            # isn't trying to be tricky.
+            db.commit = commit = c.captureOutput(['git', 'rev-parse', commit],
+                                                 cwd=b.getRepo(package)).strip()

 
             db.version = str(b.getVersion(package, commit))
 
 
             db.version = str(b.getVersion(package, commit))
 

@@ -294,7 +303,7 @@ def build():
             # raised an exception)
             if src != True:
                 db.failed_stage = 'copying package from another pocket'
             # raised an exception)
             if src != True:
                 db.failed_stage = 'copying package from another pocket'

-                aptCopy(packages, pocket, src)
+                aptCopy(package, commit, pocket, src)

             # If we can't copy the package from somewhere, but
             # validateBuild didn't raise an exception, then we need to
             # do the build ourselves
             # If we can't copy the package from somewhere, but
             # validateBuild didn't raise an exception, then we need to
             # do the build ourselves

@@ -319,18 +328,27 @@ def build():
                         db.failed_stage = 'building binary packages'
                         sbuildAll(package, commit, workdir)
                     finally:
                         db.failed_stage = 'building binary packages'
                         sbuildAll(package, commit, workdir)
                     finally:

-                        logdir = os.path.join(b._LOG_DIR, db.build_id)
+                        logdir = os.path.join(b._LOG_DIR, str(db.build_id))

                         if not os.path.exists(logdir):
                             os.makedirs(logdir)
 
                         for log in glob.glob(os.path.join(workdir, '*.build')):
                             os.copy2(log, logdir)
                         if not os.path.exists(logdir):
                             os.makedirs(logdir)
 
                         for log in glob.glob(os.path.join(workdir, '*.build')):
                             os.copy2(log, logdir)

+
+                    db.failed_stage = 'processing metadata'
+                    env = dict(os.environ)
+                    env['GIT_COMMITTER_NAME'] = config.build.tagger.name
+                    env['GIT_COMMITTER_EMAIL'] = config.build.tagger.email
+                    version = b.getVersion(package, commit)
+

                     db.failed_stage = 'tagging submodule'
                     db.failed_stage = 'tagging submodule'

-                    tagSubmodule(pocket, package, commit, principal)
+                    tagSubmodule(pocket, package, principal, version, env)

                     db.failed_stage = 'updating submodule branches'
                     updateSubmoduleBranch(pocket, package, commit)
                     db.failed_stage = 'updating superproject'
                     db.failed_stage = 'updating submodule branches'
                     updateSubmoduleBranch(pocket, package, commit)
                     db.failed_stage = 'updating superproject'

-                    updateSuperproject(pocket, package, commit, principal)
+                    updateSuperproject(pocket, package, commit, principal, version, env)
+                    db.failed_stage = 'relaxing permissions on workdir'
+                    makeReadable(workdir)

                     db.failed_stage = 'uploading packages to apt repo'
                     uploadBuild(pocket, workdir)
 
                     db.failed_stage = 'uploading packages to apt repo'
                     uploadBuild(pocket, workdir)
 

diff --git a/python/invirt/builder.py b/python/invirt/builder.py
index c1730df..f2cb421 100644 (file)
--- a/python/invirt/builder.py
+++ b/python/invirt/builder.py
@@ -28,6 +28,19 @@ def getRepo(package):
     """Return the path to the git repo for a given package."""
     return os.path.join(_REPO_DIR, 'invirt/packages', '%s.git' % package)
 
     """Return the path to the git repo for a given package."""
     return os.path.join(_REPO_DIR, 'invirt/packages', '%s.git' % package)
 

+def ensureValidRepo(package):
+    """Perform some basic sanity checks that the requested repo is in a
+    subdirectory of _REPO_DIR/invirt/packages.  This prevents weirdness
+    such as submitting a package like '../prod/...git'.  Also ensures that
+    the repo exists."""
+    # TODO: this might be easier just to regex
+    repo = os.path.abspath(getRepo(package))
+    parent_dir = os.path.dirname(repo)
+    prefix = os.path.join(_REPO_DIR, 'invirt/packages')
+    if not parent_dir.startswith(prefix):
+        raise InvalidBuild('Invalid package name %s' % package)
+    elif not os.path.exists(repo):
+        raise InvalidBuild('Nonexisting package %s' % package)

 
 def pocketToGit(pocket):
     """Map a pocket in the configuration to a git branch."""
 
 def pocketToGit(pocket):
     """Map a pocket in the configuration to a git branch."""

@@ -42,7 +55,7 @@ def pocketToApt(pocket):
 def getGitFile(package, ref, path):
     """Return the contents of a path from a git ref in a package."""
     return c.captureOutput(['git', 'cat-file', 'blob', '%s:%s' % (ref, path)],
 def getGitFile(package, ref, path):
     """Return the contents of a path from a git ref in a package."""
     return c.captureOutput(['git', 'cat-file', 'blob', '%s:%s' % (ref, path)],

-                         cwd=getRepo(package))
+                           cwd=getRepo(package))

 
 
 def getChangelog(package, ref):
 
 
 def getChangelog(package, ref):

@@ -85,6 +98,7 @@ def validateBuild(pocket, package, commit):
     another pocket, then this function returns that pocket. Otherwise,
     it returns True.
     """
     another pocket, then this function returns that pocket. Otherwise,
     it returns True.
     """

+    ensureValidRepo(package)

     package_repo = getRepo(package)
     new_version = getVersion(package, commit)
 
     package_repo = getRepo(package)
     new_version = getVersion(package, commit)
 

@@ -96,18 +110,18 @@ def validateBuild(pocket, package, commit):
 
         b = pocketToGit(p)
         current_commit = c.captureOutput(['git', 'rev-parse', b],
 
         b = pocketToGit(p)
         current_commit = c.captureOutput(['git', 'rev-parse', b],

-                                       cwd=package_repo)
+                                         cwd=package_repo).strip()

         current_version = getVersion(package, b)
 
         if current_version == new_version:
             if current_commit == commit:
                 ret = p
             else:
         current_version = getVersion(package, b)
 
         if current_version == new_version:
             if current_commit == commit:
                 ret = p
             else:

-                raise InvalidBuild('Version %s of %s already available in '
+                raise InvalidBuild('Version %s of %s already available is in '

                                    'pocket %s from commit %s' %
                                    (new_version, package, p, current_commit))
 
                                    'pocket %s from commit %s' %
                                    (new_version, package, p, current_commit))
 

-    if config.build.pockets[pocket].get('allow_backtracking', False):
+    if not config.build.pockets[pocket].get('allow_backtracking', False):

         branch = pocketToGit(pocket)
         current_version = getVersion(package, branch)
         if new_version <= current_version:
         branch = pocketToGit(pocket)
         current_version = getVersion(package, branch)
         if new_version <= current_version: