Added reporting
authorGreg Brockman <gdb@mit.edu>
Tue, 6 Jul 2010 04:27:50 +0000 (00:27 -0400)
committerGreg Brockman <gdb@mit.edu>
Tue, 6 Jul 2010 04:27:50 +0000 (00:27 -0400)
svn path=/trunk/packages/invirt-dev/; revision=3036

build-hooks/failed-build [new symlink]
build-hooks/failed-submit [new file with mode: 0755]
build-hooks/post-build [new file with mode: 0755]
build-hooks/post-submit [new file with mode: 0755]
build-hooks/pre-build [new file with mode: 0755]
debian/invirt-dev.dirs
debian/invirt-dev.install
invirt-submit-build
invirtibuilder
python/invirt/builder.py

diff --git a/build-hooks/failed-build b/build-hooks/failed-build
new file mode 120000 (symlink)
index 0000000..b3ca024
--- /dev/null
@@ -0,0 +1 @@
+post-build
\ No newline at end of file
diff --git a/build-hooks/failed-submit b/build-hooks/failed-submit
new file mode 100755 (executable)
index 0000000..6e288c8
--- /dev/null
@@ -0,0 +1,35 @@
+#!/bin/sh
+
+set -e
+set -u
+
+escape() {
+    echo "$1" | sed -e 's/@/@@/g'
+}
+
+pocket=$(escape "$1")
+package=$(escape "$2")
+commit=$(escape "$3")
+principal=$(escape "$4")
+
+base=build.hooks.failed_submit.zephyr
+class=$(invirt-getconf "$base.class" 2>/dev/null || :)
+instance=$(invirt-getconf "$base.instance" 2>/dev/null || :)
+zsig=$(invirt-getconf "$base.zsig" 2>/dev/null || :)
+
+if [ -z "$class" ]; then
+  echo "I don't know where to send a commit zephyr!" >&2
+  echo "Please provide a value for $base.class in" >&2
+  echo "your invirt config file." >&2
+  exit 1
+fi
+
+(echo "A new job has @{@color(red)failed} to be submitted to the Invirtibuilder:"
+ echo
+ echo "pocket: $pocket"
+ echo "package: $package"
+ echo "commit: $commit"k
+ echo "principal: $principal"
+ echo
+ echo -n "Failure: ";
+ ( cat | sed -e 's/@/@@/g' ) ) | zwrite -c "$class" -i "${instance:-$commit}" -s "${zsig:-failed-submit}: $pocket" -d
diff --git a/build-hooks/post-build b/build-hooks/post-build
new file mode 100755 (executable)
index 0000000..a727cf1
--- /dev/null
@@ -0,0 +1,156 @@
+#!/usr/bin/env python
+"""
+A script for reporting the results of an invirtibuild.  Supports any
+combination of the following reporting mechanisms.
+
+Note that all strings are interpolated with a dictionary containing
+the following keys:
+
+build_id, commit, failed_stage, inserted_at, package,
+pocket, principal, result, short_commit, traceback, version
+
+== zephyr ==
+
+To configure zephyr, add something like the following to your invirt config:
+
+build:
+ hooks:
+  post_build:
+   zephyr: &post_build_zepyhr
+    class: myclass [required]
+    instance: myinstance [optional]
+    zsig: myzsig [optional]
+  failed_build:
+   zephyr: *post_build_zephyr
+
+== mail ==
+
+To configure email notifications, add something like the following to your invirt config:
+
+build:
+ hooks:
+  post_build:
+   mail: &post_build_mail
+    to: myemail@example.com [required]
+    from: myemail@example.com [required]
+    subject: My Subject [optional]
+  failed_build:
+   mail: *post_build_mail
+
+post_build values will be used when this script is invoked as
+post-build, while failed_build values will be used when it is invoked
+as failed-build.
+"""
+
+import optparse
+import os
+import re
+import sys
+import textwrap
+
+from email.mime import text
+
+from invirt import common, database, builder
+from invirt.config import structs as config
+
+def make_msg(build, values, verbose=True, success=lambda x: x, failure=lambda x: x):
+    values = dict(values)
+    if not verbose and values['traceback'] is not None:
+        values['traceback'] = textwrap.fill('\n'.join(values['traceback'].split('\n')[-2:]))
+
+    if build.succeeded:
+        values['result'] = success(values['result'])
+        msg = """Build of %(package)s v%(version)s in %(pocket)s %(result)s.
+
+Branch %(pocket)s has been advanced to %(short_commit)s.
+
+(Build %(build_id)s was submitted by %(principal)s at %(inserted_at)s.)""" % values
+    else:
+        values['result'] = failure(values['result'])
+        msg = """Build of %(package)s v%(version)s in %(pocket)s %(result)s while %(failed_stage)s.
+
+%(traceback)s
+
+(Build %(build_id)s was submitted by %(principal)s at %(inserted_at)s.)""" % values
+    return msg
+
+def zephyr_escape(m):
+    m = re.sub('@', '@@', m)
+    m = re.sub('}', '@(})', m)
+    return m
+
+def zephyr_success(m):
+    return '}@{@color(green)%s}@{' % zephyr_escape(m)
+
+def zephyr_failure(m):
+    return '}@{@color(red)%s}@{' % zephyr_escape(m)
+
+def main():
+    parser = optparse.OptionParser('Usage: %prog build_id')
+    opts, args = parser.parse_args()
+    if len(args) != 1:
+        parser.print_help()
+        return 1
+    prog = os.path.basename(sys.argv[0])
+
+    database.connect()
+    build = database.Build.query().get(args[0])
+    short_commit = builder.canonicalize_commit(build.package, build.commit, shorten=True)
+    values = { 'build_id' : build.build_id,
+               'commit' : build.commit,
+               'failed_stage' : build.failed_stage,
+               'inserted_at' : build.inserted_at,
+               'package' : build.package,
+               'pocket' : build.pocket,
+               'principal' : build.principal,
+               'short_commit' : short_commit,
+               'traceback' : build.traceback,
+               'version' : build.version }
+    if build.succeeded:
+        values['result'] = 'succeeded'
+    else:
+        values['result'] = 'failed'
+
+    try:
+        if prog == 'post-build':
+            hook_config = config.build.hooks.post_build
+        elif prog == 'failed-build':
+            hook_config = config.build.hooks.failed_build
+        else:
+            print >>sys.stderr, '{post,failed}-build invoke with unrecognized name %s' % prog
+            return 2
+    except common.InvirtConfigError:
+        print >>sys.stderr, 'No hook configuration found for %s.' % prog
+        return 1
+
+    try:
+        zephyr_config = hook_config.zephyr
+        klass = zephyr_config['class'] % values
+    except common.InvirtConfigError:
+        print >>sys.stderr, 'No zephyr configuration specified for %s.' % prog
+    else:
+        msg = '@{%s}' % make_msg(build, values, verbose=False,
+                                 success=zephyr_success, failure=zephyr_failure)
+        instance = zephyr_config.get('instance', 'build_%(build_id)s') % values
+        zsig = zephyr_config.get('zsig', 'XVM Buildbot') % values
+        common.captureOutput(['zwrite', '-c', klass, '-i', instance, '-s',
+                              zsig, '-d', '-m', msg],
+                             stdout=None, stderr=None)
+
+    try:
+        mail_config = hook_config.mail
+        to = mail_config.to % values
+        sender = mail_config['from'] % values
+    except common.InvirtConfigError:
+        print >>sys.stderr, 'No email configuration specified for %s.' % prog
+    else:
+        msg = make_msg(build, values)
+        email = text.MIMEText(msg)
+        email['To'] = to % values
+        email['From'] = sender % values
+        email['Subject'] = mail_config.get('subject', 'XVM build %(build_id)s has %(result)s') % values
+        common.captureOutput(['sendmail', '-t'], email.as_string(),
+                             stdout=None, stderr=None)
+        
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/build-hooks/post-submit b/build-hooks/post-submit
new file mode 100755 (executable)
index 0000000..294818a
--- /dev/null
@@ -0,0 +1,32 @@
+#!/bin/sh
+
+set -e
+set -u
+
+escape() {
+    echo "$1" | sed -e 's/@/@@/g'
+}
+
+pocket=$(escape "$1")
+package=$(escape "$2")
+commit=$(escape "$3")
+principal=$(escape "$4")
+
+base=build.hooks.post_submit.zephyr
+class=$(invirt-getconf "$base.class" 2>/dev/null || :)
+instance=$(invirt-getconf "$base.instance" 2>/dev/null || :)
+zsig=$(invirt-getconf "$base.zsig" 2>/dev/null || :)
+
+if [ -z "$class" ]; then
+  echo "I don't know where to send a commit zephyr!" >&2
+  echo "Please provide a value for $base.class in" >&2
+  echo "your invirt config file." >&2
+  exit 1
+fi
+
+(echo "A new job has been submitted to the Invirtibuilder:";
+ echo;
+ echo "pocket: $pocket";
+ echo "package: $package";
+ echo "commit: $commit";
+ echo "principal: $principal") | zwrite -c "$class" -i "${instance:-$commit}" -s "${zsig:-Git}: $pocket" -d
diff --git a/build-hooks/pre-build b/build-hooks/pre-build
new file mode 100755 (executable)
index 0000000..68c3c57
--- /dev/null
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+set -e
+set -u
+
+escape() {
+    echo "$1" | sed -e 's/@/@@/g'
+}
+
+build_id=$(escape "$1")
+pocket=$(escape "$2")
+package=$(escape "$3")
+commit=$(escape "$4")
+principal=$(escape "$5")
+version=$(escape "$6")
+inserted_at=$(escape "$7")
+
+base=build.hooks.pre_build.zephyr
+class=$(invirt-getconf "$base.class" 2>/dev/null || :)
+instance=$(invirt-getconf "$base.instance" 2>/dev/null || :)
+zsig=$(invirt-getconf "$base.zsig" 2>/dev/null || :)
+
+if [ -z "$class" ]; then
+  echo "I don't know where to send a commit zephyr!" >&2
+  echo "Please provide a value for $base.class in" >&2
+  echo "your invirt config file." >&2
+  exit 1
+fi
+
+(echo "About to begin an Invirtibuild of $package v$version in $pocket."
+ echo "from commit $commit.";
+ echo
+ echo "(Build $build_id was submitted by $principal at $inserted_at.)") | zwrite -c "$class" -i "${instance:-build_$build_id}" -s "${zsig:-pre-build}: $pocket" -d
index a17355d..da72f77 100644 (file)
@@ -1,3 +1,3 @@
 var/lib/invirt-dev/queue
 var/log/invirt/builds
-usr/share/invirt-dev/build.d
+usr/share/invirt-dev/build-hooks
index 2982e0c..8980a61 100644 (file)
@@ -1,2 +1,3 @@
 reprepro-env usr/bin
 repository-config/* srv/repository/conf
+build-hooks usr/share/invirt-dev
\ No newline at end of file
index f9b5697..4bed2ee 100755 (executable)
@@ -46,7 +46,11 @@ def main():
         commit = b.canonicalize_commit(package, commit)
         b.validateBuild(pocket, package, commit)
     except b.InvalidBuild, e:
-        print >>sys.stderr, "E: %s" % e
+        msg = "E: %s" % e
+        print >>sys.stderr, msg
+        # Prevent an attack by submitting excessively long arguments
+        args = [arg[:min(len(arg), 80)] for arg in (pocket, package, commit)]
+        b.runHook('failed-submit', args + [principal], stdin_str=msg)
         sys.exit(1)
 
     # To keep from triggering the Invirtibuilder before we've actually
@@ -57,6 +61,9 @@ def main():
     print >>q, "%s %s %s %s" % (pocket, package, commit, principal)
     q.close()
     os.rename(q_name, q_path)
+    short_commit = b.canonicalize_commit(package, commit, shorten=True)
+    b.runHook('post-submit', [pocket, package, short_commit, principal])
+    print '%s, your job to build %s for %s:%s has been submitted!' % (principal, short_commit, package, pocket)
 
 
 if __name__ == '__main__':
index 24861f5..a42ce55 100755 (executable)
@@ -249,16 +249,6 @@ def packageWorkdir(package, commit):
     finally:
         shutil.rmtree(workdir)
 
-
-def reportBuild(build):
-    """Run hooks to report the results of a build attempt."""
-
-    c.captureOutput(['run-parts',
-                   '--arg=%s' % build.build_id,
-                   '--',
-                   b._HOOKS_DIR])
-
-
 def build():
     """Deal with items in the build queue.
 
@@ -295,6 +285,8 @@ def build():
             src = b.validateBuild(pocket, package, commit)
 
             db.version = str(b.getVersion(package, commit))
+            b.runHook('pre-build', [str(db.build_id), db.pocket, db.package,
+                                    db.commit, db.principal, db.version, str(db.inserted_at)])
 
             # If validateBuild returns something other than True, then
             # it means we should copy from that pocket to our pocket.
@@ -366,8 +358,10 @@ def build():
             # build queue item
             os.unlink(os.path.join(b._QUEUE_DIR, build))
 
-            reportBuild(db)
-
+            if db.succeeded:
+                b.runHook('post-build', [str(db.build_id)])
+            else:
+                b.runHook('failed-build', [str(db.build_id)])
 
 class Invirtibuilder(pyinotify.ProcessEvent):
     """Process inotify triggers to build new packages."""
index 68d56c8..b3aa08f 100644 (file)
@@ -17,7 +17,7 @@ from invirt.config import structs as config
 _QUEUE_DIR = '/var/lib/invirt-dev/queue'
 _REPO_DIR = '/srv/git'
 _LOG_DIR = '/var/log/invirt/builds'
-_HOOKS_DIR = '/usr/share/invirt-dev/build.d'
+_HOOKS_DIR = '/usr/share/invirt-dev/build-hooks'
 
 
 class InvalidBuild(ValueError):
@@ -74,6 +74,13 @@ def getChangelog(package, ref):
     """
     return changelog.Changelog(getGitFile(package, ref, 'debian/changelog'))
 
+def runHook(hook, args=[], stdin_str=None):
+    """Run a named hook."""
+    hook = os.path.join(_HOOKS_DIR, hook)
+    try:
+        c.captureOutput([hook] + args, stdin_str=stdin_str)
+    except OSError:
+        pass
 
 def getVersion(package, ref):
     """Get the version of a given package at a particular ref."""