From: Greg Brockman Date: Tue, 6 Jul 2010 04:27:50 +0000 (-0400) Subject: Added reporting X-Git-Tag: 0.1.5~16 X-Git-Url: http://xvm.mit.edu/gitweb/invirt/packages/invirt-dev.git/commitdiff_plain/929f1553a7e60441bee2b53fed3530445a7299cf Added reporting svn path=/trunk/packages/invirt-dev/; revision=3036 --- diff --git a/build-hooks/failed-build b/build-hooks/failed-build new file mode 120000 index 0000000..b3ca024 --- /dev/null +++ b/build-hooks/failed-build @@ -0,0 +1 @@ +post-build \ No newline at end of file diff --git a/build-hooks/failed-submit b/build-hooks/failed-submit new file mode 100755 index 0000000..6e288c8 --- /dev/null +++ b/build-hooks/failed-submit @@ -0,0 +1,35 @@ +#!/bin/sh + +set -e +set -u + +escape() { + echo "$1" | sed -e 's/@/@@/g' +} + +pocket=$(escape "$1") +package=$(escape "$2") +commit=$(escape "$3") +principal=$(escape "$4") + +base=build.hooks.failed_submit.zephyr +class=$(invirt-getconf "$base.class" 2>/dev/null || :) +instance=$(invirt-getconf "$base.instance" 2>/dev/null || :) +zsig=$(invirt-getconf "$base.zsig" 2>/dev/null || :) + +if [ -z "$class" ]; then + echo "I don't know where to send a commit zephyr!" >&2 + echo "Please provide a value for $base.class in" >&2 + echo "your invirt config file." >&2 + exit 1 +fi + +(echo "A new job has @{@color(red)failed} to be submitted to the Invirtibuilder:" + echo + echo "pocket: $pocket" + echo "package: $package" + echo "commit: $commit"k + echo "principal: $principal" + echo + echo -n "Failure: "; + ( cat | sed -e 's/@/@@/g' ) ) | zwrite -c "$class" -i "${instance:-$commit}" -s "${zsig:-failed-submit}: $pocket" -d diff --git a/build-hooks/post-build b/build-hooks/post-build new file mode 100755 index 0000000..a727cf1 --- /dev/null +++ b/build-hooks/post-build @@ -0,0 +1,156 @@ +#!/usr/bin/env python +""" +A script for reporting the results of an invirtibuild. Supports any +combination of the following reporting mechanisms. + +Note that all strings are interpolated with a dictionary containing +the following keys: + +build_id, commit, failed_stage, inserted_at, package, +pocket, principal, result, short_commit, traceback, version + +== zephyr == + +To configure zephyr, add something like the following to your invirt config: + +build: + hooks: + post_build: + zephyr: &post_build_zepyhr + class: myclass [required] + instance: myinstance [optional] + zsig: myzsig [optional] + failed_build: + zephyr: *post_build_zephyr + +== mail == + +To configure email notifications, add something like the following to your invirt config: + +build: + hooks: + post_build: + mail: &post_build_mail + to: myemail@example.com [required] + from: myemail@example.com [required] + subject: My Subject [optional] + failed_build: + mail: *post_build_mail + +post_build values will be used when this script is invoked as +post-build, while failed_build values will be used when it is invoked +as failed-build. +""" + +import optparse +import os +import re +import sys +import textwrap + +from email.mime import text + +from invirt import common, database, builder +from invirt.config import structs as config + +def make_msg(build, values, verbose=True, success=lambda x: x, failure=lambda x: x): + values = dict(values) + if not verbose and values['traceback'] is not None: + values['traceback'] = textwrap.fill('\n'.join(values['traceback'].split('\n')[-2:])) + + if build.succeeded: + values['result'] = success(values['result']) + msg = """Build of %(package)s v%(version)s in %(pocket)s %(result)s. + +Branch %(pocket)s has been advanced to %(short_commit)s. + +(Build %(build_id)s was submitted by %(principal)s at %(inserted_at)s.)""" % values + else: + values['result'] = failure(values['result']) + msg = """Build of %(package)s v%(version)s in %(pocket)s %(result)s while %(failed_stage)s. + +%(traceback)s + +(Build %(build_id)s was submitted by %(principal)s at %(inserted_at)s.)""" % values + return msg + +def zephyr_escape(m): + m = re.sub('@', '@@', m) + m = re.sub('}', '@(})', m) + return m + +def zephyr_success(m): + return '}@{@color(green)%s}@{' % zephyr_escape(m) + +def zephyr_failure(m): + return '}@{@color(red)%s}@{' % zephyr_escape(m) + +def main(): + parser = optparse.OptionParser('Usage: %prog build_id') + opts, args = parser.parse_args() + if len(args) != 1: + parser.print_help() + return 1 + prog = os.path.basename(sys.argv[0]) + + database.connect() + build = database.Build.query().get(args[0]) + short_commit = builder.canonicalize_commit(build.package, build.commit, shorten=True) + values = { 'build_id' : build.build_id, + 'commit' : build.commit, + 'failed_stage' : build.failed_stage, + 'inserted_at' : build.inserted_at, + 'package' : build.package, + 'pocket' : build.pocket, + 'principal' : build.principal, + 'short_commit' : short_commit, + 'traceback' : build.traceback, + 'version' : build.version } + if build.succeeded: + values['result'] = 'succeeded' + else: + values['result'] = 'failed' + + try: + if prog == 'post-build': + hook_config = config.build.hooks.post_build + elif prog == 'failed-build': + hook_config = config.build.hooks.failed_build + else: + print >>sys.stderr, '{post,failed}-build invoke with unrecognized name %s' % prog + return 2 + except common.InvirtConfigError: + print >>sys.stderr, 'No hook configuration found for %s.' % prog + return 1 + + try: + zephyr_config = hook_config.zephyr + klass = zephyr_config['class'] % values + except common.InvirtConfigError: + print >>sys.stderr, 'No zephyr configuration specified for %s.' % prog + else: + msg = '@{%s}' % make_msg(build, values, verbose=False, + success=zephyr_success, failure=zephyr_failure) + instance = zephyr_config.get('instance', 'build_%(build_id)s') % values + zsig = zephyr_config.get('zsig', 'XVM Buildbot') % values + common.captureOutput(['zwrite', '-c', klass, '-i', instance, '-s', + zsig, '-d', '-m', msg], + stdout=None, stderr=None) + + try: + mail_config = hook_config.mail + to = mail_config.to % values + sender = mail_config['from'] % values + except common.InvirtConfigError: + print >>sys.stderr, 'No email configuration specified for %s.' % prog + else: + msg = make_msg(build, values) + email = text.MIMEText(msg) + email['To'] = to % values + email['From'] = sender % values + email['Subject'] = mail_config.get('subject', 'XVM build %(build_id)s has %(result)s') % values + common.captureOutput(['sendmail', '-t'], email.as_string(), + stdout=None, stderr=None) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/build-hooks/post-submit b/build-hooks/post-submit new file mode 100755 index 0000000..294818a --- /dev/null +++ b/build-hooks/post-submit @@ -0,0 +1,32 @@ +#!/bin/sh + +set -e +set -u + +escape() { + echo "$1" | sed -e 's/@/@@/g' +} + +pocket=$(escape "$1") +package=$(escape "$2") +commit=$(escape "$3") +principal=$(escape "$4") + +base=build.hooks.post_submit.zephyr +class=$(invirt-getconf "$base.class" 2>/dev/null || :) +instance=$(invirt-getconf "$base.instance" 2>/dev/null || :) +zsig=$(invirt-getconf "$base.zsig" 2>/dev/null || :) + +if [ -z "$class" ]; then + echo "I don't know where to send a commit zephyr!" >&2 + echo "Please provide a value for $base.class in" >&2 + echo "your invirt config file." >&2 + exit 1 +fi + +(echo "A new job has been submitted to the Invirtibuilder:"; + echo; + echo "pocket: $pocket"; + echo "package: $package"; + echo "commit: $commit"; + echo "principal: $principal") | zwrite -c "$class" -i "${instance:-$commit}" -s "${zsig:-Git}: $pocket" -d diff --git a/build-hooks/pre-build b/build-hooks/pre-build new file mode 100755 index 0000000..68c3c57 --- /dev/null +++ b/build-hooks/pre-build @@ -0,0 +1,33 @@ +#!/bin/sh + +set -e +set -u + +escape() { + echo "$1" | sed -e 's/@/@@/g' +} + +build_id=$(escape "$1") +pocket=$(escape "$2") +package=$(escape "$3") +commit=$(escape "$4") +principal=$(escape "$5") +version=$(escape "$6") +inserted_at=$(escape "$7") + +base=build.hooks.pre_build.zephyr +class=$(invirt-getconf "$base.class" 2>/dev/null || :) +instance=$(invirt-getconf "$base.instance" 2>/dev/null || :) +zsig=$(invirt-getconf "$base.zsig" 2>/dev/null || :) + +if [ -z "$class" ]; then + echo "I don't know where to send a commit zephyr!" >&2 + echo "Please provide a value for $base.class in" >&2 + echo "your invirt config file." >&2 + exit 1 +fi + +(echo "About to begin an Invirtibuild of $package v$version in $pocket." + echo "from commit $commit."; + echo + echo "(Build $build_id was submitted by $principal at $inserted_at.)") | zwrite -c "$class" -i "${instance:-build_$build_id}" -s "${zsig:-pre-build}: $pocket" -d diff --git a/debian/invirt-dev.dirs b/debian/invirt-dev.dirs index a17355d..da72f77 100644 --- a/debian/invirt-dev.dirs +++ b/debian/invirt-dev.dirs @@ -1,3 +1,3 @@ var/lib/invirt-dev/queue var/log/invirt/builds -usr/share/invirt-dev/build.d +usr/share/invirt-dev/build-hooks diff --git a/debian/invirt-dev.install b/debian/invirt-dev.install index 2982e0c..8980a61 100644 --- a/debian/invirt-dev.install +++ b/debian/invirt-dev.install @@ -1,2 +1,3 @@ reprepro-env usr/bin repository-config/* srv/repository/conf +build-hooks usr/share/invirt-dev \ No newline at end of file diff --git a/invirt-submit-build b/invirt-submit-build index f9b5697..4bed2ee 100755 --- a/invirt-submit-build +++ b/invirt-submit-build @@ -46,7 +46,11 @@ def main(): commit = b.canonicalize_commit(package, commit) b.validateBuild(pocket, package, commit) except b.InvalidBuild, e: - print >>sys.stderr, "E: %s" % e + msg = "E: %s" % e + print >>sys.stderr, msg + # Prevent an attack by submitting excessively long arguments + args = [arg[:min(len(arg), 80)] for arg in (pocket, package, commit)] + b.runHook('failed-submit', args + [principal], stdin_str=msg) sys.exit(1) # To keep from triggering the Invirtibuilder before we've actually @@ -57,6 +61,9 @@ def main(): print >>q, "%s %s %s %s" % (pocket, package, commit, principal) q.close() os.rename(q_name, q_path) + short_commit = b.canonicalize_commit(package, commit, shorten=True) + b.runHook('post-submit', [pocket, package, short_commit, principal]) + print '%s, your job to build %s for %s:%s has been submitted!' % (principal, short_commit, package, pocket) if __name__ == '__main__': diff --git a/invirtibuilder b/invirtibuilder index 24861f5..a42ce55 100755 --- a/invirtibuilder +++ b/invirtibuilder @@ -249,16 +249,6 @@ def packageWorkdir(package, commit): finally: shutil.rmtree(workdir) - -def reportBuild(build): - """Run hooks to report the results of a build attempt.""" - - c.captureOutput(['run-parts', - '--arg=%s' % build.build_id, - '--', - b._HOOKS_DIR]) - - def build(): """Deal with items in the build queue. @@ -295,6 +285,8 @@ def build(): src = b.validateBuild(pocket, package, commit) db.version = str(b.getVersion(package, commit)) + b.runHook('pre-build', [str(db.build_id), db.pocket, db.package, + db.commit, db.principal, db.version, str(db.inserted_at)]) # If validateBuild returns something other than True, then # it means we should copy from that pocket to our pocket. @@ -366,8 +358,10 @@ def build(): # build queue item os.unlink(os.path.join(b._QUEUE_DIR, build)) - reportBuild(db) - + if db.succeeded: + b.runHook('post-build', [str(db.build_id)]) + else: + b.runHook('failed-build', [str(db.build_id)]) class Invirtibuilder(pyinotify.ProcessEvent): """Process inotify triggers to build new packages.""" diff --git a/python/invirt/builder.py b/python/invirt/builder.py index 68d56c8..b3aa08f 100644 --- a/python/invirt/builder.py +++ b/python/invirt/builder.py @@ -17,7 +17,7 @@ from invirt.config import structs as config _QUEUE_DIR = '/var/lib/invirt-dev/queue' _REPO_DIR = '/srv/git' _LOG_DIR = '/var/log/invirt/builds' -_HOOKS_DIR = '/usr/share/invirt-dev/build.d' +_HOOKS_DIR = '/usr/share/invirt-dev/build-hooks' class InvalidBuild(ValueError): @@ -74,6 +74,13 @@ def getChangelog(package, ref): """ return changelog.Changelog(getGitFile(package, ref, 'debian/changelog')) +def runHook(hook, args=[], stdin_str=None): + """Run a named hook.""" + hook = os.path.join(_HOOKS_DIR, hook) + try: + c.captureOutput([hook] + args, stdin_str=stdin_str) + except OSError: + pass def getVersion(package, ref): """Get the version of a given package at a particular ref."""