pocket package hash principal
where pocket is one of the pockets globally configured in
-git.pockets. For instance, the pockets in XVM are "prod" and "dev".
+build.pockets. For instance, the pockets in XVM are "prod" and "dev".
principal is the Kerberos principal that requested the build.
"""
+from __future__ import with_statement
+
import contextlib
import os
import re
import pyinotify
-from invirt.config import structs as config
+import invirt.builder as b
from invirt import database
-_QUEUE_DIR = '/var/lib/invirt-dev/queue'
-_REPO_DIR = '/srv/git'
-_LOG_DIR = '/var/log/invirt/builds'
-_HOOKS_DIR = '/usr/share/invirt-dev/build.d'
-
-
DISTRIBUTION = 'hardy'
-class InvalidBuild(ValueError):
- pass
-
-
-def captureOutput(popen_args, stdin_str=None, *args, **kwargs):
- """Capture stdout from a command.
-
- This method will proxy the arguments to subprocess.Popen. It
- returns the output from the command if the call succeeded and
- raises an exception if the process returns a non-0 value.
-
- This is intended to be a variant on the subprocess.check_call
- function that also allows you access to the output from the
- command.
- """
- if 'stdin' not in kwargs:
- kwargs['stdin'] = subprocess.PIPE
- if 'stdout' not in kwargs:
- kwargs['stdout'] = subprocess.PIPE
- if 'stderr' not in kwargs:
- kwargs['stderr'] = subprocess.STDOUT
- p = subprocess.Popen(popen_args, *args, **kwargs)
- out, _ = p.communicate(stdin_str)
- if p.returncode:
- raise subprocess.CalledProcessError(p.returncode, popen_args, out)
- return out
-
-
-def getRepo(package):
- """Return the path to the git repo for a given package."""
- return os.path.join(_REPO_DIR, 'packages', '%s.git' % package)
-
-
-def pocketToGit(pocket):
- """Map a pocket in the configuration to a git branch."""
- return config.git.pockets[pocket].get('git', pocket)
-
-
-def pocketToApt(pocket):
- """Map a pocket in the configuration to an apt repo pocket."""
- return config.git.pockets[pocket].get('apt', pocket)
-
-
-def getGitFile(package, ref, path):
- """Return the contents of a path from a git ref in a package."""
- return captureOutput(['git', 'cat-file', 'blob', '%s:%s' % (ref, path)],
- cwd=getRepo(package))
-
-
-def getChangelog(package, ref):
- """Get a changelog object for a given ref in a given package.
-
- This returns a debian_bundle.changelog.Changelog object for a
- given ref of a given package.
- """
- return changelog.Changelog(getGitFile(package, ref, 'debian/changelog'))
-
-
-def getVersion(package, ref):
- """Get the version of a given package at a particular ref."""
- return getChangelog(package, ref).get_version()
-
-
def getControl(package, ref):
"""Get the parsed debian/control file for a given package.
def getDscName(package, ref):
"""Return the .dsc file that will be generated for this package."""
v = getVersion(package, ref)
- return '%s_%s-%s.dsc' % (
+ if v.debian_version:
+ v_str = '%s-%s' % (v.upstream_version,
+ v.debian_version)
+ else:
+ v_str = v.upstream_version
+ return '%s_%s.dsc' % (
package,
- version.upstream_version,
- version.debian_version)
-
-
-def validateBuild(pocket, package, commit):
- """Given the parameters of a new build, validate that build.
-
- The checks this function performs vary based on whether or not the
- pocket is configured with allow_backtracking.
-
- A build of a pocket without allow_backtracking set must be a
- fast-forward of the previous revision, and the most recent version
- in the changelog most be strictly greater than the version
- currently in the repository.
-
- In all cases, this revision of the package can only have the same
- version number as any other revision currently in the apt
- repository if they have the same commit ID.
-
- If it's unspecified, it is assumed that pocket do not
- allow_backtracking.
-
- If this build request fails validation, this function will raise a
- InvalidBuild exception, with information about why the validation
- failed.
-
- If this build request can be satisfied by copying the package from
- another pocket, then this function returns that pocket. Otherwise,
- it returns True.
- """
- package_repo = getRepo(package)
- new_version = getVersion(package, commit)
-
- for p in config.git.pockets:
- if p == pocket:
- continue
-
- b = pocketToGit(p)
- current_commit = captureOutput(['git', 'rev-parse', b],
- cwd=package_repo)
- current_version = getVersion(package, b)
-
- if current_version == new_version:
- if current_commit == commit:
- return p
- else:
- raise InvalidBuild('Version %s of %s already available in '
- 'pocket %s from commit %s' %
- (new_version, package, p, current_commit))
-
- if config.git.pockets[pocket].get('allow_backtracking', False):
- branch = pocketToGit(pocket)
- current_version = getVersion(package, branch)
- if new_version <= current_version:
- raise InvalidBuild('New version %s of %s is not newer than '
- 'version %s currently in pocket %s' %
- (new_version, package, current_version, pocket))
-
- # Almost by definition, A is a fast-forward of B if B..A is
- # empty
- if not captureOutput(['git', 'rev-list', '%s..%s' % (commit, branch)]):
- raise InvalidBuild('New commit %s of %s is not a fast-forward of'
- 'commit currently in pocket %s' %
- (commit, package, pocket))
+ v_str)
def sanitizeVersion(version):
This function strips the epoch from the version number and
replaces any tildes with periods."""
- v = '%s-%s' % (version.upstream_version,
- version.debian_version)
+ if v.debian_version:
+ v = '%s-%s' % (version.upstream_version,
+ version.debian_version)
+ else:
+ v = version.upstream_version
return v.replace('~', '.')
def aptCopy(packages, dst_pocket, src_pocket):
"""Copy a package from one pocket to another."""
- binaries = []
- for line in getGitFile(package, commit, 'debian/control').split('\n'):
- m = re.match('Package: (.*)$')
- if m:
- binaries.append(m.group(1))
-
+ binaries = getBinaries(package, commit)
cpatureOutput(['reprepro-env', 'copy',
- pocketToApt(dst_pocket),
- pocketToApt(src_pocket),
+ b.pocketToApt(dst_pocket),
+ b.pocketToApt(src_pocket),
package] + binaries)
if arch_all:
args.append('-A')
args.append(getDscName(package, ref))
- captureOutput(args, cwd=workdir, stdout=None)
+ c.captureOutput(args, cwd=workdir, stdout=None)
def sbuildAll(package, ref, workdir):
I'm sure that long description gives you great confidence in teh
legitimacy of my reasoning.
"""
- if config.git.pockets[pocket].get('allow_backtracking', False):
+ if config.build.pockets[pocket].get('allow_backtracking', False):
env = dict(os.environ)
- branch = pocketToGit(pocket)
- version = getVersion(package, ref)
+ branch = b.pocketToGit(pocket)
+ version = b.getVersion(package, ref)
- env['GIT_COMMITTER_NAME'] = config.git.tagger.name
- env['GIT_COMMITTER_EMAIL'] = config.git.tagger.email
+ env['GIT_COMMITTER_NAME'] = config.build.tagger.name
+ env['GIT_COMMITTER_EMAIL'] = config.build.tagger.email
tag_msg = ('Tag %s of %s\n\n'
'Requested by %s' % (version.full_version,
package,
principal))
- captureOutput(
+ c.captureOutput(
['git', 'tag', '-m', tag_msg, commit],
stdout=None,
env=env)
def updateSubmoduleBranch(pocket, package, ref):
"""Update the appropriately named branch in the submodule."""
- branch = pocketToGit(pocket)
- captureOutput(
+ branch = b.pocketToGit(pocket)
+ c.captureOutput(
['git', 'update-ref', 'refs/heads/%s' % branch, ref])
def uploadBuild(pocket, workdir):
"""Upload all build products in the work directory."""
- apt = pocketToApt(pocket)
+ apt = b.pocketToApt(pocket)
for changes in glob.glob(os.path.join(workdir, '*.changes')):
- captureOutput(['reprepro-env',
+ c.captureOutput(['reprepro-env',
'include',
'--ignore=wrongdistribution',
apt,
Note that there's no locking issue here, because we disallow all
pushes to the superrepo.
"""
- superrepo = os.path.join(_REPO_DIR, 'packages.git')
- branch = pocketToGit(pocket)
- tree = captureOutput(['git', 'ls-tree', branch],
+ superrepo = os.path.join(b._REPO_DIR, 'invirt/packages.git')
+ branch = b.pocketToGit(pocket)
+ tree = c.captureOutput(['git', 'ls-tree', branch],
cwd=superrepo)
new_tree = re.compile(
r'\1%s\2' % commit,
tree)
- new_tree_id = captureOutput(['git', 'mktree'],
+ new_tree_id = c.captureOutput(['git', 'mktree'],
cwd=superrepo,
stdin_str=new_tree)
'Requested by %s' % (package,
version.full_version,
principal))
- new_commit = captureOutput(
+ new_commit = c.captureOutput(
['git', 'commit-tree', new_tree_hash, '-p', branch],
cwd=superrepo,
env=env,
stdin_str=commit_msg)
- captureOutput(
+ c.captureOutput(
['git', 'update-ref', 'refs/heads/%s' % branch, new_commit],
cwd=superrepo)
try:
p_archive = subprocess.Popen(
['git', 'archive',
- '--remote=file://%s' % getRepo(package),
+ '--remote=file://%s' % b.getRepo(package),
'--prefix=%s' % package,
commit,
],
def reportBuild(build):
"""Run hooks to report the results of a build attempt."""
- captureOutput(['run-parts',
+ c.captureOutput(['run-parts',
'--arg=%s' % build.build_id,
'--',
- _HOOKS_DIR])
+ b._HOOKS_DIR])
def build():
"""
while True:
stage = 'processing incoming job'
- queue = os.listdir(_QUEUE_DIR)
+ queue = os.listdir(b._QUEUE_DIR)
if not queue:
break
build = min(queue)
- job = open(os.path.join(_QUEUE_DIR, build)).read().strip()
+ job = open(os.path.join(b._QUEUE_DIR, build)).read().strip()
pocket, package, commit, principal = job.split()
database.session.begin()
db.failed_stage = 'validating job'
src = validateBuild(pocket, package, commit)
- db.version = str(getVersion(package, commit))
+ db.version = str(b.getVersion(package, commit))
# If validateBuild returns something other than True, then
# it means we should copy from that pocket to our pocket.
# If we were, we could use debuild and get nice
# environment scrubbing. Since we're not, debuild
# complains about not having an orig.tar.gz
- captureOutput(['dpkg-buildpackage', '-us', '-uc', '-S'],
+ c.captureOutput(['dpkg-buildpackage', '-us', '-uc', '-S'],
cwd=packagedir,
stdout=None)
db.failed_stage = 'building binary packages'
sbuildAll(package, commit, workdir)
finally:
- logdir = os.path.join(_LOG_DIR, db.build_id)
+ logdir = os.path.join(b._LOG_DIR, db.build_id)
if not os.path.exists(logdir):
os.makedirs(logdir)
# Finally, now that everything is done, remove the
# build queue item
- os.unlink(os.path.join(_QUEUE_DIR, build))
+ os.unlink(os.path.join(b._QUEUE_DIR, build))
except:
db.traceback = traceback.format_exc()
else:
watch_manager = pyinotify.WatchManager()
invirtibuilder = Invirtibuilder()
notifier = pyinotify.Notifier(watch_manager, invirtibuilder)
- watch_manager.add_watch(_QUEUE_DIR,
+ watch_manager.add_watch(b._QUEUE_DIR,
pyinotify.EventsCodes.ALL_FLAGS['IN_CREATE'])
# Before inotifying, run any pending builds; otherwise we won't
projects / invirt/packages/invirt-dev.git / blobdiff