import pyinotify
-from invirt.config import structs as config
+import invirt.builder as b
from invirt import database
-_QUEUE_DIR = '/var/lib/invirt-dev/queue'
-_REPO_DIR = '/srv/git'
-_LOG_DIR = '/var/log/invirt/builds'
-_HOOKS_DIR = '/usr/share/invirt-dev/build.d'
-
-
DISTRIBUTION = 'hardy'
-class InvalidBuild(ValueError):
- pass
-
-
-def captureOutput(popen_args, stdin_str=None, *args, **kwargs):
- """Capture stdout from a command.
-
- This method will proxy the arguments to subprocess.Popen. It
- returns the output from the command if the call succeeded and
- raises an exception if the process returns a non-0 value.
-
- This is intended to be a variant on the subprocess.check_call
- function that also allows you access to the output from the
- command.
- """
- if 'stdin' not in kwargs:
- kwargs['stdin'] = subprocess.PIPE
- if 'stdout' not in kwargs:
- kwargs['stdout'] = subprocess.PIPE
- if 'stderr' not in kwargs:
- kwargs['stderr'] = subprocess.STDOUT
- p = subprocess.Popen(popen_args, *args, **kwargs)
- out, _ = p.communicate(stdin_str)
- if p.returncode:
- raise subprocess.CalledProcessError(p.returncode, popen_args, out)
- return out
-
-
-def getRepo(package):
- """Return the path to the git repo for a given package."""
- return os.path.join(_REPO_DIR, 'packages', '%s.git' % package)
-
-
-def pocketToGit(pocket):
- """Map a pocket in the configuration to a git branch."""
- return config.git.pockets[pocket].get('git', pocket)
-
-
-def pocketToApt(pocket):
- """Map a pocket in the configuration to an apt repo pocket."""
- return config.git.pockets[pocket].get('apt', pocket)
-
-
-def getGitFile(package, ref, path):
- """Return the contents of a path from a git ref in a package."""
- return captureOutput(['git', 'cat-file', 'blob', '%s:%s' % (ref, path)],
- cwd=getRepo(package))
-
-
-def getChangelog(package, ref):
- """Get a changelog object for a given ref in a given package.
-
- This returns a debian_bundle.changelog.Changelog object for a
- given ref of a given package.
- """
- return changelog.Changelog(getGitFile(package, ref, 'debian/changelog'))
-
-
-def getVersion(package, ref):
- """Get the version of a given package at a particular ref."""
- return getChangelog(package, ref).get_version()
-
-
def getControl(package, ref):
"""Get the parsed debian/control file for a given package.
version.debian_version)
-def validateBuild(pocket, package, commit):
- """Given the parameters of a new build, validate that build.
-
- The checks this function performs vary based on whether or not the
- pocket is configured with allow_backtracking.
-
- A build of a pocket without allow_backtracking set must be a
- fast-forward of the previous revision, and the most recent version
- in the changelog most be strictly greater than the version
- currently in the repository.
-
- In all cases, this revision of the package can only have the same
- version number as any other revision currently in the apt
- repository if they have the same commit ID.
-
- If it's unspecified, it is assumed that pocket do not
- allow_backtracking.
-
- If this build request fails validation, this function will raise a
- InvalidBuild exception, with information about why the validation
- failed.
-
- If this build request can be satisfied by copying the package from
- another pocket, then this function returns that pocket. Otherwise,
- it returns True.
- """
- package_repo = getRepo(package)
- new_version = getVersion(package, commit)
-
- for p in config.git.pockets:
- if p == pocket:
- continue
-
- b = pocketToGit(p)
- current_commit = captureOutput(['git', 'rev-parse', b],
- cwd=package_repo)
- current_version = getVersion(package, b)
-
- if current_version == new_version:
- if current_commit == commit:
- return p
- else:
- raise InvalidBuild('Version %s of %s already available in '
- 'pocket %s from commit %s' %
- (new_version, package, p, current_commit))
-
- if config.git.pockets[pocket].get('allow_backtracking', False):
- branch = pocketToGit(pocket)
- current_version = getVersion(package, branch)
- if new_version <= current_version:
- raise InvalidBuild('New version %s of %s is not newer than '
- 'version %s currently in pocket %s' %
- (new_version, package, current_version, pocket))
-
- # Almost by definition, A is a fast-forward of B if B..A is
- # empty
- if not captureOutput(['git', 'rev-list', '%s..%s' % (commit, branch)]):
- raise InvalidBuild('New commit %s of %s is not a fast-forward of'
- 'commit currently in pocket %s' %
- (commit, package, pocket))
-
-
def sanitizeVersion(version):
"""Sanitize a Debian package version for use as a git tag.
def aptCopy(packages, dst_pocket, src_pocket):
"""Copy a package from one pocket to another."""
binaries = []
- for line in getGitFile(package, commit, 'debian/control').split('\n'):
+ for line in b.getGitFile(package, commit, 'debian/control').split('\n'):
m = re.match('Package: (.*)$')
if m:
binaries.append(m.group(1))
cpatureOutput(['reprepro-env', 'copy',
- pocketToApt(dst_pocket),
- pocketToApt(src_pocket),
+ b.pocketToApt(dst_pocket),
+ b.pocketToApt(src_pocket),
package] + binaries)
if arch_all:
args.append('-A')
args.append(getDscName(package, ref))
- captureOutput(args, cwd=workdir, stdout=None)
+ c.captureOutput(args, cwd=workdir, stdout=None)
def sbuildAll(package, ref, workdir):
"""
if config.git.pockets[pocket].get('allow_backtracking', False):
env = dict(os.environ)
- branch = pocketToGit(pocket)
- version = getVersion(package, ref)
+ branch = b.pocketToGit(pocket)
+ version = b.getVersion(package, ref)
env['GIT_COMMITTER_NAME'] = config.git.tagger.name
env['GIT_COMMITTER_EMAIL'] = config.git.tagger.email
package,
principal))
- captureOutput(
+ c.captureOutput(
['git', 'tag', '-m', tag_msg, commit],
stdout=None,
env=env)
def updateSubmoduleBranch(pocket, package, ref):
"""Update the appropriately named branch in the submodule."""
- branch = pocketToGit(pocket)
- captureOutput(
+ branch = b.pocketToGit(pocket)
+ c.captureOutput(
['git', 'update-ref', 'refs/heads/%s' % branch, ref])
def uploadBuild(pocket, workdir):
"""Upload all build products in the work directory."""
- apt = pocketToApt(pocket)
+ apt = b.pocketToApt(pocket)
for changes in glob.glob(os.path.join(workdir, '*.changes')):
- captureOutput(['reprepro-env',
+ c.captureOutput(['reprepro-env',
'include',
'--ignore=wrongdistribution',
apt,
Note that there's no locking issue here, because we disallow all
pushes to the superrepo.
"""
- superrepo = os.path.join(_REPO_DIR, 'packages.git')
- branch = pocketToGit(pocket)
- tree = captureOutput(['git', 'ls-tree', branch],
+ superrepo = os.path.join(b._REPO_DIR, 'packages.git')
+ branch = b.pocketToGit(pocket)
+ tree = c.captureOutput(['git', 'ls-tree', branch],
cwd=superrepo)
new_tree = re.compile(
r'\1%s\2' % commit,
tree)
- new_tree_id = captureOutput(['git', 'mktree'],
+ new_tree_id = c.captureOutput(['git', 'mktree'],
cwd=superrepo,
stdin_str=new_tree)
'Requested by %s' % (package,
version.full_version,
principal))
- new_commit = captureOutput(
+ new_commit = c.captureOutput(
['git', 'commit-tree', new_tree_hash, '-p', branch],
cwd=superrepo,
env=env,
stdin_str=commit_msg)
- captureOutput(
+ c.captureOutput(
['git', 'update-ref', 'refs/heads/%s' % branch, new_commit],
cwd=superrepo)
try:
p_archive = subprocess.Popen(
['git', 'archive',
- '--remote=file://%s' % getRepo(package),
+ '--remote=file://%s' % b.getRepo(package),
'--prefix=%s' % package,
commit,
],
def reportBuild(build):
"""Run hooks to report the results of a build attempt."""
- captureOutput(['run-parts',
+ c.captureOutput(['run-parts',
'--arg=%s' % build.build_id,
'--',
- _HOOKS_DIR])
+ b._HOOKS_DIR])
def build():
"""
while True:
stage = 'processing incoming job'
- queue = os.listdir(_QUEUE_DIR)
+ queue = os.listdir(b._QUEUE_DIR)
if not queue:
break
build = min(queue)
- job = open(os.path.join(_QUEUE_DIR, build)).read().strip()
+ job = open(os.path.join(b._QUEUE_DIR, build)).read().strip()
pocket, package, commit, principal = job.split()
database.session.begin()
db.failed_stage = 'validating job'
src = validateBuild(pocket, package, commit)
- db.version = str(getVersion(package, commit))
+ db.version = str(b.getVersion(package, commit))
# If validateBuild returns something other than True, then
# it means we should copy from that pocket to our pocket.
# If we were, we could use debuild and get nice
# environment scrubbing. Since we're not, debuild
# complains about not having an orig.tar.gz
- captureOutput(['dpkg-buildpackage', '-us', '-uc', '-S'],
+ c.captureOutput(['dpkg-buildpackage', '-us', '-uc', '-S'],
cwd=packagedir,
stdout=None)
db.failed_stage = 'building binary packages'
sbuildAll(package, commit, workdir)
finally:
- logdir = os.path.join(_LOG_DIR, db.build_id)
+ logdir = os.path.join(b._LOG_DIR, db.build_id)
if not os.path.exists(logdir):
os.makedirs(logdir)
# Finally, now that everything is done, remove the
# build queue item
- os.unlink(os.path.join(_QUEUE_DIR, build))
+ os.unlink(os.path.join(b._QUEUE_DIR, build))
except:
db.traceback = traceback.format_exc()
else:
watch_manager = pyinotify.WatchManager()
invirtibuilder = Invirtibuilder()
notifier = pyinotify.Notifier(watch_manager, invirtibuilder)
- watch_manager.add_watch(_QUEUE_DIR,
+ watch_manager.add_watch(b._QUEUE_DIR,
pyinotify.EventsCodes.ALL_FLAGS['IN_CREATE'])
# Before inotifying, run any pending builds; otherwise we won't
--- /dev/null
+"""Invirt build utilities.
+
+This module contains utility functions used by both the invirtibuilder
+and the remctl submission scripts that insert items into its queue.
+"""
+
+
+import os
+
+from debian_bundle import changelog
+from debian_bundle import deb822
+
+import invirt.common as c
+from invirt.config import structs as config
+
+
+_QUEUE_DIR = '/var/lib/invirt-dev/queue'
+_REPO_DIR = '/srv/git'
+_LOG_DIR = '/var/log/invirt/builds'
+_HOOKS_DIR = '/usr/share/invirt-dev/build.d'
+
+
+class InvalidBuild(ValueError):
+ pass
+
+
+def getRepo(package):
+ """Return the path to the git repo for a given package."""
+ return os.path.join(_REPO_DIR, 'packages', '%s.git' % package)
+
+
+def pocketToGit(pocket):
+ """Map a pocket in the configuration to a git branch."""
+ return config.git.pockets[pocket].get('git', pocket)
+
+
+def pocketToApt(pocket):
+ """Map a pocket in the configuration to an apt repo pocket."""
+ return config.git.pockets[pocket].get('apt', pocket)
+
+
+def getGitFile(package, ref, path):
+ """Return the contents of a path from a git ref in a package."""
+ return c.captureOutput(['git', 'cat-file', 'blob', '%s:%s' % (ref, path)],
+ cwd=getRepo(package))
+
+
+def getChangelog(package, ref):
+ """Get a changelog object for a given ref in a given package.
+
+ This returns a debian_bundle.changelog.Changelog object for a
+ given ref of a given package.
+ """
+ return changelog.Changelog(getGitFile(package, ref, 'debian/changelog'))
+
+
+def getVersion(package, ref):
+ """Get the version of a given package at a particular ref."""
+ return getChangelog(package, ref).get_version()
+
+
+def validateBuild(pocket, package, commit):
+ """Given the parameters of a new build, validate that build.
+
+ The checks this function performs vary based on whether or not the
+ pocket is configured with allow_backtracking.
+
+ A build of a pocket without allow_backtracking set must be a
+ fast-forward of the previous revision, and the most recent version
+ in the changelog most be strictly greater than the version
+ currently in the repository.
+
+ In all cases, this revision of the package can only have the same
+ version number as any other revision currently in the apt
+ repository if they have the same commit ID.
+
+ If it's unspecified, it is assumed that pocket do not
+ allow_backtracking.
+
+ If this build request fails validation, this function will raise a
+ InvalidBuild exception, with information about why the validation
+ failed.
+
+ If this build request can be satisfied by copying the package from
+ another pocket, then this function returns that pocket. Otherwise,
+ it returns True.
+ """
+ package_repo = getRepo(package)
+ new_version = getVersion(package, commit)
+
+ for p in config.git.pockets:
+ if p == pocket:
+ continue
+
+ b = pocketToGit(p)
+ current_commit = c.captureOutput(['git', 'rev-parse', b],
+ cwd=package_repo)
+ current_version = getVersion(package, b)
+
+ if current_version == new_version:
+ if current_commit == commit:
+ return p
+ else:
+ raise InvalidBuild('Version %s of %s already available in '
+ 'pocket %s from commit %s' %
+ (new_version, package, p, current_commit))
+
+ if config.git.pockets[pocket].get('allow_backtracking', False):
+ branch = pocketToGit(pocket)
+ current_version = getVersion(package, branch)
+ if new_version <= current_version:
+ raise InvalidBuild('New version %s of %s is not newer than '
+ 'version %s currently in pocket %s' %
+ (new_version, package, current_version, pocket))
+
+ # Almost by definition, A is a fast-forward of B if B..A is
+ # empty
+ if not c.captureOutput(['git', 'rev-list', '%s..%s' % (commit, branch)]):
+ raise InvalidBuild('New commit %s of %s is not a fast-forward of'
+ 'commit currently in pocket %s' %
+ (commit, package, pocket))
+
+
projects / invirt/packages/invirt-dev.git / commitdiff
