From 8b6c350df6a25b38b5b2aa9e93f328e891d71889 Mon Sep 17 00:00:00 2001 From: Peter Iannucci Date: Thu, 18 Jul 2013 02:43:38 -0400 Subject: [PATCH] Refactored invirtibuilder --- debian/changelog | 6 ++ invirtibuilder | 110 +++++++--------------------- python/invirt/builder.py | 178 +++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 192 insertions(+), 102 deletions(-) diff --git a/debian/changelog b/debian/changelog index d2febb1..2ccc675 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +invirt-dev (0.1.30) unstable; urgency=low + + * refactored invirtibuilder + + -- Peter Iannucci Thu, 18 Jul 2013 02:42:00 -0400 + invirt-dev (0.1.29) precise; urgency=low * Remove the build-source-package-in-schroot hack now that we have a diff --git a/invirtibuilder b/invirtibuilder index 43f09f1..14be290 100755 --- a/invirtibuilder +++ b/invirtibuilder @@ -65,6 +65,12 @@ def logAndRun(cmd, *args, **kwargs): logfile.write(output) return output +def getLoggingRepo(package): + return b.GitRepository(b.getRepo(package), logAndRun) + +def getLoggingSuperproject(): + return b.SuperprojectRepository(logAndRun) + def getControl(package, ref): """Get the parsed debian/control file for a given package. @@ -155,7 +161,7 @@ def sbuildAll(package, ref, distro, workdir): sbuild(package, ref, distro, 'i386', workdir) -def tagSubmodule(pocket, package, commit, principal, version, env): +def tagSubmodule(pocket, package, commit, principal, version): """Tag a new version of a submodule. If this pocket does not allow_backtracking, then this will create @@ -171,24 +177,21 @@ def tagSubmodule(pocket, package, commit, principal, version, env): legitimacy of my reasoning. """ if not config.build.pockets[pocket].get('allow_backtracking', False): - branch = b.pocketToGit(pocket) tag_msg = ('Tag %s of %s\n\n' 'Requested by %s' % (version.full_version, package, principal)) - - logAndRun( - ['git', 'tag', '-m', tag_msg, '--', sanitizeVersion(version), - commit], - env=env, - cwd=b.getRepo(package)) + getLoggingRepo(package).tag( + tag_msg, + sanitizeVersion(version), + commit, + principal) def updateSubmoduleBranch(pocket, package, commit): """Update the appropriately named branch in the submodule.""" branch = b.pocketToGit(pocket) - logAndRun( - ['git', 'update-ref', 'refs/heads/%s' % branch, commit], cwd=b.getRepo(package)) + getLoggingRepo(package).updateBranch(branch, commit) def uploadBuild(pocket, workdir): @@ -211,7 +214,7 @@ def uploadBuild(pocket, workdir): logAndRun(upload) -def updateSuperproject(pocket, package, commit, principal, version, env): +def updateSuperproject(pocket, package, commit, principal, version): """Update the superproject. This will create a new commit on the branch for the given pocket @@ -220,80 +223,21 @@ def updateSuperproject(pocket, package, commit, principal, version, env): Note that there's no locking issue here, because we disallow all pushes to the superproject. """ - superproject = os.path.join(b._REPO_DIR, 'invirt/packages.git') - - branch = b.pocketToGit(pocket) + superproject = getLoggingSuperproject() - if not b.pocketExists(pocket, superproject): - gitmodules = "\n" - gitmodules_hash = logAndRun(['git', 'hash-object', '-w', '--stdin'], - cwd=superproject, - stdin_str=gitmodules).strip() - tree_items = {'.gitmodules': "100644 blob "+gitmodules_hash} - new_tree = "\n".join("%s\t%s" % (v, k) for (k, v) in tree_items.iteritems()) - new_tree_id = logAndRun(['git', 'mktree', '--missing'], - cwd=superproject, - stdin_str=new_tree).strip() - env2 = dict(os.environ) - env2['GIT_AUTHOR_NAME'] = config.build.tagger.name - env2['GIT_AUTHOR_EMAIL'] = config.build.tagger.email - env2['GIT_COMMITTER_NAME'] = config.build.tagger.name - env2['GIT_COMMITTER_EMAIL'] = config.build.tagger.email - new_commit = logAndRun(['git', 'commit-tree', new_tree_id], - cwd=superproject, - env=env2, - stdin_str="Create new pocket").strip() - logAndRun(['git', 'update-ref', 'refs/heads/%s' % branch, new_commit], - cwd=superproject) - - tree = logAndRun(['git', 'ls-tree', branch], - cwd=superproject).strip() - - tree_items = dict((k, v) for (v, k) in (x.split("\t") for x in tree.split("\n"))) - - created = not (package in tree_items) - - tree_items[package] = "160000 commit "+commit - - # If "created" is true, we need to check if the package is - # mentioned in .gitmodules, and add it if not. - if created: - gitmodules = logAndRun(['git', 'cat-file', 'blob', '%s:.gitmodules' % (branch)], - cwd=superproject) - if ('[submodule "%s"]' % (package)) not in gitmodules.split("\n"): - gitmodules += """[submodule "%s"] -\tpath = %s -\turl = ../packages/%s.git -""" % (package, package, package) - gitmodules_hash = logAndRun(['git', 'hash-object', '-w', '--stdin'], - cwd=superproject, - stdin_str=gitmodules).strip() - tree_items['.gitmodules'] = "100644 blob "+gitmodules_hash - - new_tree = "\n".join("%s\t%s" % (v, k) for (k, v) in tree_items.iteritems()) - - new_tree_id = logAndRun(['git', 'mktree', '--missing'], - cwd=superproject, - stdin_str=new_tree).strip() + if not superproject.pocketExists(pocket): + superproject.createPocket(pocket) if created: commit_msg = 'Add %s at version %s' else: commit_msg = 'Update %s to version %s' - commit_msg = ((commit_msg + '\n\n' - 'Requested by %s') % (package, - version.full_version, - principal)) - new_commit = logAndRun( - ['git', 'commit-tree', new_tree_id, '-p', branch], - cwd=superproject, - env=env, - stdin_str=commit_msg).strip() - logAndRun( - ['git', 'update-ref', 'refs/heads/%s' % branch, new_commit], - cwd=superproject) + commit_msg += '\n\nRequested by %s' + commit_msg %= (package, version.full_version, principal) + + superproject.updatePocket(pocket, package, commit, principal, commit_msg) def makeReadable(workdir): os.chmod(workdir, 0755) @@ -378,12 +322,6 @@ def build(): db.version = str(version) b.runHook('pre-build', [str(db.build_id)]) - env = dict(os.environ) - env['GIT_COMMITTER_NAME'] = config.build.tagger.name - env['GIT_COMMITTER_EMAIL'] = config.build.tagger.email - env['GIT_AUTHOR_NAME'] = principal.split('@')[0] - env['GIT_AUTHOR_EMAIL'] = principal - # If validateBuild returns something other than True, then # it means we should copy from that pocket to our pocket. # @@ -396,7 +334,7 @@ def build(): db.failed_stage = 'updating submodule branches before copying package' updateSubmoduleBranch(pocket, package, commit) db.failed_stage = 'updating superproject before copying package' - updateSuperproject(pocket, package, commit, principal, version, env) + updateSuperproject(pocket, package, commit, principal, version) db.failed_stage = 'copying package from another pocket' aptCopy(package, commit, pocket, src) @@ -421,11 +359,11 @@ def build(): db.failed_stage = 'building binary packages' sbuildAll(package, commit, b.pocketToDistro(pocket), workdir) db.failed_stage = 'tagging submodule' - tagSubmodule(pocket, package, commit, principal, version, env) + tagSubmodule(pocket, package, commit, principal, version) db.failed_stage = 'updating submodule branches' updateSubmoduleBranch(pocket, package, commit) db.failed_stage = 'updating superproject' - updateSuperproject(pocket, package, commit, principal, version, env) + updateSuperproject(pocket, package, commit, principal, version) db.failed_stage = 'relaxing permissions on workdir' makeReadable(workdir) db.failed_stage = 'uploading packages to apt repo' diff --git a/python/invirt/builder.py b/python/invirt/builder.py index 8e24faf..7b31933 100644 --- a/python/invirt/builder.py +++ b/python/invirt/builder.py @@ -106,14 +106,6 @@ def getVersion(package, ref): """Get the version of a given package at a particular ref.""" return getChangelog(package, ref).get_version() -def pocketExists(pocket, repo): - branch = pocketToGit(pocket) - try: - c.captureOutput(['git', 'rev-parse', branch], cwd=repo) - except subprocess.CalledProcessError: - return False - return True - def validateBuild(pocket, package, commit): """Given the parameters of a new build, validate that build. @@ -141,7 +133,7 @@ def validateBuild(pocket, package, commit): it returns True. """ ensureValidPackage(package) - package_repo = getRepo(package) + package_repo = GitRepository(getRepo(package)) new_version = getVersion(package, commit) new_distro = pocketToDistro(pocket) @@ -152,10 +144,8 @@ def validateBuild(pocket, package, commit): continue b = pocketToGit(p) - try: - current_commit = c.captureOutput(['git', 'rev-parse', b], - cwd=package_repo).strip() - except subprocess.CalledProcessError: + current_commit = package_repo.revParse(b) + if not current_commit: # Guess we haven't created this pocket yet continue @@ -174,7 +164,7 @@ def validateBuild(pocket, package, commit): (new_version, package, p, current_commit)) if not config.build.pockets[pocket].get('allow_backtracking', False): - if not pocketExists(pocket, package_repo): + if not package_repo.pocketExists(pocket): return True branch = pocketToGit(pocket) @@ -186,10 +176,166 @@ def validateBuild(pocket, package, commit): # Almost by definition, A is a fast-forward of B if B..A is # empty - if c.captureOutput(['git', 'rev-list', '%s..%s' % (commit, branch)], - cwd=package_repo): + if package_repo.revList(commit, branch): raise InvalidBuild('New commit %s of %s is not a fast-forward of' 'commit currently in pocket %s' % (commit, package, pocket)) return ret + + +class GitRepository(object): + def __init__(self, repository, subprocess_command=None): + super(GitRepository, self).__init__() + self.repository = repository + self.captureOutput = subprocess_command or c.captureOutput + + def commitEnvironment(principal=None): + tagger_name = config.build.tagger.name + tagger_email = config.build.tagger.email + + env = dict(os.environ) + env.update({ + 'GIT_COMMITTER_NAME': tagger_name, + 'GIT_COMMITTER_EMAIL': tagger_email, + 'GIT_AUTHOR_NAME': principal.split('@')[0] if principal else \ + tagger_name, + 'GIT_AUTHOR_EMAIL': principal if principal else tagger_email, + }) + return env + + def listTree(self, ref): + tree = self.captureOutput(['git', 'ls-tree', ref], + cwd=self.repository) + return dict(x.split("\t")[::-1] for x in tree.strip().split("\n")) + + def makeTree(self, tree): + new_tree = "\n".join(v + "\t" + k for (k, v) in tree.iteritems()) + return self.captureOutput(['git', 'mktree', '--missing'], + cwd=self.repository, + stdin_str=new_tree).strip() + + def hashObject(self, contents): + return self.captureOutput(['git', 'hash-object', '-w', '--stdin'], + cwd=self.repository, + stdin_str=contents).strip() + + def catFile(self, ref, filename): + object_name = '%s:%s' % (ref, filename) + return self.captureOutput(['git', 'cat-file', 'blob', object_name], + cwd=self.repository) + + def commitTree(self, tree_id, parents, message, principal): + parent_args = ['-p', None] * len(parents) + parent_args[1::2] = parents + command = ['git', 'commit-tree', tree_id] + parent_args + return self.captureOutput(command, + cwd=self.repository, + env=self.commitEnvironment(principal), + stdin_str=message).strip() + + def updateRef(self, ref, new_commit): + self.captureOutput(['git', 'update-ref', ref, new_commit], + cwd=self.repository) + + def updateBranch(self, branch, new_commit): + self.updateRef('refs/heads/%s' % branch, new_commit) + + def updateRefWithTree(self, tree, parents, message, principal, ref): + tree_id = self.makeTree(tree) + commit = self.commitTree(tree_id, parents, message, principal) + self.updateRef(ref, commit) + + def updateBranchWithTree(self, tree, message, principal, branch, orphan): + ref = 'refs/heads/%s' % branch + parents = () if orphan else (branch,) + self.updateRefWithTree(tree, parents, message, principal, ref) + + def revParse(self, ref): + try: + command = ['git', 'rev-parse', '--verify', '--quiet', ref] + return self.captureOutput(command, + cwd=self.repository).strip() + except subprocess.CalledProcessError: + return None + + def revList(self, a, b): + return self.captureOutput(['git', 'rev-list', '%s..%s' % (a, b)], + cwd=self.repository) + + def tag(self, tag_msg, tag_name, commit, principal): + command = ['git', 'tag', '-m', tag_msg, '--', tag_name, commit] + self.captureOutput(command, + env=self.commitEnvironment(principal), + cwd=self.repository) + + +class Gitmodules(object): + def __init__(self, text): + super(Gitmodules, self).__init__() + self.text = text + + def __contains__(self, module): + return '[submodule "%s"]' % module in gitmodules.split("\n") + + def add(self, submodule, path, url): + self.text += '[submodule "%s"]\n'\ + '\tpath = %s\n'\ + '\turl = %s\n' % (submodule, path, url) + + def __str__(self): + return self.text + + +class SuperprojectRepository(GitRepository): + def __init__(self, subprocess_command=None): + path = os.path.join(_REPO_DIR, 'invirt/packages.git') + super(SuperprojectRepository, self).__init__(path, subprocess_command) + + def pocketExists(self, pocket): + return self.revParse(pocketToGit(pocket)) is not None + + def gitmodules(self, ref): + return Gitmodules(self.catFile(ref, '.gitmodules')) + + def createPocket(self, pocket): + branch = pocketToGit(pocket) + ref = 'refs/heads/%s' % branch + + gitmodules = "\n" + gitmodules_hash = self.hashObject(gitmodules) + + tree = {'.gitmodules': "100644 blob "+gitmodules_hash} + + self.updateBranchWithTree(tree, + message="Create new pocket", + principal=None, + branch=branch, + orphan=True) + + def updatePocket(self, pocket, package, commit, message, principal): + branch = pocketToGit(pocket) + ref = 'refs/heads/%s' % branch + + tree = self.listTree(branch) + + created = package not in tree + + tree[package] = "160000 commit "+commit + + # If "created" is true, we need to check if the package is + # mentioned in .gitmodules, and add it if not. + if created: + gitmodules = self.gitmodules(branch) + if package not in gitmodules: + gitmodules.add(submodule=package, + path=package, + url='../packages/%s.git' % package) + gitmodules_hash = self.hashObject(self, gitmodules) + tree['.gitmodules'] = "100644 blob %s" % gitmodules_hash + + self.updateBranchWithTree(tree, + message=message, + principal=principal, + branch=branch, + orphan=False) -- 1.7.9.5