Add pylorax.api.recipes code for handling the Recipe's Git repository

This commit is contained in:
Brian C. Lane 2017-10-13 08:48:11 -07:00
parent d71290153e
commit bfc9c6913f
4 changed files with 658 additions and 2 deletions

656
src/pylorax/api/recipes.py Normal file
View File

@ -0,0 +1,656 @@
#
# Copyright (C) 2017 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import gi
gi.require_version("Ggit", "1.0")
from gi.repository import Ggit as Git
from gi.repository import Gio
from gi.repository import GLib
import os
import pytoml as toml
import semantic_version as semver
from pylorax.base import DataHolder
from pylorax.sysutils import joinpaths
class CommitTimeValError(Exception):
pass
class RecipeFileError(Exception):
pass
class RecipeTOMLError(Exception):
pass
class Recipe(dict):
"""A Recipe of package and modules
This is a subclass of dict that enforces the constructor arguments
and adds a .filename property to return the recipe's filename,
and a .toml() function to return the recipe as a TOML string.
"""
def __init__(self, name, description, version, modules, packages):
# Check that version is empty or semver compatible
if version:
semver.Version(version)
# Make sure modules and packages are listed by their case-insensitive names
if modules is not None:
modules = sorted(modules, key=lambda m: m["name"].lower())
if packages is not None:
packages = sorted(packages, key=lambda p: p["name"].lower())
dict.__init__(self, name=name,
description=description,
version=version,
modules=modules,
packages=packages)
@property
def filename(self):
"""Return the Recipe's filename
Replaces spaces in the name with '-' and appends .toml
"""
return recipe_filename(self.get("name"))
def toml(self):
"""Return the Recipe in TOML format"""
return toml.dumps(self).encode("UTF-8")
def bump_version(self, new_version=None):
"""semver recipe version number bump
:param new_version: An optional new version number
:type new_version: str
:returns: The new version number or None
:rtype: str
:raises: ValueError
If neither have a version, 0.0.1 is returned
If there is no previous version the new version is checked and returned
If there is no new version, but there is a previous one, bump its patch level
If the previous and new versions are the same, bump the patch level
If they are different, check and return the new version
"""
old_version = self.get("version")
if not new_version and not old_version:
self["version"] = "0.0.1"
elif new_version and not old_version:
semver.Version(new_version)
self["version"] = new_version
elif not new_version or new_version == old_version:
new_version = str(semver.Version(old_version).next_patch())
self["version"] = new_version
else:
semver.Version(new_version)
self["version"] = new_version
# Return the new version
return str(semver.Version(self["version"]))
class RecipeModule(dict):
def __init__(self, name, version):
dict.__init__(self, name=name, version=version)
class RecipePackage(RecipeModule):
pass
def recipe_from_toml(recipe_str):
"""Create a Recipe object from a toml string.
:param recipe_str: The Recipe TOML string
:type recipe_str: str
:returns: A Recipe object
:rtype: Recipe
:raises: TomlError
"""
recipe_toml = toml.loads(recipe_str)
# Make RecipeModule objects from the toml
# The TOML may not have modules or packages in it. Set them to None in this case
try:
if recipe_toml.get("modules"):
modules = [RecipeModule(m.get("name"), m.get("version")) for m in recipe_toml["modules"]]
else:
modules = None
if recipe_toml.get("packages"):
packages = [RecipePackage(p.get("name"), p.get("version")) for p in recipe_toml["packages"]]
else:
packages = None
name = recipe_toml["name"]
description = recipe_toml["description"]
version = recipe_toml.get("version", None)
except KeyError:
raise RecipeTOMLError
return Recipe(name, description, version, modules, packages)
def gfile(path):
"""Convert a string path to GFile for use with Git"""
return Gio.file_new_for_path(path)
def recipe_filename(name):
"""Return the toml filename for a recipe
Replaces spaces with '-' and appends '.toml'
"""
# XXX Raise and error if this is empty?
return name.replace(" ", "-") + ".toml"
def head_commit(repo, branch):
"""Get the branch's HEAD Commit Object
:param repo: Open repository
:type repo: Git.Repository
:param branch: Branch name
:type branch: str
:returns: Branch's head commit
:rtype: Git.Commit
:raises: Can raise errors from Ggit
"""
branch_obj = repo.lookup_branch(branch, Git.BranchType.LOCAL)
commit_id = branch_obj.get_target()
return repo.lookup(commit_id, Git.Commit)
def prepare_commit(repo, branch, builder):
"""Prepare for a commit
:param repo: Open repository
:type repo: Git.Repository
:param branch: Branch name
:type branch: str
:param builder: instance of TreeBuilder
:type builder: TreeBuilder
:returns: (Tree, Sig, Ref)
:rtype: tuple
:raises: Can raise errors from Ggit
"""
tree_id = builder.write()
tree = repo.lookup(tree_id, Git.Tree)
sig = Git.Signature.new_now("bdcs-api-server", "user-email")
ref = "refs/heads/%s" % branch
return (tree, sig, ref)
def open_or_create_repo(path):
"""Open an existing repo, or create a new one
:param path: path to recipe directory
:type path: string
:returns: A repository object
:rtype: Git.Repository
:raises: Can raise errors from Ggit
A bare git repo will be created in at the specified path.
If a repo already exists it will be opened and returned instead of
creating a new one.
"""
Git.init()
if os.path.exists(joinpaths(path, "HEAD")):
return Git.Repository.open(gfile(path))
repo = Git.Repository.init_repository(gfile(path), True)
# Make an initial empty commit
sig = Git.Signature.new_now("bdcs-api-server", "user-email")
tree_id = repo.get_index().write_tree()
tree = repo.lookup(tree_id, Git.Tree)
repo.create_commit("HEAD", sig, sig, "UTF-8", "Initial Recipe repository commit", tree, [])
return repo
def write_commit(repo, branch, filename, message, content):
"""Make a new commit to a repository's branch
:param repo: Open repository
:type repo: Git.Repository
:param branch: Branch name
:type branch: str
:param filename: full path of the file to add
:type filename: str
:param message: The commit message
:type message: str
:param content: The data to write
:type content: str
:returns: OId of the new commit
:rtype: Git.OId
:raises: Can raise errors from Ggit
"""
parent_commit = head_commit(repo, branch)
blob_id = repo.create_blob_from_buffer(content)
# Use treebuilder to make a new entry for this filename and blob
parent_tree = parent_commit.get_tree()
builder = repo.create_tree_builder_from_tree(parent_tree)
builder.insert(filename, blob_id, Git.FileMode.BLOB)
(tree, sig, ref) = prepare_commit(repo, branch, builder)
return repo.create_commit(ref, sig, sig, "UTF-8", message, tree, [parent_commit])
def read_commit_spec(repo, spec):
"""Return the raw content of the blob specified by the spec
:param repo: Open repository
:type repo: Git.Repository
:param spec: Git revparse spec
:type spec: str
:returns: Contents of the commit
:rtype: str
:raises: Can raise errors from Ggit
eg. To read the README file from master the spec is "master:README"
"""
commit_id = repo.revparse(spec).get_id()
blob = repo.lookup(commit_id, Git.Blob)
return blob.get_raw_content()
def read_commit(repo, branch, filename, commit=None):
"""Return the contents of a file on a specific branch or commit.
:param repo: Open repository
:type repo: Git.Repository
:param branch: Branch name
:type branch: str
:param filename: filename to read
:type filename: str
:param commit: Optional commit hash
:type commit: str
:returns: Contents of the commit
:rtype: str
:raises: Can raise errors from Ggit
If no commit is passed the master:filename is returned, otherwise it will be
commit:filename
"""
return read_commit_spec(repo, "%s:%s" % (commit or branch, filename))
def read_recipe_commit(repo, branch, filename, commit=None):
"""Read a recipe commit from git and return a Recipe object
:param repo: Open repository
:type repo: Git.Repository
:param branch: Branch name
:type branch: str
:param filename: filename to read
:type filename: str
:param commit: Optional commit hash
:type commit: str
:returns: A Recipe object
:rtype: Recipe
:raises: Can raise errors from Ggit
If no commit is passed the master:filename is returned, otherwise it will be
commit:filename
"""
recipe_toml = read_commit(repo, branch, filename, commit)
return recipe_from_toml(recipe_toml)
def list_branch_files(repo, branch):
"""Return a sorted list of the files on the branch HEAD
:param repo: Open repository
:type repo: Git.Repository
:param branch: Branch name
:type branch: str
:returns: A sorted list of the filenames
:rtype: list(str)
:raises: Can raise errors from Ggit
"""
commit = head_commit(repo, branch).get_id().to_string()
return list_commit_files(repo, commit)
def list_commit_files(repo, commit):
"""Return a sorted list of the files on a commit
:param repo: Open repository
:type repo: Git.Repository
:param commit: The commit hash to list
:type commit: str
:returns: A sorted list of the filenames
:rtype: list(str)
:raises: Can raise errors from Ggit
"""
commit_id = Git.OId.new_from_string(commit)
commit_obj = repo.lookup(commit_id, Git.Commit)
tree = commit_obj.get_tree()
return sorted([tree.get(i).get_name() for i in range(0,tree.size())])
def delete_file(repo, branch, filename):
"""Delete a file from a branch.
:param repo: Open repository
:type repo: Git.Repository
:param branch: Branch name
:type branch: str
:param filename: filename to delete
:type filename: str
:returns: OId of the new commit
:rtype: Git.OId
:raises: Can raise errors from Ggit
"""
parent_commit = head_commit(repo, branch)
parent_tree = parent_commit.get_tree()
builder = repo.create_tree_builder_from_tree(parent_tree)
builder.remove(filename)
(tree, sig, ref) = prepare_commit(repo, branch, builder)
message = "Recipe %s deleted" % filename
return repo.create_commit(ref, sig, sig, "UTF-8", message, tree, [parent_commit])
def revert_file(repo, branch, filename, commit):
"""Revert the contents of a file to that of a previous commit
:param repo: Open repository
:type repo: Git.Repository
:param branch: Branch name
:type branch: str
:param filename: filename to revert
:type filename: str
:param commit: Commit hash
:type commit: str
:returns: OId of the new commit
:rtype: Git.OId
:raises: Can raise errors from Ggit
"""
commit_id = Git.OId.new_from_string(commit)
commit_obj = repo.lookup(commit_id, Git.Commit)
revert_tree = commit_obj.get_tree()
entry = revert_tree.get_by_name(filename)
blob_id = entry.get_id()
parent_commit = head_commit(repo, branch)
# Use treebuilder to modify the tree
parent_tree = parent_commit.get_tree()
builder = repo.create_tree_builder_from_tree(parent_tree)
builder.insert(filename, blob_id, Git.FileMode.BLOB)
(tree, sig, ref) = prepare_commit(repo, branch, builder)
commit_hash = commit_id.to_string()
message = "Recipe %s reverted to commit %s" % (filename, commit_hash)
return repo.create_commit(ref, sig, sig, "UTF-8", message, tree, [parent_commit])
def commit_recipe(repo, branch, recipe, new_version=None):
"""Commit a recipe to a branch
:param repo: Open repository
:type repo: Git.Repository
:param branch: Branch name
:type branch: str
:param recipe: Recipe to commit
:type recipe: Recipe
:returns: OId of the new commit
:rtype: Git.OId
:raises: Can raise errors from Ggit
"""
recipe.bump_version(new_version)
recipe_toml = recipe.toml()
message = "Recipe %s, version %s saved." % (recipe["name"], recipe["version"])
return write_commit(repo, branch, recipe.filename, message, recipe_toml)
def commit_recipe_file(repo, branch, filename):
"""Commit a recipe file to a branch
:param repo: Open repository
:type repo: Git.Repository
:param branch: Branch name
:type branch: str
:param filename: Path to the recipe file to commit
:type filename: str
:returns: OId of the new commit
:rtype: Git.OId
:raises: Can raise errors from Ggit or RecipeFileError
"""
try:
f = open(filename, 'rb')
recipe = recipe_from_toml(f.read())
except IOError:
raise RecipeFileError
return commit_recipe(repo, branch, recipe, new_version=recipe["version"])
def commit_recipe_directory(repo, branch, directory):
"""Commit all *.toml files from a directory, if they aren't already in git.
:param repo: Open repository
:type repo: Git.Repository
:param branch: Branch name
:type branch: str
:param directory: The directory of *.toml recipes to commit
:type directory: str
:returns: None
:raises: Can raise errors from Ggit or RecipeFileError
Files with Toml or RecipeFileErrors will be skipped, and the remainder will
be tried.
"""
dir_files = set([e for e in os.listdir(directory) if e.endswith(".toml")])
branch_files = set(list_branch_files(repo, branch))
new_files = dir_files.difference(branch_files)
for f in new_files:
# Skip files with errors, but try the others
try:
commit_recipe_file(repo, branch, f)
except (RecipeFileError, toml.TomlError):
pass
def tag_file_commit(repo, branch, filename):
"""Tag a file's most recent commit
:param repo: Open repository
:type repo: Git.Repository
:param branch: Branch name
:type branch: str
:param filename: Filename to tag
:type filename: str
:returns: Tag id or None if it failed.
:rtype: Git.OId
:raises: Can raise errors from Ggit
This uses git tags, of the form `refs/tags/<branch>/<filename>/r<revision>`
Only the most recent recipe commit can be tagged to prevent out of order tagging.
Revisions start at 1 and increment for each new commit that is tagged.
If the commit has already been tagged it will return false.
"""
file_commits = list_commits(repo, branch, filename)
if not file_commits:
return None
# Find the most recently tagged version (may not be one) and add 1 to it.
for details in file_commits:
if details.revision is not None:
new_revision = details.revision + 1
break
else:
new_revision = 1
name = "%s/%s/r%d" % (branch, filename, new_revision)
sig = Git.Signature.new_now("bdcs-api-server", "user-email")
commit_id = Git.OId.new_from_string(file_commits[0].commit)
commit = repo.lookup(commit_id, Git.Commit)
return repo.create_tag(name, commit, sig, name, Git.CreateFlags.NONE)
def find_commit_tag(repo, branch, filename, commit_id):
"""Find the tag that matches the commit_id
:param repo: Open repository
:type repo: Git.Repository
:param branch: Branch name
:type branch: str
:param filename: filename to revert
:type filename: str
:param commit_id: The commit id to check
:type commit_id: Git.OId
:returns: The tag or None if there isn't one
:rtype: str or None
There should be only 1 tag pointing to a commit, but there may not
be a tag at all.
The tag will look like: 'refs/tags/<branch>/<filename>/r<revision>'
"""
pattern = "%s/%s/r*" % (branch, filename)
tags = [t for t in repo.list_tags_match(pattern) if is_commit_tag(repo, commit_id, t)]
if len(tags) != 1:
return None
else:
return tags[0]
def is_commit_tag(repo, commit_id, tag):
"""Check to see if a tag points to a specific commit.
:param repo: Open repository
:type repo: Git.Repository
:param commit_id: The commit id to check
:type commit_id: Git.OId
:param tag: The tag to check
:type tag: str
:returns: True if the tag points to the commit, False otherwise
:rtype: bool
"""
ref = repo.lookup_reference("refs/tags/" + tag)
tag_id = ref.get_target()
tag = repo.lookup(tag_id, Git.Tag)
target_id = tag.get_target_id()
return commit_id.compare(target_id) == 0
def get_revision_from_tag(tag):
"""Return the revision number from a tag
:param tag: The tag to exract the revision from
:type tag: str
:returns: The integer revision or None
:rtype: int or None
The revision is the part after the r in 'branch/filename/rXXX'
"""
if tag is None:
return None
try:
return int(tag.rsplit('r',2)[-1])
except (ValueError, IndexError):
return None
class CommitDetails(DataHolder):
def __init__(self, commit, timestamp, message, revision=None):
DataHolder.__init__(self,
commit = commit,
timestamp = timestamp,
message = message,
revision = revision)
def list_commits(repo, branch, filename):
"""List the commit history of a file on a branch.
:param repo: Open repository
:type repo: Git.Repository
:param branch: Branch name
:type branch: str
:param filename: filename to revert
:type filename: str
:returns: A list of commit details
:rtype: list(CommitDetails)
:raises: Can raise errors from Ggit
"""
revwalk = Git.RevisionWalker.new(repo)
revwalk.set_sort_mode(Git.SortMode.TIME | Git.SortMode.REVERSE)
branch_ref = "refs/heads/%s" % branch
revwalk.push_ref(branch_ref)
commits = []
while True:
commit_id = revwalk.next()
if not commit_id:
break
commit = repo.lookup(commit_id, Git.Commit)
parents = commit.get_parents()
# No parents? Must be the first commit.
if parents.get_size() == 0:
continue
tree = commit.get_tree()
# Is the filename in this tree? If not, move on.
if not tree.get_by_name(filename):
continue
# Is filename different in all of the parent commits?
parent_commits = map(parents.get, xrange(0, parents.get_size()))
is_diff = all(map(lambda pc: is_parent_diff(repo, filename, tree, pc), parent_commits))
# No changes from parents, skip it.
if not is_diff:
continue
tag = find_commit_tag(repo, branch, filename, commit.get_id())
try:
commits.append(get_commit_details(commit, get_revision_from_tag(tag)))
except CommitTimeValError:
# Skip any commits that have trouble converting the time
# TODO - log details about this failure
pass
# These will be in reverse time sort order thanks to revwalk
return commits
def get_commit_details(commit, revision=None):
"""Return the details about a specific commit.
:param commit: The commit to get details from
:type commit: Git.Commit
:param revision: Optional commit revision
:type revision: int
:returns: Details about the commit
:rtype: CommitDetails
:raises: CommitTimeValError or Ggit exceptions
"""
message = commit.get_message()
commit_str = commit.get_id().to_string()
sig = commit.get_committer()
datetime = sig.get_time()
# XXX What do we do with timezone?
_timezone = sig.get_time_zone()
timeval = GLib.TimeVal()
ok = datetime.to_timeval(timeval)
if not ok:
raise CommitTimeValError
time_str = timeval.to_iso8601()
return CommitDetails(commit_str, time_str, message, revision)
def is_parent_diff(repo, filename, tree, parent):
"""Check to see if the commit is different from its parents
:param repo: Open repository
:type repo: Git.Repository
:param filename: filename to revert
:type filename: str
:param tree: The commit's tree
:type tree: Git.Tree
:param parent: The commit's parent commit
:type parent: Git.Commit
:retuns: True if filename in the commit is different from its parents
:rtype: bool
"""
diff_opts = Git.DiffOptions.new()
diff_opts.set_pathspec([filename])
diff = Git.Diff.new_tree_to_tree(repo, parent.get_tree(), tree, diff_opts)
return diff.get_num_deltas() > 0

View File

@ -17,7 +17,7 @@
from flask import Flask
from pylorax.api import crossdomain
from pylorax.api.crossdomain import crossdomain
from pylorax.api.v0 import v0_api
server = Flask(__name__)

View File

@ -20,7 +20,7 @@ from flask import jsonify
from pykickstart.parser import KickstartParser
from pykickstart.version import makeVersion, RHEL7
from pylorax.api import crossdomain
from pylorax.api.crossdomain import crossdomain
from pylorax.creator import DRACUT_DEFAULT, mount_boot_part_over_root
from pylorax.creator import make_appliance, make_image, make_livecd, make_live_images
from pylorax.creator import make_runtime, make_squashfs

View File