Add pylorax.api.recipes code for handling the Recipe's Git repository

src/pylorax/api/recipes.py

@@ -0,0 +1,656 @@
+#
+# Copyright (C) 2017  Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import gi
+gi.require_version("Ggit", "1.0")
+from gi.repository import Ggit as Git
+from gi.repository import Gio
+from gi.repository import GLib
+
+import os
+import pytoml as toml
+import semantic_version as semver
+
+from pylorax.base import DataHolder
+from pylorax.sysutils import joinpaths
+
+
+class CommitTimeValError(Exception):
+    pass
+
+class RecipeFileError(Exception):
+    pass
+
+class RecipeTOMLError(Exception):
+    pass
+
+
+class Recipe(dict):
+    """A Recipe of package and modules
+
+    This is a subclass of dict that enforces the constructor arguments
+    and adds a .filename property to return the recipe's filename,
+    and a .toml() function to return the recipe as a TOML string.
+    """
+    def __init__(self, name, description, version, modules, packages):
+        # Check that version is empty or semver compatible
+        if version:
+            semver.Version(version)
+
+        # Make sure modules and packages are listed by their case-insensitive names
+        if modules is not None:
+            modules = sorted(modules, key=lambda m: m["name"].lower())
+        if packages is not None:
+            packages = sorted(packages, key=lambda p: p["name"].lower())
+        dict.__init__(self, name=name,
+                            description=description,
+                            version=version,
+                            modules=modules,
+                            packages=packages)
+
+    @property
+    def filename(self):
+        """Return the Recipe's filename
+
+        Replaces spaces in the name with '-' and appends .toml
+        """
+        return recipe_filename(self.get("name"))
+
+    def toml(self):
+        """Return the Recipe in TOML format"""
+        return toml.dumps(self).encode("UTF-8")
+
+    def bump_version(self, new_version=None):
+        """semver recipe version number bump
+
+        :param new_version: An optional new version number
+        :type new_version: str
+        :returns: The new version number or None
+        :rtype: str
+        :raises: ValueError
+
+        If neither have a version, 0.0.1 is returned
+        If there is no previous version the new version is checked and returned
+        If there is no new version, but there is a previous one, bump its patch level
+        If the previous and new versions are the same, bump the patch level
+        If they are different, check and return the new version
+        """
+        old_version = self.get("version")
+        if not new_version and not old_version:
+            self["version"] = "0.0.1"
+
+        elif new_version and not old_version:
+            semver.Version(new_version)
+            self["version"] = new_version
+
+        elif not new_version or new_version == old_version:
+            new_version = str(semver.Version(old_version).next_patch())
+            self["version"] = new_version
+
+        else:
+            semver.Version(new_version)
+            self["version"] = new_version
+
+        # Return the new version
+        return str(semver.Version(self["version"]))
+
+class RecipeModule(dict):
+    def __init__(self, name, version):
+        dict.__init__(self, name=name, version=version)
+
+class RecipePackage(RecipeModule):
+    pass
+
+def recipe_from_toml(recipe_str):
+    """Create a Recipe object from a toml string.
+
+    :param recipe_str: The Recipe TOML string
+    :type recipe_str: str
+    :returns: A Recipe object
+    :rtype: Recipe
+    :raises: TomlError
+    """
+    recipe_toml = toml.loads(recipe_str)
+
+    # Make RecipeModule objects from the toml
+    # The TOML may not have modules or packages in it. Set them to None in this case
+    try:
+        if recipe_toml.get("modules"):
+            modules = [RecipeModule(m.get("name"), m.get("version")) for m in recipe_toml["modules"]]
+        else:
+            modules = None
+        if recipe_toml.get("packages"):
+            packages = [RecipePackage(p.get("name"), p.get("version")) for p in recipe_toml["packages"]]
+        else:
+            packages = None
+        name = recipe_toml["name"]
+        description = recipe_toml["description"]
+        version = recipe_toml.get("version", None)
+    except KeyError:
+        raise RecipeTOMLError
+
+    return Recipe(name, description, version, modules, packages)
+
+def gfile(path):
+    """Convert a string path to GFile for use with Git"""
+    return Gio.file_new_for_path(path)
+
+def recipe_filename(name):
+    """Return the toml filename for a recipe
+
+    Replaces spaces with '-' and appends '.toml'
+    """
+    # XXX Raise and error if this is empty?
+    return name.replace(" ", "-") + ".toml"
+
+def head_commit(repo, branch):
+    """Get the branch's HEAD Commit Object
+
+    :param repo: Open repository
+    :type repo: Git.Repository
+    :param branch: Branch name
+    :type branch: str
+    :returns: Branch's head commit
+    :rtype: Git.Commit
+    :raises: Can raise errors from Ggit
+    """
+    branch_obj = repo.lookup_branch(branch, Git.BranchType.LOCAL)
+    commit_id = branch_obj.get_target()
+    return repo.lookup(commit_id, Git.Commit)
+
+def prepare_commit(repo, branch, builder):
+    """Prepare for a commit
+
+    :param repo: Open repository
+    :type repo: Git.Repository
+    :param branch: Branch name
+    :type branch: str
+    :param builder: instance of TreeBuilder
+    :type builder: TreeBuilder
+    :returns: (Tree, Sig, Ref)
+    :rtype: tuple
+    :raises: Can raise errors from Ggit
+    """
+    tree_id = builder.write()
+    tree = repo.lookup(tree_id, Git.Tree)
+    sig = Git.Signature.new_now("bdcs-api-server", "user-email")
+    ref = "refs/heads/%s" % branch
+    return (tree, sig, ref)
+
+def open_or_create_repo(path):
+    """Open an existing repo, or create a new one
+
+    :param path: path to recipe directory
+    :type path: string
+    :returns: A repository object
+    :rtype: Git.Repository
+    :raises: Can raise errors from Ggit
+
+    A bare git repo will be created in at the specified path.
+    If a repo already exists it will be opened and returned instead of
+    creating a new one.
+    """
+    Git.init()
+    if os.path.exists(joinpaths(path, "HEAD")):
+        return Git.Repository.open(gfile(path))
+
+    repo = Git.Repository.init_repository(gfile(path), True)
+
+    # Make an initial empty commit
+    sig = Git.Signature.new_now("bdcs-api-server", "user-email")
+    tree_id = repo.get_index().write_tree()
+    tree = repo.lookup(tree_id, Git.Tree)
+    repo.create_commit("HEAD", sig, sig, "UTF-8", "Initial Recipe repository commit", tree, [])
+    return repo
+
+def write_commit(repo, branch, filename, message, content):
+    """Make a new commit to a repository's branch
+
+    :param repo: Open repository
+    :type repo: Git.Repository
+    :param branch: Branch name
+    :type branch: str
+    :param filename: full path of the file to add
+    :type filename: str
+    :param message: The commit message
+    :type message: str
+    :param content: The data to write
+    :type content: str
+    :returns: OId of the new commit
+    :rtype: Git.OId
+    :raises: Can raise errors from Ggit
+    """
+    parent_commit = head_commit(repo, branch)
+    blob_id = repo.create_blob_from_buffer(content)
+
+    # Use treebuilder to make a new entry for this filename and blob
+    parent_tree = parent_commit.get_tree()
+    builder = repo.create_tree_builder_from_tree(parent_tree)
+    builder.insert(filename, blob_id, Git.FileMode.BLOB)
+    (tree, sig, ref) = prepare_commit(repo, branch, builder)
+    return repo.create_commit(ref, sig, sig, "UTF-8", message, tree, [parent_commit])
+
+def read_commit_spec(repo, spec):
+    """Return the raw content of the blob specified by the spec
+
+    :param repo: Open repository
+    :type repo: Git.Repository
+    :param spec: Git revparse spec
+    :type spec: str
+    :returns: Contents of the commit
+    :rtype: str
+    :raises: Can raise errors from Ggit
+
+    eg. To read the README file from master the spec is "master:README"
+    """
+    commit_id = repo.revparse(spec).get_id()
+    blob = repo.lookup(commit_id, Git.Blob)
+    return blob.get_raw_content()
+
+def read_commit(repo, branch, filename, commit=None):
+    """Return the contents of a file on a specific branch or commit.
+
+    :param repo: Open repository
+    :type repo: Git.Repository
+    :param branch: Branch name
+    :type branch: str
+    :param filename: filename to read
+    :type filename: str
+    :param commit: Optional commit hash
+    :type commit: str
+    :returns: Contents of the commit
+    :rtype: str
+    :raises: Can raise errors from Ggit
+
+    If no commit is passed the master:filename is returned, otherwise it will be
+    commit:filename
+    """
+    return read_commit_spec(repo, "%s:%s" % (commit or branch, filename))
+
+def read_recipe_commit(repo, branch, filename, commit=None):
+    """Read a recipe commit from git and return a Recipe object
+
+    :param repo: Open repository
+    :type repo: Git.Repository
+    :param branch: Branch name
+    :type branch: str
+    :param filename: filename to read
+    :type filename: str
+    :param commit: Optional commit hash
+    :type commit: str
+    :returns: A Recipe object
+    :rtype: Recipe
+    :raises: Can raise errors from Ggit
+
+    If no commit is passed the master:filename is returned, otherwise it will be
+    commit:filename
+    """
+    recipe_toml = read_commit(repo, branch, filename, commit)
+    return recipe_from_toml(recipe_toml)
+
+def list_branch_files(repo, branch):
+    """Return a sorted list of the files on the branch HEAD
+
+    :param repo: Open repository
+    :type repo: Git.Repository
+    :param branch: Branch name
+    :type branch: str
+    :returns: A sorted list of the filenames
+    :rtype: list(str)
+    :raises: Can raise errors from Ggit
+    """
+    commit = head_commit(repo, branch).get_id().to_string()
+    return list_commit_files(repo, commit)
+
+def list_commit_files(repo, commit):
+    """Return a sorted list of the files on a commit
+
+    :param repo: Open repository
+    :type repo: Git.Repository
+    :param commit: The commit hash to list
+    :type commit: str
+    :returns: A sorted list of the filenames
+    :rtype: list(str)
+    :raises: Can raise errors from Ggit
+    """
+    commit_id = Git.OId.new_from_string(commit)
+    commit_obj = repo.lookup(commit_id, Git.Commit)
+    tree = commit_obj.get_tree()
+    return sorted([tree.get(i).get_name() for i in range(0,tree.size())])
+
+def delete_file(repo, branch, filename):
+    """Delete a file from a branch.
+
+    :param repo: Open repository
+    :type repo: Git.Repository
+    :param branch: Branch name
+    :type branch: str
+    :param filename: filename to delete
+    :type filename: str
+    :returns: OId of the new commit
+    :rtype: Git.OId
+    :raises: Can raise errors from Ggit
+    """
+    parent_commit = head_commit(repo, branch)
+    parent_tree = parent_commit.get_tree()
+    builder = repo.create_tree_builder_from_tree(parent_tree)
+    builder.remove(filename)
+    (tree, sig, ref) = prepare_commit(repo, branch, builder)
+    message = "Recipe %s deleted" % filename
+    return repo.create_commit(ref, sig, sig, "UTF-8", message, tree, [parent_commit])
+
+def revert_file(repo, branch, filename, commit):
+    """Revert the contents of a file to that of a previous commit
+
+    :param repo: Open repository
+    :type repo: Git.Repository
+    :param branch: Branch name
+    :type branch: str
+    :param filename: filename to revert
+    :type filename: str
+    :param commit: Commit hash
+    :type commit: str
+    :returns: OId of the new commit
+    :rtype: Git.OId
+    :raises: Can raise errors from Ggit
+    """
+    commit_id = Git.OId.new_from_string(commit)
+    commit_obj = repo.lookup(commit_id, Git.Commit)
+    revert_tree = commit_obj.get_tree()
+    entry = revert_tree.get_by_name(filename)
+    blob_id = entry.get_id()
+    parent_commit = head_commit(repo, branch)
+
+    # Use treebuilder to modify the tree
+    parent_tree = parent_commit.get_tree()
+    builder = repo.create_tree_builder_from_tree(parent_tree)
+    builder.insert(filename, blob_id, Git.FileMode.BLOB)
+    (tree, sig, ref) = prepare_commit(repo, branch, builder)
+    commit_hash = commit_id.to_string()
+    message = "Recipe %s reverted to commit %s" % (filename, commit_hash)
+    return repo.create_commit(ref, sig, sig, "UTF-8", message, tree, [parent_commit])
+
+def commit_recipe(repo, branch, recipe, new_version=None):
+    """Commit a recipe to a branch
+
+    :param repo: Open repository
+    :type repo: Git.Repository
+    :param branch: Branch name
+    :type branch: str
+    :param recipe: Recipe to commit
+    :type recipe: Recipe
+    :returns: OId of the new commit
+    :rtype: Git.OId
+    :raises: Can raise errors from Ggit
+    """
+    recipe.bump_version(new_version)
+    recipe_toml = recipe.toml()
+    message = "Recipe %s, version %s saved." % (recipe["name"], recipe["version"])
+    return write_commit(repo, branch, recipe.filename, message, recipe_toml)
+
+def commit_recipe_file(repo, branch, filename):
+    """Commit a recipe file to a branch
+
+    :param repo: Open repository
+    :type repo: Git.Repository
+    :param branch: Branch name
+    :type branch: str
+    :param filename: Path to the recipe file to commit
+    :type filename: str
+    :returns: OId of the new commit
+    :rtype: Git.OId
+    :raises: Can raise errors from Ggit or RecipeFileError
+    """
+    try:
+        f = open(filename, 'rb')
+        recipe = recipe_from_toml(f.read())
+    except IOError:
+        raise RecipeFileError
+
+    return commit_recipe(repo, branch, recipe, new_version=recipe["version"])
+
+def commit_recipe_directory(repo, branch, directory):
+    """Commit all *.toml files from a directory, if they aren't already in git.
+
+    :param repo: Open repository
+    :type repo: Git.Repository
+    :param branch: Branch name
+    :type branch: str
+    :param directory: The directory of *.toml recipes to commit
+    :type directory: str
+    :returns: None
+    :raises: Can raise errors from Ggit or RecipeFileError
+
+    Files with Toml or RecipeFileErrors will be skipped, and the remainder will
+    be tried.
+    """
+    dir_files = set([e for e in os.listdir(directory) if e.endswith(".toml")])
+    branch_files = set(list_branch_files(repo, branch))
+    new_files = dir_files.difference(branch_files)
+
+    for f in new_files:
+        # Skip files with errors, but try the others
+        try:
+            commit_recipe_file(repo, branch, f)
+        except (RecipeFileError, toml.TomlError):
+            pass
+
+def tag_file_commit(repo, branch, filename):
+    """Tag a file's most recent commit
+
+    :param repo: Open repository
+    :type repo: Git.Repository
+    :param branch: Branch name
+    :type branch: str
+    :param filename: Filename to tag
+    :type filename: str
+    :returns: Tag id or None if it failed.
+    :rtype: Git.OId
+    :raises: Can raise errors from Ggit
+
+    This uses git tags, of the form `refs/tags/<branch>/<filename>/r<revision>`
+    Only the most recent recipe commit can be tagged to prevent out of order tagging.
+    Revisions start at 1 and increment for each new commit that is tagged.
+    If the commit has already been tagged it will return false.
+    """
+    file_commits = list_commits(repo, branch, filename)
+    if not file_commits:
+        return None
+
+    # Find the most recently tagged version (may not be one) and add 1 to it.
+    for details in file_commits:
+        if details.revision is not None:
+            new_revision = details.revision + 1
+            break
+    else:
+        new_revision = 1
+
+    name = "%s/%s/r%d" % (branch, filename, new_revision)
+    sig = Git.Signature.new_now("bdcs-api-server", "user-email")
+    commit_id = Git.OId.new_from_string(file_commits[0].commit)
+    commit = repo.lookup(commit_id, Git.Commit)
+    return repo.create_tag(name, commit, sig, name, Git.CreateFlags.NONE)
+
+def find_commit_tag(repo, branch, filename, commit_id):
+    """Find the tag that matches the commit_id
+
+    :param repo: Open repository
+    :type repo: Git.Repository
+    :param branch: Branch name
+    :type branch: str
+    :param filename: filename to revert
+    :type filename: str
+    :param commit_id: The commit id to check
+    :type commit_id: Git.OId
+    :returns: The tag or None if there isn't one
+    :rtype: str or None
+
+    There should be only 1 tag pointing to a commit, but there may not
+    be a tag at all.
+
+    The tag will look like: 'refs/tags/<branch>/<filename>/r<revision>'
+    """
+    pattern = "%s/%s/r*" % (branch, filename)
+    tags = [t for t in repo.list_tags_match(pattern) if is_commit_tag(repo, commit_id, t)]
+    if len(tags) != 1:
+        return None
+    else:
+        return tags[0]
+
+def is_commit_tag(repo, commit_id, tag):
+    """Check to see if a tag points to a specific commit.
+
+    :param repo: Open repository
+    :type repo: Git.Repository
+    :param commit_id: The commit id to check
+    :type commit_id: Git.OId
+    :param tag: The tag to check
+    :type tag: str
+    :returns: True if the tag points to the commit, False otherwise
+    :rtype: bool
+    """
+    ref = repo.lookup_reference("refs/tags/" + tag)
+    tag_id = ref.get_target()
+    tag = repo.lookup(tag_id, Git.Tag)
+    target_id = tag.get_target_id()
+    return commit_id.compare(target_id) == 0
+
+def get_revision_from_tag(tag):
+    """Return the revision number from a tag
+
+    :param tag: The tag to exract the revision from
+    :type tag: str
+    :returns: The integer revision or None
+    :rtype: int or None
+
+    The revision is the part after the r in 'branch/filename/rXXX'
+    """
+    if tag is None:
+        return None
+    try:
+        return int(tag.rsplit('r',2)[-1])
+    except (ValueError, IndexError):
+        return None
+
+class CommitDetails(DataHolder):
+    def __init__(self, commit, timestamp, message, revision=None):
+        DataHolder.__init__(self,
+                            commit = commit,
+                            timestamp = timestamp,
+                            message = message,
+                            revision = revision)
+
+def list_commits(repo, branch, filename):
+    """List the commit history of a file on a branch.
+
+    :param repo: Open repository
+    :type repo: Git.Repository
+    :param branch: Branch name
+    :type branch: str
+    :param filename: filename to revert
+    :type filename: str
+    :returns: A list of commit details
+    :rtype: list(CommitDetails)
+    :raises: Can raise errors from Ggit
+    """
+    revwalk = Git.RevisionWalker.new(repo)
+    revwalk.set_sort_mode(Git.SortMode.TIME | Git.SortMode.REVERSE)
+    branch_ref = "refs/heads/%s" % branch
+    revwalk.push_ref(branch_ref)
+
+    commits = []
+    while True:
+        commit_id = revwalk.next()
+        if not commit_id:
+            break
+        commit = repo.lookup(commit_id, Git.Commit)
+
+        parents = commit.get_parents()
+        # No parents? Must be the first commit.
+        if parents.get_size() == 0:
+            continue
+
+        tree = commit.get_tree()
+        # Is the filename in this tree? If not, move on.
+        if not tree.get_by_name(filename):
+            continue
+
+        # Is filename different in all of the parent commits?
+        parent_commits = map(parents.get, xrange(0, parents.get_size()))
+        is_diff = all(map(lambda pc: is_parent_diff(repo, filename, tree, pc), parent_commits))
+        # No changes from parents, skip it.
+        if not is_diff:
+            continue
+
+        tag = find_commit_tag(repo, branch, filename, commit.get_id())
+        try:
+            commits.append(get_commit_details(commit, get_revision_from_tag(tag)))
+        except CommitTimeValError:
+            # Skip any commits that have trouble converting the time
+            # TODO - log details about this failure
+            pass
+
+    # These will be in reverse time sort order thanks to revwalk
+    return commits
+
+def get_commit_details(commit, revision=None):
+    """Return the details about a specific commit.
+
+    :param commit: The commit to get details from
+    :type commit: Git.Commit
+    :param revision: Optional commit revision
+    :type revision: int
+    :returns: Details about the commit
+    :rtype: CommitDetails
+    :raises: CommitTimeValError or Ggit exceptions
+
+    """
+    message = commit.get_message()
+    commit_str = commit.get_id().to_string()
+    sig = commit.get_committer()
+
+    datetime = sig.get_time()
+    # XXX What do we do with timezone?
+    _timezone = sig.get_time_zone()
+    timeval = GLib.TimeVal()
+    ok = datetime.to_timeval(timeval)
+    if not ok:
+        raise CommitTimeValError
+    time_str = timeval.to_iso8601()
+
+    return CommitDetails(commit_str, time_str, message, revision)
+
+def is_parent_diff(repo, filename, tree, parent):
+    """Check to see if the commit is different from its parents
+
+    :param repo: Open repository
+    :type repo: Git.Repository
+    :param filename: filename to revert
+    :type filename: str
+    :param tree: The commit's tree
+    :type tree: Git.Tree
+    :param parent: The commit's parent commit
+    :type parent: Git.Commit
+    :retuns: True if filename in the commit is different from its parents
+    :rtype: bool
+    """
+    diff_opts = Git.DiffOptions.new()
+    diff_opts.set_pathspec([filename])
+    diff = Git.Diff.new_tree_to_tree(repo, parent.get_tree(), tree, diff_opts)
+    return diff.get_num_deltas() > 0
+
+

src/pylorax/api/server.py

@@ -17,7 +17,7 @@
 
 from flask import Flask
 
-from pylorax.api import crossdomain
+from pylorax.api.crossdomain import crossdomain
 from pylorax.api.v0 import v0_api
 
 server = Flask(__name__)

src/pylorax/api/v0.py

@@ -20,7 +20,7 @@ from flask import jsonify
 from pykickstart.parser import KickstartParser
 from pykickstart.version import makeVersion, RHEL7
 
-from pylorax.api import crossdomain
+from pylorax.api.crossdomain import crossdomain
 from pylorax.creator import DRACUT_DEFAULT, mount_boot_part_over_root
 from pylorax.creator import make_appliance, make_image, make_livecd, make_live_images
 from pylorax.creator import make_runtime, make_squashfs