Add support for [[repos.git]] section to blueprints

This adds support, documentation, and testing for a [[repos.git]]
blueprint section that can be used to install files from a git
repository. It will create an rpm that will be added to the build,
and included in the metadata that can be downloaded. This allows you to
accurately keep track of the source of configuration files and extra
metadata that is added to the build.

The source repo and reference will be listed in the rpm's summary making
it easy to discover on the installed system.

(cherry picked from commit d7b96c8f0f)
(cherry picked from commit 047f174dcf)

docs/lorax-composer.rst

@@ -244,6 +244,37 @@ Add a group to the image. ``name`` is required and ``gid`` is optional::
     gid = 1130
 
 
+[[repos.git]]
+~~~~~~~~~~~~~
+
+The ``[[repos.git]]`` entries are used to add files from a `git repository<https://git-scm.com/>`
+repository to the created image. The repository is cloned, the specified ``ref`` is checked out
+and an rpm is created to install the files to a ``destination`` path. The rpm includes a summary
+with the details of the repository and reference used to create it. The rpm is also included in the
+image build metadata.
+
+To create an rpm named ``server-config-1.0-1.noarch.rpm`` you would add this to your blueprint::
+
+    [[repos.git]]
+    rpmname="server-config"
+    rpmversion="1.0"
+    rpmrelease="1"
+    summary="Setup files for server deployment"
+    repo="PATH OF GIT REPO TO CLONE"
+    ref="v1.0"
+    destination="/opt/server/"
+
+An rpm will be created with the contents of the git repository referenced, with the files
+being installed under ``/opt/server/`` in this case.
+
+``ref`` can be any valid git reference for use with ``git archive``. eg. to use the head
+of a branch set it to ``origin/branch-name``, a tag name, or a commit hash.
+
+Note that the repository is cloned in full each time a build is started, so pointing to a
+repository with a large amount of history may take a while to clone and use a significant
+amount of disk space. The clone is temporary and is removed once the rpm is created.
+
+
 Adding Output Types
 -------------------
 

src/pylorax/api/recipes.py

@@ -47,7 +47,7 @@ class Recipe(dict):
     and adds a .filename property to return the recipe's filename,
     and a .toml() function to return the recipe as a TOML string.
     """
-    def __init__(self, name, description, version, modules, packages, groups, customizations=None):
+    def __init__(self, name, description, version, modules, packages, groups, customizations=None, gitrepos=None):
         # Check that version is empty or semver compatible
         if version:
             semver.Version(version)
@@ -59,18 +59,29 @@ class Recipe(dict):
             packages = sorted(packages, key=lambda p: p["name"].lower())
         if groups is not None:
             groups = sorted(groups, key=lambda g: g["name"].lower())
+
+        # Only support [[repos.git]] for now
+        if gitrepos is not None:
+            repos = {"git": sorted(gitrepos, key=lambda g: g["repo"].lower())}
+        else:
+            repos = None
         dict.__init__(self, name=name,
                             description=description,
                             version=version,
                             modules=modules,
                             packages=packages,
                             groups=groups,
-                            customizations=customizations)
+                            customizations=customizations,
+                            repos=repos)
 
         # We don't want customizations=None to show up in the TOML so remove it
         if customizations is None:
             del self["customizations"]
 
+        # Don't include empty repos or repos.git
+        if repos is None or not repos["git"]:
+            del self["repos"]
+
     @property
     def package_names(self):
         """Return the names of the packages"""
@@ -168,9 +179,13 @@ class Recipe(dict):
             customizations = self["customizations"]
         else:
             customizations = None
+        if "repos" in self and "git" in self["repos"]:
+            gitrepos = self["repos"]["git"]
+        else:
+            gitrepos = None
 
         return Recipe(self["name"], self["description"], self["version"],
-                      new_modules, new_packages, new_groups, customizations)
+                      new_modules, new_packages, new_groups, customizations, gitrepos)
 
 class RecipeModule(dict):
     def __init__(self, name, version):
@@ -183,6 +198,54 @@ class RecipeGroup(dict):
     def __init__(self, name):
         dict.__init__(self, name=name)
 
+def NewRecipeGit(toml_dict):
+    """Create a RecipeGit object from fields in a TOML dict
+
+    :param rpmname: Name of the rpm to create, also used as the prefix name in the tar archive
+    :type rpmname: str
+    :param rpmversion: Version of the rpm, eg. "1.0.0"
+    :type rpmversion: str
+    :param rpmrelease: Release of the rpm, eg. "1"
+    :type rpmrelease: str
+    :param summary: Summary string for the rpm
+    :type summary: str
+    :param repo: URL of the get repo to clone and create the archive from
+    :type repo: str
+    :param ref: Git reference to check out. eg. origin/branch-name, git tag, or git commit hash
+    :type ref: str
+    :param destination: Path to install the / of the git repo at when installing the rpm
+    :type destination: str
+    :returns: A populated RecipeGit object
+    :rtype: RecipeGit
+
+    The TOML should look like this::
+
+        [[repos.git]]
+        rpmname="server-config"
+        rpmversion="1.0"
+        rpmrelease="1"
+        summary="Setup files for server deployment"
+        repo="PATH OF GIT REPO TO CLONE"
+        ref="v1.0"
+        destination="/opt/server/"
+
+    Note that the repo path supports anything that git supports, file://, https://, http://
+
+    Currently there is no support for authentication
+    """
+    return RecipeGit(toml_dict.get("rpmname"),
+                     toml_dict.get("rpmversion"),
+                     toml_dict.get("rpmrelease"),
+                     toml_dict.get("summary", ""),
+                     toml_dict.get("repo"),
+                     toml_dict.get("ref"),
+                     toml_dict.get("destination"))
+
+class RecipeGit(dict):
+    def __init__(self, rpmname, rpmversion, rpmrelease, summary, repo, ref, destination):
+        dict.__init__(self, rpmname=rpmname, rpmversion=rpmversion, rpmrelease=rpmrelease,
+                      summary=summary, repo=repo, ref=ref, destination=destination)
+
 def recipe_from_file(recipe_path):
     """Return a recipe file as a Recipe object
 
@@ -230,6 +293,10 @@ def recipe_from_dict(recipe_dict):
             groups = [RecipeGroup(g.get("name")) for g in recipe_dict["groups"]]
         else:
             groups = []
+        if recipe_dict.get("repos") and recipe_dict.get("repos").get("git"):
+            gitrepos = [NewRecipeGit(r) for r in recipe_dict["repos"]["git"]]
+        else:
+            gitrepos = []
         name = recipe_dict["name"]
         description = recipe_dict["description"]
         version = recipe_dict.get("version", None)
@@ -237,7 +304,7 @@ def recipe_from_dict(recipe_dict):
     except KeyError as e:
         raise RecipeError("There was a problem parsing the recipe: %s" % str(e))
 
-    return Recipe(name, description, version, modules, packages, groups, customizations)
+    return Recipe(name, description, version, modules, packages, groups, customizations, gitrepos)
 
 def gfile(path):
     """Convert a string path to GFile for use with Git"""

tests/pylorax/results/repos-git.dict

@@ -0,0 +1 @@
+{'description': u'An example http server with PHP and MySQL support.', 'packages': [], 'groups': [], 'modules': [], 'version': u'0.0.1', 'name': u'http-server', 'repos': {'git': [{"rpmname": "server-config-files", "rpmversion": "1.0", "rpmrelease": "1", "summary": "Setup files for server deployment", "repo": "https://github.com/bcl/server-config-files", "ref": "v3.0", "destination": "/srv/config/"}]}}

tests/pylorax/results/repos-git.toml

@@ -0,0 +1,12 @@
+name = "http-server"
+description = "An example http server with PHP and MySQL support."
+version = "0.0.1"
+
+[[repos.git]]
+rpmname="server-config-files"
+rpmversion="1.0"
+rpmrelease="1"
+summary="Setup files for server deployment"
+repo="https://github.com/bcl/server-config-files"
+ref="v3.0"
+destination="/srv/config/"

tests/pylorax/test_recipes.py

@@ -37,7 +37,8 @@ class BasicRecipeTest(unittest.TestCase):
                          ("modules-only.toml", "modules-only.dict"),
                          ("packages-only.toml", "packages-only.dict"),
                          ("groups-only.toml", "groups-only.dict"),
-                         ("custom-base.toml", "custom-base.dict")]
+                         ("custom-base.toml", "custom-base.dict"),
+                         ("repos-git.toml", "repos-git.dict")]
         results_path = "./tests/pylorax/results/"
         self.input_toml = []
         for (recipe_toml, recipe_dict) in input_recipes: