From 21b03c21086ffa754c8d25dc063d39ce1b54c669 Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Tue, 5 Feb 2019 13:26:58 -0800 Subject: [PATCH] Add support for [[repos.git]] section to blueprints This adds support, documentation, and testing for a [[repos.git]] blueprint section that can be used to install files from a git repository. It will create an rpm that will be added to the build, and included in the metadata that can be downloaded. This allows you to accurately keep track of the source of configuration files and extra metadata that is added to the build. The source repo and reference will be listed in the rpm's summary making it easy to discover on the installed system. (cherry picked from commit d7b96c8f0fbdc87f90dfea25a92750c5bbab8265) (cherry picked from commit 047f174dcff326f7319592b4c4bd4d8d79e4ac1d) --- docs/lorax-composer.rst | 31 ++++++++++++ src/pylorax/api/recipes.py | 75 ++++++++++++++++++++++++++-- tests/pylorax/results/repos-git.dict | 1 + tests/pylorax/results/repos-git.toml | 12 +++++ tests/pylorax/test_recipes.py | 3 +- 5 files changed, 117 insertions(+), 5 deletions(-) create mode 100644 tests/pylorax/results/repos-git.dict create mode 100644 tests/pylorax/results/repos-git.toml diff --git a/docs/lorax-composer.rst b/docs/lorax-composer.rst index d349f787..f38e651b 100644 --- a/docs/lorax-composer.rst +++ b/docs/lorax-composer.rst @@ -244,6 +244,37 @@ Add a group to the image. ``name`` is required and ``gid`` is optional:: gid = 1130 +[[repos.git]] +~~~~~~~~~~~~~ + +The ``[[repos.git]]`` entries are used to add files from a `git repository` +repository to the created image. The repository is cloned, the specified ``ref`` is checked out +and an rpm is created to install the files to a ``destination`` path. The rpm includes a summary +with the details of the repository and reference used to create it. The rpm is also included in the +image build metadata. + +To create an rpm named ``server-config-1.0-1.noarch.rpm`` you would add this to your blueprint:: + + [[repos.git]] + rpmname="server-config" + rpmversion="1.0" + rpmrelease="1" + summary="Setup files for server deployment" + repo="PATH OF GIT REPO TO CLONE" + ref="v1.0" + destination="/opt/server/" + +An rpm will be created with the contents of the git repository referenced, with the files +being installed under ``/opt/server/`` in this case. + +``ref`` can be any valid git reference for use with ``git archive``. eg. to use the head +of a branch set it to ``origin/branch-name``, a tag name, or a commit hash. + +Note that the repository is cloned in full each time a build is started, so pointing to a +repository with a large amount of history may take a while to clone and use a significant +amount of disk space. The clone is temporary and is removed once the rpm is created. + + Adding Output Types ------------------- diff --git a/src/pylorax/api/recipes.py b/src/pylorax/api/recipes.py index d250e28c..64a6db87 100644 --- a/src/pylorax/api/recipes.py +++ b/src/pylorax/api/recipes.py @@ -47,7 +47,7 @@ class Recipe(dict): and adds a .filename property to return the recipe's filename, and a .toml() function to return the recipe as a TOML string. """ - def __init__(self, name, description, version, modules, packages, groups, customizations=None): + def __init__(self, name, description, version, modules, packages, groups, customizations=None, gitrepos=None): # Check that version is empty or semver compatible if version: semver.Version(version) @@ -59,18 +59,29 @@ class Recipe(dict): packages = sorted(packages, key=lambda p: p["name"].lower()) if groups is not None: groups = sorted(groups, key=lambda g: g["name"].lower()) + + # Only support [[repos.git]] for now + if gitrepos is not None: + repos = {"git": sorted(gitrepos, key=lambda g: g["repo"].lower())} + else: + repos = None dict.__init__(self, name=name, description=description, version=version, modules=modules, packages=packages, groups=groups, - customizations=customizations) + customizations=customizations, + repos=repos) # We don't want customizations=None to show up in the TOML so remove it if customizations is None: del self["customizations"] + # Don't include empty repos or repos.git + if repos is None or not repos["git"]: + del self["repos"] + @property def package_names(self): """Return the names of the packages""" @@ -168,9 +179,13 @@ class Recipe(dict): customizations = self["customizations"] else: customizations = None + if "repos" in self and "git" in self["repos"]: + gitrepos = self["repos"]["git"] + else: + gitrepos = None return Recipe(self["name"], self["description"], self["version"], - new_modules, new_packages, new_groups, customizations) + new_modules, new_packages, new_groups, customizations, gitrepos) class RecipeModule(dict): def __init__(self, name, version): @@ -183,6 +198,54 @@ class RecipeGroup(dict): def __init__(self, name): dict.__init__(self, name=name) +def NewRecipeGit(toml_dict): + """Create a RecipeGit object from fields in a TOML dict + + :param rpmname: Name of the rpm to create, also used as the prefix name in the tar archive + :type rpmname: str + :param rpmversion: Version of the rpm, eg. "1.0.0" + :type rpmversion: str + :param rpmrelease: Release of the rpm, eg. "1" + :type rpmrelease: str + :param summary: Summary string for the rpm + :type summary: str + :param repo: URL of the get repo to clone and create the archive from + :type repo: str + :param ref: Git reference to check out. eg. origin/branch-name, git tag, or git commit hash + :type ref: str + :param destination: Path to install the / of the git repo at when installing the rpm + :type destination: str + :returns: A populated RecipeGit object + :rtype: RecipeGit + + The TOML should look like this:: + + [[repos.git]] + rpmname="server-config" + rpmversion="1.0" + rpmrelease="1" + summary="Setup files for server deployment" + repo="PATH OF GIT REPO TO CLONE" + ref="v1.0" + destination="/opt/server/" + + Note that the repo path supports anything that git supports, file://, https://, http:// + + Currently there is no support for authentication + """ + return RecipeGit(toml_dict.get("rpmname"), + toml_dict.get("rpmversion"), + toml_dict.get("rpmrelease"), + toml_dict.get("summary", ""), + toml_dict.get("repo"), + toml_dict.get("ref"), + toml_dict.get("destination")) + +class RecipeGit(dict): + def __init__(self, rpmname, rpmversion, rpmrelease, summary, repo, ref, destination): + dict.__init__(self, rpmname=rpmname, rpmversion=rpmversion, rpmrelease=rpmrelease, + summary=summary, repo=repo, ref=ref, destination=destination) + def recipe_from_file(recipe_path): """Return a recipe file as a Recipe object @@ -230,6 +293,10 @@ def recipe_from_dict(recipe_dict): groups = [RecipeGroup(g.get("name")) for g in recipe_dict["groups"]] else: groups = [] + if recipe_dict.get("repos") and recipe_dict.get("repos").get("git"): + gitrepos = [NewRecipeGit(r) for r in recipe_dict["repos"]["git"]] + else: + gitrepos = [] name = recipe_dict["name"] description = recipe_dict["description"] version = recipe_dict.get("version", None) @@ -237,7 +304,7 @@ def recipe_from_dict(recipe_dict): except KeyError as e: raise RecipeError("There was a problem parsing the recipe: %s" % str(e)) - return Recipe(name, description, version, modules, packages, groups, customizations) + return Recipe(name, description, version, modules, packages, groups, customizations, gitrepos) def gfile(path): """Convert a string path to GFile for use with Git""" diff --git a/tests/pylorax/results/repos-git.dict b/tests/pylorax/results/repos-git.dict new file mode 100644 index 00000000..7ae43135 --- /dev/null +++ b/tests/pylorax/results/repos-git.dict @@ -0,0 +1 @@ +{'description': u'An example http server with PHP and MySQL support.', 'packages': [], 'groups': [], 'modules': [], 'version': u'0.0.1', 'name': u'http-server', 'repos': {'git': [{"rpmname": "server-config-files", "rpmversion": "1.0", "rpmrelease": "1", "summary": "Setup files for server deployment", "repo": "https://github.com/bcl/server-config-files", "ref": "v3.0", "destination": "/srv/config/"}]}} diff --git a/tests/pylorax/results/repos-git.toml b/tests/pylorax/results/repos-git.toml new file mode 100644 index 00000000..832dc948 --- /dev/null +++ b/tests/pylorax/results/repos-git.toml @@ -0,0 +1,12 @@ +name = "http-server" +description = "An example http server with PHP and MySQL support." +version = "0.0.1" + +[[repos.git]] +rpmname="server-config-files" +rpmversion="1.0" +rpmrelease="1" +summary="Setup files for server deployment" +repo="https://github.com/bcl/server-config-files" +ref="v3.0" +destination="/srv/config/" diff --git a/tests/pylorax/test_recipes.py b/tests/pylorax/test_recipes.py index 9156934e..a5b7df0c 100644 --- a/tests/pylorax/test_recipes.py +++ b/tests/pylorax/test_recipes.py @@ -37,7 +37,8 @@ class BasicRecipeTest(unittest.TestCase): ("modules-only.toml", "modules-only.dict"), ("packages-only.toml", "packages-only.dict"), ("groups-only.toml", "groups-only.dict"), - ("custom-base.toml", "custom-base.dict")] + ("custom-base.toml", "custom-base.dict"), + ("repos-git.toml", "repos-git.dict")] results_path = "./tests/pylorax/results/" self.input_toml = [] for (recipe_toml, recipe_dict) in input_recipes: