Add repos.git support to lorax-composer builds

This hooks up creation of the rpm to the build, adds it to the
kickstart, and passes the url to Anaconda. The dnf repo with the rpms is
created under the results directory so it will be included when
downloading the build's results.

lorax.spec

@@ -139,6 +139,10 @@ Requires: python3-gevent
 Requires: anaconda-tui >= 29.19-1
 Requires: qemu-img
 Requires: tar
+Requires: python3-rpmfluff
+Requires: git
+Requires: xz
+Requires: createrepo_c
 
 %{?systemd_requires}
 BuildRequires: systemd

src/pylorax/api/compose.py

@@ -45,6 +45,7 @@ from pykickstart.parser import KickstartParser
 from pykickstart.version import makeVersion
 
 from pylorax import ArchData, find_templates, get_buildarch
+from pylorax.api.gitrpm import create_gitrpm_repo
 from pylorax.api.projects import projects_depsolve, projects_depsolve_with_size, dep_nevra
 from pylorax.api.projects import ProjectsError
 from pylorax.api.recipes import read_recipe_and_id
@@ -421,6 +422,9 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m
     if not repos:
         raise RuntimeError("No enabled repos, canceling build.")
 
+    # Create the git rpms, if any, and return the path to the repo under results_dir
+    gitrpm_repo = create_gitrpm_repo(results_dir, recipe)
+
     # Create the final kickstart with repos and package list
     ks_path = joinpaths(results_dir, "final-kickstart.ks")
     with open(ks_path, "w") as f:
@@ -432,6 +436,10 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m
             log.debug("repo composer-%s = %s", idx, ks_repo)
             f.write('repo --name="composer-%s" %s\n' % (idx, ks_repo))
 
+        if gitrpm_repo:
+            log.debug("repo gitrpms = %s", gitrpm_repo)
+            f.write('repo --name="gitrpms" --baseurl="file://%s"\n' % gitrpm_repo)
+
         # Setup the disk for booting
         # TODO Add GPT and UEFI boot support
         f.write('clearpart --all --initlabel\n')
@@ -443,6 +451,12 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m
 
         for d in deps:
             f.write(dep_nevra(d)+"\n")
+
+        # Include the rpms from the gitrpm repo directory
+        if gitrpm_repo:
+            for rpm in glob(os.path.join(gitrpm_repo, "*.rpm")):
+                f.write(os.path.basename(rpm)[:-4]+"\n")
+
         f.write("%end\n")
 
         add_customizations(f, recipe)

src/pylorax/api/gitrpm.py

@@ -29,6 +29,7 @@ import subprocess
 import tempfile
 import time
 
+from pylorax.sysutils import joinpaths
 
 def get_repo_description(gitRepo):
     """ Return a description including the git repo and reference
@@ -56,26 +57,26 @@ class GitArchiveTarball:
         The result is in RPMNAME.tar.xz under the sourcesDir
         """
         # Clone the repository into a temporary location
-        cmd = ["git", "clone", self._gitRepo["repo"], os.path.join(sourcesDir, "gitrepo")]
+        cmd = ["git", "clone", self._gitRepo["repo"], joinpaths(sourcesDir, "gitrepo")]
         log.debug(cmd)
         subprocess.check_call(cmd)
 
         oldcwd = os.getcwd()
         try:
-            os.chdir(os.path.join(sourcesDir, "gitrepo"))
+            os.chdir(joinpaths(sourcesDir, "gitrepo"))
 
             # Configure archive to create a .tar.xz
             cmd = ["git", "config", "tar.tar.xz.command", "xz -c"]
             log.debug(cmd)
             subprocess.check_call(cmd)
 
-            cmd = ["git", "archive", "--prefix", self._gitRepo["rpmname"] + "/", "-o", os.path.join(sourcesDir, self.sourceName), self._gitRepo["ref"]]
+            cmd = ["git", "archive", "--prefix", self._gitRepo["rpmname"] + "/", "-o", joinpaths(sourcesDir, self.sourceName), self._gitRepo["ref"]]
             log.debug(cmd)
             subprocess.check_call(cmd)
         finally:
             # Cleanup even if there was an error
             os.chdir(oldcwd)
-            shutil.rmtree(os.path.join(sourcesDir, "gitrepo"))
+            shutil.rmtree(joinpaths(sourcesDir, "gitrepo"))
 
 class GitRpmBuild(SimpleRpmBuild):
     """Build an rpm containing files from a git repository"""
@@ -91,7 +92,7 @@ class GitRpmBuild(SimpleRpmBuild):
         """
         if not self._base_dir:
             self._base_dir = tempfile.mkdtemp(prefix="lorax-git-rpm.")
-        return os.path.join(self._base_dir, "rpmbuild")
+        return joinpaths(self._base_dir, "rpmbuild")
 
     def cleanup_tmpdir(self):
         """Remove the temporary directory and all of its contents
@@ -169,3 +170,33 @@ def make_git_rpm(gitRepo, dest):
         gitRpm.cleanup_tmpdir()
 
     return os.path.basename(rpmfile)
+
+# Create the git rpms, if any, and return the path to the repo under results_dir
+def create_gitrpm_repo(results_dir, recipe):
+    """Create a dnf repository with the rpms from the recipe
+
+    :param results_dir: Path to create the repository under
+    :type results_dir: str
+    :param recipe: The recipe to get the repos.git entries from
+    :type recipe: Recipe
+    :returns: Path to the dnf repository or ""
+    :rtype: str
+
+    This function creates a dnf repository directory at results_dir+"repo/",
+    creates rpms for all of the repos.git entries in the recipe, runs createrepo_c
+    on the dnf repository so that Anaconda can use it, and returns the path to the
+    repository to the caller.
+    """
+    if "repos" not in recipe or "git" not in recipe["repos"]:
+        return ""
+
+    gitrepo = joinpaths(results_dir, "repo/")
+    if not os.path.exists(gitrepo):
+        os.makedirs(gitrepo)
+    for r in recipe["repos"]["git"]:
+        make_git_rpm(r, gitrepo)
+    cmd = ["createrepo_c", gitrepo]
+    log.debug(cmd)
+    subprocess.check_call(cmd)
+
+    return gitrepo

tests/pylorax/test_gitrpm.py

@@ -24,8 +24,8 @@ import tarfile
 import tempfile
 import unittest
 
-from pylorax.api.gitrpm import GitArchiveTarball, make_git_rpm
+from pylorax.api.gitrpm import GitArchiveTarball, make_git_rpm, create_gitrpm_repo
-
+from pylorax.sysutils import joinpaths
 
 def _setup_git_repo(self):
     """Setup a git repo in a tmpdir, storing details into self
@@ -261,3 +261,35 @@ class GitRpmTest(unittest.TestCase):
             self._check_rpm(git_repo["repos"]["git"][0], rpm_dir, rpm_file, "second")
         finally:
             shutil.rmtree(rpm_dir)
+
+    def gitrpm_repo_test(self):
+        """Test creating a dnf repo of the git rpms"""
+        recipe = toml.loads("""
+            [[repos.git]]
+            rpmname="repo-test-alpha"
+            rpmversion="1.1.0"
+            rpmrelease="1"
+            summary="Testing the git rpm code"
+            repo="file://%s"
+            ref="v1.1.0"
+            destination="/srv/testing-alpha/"
+
+            [[repos.git]]
+            rpmname="repo-test-beta"
+            rpmversion="1.0.0"
+            rpmrelease="1"
+            summary="Testing the git rpm code"
+            repo="file://%s"
+            ref="v1.0.0"
+            destination="/srv/testing-beta/"
+        """ % (self.repodir, self.repodir))
+        try:
+            temp_dir = tempfile.mkdtemp(prefix="git-rpm-test.")
+            repo_dir = create_gitrpm_repo(temp_dir, recipe)
+
+            self.assertTrue(len(repo_dir) > 0)
+            self.assertTrue(os.path.exists(joinpaths(repo_dir, "repo-test-alpha-1.1.0-1.noarch.rpm")))
+            self.assertTrue(os.path.exists(joinpaths(repo_dir, "repo-test-beta-1.0.0-1.noarch.rpm")))
+
+        finally:
+            shutil.rmtree(temp_dir)