From 2e596010d3f2a01d9bf360e6001bc63361c13f22 Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Wed, 6 Feb 2019 16:44:51 -0800 Subject: [PATCH] Add repos.git support to lorax-composer builds This hooks up creation of the rpm to the build, adds it to the kickstart, and passes the url to Anaconda. The dnf repo with the rpms is created under the results directory so it will be included when downloading the build's results. (cherry picked from commit cd8c884adb32e92052722c6ee6188dc53710744f) --- lorax.spec | 4 ++++ src/pylorax/api/compose.py | 14 ++++++++++++ src/pylorax/api/gitrpm.py | 41 +++++++++++++++++++++++++++++++----- tests/pylorax/test_gitrpm.py | 36 +++++++++++++++++++++++++++++-- 4 files changed, 88 insertions(+), 7 deletions(-) diff --git a/lorax.spec b/lorax.spec index 0127add3..49fd0479 100644 --- a/lorax.spec +++ b/lorax.spec @@ -139,6 +139,10 @@ Requires: python3-gevent Requires: anaconda-tui >= 29.19-1 Requires: qemu-img Requires: tar +Requires: python3-rpmfluff +Requires: git +Requires: xz +Requires: createrepo_c %{?systemd_requires} BuildRequires: systemd diff --git a/src/pylorax/api/compose.py b/src/pylorax/api/compose.py index 519e9be7..3ac3061c 100644 --- a/src/pylorax/api/compose.py +++ b/src/pylorax/api/compose.py @@ -46,6 +46,7 @@ from pykickstart.parser import KickstartParser from pykickstart.version import makeVersion from pylorax import ArchData, find_templates, get_buildarch +from pylorax.api.gitrpm import create_gitrpm_repo from pylorax.api.projects import projects_depsolve, projects_depsolve_with_size, dep_nevra from pylorax.api.projects import ProjectsError from pylorax.api.recipes import read_recipe_and_id @@ -491,6 +492,9 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m if not repos: raise RuntimeError("No enabled repos, canceling build.") + # Create the git rpms, if any, and return the path to the repo under results_dir + gitrpm_repo = create_gitrpm_repo(results_dir, recipe) + # Create the final kickstart with repos and package list ks_path = joinpaths(results_dir, "final-kickstart.ks") with open(ks_path, "w") as f: @@ -502,6 +506,10 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m log.debug("repo composer-%s = %s", idx, ks_repo) f.write('repo --name="composer-%s" %s\n' % (idx, ks_repo)) + if gitrpm_repo: + log.debug("repo gitrpms = %s", gitrpm_repo) + f.write('repo --name="gitrpms" --baseurl="file://%s"\n' % gitrpm_repo) + # Setup the disk for booting # TODO Add GPT and UEFI boot support f.write('clearpart --all --initlabel\n') @@ -514,6 +522,12 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m for d in deps: f.write(dep_nevra(d)+"\n") + + # Include the rpms from the gitrpm repo directory + if gitrpm_repo: + for rpm in glob(os.path.join(gitrpm_repo, "*.rpm")): + f.write(os.path.basename(rpm)[:-4]+"\n") + f.write("%end\n") # Other customizations can be appended to the kickstart diff --git a/src/pylorax/api/gitrpm.py b/src/pylorax/api/gitrpm.py index 069486cb..04daa400 100644 --- a/src/pylorax/api/gitrpm.py +++ b/src/pylorax/api/gitrpm.py @@ -29,6 +29,7 @@ import subprocess import tempfile import time +from pylorax.sysutils import joinpaths def get_repo_description(gitRepo): """ Return a description including the git repo and reference @@ -56,26 +57,26 @@ class GitArchiveTarball: The result is in RPMNAME.tar.xz under the sourcesDir """ # Clone the repository into a temporary location - cmd = ["git", "clone", self._gitRepo["repo"], os.path.join(sourcesDir, "gitrepo")] + cmd = ["git", "clone", self._gitRepo["repo"], joinpaths(sourcesDir, "gitrepo")] log.debug(cmd) subprocess.check_call(cmd) oldcwd = os.getcwd() try: - os.chdir(os.path.join(sourcesDir, "gitrepo")) + os.chdir(joinpaths(sourcesDir, "gitrepo")) # Configure archive to create a .tar.xz cmd = ["git", "config", "tar.tar.xz.command", "xz -c"] log.debug(cmd) subprocess.check_call(cmd) - cmd = ["git", "archive", "--prefix", self._gitRepo["rpmname"] + "/", "-o", os.path.join(sourcesDir, self.sourceName), self._gitRepo["ref"]] + cmd = ["git", "archive", "--prefix", self._gitRepo["rpmname"] + "/", "-o", joinpaths(sourcesDir, self.sourceName), self._gitRepo["ref"]] log.debug(cmd) subprocess.check_call(cmd) finally: # Cleanup even if there was an error os.chdir(oldcwd) - shutil.rmtree(os.path.join(sourcesDir, "gitrepo")) + shutil.rmtree(joinpaths(sourcesDir, "gitrepo")) class GitRpmBuild(SimpleRpmBuild): """Build an rpm containing files from a git repository""" @@ -91,7 +92,7 @@ class GitRpmBuild(SimpleRpmBuild): """ if not self._base_dir: self._base_dir = tempfile.mkdtemp(prefix="lorax-git-rpm.") - return os.path.join(self._base_dir, "rpmbuild") + return joinpaths(self._base_dir, "rpmbuild") def cleanup_tmpdir(self): """Remove the temporary directory and all of its contents @@ -169,3 +170,33 @@ def make_git_rpm(gitRepo, dest): gitRpm.cleanup_tmpdir() return os.path.basename(rpmfile) + +# Create the git rpms, if any, and return the path to the repo under results_dir +def create_gitrpm_repo(results_dir, recipe): + """Create a dnf repository with the rpms from the recipe + + :param results_dir: Path to create the repository under + :type results_dir: str + :param recipe: The recipe to get the repos.git entries from + :type recipe: Recipe + :returns: Path to the dnf repository or "" + :rtype: str + + This function creates a dnf repository directory at results_dir+"repo/", + creates rpms for all of the repos.git entries in the recipe, runs createrepo_c + on the dnf repository so that Anaconda can use it, and returns the path to the + repository to the caller. + """ + if "repos" not in recipe or "git" not in recipe["repos"]: + return "" + + gitrepo = joinpaths(results_dir, "repo/") + if not os.path.exists(gitrepo): + os.makedirs(gitrepo) + for r in recipe["repos"]["git"]: + make_git_rpm(r, gitrepo) + cmd = ["createrepo_c", gitrepo] + log.debug(cmd) + subprocess.check_call(cmd) + + return gitrepo diff --git a/tests/pylorax/test_gitrpm.py b/tests/pylorax/test_gitrpm.py index b947a107..9a33a80d 100644 --- a/tests/pylorax/test_gitrpm.py +++ b/tests/pylorax/test_gitrpm.py @@ -24,8 +24,8 @@ import tarfile import tempfile import unittest -from pylorax.api.gitrpm import GitArchiveTarball, make_git_rpm - +from pylorax.api.gitrpm import GitArchiveTarball, make_git_rpm, create_gitrpm_repo +from pylorax.sysutils import joinpaths def _setup_git_repo(self): """Setup a git repo in a tmpdir, storing details into self @@ -261,3 +261,35 @@ class GitRpmTest(unittest.TestCase): self._check_rpm(git_repo["repos"]["git"][0], rpm_dir, rpm_file, "second") finally: shutil.rmtree(rpm_dir) + + def gitrpm_repo_test(self): + """Test creating a dnf repo of the git rpms""" + recipe = toml.loads(""" + [[repos.git]] + rpmname="repo-test-alpha" + rpmversion="1.1.0" + rpmrelease="1" + summary="Testing the git rpm code" + repo="file://%s" + ref="v1.1.0" + destination="/srv/testing-alpha/" + + [[repos.git]] + rpmname="repo-test-beta" + rpmversion="1.0.0" + rpmrelease="1" + summary="Testing the git rpm code" + repo="file://%s" + ref="v1.0.0" + destination="/srv/testing-beta/" + """ % (self.repodir, self.repodir)) + try: + temp_dir = tempfile.mkdtemp(prefix="git-rpm-test.") + repo_dir = create_gitrpm_repo(temp_dir, recipe) + + self.assertTrue(len(repo_dir) > 0) + self.assertTrue(os.path.exists(joinpaths(repo_dir, "repo-test-alpha-1.1.0-1.noarch.rpm"))) + self.assertTrue(os.path.exists(joinpaths(repo_dir, "repo-test-beta-1.0.0-1.noarch.rpm"))) + + finally: + shutil.rmtree(temp_dir)