diff --git a/src/pylorax/api/gitrpm.py b/src/pylorax/api/gitrpm.py new file mode 100644 index 00000000..069486cb --- /dev/null +++ b/src/pylorax/api/gitrpm.py @@ -0,0 +1,171 @@ +# Copyright (C) 2019 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +""" Clone a git repository and package it as an rpm + +This module contains functions for cloning a git repo, creating a tar archive of +the selected commit, branch, or tag, and packaging the files into an rpm that will +be installed by anaconda when creating the image. +""" +import logging +log = logging.getLogger("lorax-composer") + +import os +from rpmfluff import SimpleRpmBuild +import shutil +import subprocess +import tempfile +import time + + +def get_repo_description(gitRepo): + """ Return a description including the git repo and reference + + :param gitRepo: A dict with the repository details + :type gitRepo: dict + :returns: A string with the git repo url and reference + :rtype: str + """ + return "Created from %s, reference '%s', on %s" % (gitRepo["repo"], gitRepo["ref"], time.ctime()) + +class GitArchiveTarball: + """Create a git archive of the selected git repo and reference""" + def __init__(self, gitRepo): + self._gitRepo = gitRepo + self.sourceName = self._gitRepo["rpmname"]+".tar.xz" + + def write_file(self, sourcesDir): + """ Create the tar archive + + :param sourcesDir: Path to use for creating the archive + :type sourcesDir: str + + This clones the git repository and creates a git archive from the specified reference. + The result is in RPMNAME.tar.xz under the sourcesDir + """ + # Clone the repository into a temporary location + cmd = ["git", "clone", self._gitRepo["repo"], os.path.join(sourcesDir, "gitrepo")] + log.debug(cmd) + subprocess.check_call(cmd) + + oldcwd = os.getcwd() + try: + os.chdir(os.path.join(sourcesDir, "gitrepo")) + + # Configure archive to create a .tar.xz + cmd = ["git", "config", "tar.tar.xz.command", "xz -c"] + log.debug(cmd) + subprocess.check_call(cmd) + + cmd = ["git", "archive", "--prefix", self._gitRepo["rpmname"] + "/", "-o", os.path.join(sourcesDir, self.sourceName), self._gitRepo["ref"]] + log.debug(cmd) + subprocess.check_call(cmd) + finally: + # Cleanup even if there was an error + os.chdir(oldcwd) + shutil.rmtree(os.path.join(sourcesDir, "gitrepo")) + +class GitRpmBuild(SimpleRpmBuild): + """Build an rpm containing files from a git repository""" + def __init__(self, *args, **kwargs): + self._base_dir = None + super().__init__(*args, **kwargs) + + def check(self): + raise NotImplementedError + + def get_base_dir(self): + """Place all the files under a temporary directory + rpmbuild/ + """ + if not self._base_dir: + self._base_dir = tempfile.mkdtemp(prefix="lorax-git-rpm.") + return os.path.join(self._base_dir, "rpmbuild") + + def cleanup_tmpdir(self): + """Remove the temporary directory and all of its contents + """ + if len(self._base_dir) < 5: + raise RuntimeError("Invalid base_dir: %s" % self.get_base_dir()) + + shutil.rmtree(self._base_dir) + + def clean(self): + """Remove the base directory from inside the tmpdir""" + if len(self.get_base_dir()) < 5: + raise RuntimeError("Invalid base_dir: %s" % self.get_base_dir()) + shutil.rmtree(self.get_base_dir(), ignore_errors=True) + + def add_git_tarball(self, gitRepo): + """Add a tar archive of a git repository to the rpm + + :param gitRepo: A dict with the repository details + :type gitRepo: dict + + This populates the rpm with the URL of the git repository, the summary + describing the repo, the description of the repository and reference used, + and sets up the rpm to install the archive contents into the destination + path. + """ + self.addUrl(gitRepo["repo"]) + self.add_summary(gitRepo["summary"]) + self.add_description(get_repo_description(gitRepo)) + self.addLicense("Unknown") + sourceIndex = self.add_source(GitArchiveTarball(gitRepo)) + self.section_build += "tar -xvf %s\n" % self.sources[sourceIndex].sourceName + dest = os.path.normpath(gitRepo["destination"]) + self.create_parent_dirs(dest) + self.section_install += "cp -r %s $RPM_BUILD_ROOT/%s\n" % (gitRepo["rpmname"], dest) + sub = self.get_subpackage(None) + sub.section_files += "%s/" % dest + +def make_git_rpm(gitRepo, dest): + """ Create an rpm from the specified git repo + + :param gitRepo: A dict with the repository details + :type gitRepo: dict + + This will clone the git repository, create an archive of the selected reference, + and build an rpm that will install the files from the repository under the destination + directory. The gitRepo dict should have the following fields:: + + rpmname: "server-config" + rpmversion: "1.0" + rpmrelease: "1" + summary: "Setup files for server deployment" + repo: "PATH OF GIT REPO TO CLONE" + ref: "v1.0" + destination: "/opt/server/" + + * rpmname: Name of the rpm to create, also used as the prefix name in the tar archive + * rpmversion: Version of the rpm, eg. "1.0.0" + * rpmrelease: Release of the rpm, eg. "1" + * summary: Summary string for the rpm + * repo: URL of the get repo to clone and create the archive from + * ref: Git reference to check out. eg. origin/branch-name, git tag, or git commit hash + * destination: Path to install the / of the git repo at when installing the rpm + """ + gitRpm = GitRpmBuild(gitRepo["rpmname"], gitRepo["rpmversion"], gitRepo["rpmrelease"], ["noarch"]) + try: + gitRpm.add_git_tarball(gitRepo) + gitRpm.do_make() + rpmfile = gitRpm.get_built_rpm("noarch") + shutil.move(rpmfile, dest) + except Exception as e: + log.error("Creating git repo rpm: %s", e) + raise RuntimeError("Creating git repo rpm: %s" % e) + finally: + gitRpm.cleanup_tmpdir() + + return os.path.basename(rpmfile) diff --git a/tests/pylorax/test_gitrpm.py b/tests/pylorax/test_gitrpm.py new file mode 100644 index 00000000..b947a107 --- /dev/null +++ b/tests/pylorax/test_gitrpm.py @@ -0,0 +1,263 @@ +# +# Copyright (C) 2019 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import os +import pytoml as toml +import rpm +import shutil +import stat +import subprocess +import tarfile +import tempfile +import unittest + +from pylorax.api.gitrpm import GitArchiveTarball, make_git_rpm + + +def _setup_git_repo(self): + """Setup a git repo in a tmpdir, storing details into self + + Call this from setUpClass() + """ + self.repodir = tempfile.mkdtemp(prefix="git-rpm-test.") + # Create a local git repo in a temporary directory, populate it with files. + cmd = ["git", "init", self.repodir] + subprocess.check_call(cmd) + + oldcwd = os.getcwd() + os.chdir(self.repodir) + cmd = ["git", "config", "user.email", "test@testing.localhost"] + subprocess.check_call(cmd) + + # Hold the expected file paths for the tests + self.test_results = {"first": [], "second": [], "branch": []} + # Add some files + results_path = "./tests/pylorax/results/" + for f in ["full-recipe.toml", "minimal.toml", "modules-only.toml"]: + shutil.copy2(os.path.join(oldcwd, results_path, f), self.repodir) + self.test_results["first"].append(f) + + cmd = ["git", "add", "*.toml"] + subprocess.check_call(cmd) + cmd = ["git", "commit", "-m", "first files"] + subprocess.check_call(cmd) + cmd = ["git", "tag", "v1.0.0"] + subprocess.check_call(cmd) + + # Get the commit hash + cmd = ["git", "log", "--pretty=%H"] + self.first_commit = subprocess.check_output(cmd).decode("UTF-8").strip() + + # 2nd commit adds to 1st commit + self.test_results["second"] = self.test_results["first"].copy() + + # Add some more files + os.makedirs(os.path.join(self.repodir, "only-bps/")) + for f in ["packages-only.toml", "groups-only.toml"]: + shutil.copy2(os.path.join(oldcwd, results_path, f), os.path.join(self.repodir, "only-bps/")) + self.test_results["second"].append(os.path.join("only-bps/", f)) + self.test_results["second"] = sorted(self.test_results["second"]) + + cmd = ["git", "add", "*.toml"] + subprocess.check_call(cmd) + cmd = ["git", "commit", "-m", "second files"] + subprocess.check_call(cmd) + cmd = ["git", "tag", "v1.1.0"] + subprocess.check_call(cmd) + + # Make a branch for some other files + cmd = ["git", "checkout", "-b", "custom-branch"] + subprocess.check_call(cmd) + + # 3nd commit adds to 2nd commit + self.test_results["branch"] = self.test_results["second"].copy() + + # Add some files to the new branch + for f in ["custom-base.toml", "repos-git.toml"]: + shutil.copy2(os.path.join(oldcwd, results_path, f), self.repodir) + self.test_results["branch"].append(f) + self.test_results["branch"] = sorted(self.test_results["branch"]) + + cmd = ["git", "add", "*.toml"] + subprocess.check_call(cmd) + cmd = ["git", "commit", "-m", "branch files"] + subprocess.check_call(cmd) + + os.chdir(oldcwd) + + +class GitArchiveTest(unittest.TestCase): + @classmethod + def setUpClass(self): + self.repodir = None + self.first_commit = None + self.test_results = {} + + _setup_git_repo(self) + + @classmethod + def tearDownClass(self): + shutil.rmtree(self.repodir) + + def _check_tar(self, archive, prefix, test_name): + """Check the file list of the created archive against the expected list in self.test_results""" + try: + tardir = tempfile.mkdtemp(prefix="git-rpm-test.") + archive.write_file(tardir) + tarpath = os.path.join(tardir, archive.sourceName) + + # Archive is in rpmdir + archive.sourceName + self.assertTrue(os.path.exists(tarpath)) + + # Examine contents of the tarfile + tar = tarfile.open(tarpath, "r") + files = sorted(i.name for i in tar if i.isreg()) + self.assertEqual(files, [os.path.join(prefix, f) for f in self.test_results[test_name]]) + tar.close() + finally: + shutil.rmtree(tardir) + + def git_branch_test(self): + """Test creating an archive from a git branch""" + git_repo = toml.loads(""" + [[repos.git]] + rpmname="git-rpm-test" + rpmversion="1.0.0" + rpmrelease="1" + summary="Testing the git rpm code" + repo="file://%s" + ref="origin/custom-branch" + destination="/srv/testing-rpm/" + """ % self.repodir) + archive = GitArchiveTarball(git_repo["repos"]["git"][0]) + self._check_tar(archive, "git-rpm-test/", "branch") + + def git_commit_test(self): + """Test creating an archive from a git commit hash""" + git_repo = toml.loads(""" + [[repos.git]] + rpmname="git-rpm-test" + rpmversion="1.0.0" + rpmrelease="1" + summary="Testing the git rpm code" + repo="file://%s" + ref="%s" + destination="/srv/testing-rpm/" + """ % (self.repodir, self.first_commit)) + archive = GitArchiveTarball(git_repo["repos"]["git"][0]) + self._check_tar(archive, "git-rpm-test/", "first") + + def git_tag_test(self): + """Test creating an archive from a git tag""" + git_repo = toml.loads(""" + [[repos.git]] + rpmname="git-rpm-test" + rpmversion="1.0.0" + rpmrelease="1" + summary="Testing the git rpm code" + repo="file://%s" + ref="v1.1.0" + destination="/srv/testing-rpm/" + """ % (self.repodir)) + archive = GitArchiveTarball(git_repo["repos"]["git"][0]) + self._check_tar(archive, "git-rpm-test/", "second") + + +class GitRpmTest(unittest.TestCase): + @classmethod + def setUpClass(self): + self.repodir = None + self.first_commit = None + self.test_results = {} + + _setup_git_repo(self) + + @classmethod + def tearDownClass(self): + shutil.rmtree(self.repodir) + + def _check_rpm(self, repo, rpm_dir, rpm_file, test_name): + """Check the contents of the rpm against the expected test results + """ + ts = rpm.TransactionSet() + fd = os.open(os.path.join(rpm_dir, rpm_file), os.O_RDONLY) + hdr = ts.hdrFromFdno(fd) + os.close(fd) + + self.assertEqual(hdr[rpm.RPMTAG_NAME].decode("UTF-8"), repo["rpmname"]) + self.assertEqual(hdr[rpm.RPMTAG_VERSION].decode("UTF-8"), repo["rpmversion"]) + self.assertEqual(hdr[rpm.RPMTAG_RELEASE].decode("UTF-8"), repo["rpmrelease"]) + self.assertEqual(hdr[rpm.RPMTAG_URL].decode("UTF-8"), repo["repo"]) + + files = sorted(f.name for f in rpm.files(hdr) if stat.S_ISREG(f.mode)) + self.assertEqual(files, [os.path.join(repo["destination"], f) for f in self.test_results[test_name]]) + + def git_branch_test(self): + """Test creating an rpm from a git branch""" + git_repo = toml.loads(""" + [[repos.git]] + rpmname="git-rpm-test" + rpmversion="1.0.0" + rpmrelease="1" + summary="Testing the git rpm code" + repo="file://%s" + ref="origin/custom-branch" + destination="/srv/testing-rpm/" + """ % self.repodir) + try: + rpm_dir = tempfile.mkdtemp(prefix="git-rpm-test.") + rpm_file = make_git_rpm(git_repo["repos"]["git"][0], rpm_dir) + self._check_rpm(git_repo["repos"]["git"][0], rpm_dir, rpm_file, "branch") + finally: + shutil.rmtree(rpm_dir) + + def git_commit_test(self): + """Test creating an rpm from a git commit hash""" + git_repo = toml.loads(""" + [[repos.git]] + rpmname="git-rpm-test" + rpmversion="1.0.0" + rpmrelease="1" + summary="Testing the git rpm code" + repo="file://%s" + ref="%s" + destination="/srv/testing-rpm/" + """ % (self.repodir, self.first_commit)) + try: + rpm_dir = tempfile.mkdtemp(prefix="git-rpm-test.") + rpm_file = make_git_rpm(git_repo["repos"]["git"][0], rpm_dir) + self._check_rpm(git_repo["repos"]["git"][0], rpm_dir, rpm_file, "first") + finally: + shutil.rmtree(rpm_dir) + + def git_tag_test(self): + """Test creating an rpm from a git tag""" + git_repo = toml.loads(""" + [[repos.git]] + rpmname="git-rpm-test" + rpmversion="1.0.0" + rpmrelease="1" + summary="Testing the git rpm code" + repo="file://%s" + ref="v1.1.0" + destination="/srv/testing-rpm/" + """ % (self.repodir)) + try: + rpm_dir = tempfile.mkdtemp(prefix="git-rpm-test.") + rpm_file = make_git_rpm(git_repo["repos"]["git"][0], rpm_dir) + self._check_rpm(git_repo["repos"]["git"][0], rpm_dir, rpm_file, "second") + finally: + shutil.rmtree(rpm_dir)