Add pylorax.api.gitrpm module and tests
This handles creating the rpm from the dictionary describing the
repository and rpm. Also adds tests for archive and rpm creation.
(cherry picked from commit f6f2308765)
This commit is contained in:
Brian C. Lane
parent
047f174dcf
commit
efc77c1d71
|
|
@ -264,6 +264,14 @@ To create an rpm named ``server-config-1.0-1.noarch.rpm`` you would add this to
|
|||
ref="v1.0"
|
||||
destination="/opt/server/"
|
||||
|
||||
* rpmname: Name of the rpm to create, also used as the prefix name in the tar archive
|
||||
* rpmversion: Version of the rpm, eg. "1.0.0"
|
||||
* rpmrelease: Release of the rpm, eg. "1"
|
||||
* summary: Summary string for the rpm
|
||||
* repo: URL of the get repo to clone and create the archive from
|
||||
* ref: Git reference to check out. eg. origin/branch-name, git tag, or git commit hash
|
||||
* destination: Path to install the / of the git repo at when installing the rpm
|
||||
|
||||
An rpm will be created with the contents of the git repository referenced, with the files
|
||||
being installed under ``/opt/server/`` in this case.
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,171 @@
|
|||
# Copyright (C) 2019 Red Hat, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
""" Clone a git repository and package it as an rpm
|
||||
|
||||
This module contains functions for cloning a git repo, creating a tar archive of
|
||||
the selected commit, branch, or tag, and packaging the files into an rpm that will
|
||||
be installed by anaconda when creating the image.
|
||||
"""
|
||||
import logging
|
||||
log = logging.getLogger("lorax-composer")
|
||||
|
||||
import os
|
||||
from rpmfluff import SimpleRpmBuild
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
|
||||
def get_repo_description(gitRepo):
|
||||
""" Return a description including the git repo and reference
|
||||
|
||||
:param gitRepo: A dict with the repository details
|
||||
:type gitRepo: dict
|
||||
:returns: A string with the git repo url and reference
|
||||
:rtype: str
|
||||
"""
|
||||
return "Created from %s, reference '%s', on %s" % (gitRepo["repo"], gitRepo["ref"], time.ctime())
|
||||
|
||||
class GitArchiveTarball:
|
||||
"""Create a git archive of the selected git repo and reference"""
|
||||
def __init__(self, gitRepo):
|
||||
self._gitRepo = gitRepo
|
||||
self.sourceName = self._gitRepo["rpmname"]+".tar.xz"
|
||||
|
||||
def write_file(self, sourcesDir):
|
||||
""" Create the tar archive
|
||||
|
||||
:param sourcesDir: Path to use for creating the archive
|
||||
:type sourcesDir: str
|
||||
|
||||
This clones the git repository and creates a git archive from the specified reference.
|
||||
The result is in RPMNAME.tar.xz under the sourcesDir
|
||||
"""
|
||||
# Clone the repository into a temporary location
|
||||
cmd = ["git", "clone", self._gitRepo["repo"], os.path.join(sourcesDir, "gitrepo")]
|
||||
log.debug(cmd)
|
||||
subprocess.check_call(cmd)
|
||||
|
||||
oldcwd = os.getcwd()
|
||||
try:
|
||||
os.chdir(os.path.join(sourcesDir, "gitrepo"))
|
||||
|
||||
# Configure archive to create a .tar.xz
|
||||
cmd = ["git", "config", "tar.tar.xz.command", "xz -c"]
|
||||
log.debug(cmd)
|
||||
subprocess.check_call(cmd)
|
||||
|
||||
cmd = ["git", "archive", "--prefix", self._gitRepo["rpmname"] + "/", "-o", os.path.join(sourcesDir, self.sourceName), self._gitRepo["ref"]]
|
||||
log.debug(cmd)
|
||||
subprocess.check_call(cmd)
|
||||
finally:
|
||||
# Cleanup even if there was an error
|
||||
os.chdir(oldcwd)
|
||||
shutil.rmtree(os.path.join(sourcesDir, "gitrepo"))
|
||||
|
||||
class GitRpmBuild(SimpleRpmBuild):
|
||||
"""Build an rpm containing files from a git repository"""
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._base_dir = None
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def check(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def get_base_dir(self):
|
||||
"""Place all the files under a temporary directory + rpmbuild/
|
||||
"""
|
||||
if not self._base_dir:
|
||||
self._base_dir = tempfile.mkdtemp(prefix="lorax-git-rpm.")
|
||||
return os.path.join(self._base_dir, "rpmbuild")
|
||||
|
||||
def cleanup_tmpdir(self):
|
||||
"""Remove the temporary directory and all of its contents
|
||||
"""
|
||||
if len(self._base_dir) < 5:
|
||||
raise RuntimeError("Invalid base_dir: %s" % self.get_base_dir())
|
||||
|
||||
shutil.rmtree(self._base_dir)
|
||||
|
||||
def clean(self):
|
||||
"""Remove the base directory from inside the tmpdir"""
|
||||
if len(self.get_base_dir()) < 5:
|
||||
raise RuntimeError("Invalid base_dir: %s" % self.get_base_dir())
|
||||
shutil.rmtree(self.get_base_dir(), ignore_errors=True)
|
||||
|
||||
def add_git_tarball(self, gitRepo):
|
||||
"""Add a tar archive of a git repository to the rpm
|
||||
|
||||
:param gitRepo: A dict with the repository details
|
||||
:type gitRepo: dict
|
||||
|
||||
This populates the rpm with the URL of the git repository, the summary
|
||||
describing the repo, the description of the repository and reference used,
|
||||
and sets up the rpm to install the archive contents into the destination
|
||||
path.
|
||||
"""
|
||||
self.addUrl(gitRepo["repo"])
|
||||
self.add_summary(gitRepo["summary"])
|
||||
self.add_description(get_repo_description(gitRepo))
|
||||
self.addLicense("Unknown")
|
||||
sourceIndex = self.add_source(GitArchiveTarball(gitRepo))
|
||||
self.section_build += "tar -xvf %s\n" % self.sources[sourceIndex].sourceName
|
||||
dest = os.path.normpath(gitRepo["destination"])
|
||||
self.create_parent_dirs(dest)
|
||||
self.section_install += "cp -r %s $RPM_BUILD_ROOT/%s\n" % (gitRepo["rpmname"], dest)
|
||||
sub = self.get_subpackage(None)
|
||||
sub.section_files += "%s/" % dest
|
||||
|
||||
def make_git_rpm(gitRepo, dest):
|
||||
""" Create an rpm from the specified git repo
|
||||
|
||||
:param gitRepo: A dict with the repository details
|
||||
:type gitRepo: dict
|
||||
|
||||
This will clone the git repository, create an archive of the selected reference,
|
||||
and build an rpm that will install the files from the repository under the destination
|
||||
directory. The gitRepo dict should have the following fields::
|
||||
|
||||
rpmname: "server-config"
|
||||
rpmversion: "1.0"
|
||||
rpmrelease: "1"
|
||||
summary: "Setup files for server deployment"
|
||||
repo: "PATH OF GIT REPO TO CLONE"
|
||||
ref: "v1.0"
|
||||
destination: "/opt/server/"
|
||||
|
||||
* rpmname: Name of the rpm to create, also used as the prefix name in the tar archive
|
||||
* rpmversion: Version of the rpm, eg. "1.0.0"
|
||||
* rpmrelease: Release of the rpm, eg. "1"
|
||||
* summary: Summary string for the rpm
|
||||
* repo: URL of the get repo to clone and create the archive from
|
||||
* ref: Git reference to check out. eg. origin/branch-name, git tag, or git commit hash
|
||||
* destination: Path to install the / of the git repo at when installing the rpm
|
||||
"""
|
||||
gitRpm = GitRpmBuild(gitRepo["rpmname"], gitRepo["rpmversion"], gitRepo["rpmrelease"], ["noarch"])
|
||||
try:
|
||||
gitRpm.add_git_tarball(gitRepo)
|
||||
gitRpm.do_make()
|
||||
rpmfile = gitRpm.get_built_rpm("noarch")
|
||||
shutil.move(rpmfile, dest)
|
||||
except Exception as e:
|
||||
log.error("Creating git repo rpm: %s", e)
|
||||
raise RuntimeError("Creating git repo rpm: %s" % e)
|
||||
finally:
|
||||
gitRpm.cleanup_tmpdir()
|
||||
|
||||
return os.path.basename(rpmfile)
|
||||
|
|
@ -0,0 +1,263 @@
|
|||
#
|
||||
# Copyright (C) 2019 Red Hat, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import os
|
||||
import pytoml as toml
|
||||
import rpm
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from pylorax.api.gitrpm import GitArchiveTarball, make_git_rpm
|
||||
|
||||
|
||||
def _setup_git_repo(self):
|
||||
"""Setup a git repo in a tmpdir, storing details into self
|
||||
|
||||
Call this from setUpClass()
|
||||
"""
|
||||
self.repodir = tempfile.mkdtemp(prefix="git-rpm-test.")
|
||||
# Create a local git repo in a temporary directory, populate it with files.
|
||||
cmd = ["git", "init", self.repodir]
|
||||
subprocess.check_call(cmd)
|
||||
|
||||
oldcwd = os.getcwd()
|
||||
os.chdir(self.repodir)
|
||||
cmd = ["git", "config", "user.email", "test@testing.localhost"]
|
||||
subprocess.check_call(cmd)
|
||||
|
||||
# Hold the expected file paths for the tests
|
||||
self.test_results = {"first": [], "second": [], "branch": []}
|
||||
# Add some files
|
||||
results_path = "./tests/pylorax/results/"
|
||||
for f in ["full-recipe.toml", "minimal.toml", "modules-only.toml"]:
|
||||
shutil.copy2(os.path.join(oldcwd, results_path, f), self.repodir)
|
||||
self.test_results["first"].append(f)
|
||||
|
||||
cmd = ["git", "add", "*.toml"]
|
||||
subprocess.check_call(cmd)
|
||||
cmd = ["git", "commit", "-m", "first files"]
|
||||
subprocess.check_call(cmd)
|
||||
cmd = ["git", "tag", "v1.0.0"]
|
||||
subprocess.check_call(cmd)
|
||||
|
||||
# Get the commit hash
|
||||
cmd = ["git", "log", "--pretty=%H"]
|
||||
self.first_commit = subprocess.check_output(cmd).decode("UTF-8").strip()
|
||||
|
||||
# 2nd commit adds to 1st commit
|
||||
self.test_results["second"] = self.test_results["first"].copy()
|
||||
|
||||
# Add some more files
|
||||
os.makedirs(os.path.join(self.repodir, "only-bps/"))
|
||||
for f in ["packages-only.toml", "groups-only.toml"]:
|
||||
shutil.copy2(os.path.join(oldcwd, results_path, f), os.path.join(self.repodir, "only-bps/"))
|
||||
self.test_results["second"].append(os.path.join("only-bps/", f))
|
||||
self.test_results["second"] = sorted(self.test_results["second"])
|
||||
|
||||
cmd = ["git", "add", "*.toml"]
|
||||
subprocess.check_call(cmd)
|
||||
cmd = ["git", "commit", "-m", "second files"]
|
||||
subprocess.check_call(cmd)
|
||||
cmd = ["git", "tag", "v1.1.0"]
|
||||
subprocess.check_call(cmd)
|
||||
|
||||
# Make a branch for some other files
|
||||
cmd = ["git", "checkout", "-b", "custom-branch"]
|
||||
subprocess.check_call(cmd)
|
||||
|
||||
# 3nd commit adds to 2nd commit
|
||||
self.test_results["branch"] = self.test_results["second"].copy()
|
||||
|
||||
# Add some files to the new branch
|
||||
for f in ["custom-base.toml", "repos-git.toml"]:
|
||||
shutil.copy2(os.path.join(oldcwd, results_path, f), self.repodir)
|
||||
self.test_results["branch"].append(f)
|
||||
self.test_results["branch"] = sorted(self.test_results["branch"])
|
||||
|
||||
cmd = ["git", "add", "*.toml"]
|
||||
subprocess.check_call(cmd)
|
||||
cmd = ["git", "commit", "-m", "branch files"]
|
||||
subprocess.check_call(cmd)
|
||||
|
||||
os.chdir(oldcwd)
|
||||
|
||||
|
||||
class GitArchiveTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(self):
|
||||
self.repodir = None
|
||||
self.first_commit = None
|
||||
self.test_results = {}
|
||||
|
||||
_setup_git_repo(self)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(self):
|
||||
shutil.rmtree(self.repodir)
|
||||
|
||||
def _check_tar(self, archive, prefix, test_name):
|
||||
"""Check the file list of the created archive against the expected list in self.test_results"""
|
||||
try:
|
||||
tardir = tempfile.mkdtemp(prefix="git-rpm-test.")
|
||||
archive.write_file(tardir)
|
||||
tarpath = os.path.join(tardir, archive.sourceName)
|
||||
|
||||
# Archive is in rpmdir + archive.sourceName
|
||||
self.assertTrue(os.path.exists(tarpath))
|
||||
|
||||
# Examine contents of the tarfile
|
||||
tar = tarfile.open(tarpath, "r")
|
||||
files = sorted(i.name for i in tar if i.isreg())
|
||||
self.assertEqual(files, [os.path.join(prefix, f) for f in self.test_results[test_name]])
|
||||
tar.close()
|
||||
finally:
|
||||
shutil.rmtree(tardir)
|
||||
|
||||
def git_branch_test(self):
|
||||
"""Test creating an archive from a git branch"""
|
||||
git_repo = toml.loads("""
|
||||
[[repos.git]]
|
||||
rpmname="git-rpm-test"
|
||||
rpmversion="1.0.0"
|
||||
rpmrelease="1"
|
||||
summary="Testing the git rpm code"
|
||||
repo="file://%s"
|
||||
ref="origin/custom-branch"
|
||||
destination="/srv/testing-rpm/"
|
||||
""" % self.repodir)
|
||||
archive = GitArchiveTarball(git_repo["repos"]["git"][0])
|
||||
self._check_tar(archive, "git-rpm-test/", "branch")
|
||||
|
||||
def git_commit_test(self):
|
||||
"""Test creating an archive from a git commit hash"""
|
||||
git_repo = toml.loads("""
|
||||
[[repos.git]]
|
||||
rpmname="git-rpm-test"
|
||||
rpmversion="1.0.0"
|
||||
rpmrelease="1"
|
||||
summary="Testing the git rpm code"
|
||||
repo="file://%s"
|
||||
ref="%s"
|
||||
destination="/srv/testing-rpm/"
|
||||
""" % (self.repodir, self.first_commit))
|
||||
archive = GitArchiveTarball(git_repo["repos"]["git"][0])
|
||||
self._check_tar(archive, "git-rpm-test/", "first")
|
||||
|
||||
def git_tag_test(self):
|
||||
"""Test creating an archive from a git tag"""
|
||||
git_repo = toml.loads("""
|
||||
[[repos.git]]
|
||||
rpmname="git-rpm-test"
|
||||
rpmversion="1.0.0"
|
||||
rpmrelease="1"
|
||||
summary="Testing the git rpm code"
|
||||
repo="file://%s"
|
||||
ref="v1.1.0"
|
||||
destination="/srv/testing-rpm/"
|
||||
""" % (self.repodir))
|
||||
archive = GitArchiveTarball(git_repo["repos"]["git"][0])
|
||||
self._check_tar(archive, "git-rpm-test/", "second")
|
||||
|
||||
|
||||
class GitRpmTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(self):
|
||||
self.repodir = None
|
||||
self.first_commit = None
|
||||
self.test_results = {}
|
||||
|
||||
_setup_git_repo(self)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(self):
|
||||
shutil.rmtree(self.repodir)
|
||||
|
||||
def _check_rpm(self, repo, rpm_dir, rpm_file, test_name):
|
||||
"""Check the contents of the rpm against the expected test results
|
||||
"""
|
||||
ts = rpm.TransactionSet()
|
||||
fd = os.open(os.path.join(rpm_dir, rpm_file), os.O_RDONLY)
|
||||
hdr = ts.hdrFromFdno(fd)
|
||||
os.close(fd)
|
||||
|
||||
self.assertEqual(hdr[rpm.RPMTAG_NAME].decode("UTF-8"), repo["rpmname"])
|
||||
self.assertEqual(hdr[rpm.RPMTAG_VERSION].decode("UTF-8"), repo["rpmversion"])
|
||||
self.assertEqual(hdr[rpm.RPMTAG_RELEASE].decode("UTF-8"), repo["rpmrelease"])
|
||||
self.assertEqual(hdr[rpm.RPMTAG_URL].decode("UTF-8"), repo["repo"])
|
||||
|
||||
files = sorted(f.name for f in rpm.files(hdr) if stat.S_ISREG(f.mode))
|
||||
self.assertEqual(files, [os.path.join(repo["destination"], f) for f in self.test_results[test_name]])
|
||||
|
||||
def git_branch_test(self):
|
||||
"""Test creating an rpm from a git branch"""
|
||||
git_repo = toml.loads("""
|
||||
[[repos.git]]
|
||||
rpmname="git-rpm-test"
|
||||
rpmversion="1.0.0"
|
||||
rpmrelease="1"
|
||||
summary="Testing the git rpm code"
|
||||
repo="file://%s"
|
||||
ref="origin/custom-branch"
|
||||
destination="/srv/testing-rpm/"
|
||||
""" % self.repodir)
|
||||
try:
|
||||
rpm_dir = tempfile.mkdtemp(prefix="git-rpm-test.")
|
||||
rpm_file = make_git_rpm(git_repo["repos"]["git"][0], rpm_dir)
|
||||
self._check_rpm(git_repo["repos"]["git"][0], rpm_dir, rpm_file, "branch")
|
||||
finally:
|
||||
shutil.rmtree(rpm_dir)
|
||||
|
||||
def git_commit_test(self):
|
||||
"""Test creating an rpm from a git commit hash"""
|
||||
git_repo = toml.loads("""
|
||||
[[repos.git]]
|
||||
rpmname="git-rpm-test"
|
||||
rpmversion="1.0.0"
|
||||
rpmrelease="1"
|
||||
summary="Testing the git rpm code"
|
||||
repo="file://%s"
|
||||
ref="%s"
|
||||
destination="/srv/testing-rpm/"
|
||||
""" % (self.repodir, self.first_commit))
|
||||
try:
|
||||
rpm_dir = tempfile.mkdtemp(prefix="git-rpm-test.")
|
||||
rpm_file = make_git_rpm(git_repo["repos"]["git"][0], rpm_dir)
|
||||
self._check_rpm(git_repo["repos"]["git"][0], rpm_dir, rpm_file, "first")
|
||||
finally:
|
||||
shutil.rmtree(rpm_dir)
|
||||
|
||||
def git_tag_test(self):
|
||||
"""Test creating an rpm from a git tag"""
|
||||
git_repo = toml.loads("""
|
||||
[[repos.git]]
|
||||
rpmname="git-rpm-test"
|
||||
rpmversion="1.0.0"
|
||||
rpmrelease="1"
|
||||
summary="Testing the git rpm code"
|
||||
repo="file://%s"
|
||||
ref="v1.1.0"
|
||||
destination="/srv/testing-rpm/"
|
||||
""" % (self.repodir))
|
||||
try:
|
||||
rpm_dir = tempfile.mkdtemp(prefix="git-rpm-test.")
|
||||
rpm_file = make_git_rpm(git_repo["repos"]["git"][0], rpm_dir)
|
||||
self._check_rpm(git_repo["repos"]["git"][0], rpm_dir, rpm_file, "second")
|
||||
finally:
|
||||
shutil.rmtree(rpm_dir)
|
||||
Loading…
Reference in New Issue