almalinux/lorax
13
0
Fork 0
Code Issues Pull Requests Releases Activity

Add pylorax.api.gitrpm module and tests

Browse Source 
This handles creating the rpm from the dictionary describing the
repository and rpm. Also adds tests for archive and rpm creation.

(cherry picked from commit f6f2308765)
(cherry picked from commit efc77c1d71)
This commit is contained in:
Brian C. Lane 2019-02-06 12:09:11 -08:00
parent 21b03c2108
commit 0169422746
3 changed files with 442 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docs/lorax-composer.rst
View File

@ -264,6 +264,14 @@ To create an rpm named ``server-config-1.0-1.noarch.rpm`` you would add this to
ref="v1.0"
destination="/opt/server/"
* rpmname: Name of the rpm to create, also used as the prefix name in the tar archive
* rpmversion: Version of the rpm, eg. "1.0.0"
* rpmrelease: Release of the rpm, eg. "1"
* summary: Summary string for the rpm
* repo: URL of the get repo to clone and create the archive from
* ref: Git reference to check out. eg. origin/branch-name, git tag, or git commit hash
* destination: Path to install the / of the git repo at when installing the rpm
An rpm will be created with the contents of the git repository referenced, with the files
being installed under ``/opt/server/`` in this case.

171
src/pylorax/api/gitrpm.py Normal file
View File

@ -0,0 +1,171 @@
# Copyright (C) 2019 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
""" Clone a git repository and package it as an rpm
This module contains functions for cloning a git repo, creating a tar archive of
the selected commit, branch, or tag, and packaging the files into an rpm that will
be installed by anaconda when creating the image.
"""
import logging
log = logging.getLogger("lorax-composer")
import os
from rpmfluff import SimpleRpmBuild
import shutil
import subprocess
import tempfile
import time
def get_repo_description(gitRepo):
""" Return a description including the git repo and reference
:param gitRepo: A dict with the repository details
:type gitRepo: dict
:returns: A string with the git repo url and reference
:rtype: str
"""
return "Created from %s, reference '%s', on %s" % (gitRepo["repo"], gitRepo["ref"], time.ctime())
class GitArchiveTarball:
"""Create a git archive of the selected git repo and reference"""
def __init__(self, gitRepo):
self._gitRepo = gitRepo
self.sourceName = self._gitRepo["rpmname"]+".tar.xz"
def write_file(self, sourcesDir):
""" Create the tar archive
:param sourcesDir: Path to use for creating the archive
:type sourcesDir: str
This clones the git repository and creates a git archive from the specified reference.
The result is in RPMNAME.tar.xz under the sourcesDir
"""
# Clone the repository into a temporary location
cmd = ["git", "clone", self._gitRepo["repo"], os.path.join(sourcesDir, "gitrepo")]
log.debug(cmd)
subprocess.check_call(cmd)
oldcwd = os.getcwd()
try:
os.chdir(os.path.join(sourcesDir, "gitrepo"))
# Configure archive to create a .tar.xz
cmd = ["git", "config", "tar.tar.xz.command", "xz -c"]
log.debug(cmd)
subprocess.check_call(cmd)
cmd = ["git", "archive", "--prefix", self._gitRepo["rpmname"] + "/", "-o", os.path.join(sourcesDir, self.sourceName), self._gitRepo["ref"]]
log.debug(cmd)
subprocess.check_call(cmd)
finally:
# Cleanup even if there was an error
os.chdir(oldcwd)
shutil.rmtree(os.path.join(sourcesDir, "gitrepo"))
class GitRpmBuild(SimpleRpmBuild):
"""Build an rpm containing files from a git repository"""
def __init__(self, *args, **kwargs):
self._base_dir = None
super().__init__(*args, **kwargs)
def check(self):
raise NotImplementedError
def get_base_dir(self):
"""Place all the files under a temporary directory + rpmbuild/
"""
if not self._base_dir:
self._base_dir = tempfile.mkdtemp(prefix="lorax-git-rpm.")
return os.path.join(self._base_dir, "rpmbuild")
def cleanup_tmpdir(self):
"""Remove the temporary directory and all of its contents
"""
if len(self._base_dir) < 5:
raise RuntimeError("Invalid base_dir: %s" % self.get_base_dir())
shutil.rmtree(self._base_dir)
def clean(self):
"""Remove the base directory from inside the tmpdir"""
if len(self.get_base_dir()) < 5:
raise RuntimeError("Invalid base_dir: %s" % self.get_base_dir())
shutil.rmtree(self.get_base_dir(), ignore_errors=True)
def add_git_tarball(self, gitRepo):
"""Add a tar archive of a git repository to the rpm
:param gitRepo: A dict with the repository details
:type gitRepo: dict
This populates the rpm with the URL of the git repository, the summary
describing the repo, the description of the repository and reference used,
and sets up the rpm to install the archive contents into the destination
path.
"""
self.addUrl(gitRepo["repo"])
self.add_summary(gitRepo["summary"])
self.add_description(get_repo_description(gitRepo))
self.addLicense("Unknown")
sourceIndex = self.add_source(GitArchiveTarball(gitRepo))
self.section_build += "tar -xvf %s\n" % self.sources[sourceIndex].sourceName
dest = os.path.normpath(gitRepo["destination"])
self.create_parent_dirs(dest)
self.section_install += "cp -r %s $RPM_BUILD_ROOT/%s\n" % (gitRepo["rpmname"], dest)
sub = self.get_subpackage(None)
sub.section_files += "%s/" % dest
def make_git_rpm(gitRepo, dest):
""" Create an rpm from the specified git repo
:param gitRepo: A dict with the repository details
:type gitRepo: dict
This will clone the git repository, create an archive of the selected reference,
and build an rpm that will install the files from the repository under the destination
directory. The gitRepo dict should have the following fields::
rpmname: "server-config"
rpmversion: "1.0"
rpmrelease: "1"
summary: "Setup files for server deployment"
repo: "PATH OF GIT REPO TO CLONE"
ref: "v1.0"
destination: "/opt/server/"
* rpmname: Name of the rpm to create, also used as the prefix name in the tar archive
* rpmversion: Version of the rpm, eg. "1.0.0"
* rpmrelease: Release of the rpm, eg. "1"
* summary: Summary string for the rpm
* repo: URL of the get repo to clone and create the archive from
* ref: Git reference to check out. eg. origin/branch-name, git tag, or git commit hash
* destination: Path to install the / of the git repo at when installing the rpm
"""
gitRpm = GitRpmBuild(gitRepo["rpmname"], gitRepo["rpmversion"], gitRepo["rpmrelease"], ["noarch"])
try:
gitRpm.add_git_tarball(gitRepo)
gitRpm.do_make()
rpmfile = gitRpm.get_built_rpm("noarch")
shutil.move(rpmfile, dest)
except Exception as e:
log.error("Creating git repo rpm: %s", e)
raise RuntimeError("Creating git repo rpm: %s" % e)
finally:
gitRpm.cleanup_tmpdir()
return os.path.basename(rpmfile)

263
tests/pylorax/test_gitrpm.py Normal file
View File

@ -0,0 +1,263 @@
#
# Copyright (C) 2019 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import pytoml as toml
import rpm
import shutil
import stat
import subprocess
import tarfile
import tempfile
import unittest
from pylorax.api.gitrpm import GitArchiveTarball, make_git_rpm
def _setup_git_repo(self):
"""Setup a git repo in a tmpdir, storing details into self
Call this from setUpClass()
"""
self.repodir = tempfile.mkdtemp(prefix="git-rpm-test.")
# Create a local git repo in a temporary directory, populate it with files.
cmd = ["git", "init", self.repodir]
subprocess.check_call(cmd)
oldcwd = os.getcwd()
os.chdir(self.repodir)
cmd = ["git", "config", "user.email", "test@testing.localhost"]
subprocess.check_call(cmd)
# Hold the expected file paths for the tests
self.test_results = {"first": [], "second": [], "branch": []}
# Add some files
results_path = "./tests/pylorax/results/"
for f in ["full-recipe.toml", "minimal.toml", "modules-only.toml"]:
shutil.copy2(os.path.join(oldcwd, results_path, f), self.repodir)
self.test_results["first"].append(f)
cmd = ["git", "add", "*.toml"]
subprocess.check_call(cmd)
cmd = ["git", "commit", "-m", "first files"]
subprocess.check_call(cmd)
cmd = ["git", "tag", "v1.0.0"]
subprocess.check_call(cmd)
# Get the commit hash
cmd = ["git", "log", "--pretty=%H"]
self.first_commit = subprocess.check_output(cmd).decode("UTF-8").strip()
# 2nd commit adds to 1st commit
self.test_results["second"] = self.test_results["first"].copy()
# Add some more files
os.makedirs(os.path.join(self.repodir, "only-bps/"))
for f in ["packages-only.toml", "groups-only.toml"]:
shutil.copy2(os.path.join(oldcwd, results_path, f), os.path.join(self.repodir, "only-bps/"))
self.test_results["second"].append(os.path.join("only-bps/", f))
self.test_results["second"] = sorted(self.test_results["second"])
cmd = ["git", "add", "*.toml"]
subprocess.check_call(cmd)
cmd = ["git", "commit", "-m", "second files"]
subprocess.check_call(cmd)
cmd = ["git", "tag", "v1.1.0"]
subprocess.check_call(cmd)
# Make a branch for some other files
cmd = ["git", "checkout", "-b", "custom-branch"]
subprocess.check_call(cmd)
# 3nd commit adds to 2nd commit
self.test_results["branch"] = self.test_results["second"].copy()
# Add some files to the new branch
for f in ["custom-base.toml", "repos-git.toml"]:
shutil.copy2(os.path.join(oldcwd, results_path, f), self.repodir)
self.test_results["branch"].append(f)
self.test_results["branch"] = sorted(self.test_results["branch"])
cmd = ["git", "add", "*.toml"]
subprocess.check_call(cmd)
cmd = ["git", "commit", "-m", "branch files"]
subprocess.check_call(cmd)
os.chdir(oldcwd)
class GitArchiveTest(unittest.TestCase):
@classmethod
def setUpClass(self):
self.repodir = None
self.first_commit = None
self.test_results = {}
_setup_git_repo(self)
@classmethod
def tearDownClass(self):
shutil.rmtree(self.repodir)
def _check_tar(self, archive, prefix, test_name):
"""Check the file list of the created archive against the expected list in self.test_results"""
try:
tardir = tempfile.mkdtemp(prefix="git-rpm-test.")
archive.write_file(tardir)
tarpath = os.path.join(tardir, archive.sourceName)
# Archive is in rpmdir + archive.sourceName
self.assertTrue(os.path.exists(tarpath))
# Examine contents of the tarfile
tar = tarfile.open(tarpath, "r")
files = sorted(i.name for i in tar if i.isreg())
self.assertEqual(files, [os.path.join(prefix, f) for f in self.test_results[test_name]])
tar.close()
finally:
shutil.rmtree(tardir)
def git_branch_test(self):
"""Test creating an archive from a git branch"""
git_repo = toml.loads("""
[[repos.git]]
rpmname="git-rpm-test"
rpmversion="1.0.0"
rpmrelease="1"
summary="Testing the git rpm code"
repo="file://%s"
ref="origin/custom-branch"
destination="/srv/testing-rpm/"
""" % self.repodir)
archive = GitArchiveTarball(git_repo["repos"]["git"][0])
self._check_tar(archive, "git-rpm-test/", "branch")
def git_commit_test(self):
"""Test creating an archive from a git commit hash"""
git_repo = toml.loads("""
[[repos.git]]
rpmname="git-rpm-test"
rpmversion="1.0.0"
rpmrelease="1"
summary="Testing the git rpm code"
repo="file://%s"
ref="%s"
destination="/srv/testing-rpm/"
""" % (self.repodir, self.first_commit))
archive = GitArchiveTarball(git_repo["repos"]["git"][0])
self._check_tar(archive, "git-rpm-test/", "first")
def git_tag_test(self):
"""Test creating an archive from a git tag"""
git_repo = toml.loads("""
[[repos.git]]
rpmname="git-rpm-test"
rpmversion="1.0.0"
rpmrelease="1"
summary="Testing the git rpm code"
repo="file://%s"
ref="v1.1.0"
destination="/srv/testing-rpm/"
""" % (self.repodir))
archive = GitArchiveTarball(git_repo["repos"]["git"][0])
self._check_tar(archive, "git-rpm-test/", "second")
class GitRpmTest(unittest.TestCase):
@classmethod
def setUpClass(self):
self.repodir = None
self.first_commit = None
self.test_results = {}
_setup_git_repo(self)
@classmethod
def tearDownClass(self):
shutil.rmtree(self.repodir)
def _check_rpm(self, repo, rpm_dir, rpm_file, test_name):
"""Check the contents of the rpm against the expected test results
"""
ts = rpm.TransactionSet()
fd = os.open(os.path.join(rpm_dir, rpm_file), os.O_RDONLY)
hdr = ts.hdrFromFdno(fd)
os.close(fd)
self.assertEqual(hdr[rpm.RPMTAG_NAME].decode("UTF-8"), repo["rpmname"])
self.assertEqual(hdr[rpm.RPMTAG_VERSION].decode("UTF-8"), repo["rpmversion"])
self.assertEqual(hdr[rpm.RPMTAG_RELEASE].decode("UTF-8"), repo["rpmrelease"])
self.assertEqual(hdr[rpm.RPMTAG_URL].decode("UTF-8"), repo["repo"])
files = sorted(f.name for f in rpm.files(hdr) if stat.S_ISREG(f.mode))
self.assertEqual(files, [os.path.join(repo["destination"], f) for f in self.test_results[test_name]])
def git_branch_test(self):
"""Test creating an rpm from a git branch"""
git_repo = toml.loads("""
[[repos.git]]
rpmname="git-rpm-test"
rpmversion="1.0.0"
rpmrelease="1"
summary="Testing the git rpm code"
repo="file://%s"
ref="origin/custom-branch"
destination="/srv/testing-rpm/"
""" % self.repodir)
try:
rpm_dir = tempfile.mkdtemp(prefix="git-rpm-test.")
rpm_file = make_git_rpm(git_repo["repos"]["git"][0], rpm_dir)
self._check_rpm(git_repo["repos"]["git"][0], rpm_dir, rpm_file, "branch")
finally:
shutil.rmtree(rpm_dir)
def git_commit_test(self):
"""Test creating an rpm from a git commit hash"""
git_repo = toml.loads("""
[[repos.git]]
rpmname="git-rpm-test"
rpmversion="1.0.0"
rpmrelease="1"
summary="Testing the git rpm code"
repo="file://%s"
ref="%s"
destination="/srv/testing-rpm/"
""" % (self.repodir, self.first_commit))
try:
rpm_dir = tempfile.mkdtemp(prefix="git-rpm-test.")
rpm_file = make_git_rpm(git_repo["repos"]["git"][0], rpm_dir)
self._check_rpm(git_repo["repos"]["git"][0], rpm_dir, rpm_file, "first")
finally:
shutil.rmtree(rpm_dir)
def git_tag_test(self):
"""Test creating an rpm from a git tag"""
git_repo = toml.loads("""
[[repos.git]]
rpmname="git-rpm-test"
rpmversion="1.0.0"
rpmrelease="1"
summary="Testing the git rpm code"
repo="file://%s"
ref="v1.1.0"
destination="/srv/testing-rpm/"
""" % (self.repodir))
try:
rpm_dir = tempfile.mkdtemp(prefix="git-rpm-test.")
rpm_file = make_git_rpm(git_repo["repos"]["git"][0], rpm_dir)
self._check_rpm(git_repo["repos"]["git"][0], rpm_dir, rpm_file, "second")
finally:
shutil.rmtree(rpm_dir)