almalinux/pungi
15
2
Fork 0
Code Issues Pull Requests Releases Activity

scm: Add backend for downloading archives from Koji

Browse Source 
Tests and documentation included.

JIRA: COMPOSE-3816
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2019-10-07 10:37:06 +02:00
parent 39e8f6f710
commit 20c3614fb3
4 changed files with 206 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
doc/scm_support.rst
View File

@ -17,13 +17,21 @@ which can contain following keys.
* ``git`` -- copies files from a Git repository
* ``cvs`` -- copies files from a CVS repository
* ``rpm`` -- copies files from a package in the compose
* ``koji`` -- downloads archives from a given build in Koji build system
* ``repo`` -- for Git and CVS backends URL to the repository, for RPM a shell
glob for matching package names (or a list of such globs); for ``file``
backend this option should be empty
* ``repo``
* ``branch`` -- branch name for Git and CVS backends, with ``master`` and
``HEAD`` as defaults. Ignored for other backends.
* for Git and CVS backends this should be URL to the repository
* for RPM backend this should be a shell style glob matching package names
(or a list of such globs)
* for file backend this should be empty
* for Koji backend this should be an NVR or package name
* ``branch``
* branch name for Git and CVS backends, with ``master`` and ``HEAD`` as defaults
* Koji tag for koji backend if only package name is given
* otherwise should not be specified
* ``file`` -- a list of files that should be exported.
@ -34,6 +42,31 @@ which can contain following keys.
needed file (for example to run ``make``). Only supported in Git backend.
Koji examples
-------------
There are two different ways how to configure the Koji backend. ::
{
# Download all *.tar files from build my-image-1.0-1.
"scm": "koji",
"repo": "my-image-1.0-1",
"file": "*.tar",
}
{
# Find latest build of my-image in tag my-tag and take files from
# there.
"scm": "koji",
"repo": "my-image",
"branch": "my-tag",
"file": "*.tar",
}
Using both tag name and exact NVR will result in error: the NVR would be
interpreted as a package name, and would not match anything.
``file`` vs. ``dir``
--------------------
@ -53,3 +86,7 @@ after ``pkgset`` phase finished. You can't get comps file from a package.
Depending on Git repository URL configuration Pungi can only export the
requested content using ``git archive``. When a command should run this is not
possible and a clone is always needed.
When using ``koji`` backend, it is required to provide configuration for Koji
profile to be used (``koji_profile``). It is not possible to contact multiple
different Koji instances.

2
pungi/checks.py
View File

@ -446,7 +446,7 @@ def make_schema():
"properties": {
"scm": {
"type": "string",
"enum": ["file", "cvs", "git", "rpm"],
"enum": ["file", "cvs", "git", "rpm", "koji"],
},
"repo": {"type": "string"},
"branch": {"$ref": "#/definitions/optional_string"},

64
pungi/wrappers/scm.py
View File

@ -20,15 +20,18 @@ import shutil
import glob
import six
from six.moves import shlex_quote
from six.moves.urllib.request import urlretrieve
from fnmatch import fnmatch
import kobo.log
from kobo.shortcuts import run, force_list
from pungi.util import (explode_rpm_package, makedirs, copy_all, temp_dir,
retry)
from .kojiwrapper import KojiWrapper
class ScmBase(kobo.log.LoggingBase):
def __init__(self, logger=None, command=None):
def __init__(self, logger=None, command=None, compose=None):
kobo.log.LoggingBase.__init__(self, logger=logger)
self.command = command
@ -196,17 +199,70 @@ class RpmScmWrapper(ScmBase):
shutil.copy2(src, dst)
class KojiScmWrapper(ScmBase):
def __init__(self, *args, **kwargs):
super(KojiScmWrapper, self).__init__(*args, **kwargs)
try:
profile = kwargs["compose"].conf["koji_profile"]
except KeyError:
raise RuntimeError("Koji profile must be configured")
wrapper = KojiWrapper(profile)
self.koji = wrapper.koji_module
self.proxy = wrapper.koji_proxy
def export_dir(self, *args, **kwargs):
raise RuntimeError("Only files can be exported from Koji")
def export_file(self, scm_root, scm_file, target_dir, scm_branch=None):
if scm_branch:
self._get_latest_from_tag(scm_branch, scm_root, scm_file, target_dir)
else:
self._get_from_build(scm_root, scm_file, target_dir)
def _get_latest_from_tag(self, koji_tag, package, file_pattern, target_dir):
self.log_debug(
"Exporting file %s from latest Koji package %s in tag %s",
file_pattern,
package,
koji_tag,
)
builds = self.proxy.listTagged(koji_tag, package=package, latest=True)
if len(builds) != 1:
raise RuntimeError("No package %s in tag %s", package, koji_tag)
self._download_build(builds[0], file_pattern, target_dir)
def _get_from_build(self, build_id, file_pattern, target_dir):
self.log_debug(
"Exporting file %s from Koji build %s", file_pattern, build_id
)
build = self.proxy.getBuild(build_id)
self._download_build(build, file_pattern, target_dir)
def _download_build(self, build, file_pattern, target_dir):
for archive in self.proxy.listArchives(build["build_id"]):
filename = archive["filename"]
if not fnmatch(filename, file_pattern):
continue
typedir = self.koji.pathinfo.typedir(build, archive["btype"])
file_path = os.path.join(typedir, filename)
url = file_path.replace(self.koji.config.topdir, self.koji.config.topurl)
target_file = os.path.join(target_dir, filename)
urlretrieve(url, target_file)
def _get_wrapper(scm_type, *args, **kwargs):
SCM_WRAPPERS = {
"file": FileWrapper,
"cvs": CvsWrapper,
"git": GitWrapper,
"rpm": RpmScmWrapper,
"koji": KojiScmWrapper,
}
try:
return SCM_WRAPPERS[scm_type](*args, **kwargs)
cls = SCM_WRAPPERS[scm_type]
except KeyError:
raise ValueError("Unknown SCM type: %s" % scm_type)
return cls(*args, **kwargs)
def get_file_from_scm(scm_dict, target_path, compose=None):
@ -254,7 +310,7 @@ def get_file_from_scm(scm_dict, target_path, compose=None):
command = scm_dict.get('command')
logger = compose._logger if compose else None
scm = _get_wrapper(scm_type, logger=logger, command=command)
scm = _get_wrapper(scm_type, logger=logger, command=command, compose=compose)
files_copied = []
for i in force_list(scm_file):
@ -308,7 +364,7 @@ def get_dir_from_scm(scm_dict, target_path, compose=None):
command = scm_dict.get("command")
logger = compose._logger if compose else None
scm = _get_wrapper(scm_type, logger=logger, command=command)
scm = _get_wrapper(scm_type, logger=logger, command=command, compose=compose)
with temp_dir(prefix="scm_checkout_") as tmp_dir:
scm.export_dir(scm_repo, scm_dir, scm_branch=scm_branch, target_dir=tmp_dir)

103
tests/test_scm.py
View File

@ -399,3 +399,106 @@ class CvsSCMTestCase(SCMBaseTest):
self.assertEqual(
commands,
['/usr/bin/cvs -q -d http://example.com/cvs export -r HEAD subdir'])
@mock.patch("pungi.wrappers.scm.urlretrieve")
@mock.patch("pungi.wrappers.scm.KojiWrapper")
class KojiSCMTestCase(SCMBaseTest):
def test_without_koji_profile(self, KW, dl):
compose = mock.Mock(conf={})
with self.assertRaises(RuntimeError) as ctx:
scm.get_file_from_scm(
{"scm": "koji", "repo": "my-build-1.0-2", "file": "*"},
self.destdir,
compose=compose,
)
self.assertIn("Koji profile must be configured", str(ctx.exception))
self.assertEqual(KW.mock_calls, [])
self.assertEqual(dl.mock_calls, [])
def test_doesnt_get_dirs(self, KW, dl):
compose = mock.Mock(conf={"koji_profile": "koji"})
with self.assertRaises(RuntimeError) as ctx:
scm.get_dir_from_scm(
{"scm": "koji", "repo": "my-build-1.0-2", "dir": "*"},
self.destdir,
compose=compose,
)
self.assertIn("Only files can be exported", str(ctx.exception))
self.assertEqual(KW.mock_calls, [mock.call("koji")])
self.assertEqual(dl.mock_calls, [])
def _setup_koji_wrapper(self, KW, build_id, files):
KW.return_value.koji_module.config.topdir = "/mnt/koji"
KW.return_value.koji_module.config.topurl = "http://koji.local/koji"
KW.return_value.koji_module.pathinfo.typedir.return_value = "/mnt/koji/images"
buildinfo = {"build_id": build_id}
KW.return_value.koji_proxy.getBuild.return_value = buildinfo
KW.return_value.koji_proxy.listArchives.return_value = [
{"filename": f, "btype": "image"} for f in files
]
KW.return_value.koji_proxy.listTagged.return_value = [buildinfo]
def test_get_from_build(self, KW, dl):
compose = mock.Mock(conf={"koji_profile": "koji"})
def download(src, dst):
touch(dst)
dl.side_effect = download
self._setup_koji_wrapper(KW, 123, ["abc.out", "abc.tar"])
retval = scm.get_file_from_scm(
{"scm": "koji", "repo": "my-build-1.0-2", "file": "*.tar"},
self.destdir,
compose=compose,
)
self.assertStructure(retval, ["abc.tar"])
self.assertEqual(
KW.mock_calls,
[
mock.call("koji"),
mock.call().koji_proxy.getBuild("my-build-1.0-2"),
mock.call().koji_proxy.listArchives(123),
mock.call().koji_module.pathinfo.typedir({"build_id": 123}, "image"),
],
)
self.assertEqual(
dl.call_args_list,
[mock.call("http://koji.local/koji/images/abc.tar", mock.ANY)],
)
def test_get_from_latest_build(self, KW, dl):
compose = mock.Mock(conf={"koji_profile": "koji"})
def download(src, dst):
touch(dst)
dl.side_effect = download
self._setup_koji_wrapper(KW, 123, ["abc.out", "abc.tar"])
retval = scm.get_file_from_scm(
{"scm": "koji", "repo": "my-build", "file": "*.tar", "branch": "images"},
self.destdir,
compose=compose,
)
self.assertStructure(retval, ["abc.tar"])
self.assertEqual(
KW.mock_calls,
[
mock.call("koji"),
mock.call().koji_proxy.listTagged(
"images", package="my-build", latest=True
),
mock.call().koji_proxy.listArchives(123),
mock.call().koji_module.pathinfo.typedir({"build_id": 123}, "image"),
],
)
self.assertEqual(
dl.call_args_list,
[mock.call("http://koji.local/koji/images/abc.tar", mock.ANY)],
)