scm: Add backend for downloading archives from Koji

Tests and documentation included.

JIRA: COMPOSE-3816
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

doc/scm_support.rst

@@ -17,13 +17,21 @@ which can contain following keys.
   * ``git`` -- copies files from a Git repository
   * ``cvs`` -- copies files from a CVS repository
   * ``rpm`` -- copies files from a package in the compose
+  * ``koji`` -- downloads archives from a given build in Koji build system
 
-* ``repo`` -- for Git and CVS backends URL to the repository, for RPM a shell
-  glob for matching package names (or a list of such globs); for ``file``
-  backend this option should be empty
+* ``repo``
 
-* ``branch`` -- branch name for Git and CVS backends, with ``master`` and
-  ``HEAD`` as defaults. Ignored for other backends.
+  * for Git and CVS backends this should be URL to the repository
+  * for RPM backend this should be a shell style glob matching package names
+    (or a list of such globs)
+  * for file backend this should be empty
+  * for Koji backend this should be an NVR or package name
+
+* ``branch``
+
+  * branch name for Git and CVS backends, with ``master`` and ``HEAD`` as defaults
+  * Koji tag for koji backend if only package name is given
+  * otherwise should not be specified
 
 * ``file`` -- a list of files that should be exported.
 
@@ -34,6 +42,31 @@ which can contain following keys.
   needed file (for example to run ``make``). Only supported in Git backend.
 
 
+Koji examples
+-------------
+
+There are two different ways how to configure the Koji backend. ::
+
+    {
+        # Download all *.tar files from build my-image-1.0-1.
+        "scm": "koji",
+        "repo": "my-image-1.0-1",
+        "file": "*.tar",
+    }
+
+    {
+        # Find latest build of my-image in tag my-tag and take files from
+        # there.
+        "scm": "koji",
+        "repo": "my-image",
+        "branch": "my-tag",
+        "file": "*.tar",
+    }
+
+Using both tag name and exact NVR will result in error: the NVR would be
+interpreted as a package name, and would not match anything.
+
+
 ``file`` vs. ``dir``
 --------------------
 
@@ -53,3 +86,7 @@ after ``pkgset`` phase finished. You can't get comps file from a package.
 Depending on Git repository URL configuration Pungi can only export the
 requested content using ``git archive``. When a command should run this is not
 possible and a clone is always needed.
+
+When using ``koji`` backend, it is required to provide configuration for Koji
+profile to be used (``koji_profile``). It is not possible to contact multiple
+different Koji instances.

pungi/checks.py

@@ -446,7 +446,7 @@ def make_schema():
                 "properties": {
                     "scm": {
                         "type": "string",
-                        "enum": ["file", "cvs", "git", "rpm"],
+                        "enum": ["file", "cvs", "git", "rpm", "koji"],
                     },
                     "repo": {"type": "string"},
                     "branch": {"$ref": "#/definitions/optional_string"},

pungi/wrappers/scm.py

@@ -20,15 +20,18 @@ import shutil
 import glob
 import six
 from six.moves import shlex_quote
+from six.moves.urllib.request import urlretrieve
+from fnmatch import fnmatch
 
 import kobo.log
 from kobo.shortcuts import run, force_list
 from pungi.util import (explode_rpm_package, makedirs, copy_all, temp_dir,
                         retry)
+from .kojiwrapper import KojiWrapper
 
 
 class ScmBase(kobo.log.LoggingBase):
-    def __init__(self, logger=None, command=None):
+    def __init__(self, logger=None, command=None, compose=None):
         kobo.log.LoggingBase.__init__(self, logger=logger)
         self.command = command
 
@@ -196,17 +199,70 @@ class RpmScmWrapper(ScmBase):
                     shutil.copy2(src, dst)
 
 
+class KojiScmWrapper(ScmBase):
+    def __init__(self, *args, **kwargs):
+        super(KojiScmWrapper, self).__init__(*args, **kwargs)
+        try:
+            profile = kwargs["compose"].conf["koji_profile"]
+        except KeyError:
+            raise RuntimeError("Koji profile must be configured")
+        wrapper = KojiWrapper(profile)
+        self.koji = wrapper.koji_module
+        self.proxy = wrapper.koji_proxy
+
+    def export_dir(self, *args, **kwargs):
+        raise RuntimeError("Only files can be exported from Koji")
+
+    def export_file(self, scm_root, scm_file, target_dir, scm_branch=None):
+        if scm_branch:
+            self._get_latest_from_tag(scm_branch, scm_root, scm_file, target_dir)
+        else:
+            self._get_from_build(scm_root, scm_file, target_dir)
+
+    def _get_latest_from_tag(self, koji_tag, package, file_pattern, target_dir):
+        self.log_debug(
+            "Exporting file %s from latest Koji package %s in tag %s",
+            file_pattern,
+            package,
+            koji_tag,
+        )
+        builds = self.proxy.listTagged(koji_tag, package=package, latest=True)
+        if len(builds) != 1:
+            raise RuntimeError("No package %s in tag %s", package, koji_tag)
+        self._download_build(builds[0], file_pattern, target_dir)
+
+    def _get_from_build(self, build_id, file_pattern, target_dir):
+        self.log_debug(
+            "Exporting file %s from Koji build %s", file_pattern, build_id
+        )
+        build = self.proxy.getBuild(build_id)
+        self._download_build(build, file_pattern, target_dir)
+
+    def _download_build(self, build, file_pattern, target_dir):
+        for archive in self.proxy.listArchives(build["build_id"]):
+            filename = archive["filename"]
+            if not fnmatch(filename, file_pattern):
+                continue
+            typedir = self.koji.pathinfo.typedir(build, archive["btype"])
+            file_path = os.path.join(typedir, filename)
+            url = file_path.replace(self.koji.config.topdir, self.koji.config.topurl)
+            target_file = os.path.join(target_dir, filename)
+            urlretrieve(url, target_file)
+
+
 def _get_wrapper(scm_type, *args, **kwargs):
     SCM_WRAPPERS = {
         "file": FileWrapper,
         "cvs": CvsWrapper,
         "git": GitWrapper,
         "rpm": RpmScmWrapper,
+        "koji": KojiScmWrapper,
     }
     try:
-        return SCM_WRAPPERS[scm_type](*args, **kwargs)
+        cls = SCM_WRAPPERS[scm_type]
     except KeyError:
         raise ValueError("Unknown SCM type: %s" % scm_type)
+    return cls(*args, **kwargs)
 
 
 def get_file_from_scm(scm_dict, target_path, compose=None):
@@ -254,7 +310,7 @@ def get_file_from_scm(scm_dict, target_path, compose=None):
         command = scm_dict.get('command')
 
     logger = compose._logger if compose else None
-    scm = _get_wrapper(scm_type, logger=logger, command=command)
+    scm = _get_wrapper(scm_type, logger=logger, command=command, compose=compose)
 
     files_copied = []
     for i in force_list(scm_file):
@@ -308,7 +364,7 @@ def get_dir_from_scm(scm_dict, target_path, compose=None):
         command = scm_dict.get("command")
 
     logger = compose._logger if compose else None
-    scm = _get_wrapper(scm_type, logger=logger, command=command)
+    scm = _get_wrapper(scm_type, logger=logger, command=command, compose=compose)
 
     with temp_dir(prefix="scm_checkout_") as tmp_dir:
         scm.export_dir(scm_repo, scm_dir, scm_branch=scm_branch, target_dir=tmp_dir)

tests/test_scm.py

@@ -399,3 +399,106 @@ class CvsSCMTestCase(SCMBaseTest):
         self.assertEqual(
             commands,
             ['/usr/bin/cvs -q -d http://example.com/cvs export -r HEAD subdir'])
+
+
+@mock.patch("pungi.wrappers.scm.urlretrieve")
+@mock.patch("pungi.wrappers.scm.KojiWrapper")
+class KojiSCMTestCase(SCMBaseTest):
+    def test_without_koji_profile(self, KW, dl):
+        compose = mock.Mock(conf={})
+
+        with self.assertRaises(RuntimeError) as ctx:
+            scm.get_file_from_scm(
+                {"scm": "koji", "repo": "my-build-1.0-2", "file": "*"},
+                self.destdir,
+                compose=compose,
+            )
+        self.assertIn("Koji profile must be configured", str(ctx.exception))
+        self.assertEqual(KW.mock_calls, [])
+        self.assertEqual(dl.mock_calls, [])
+
+    def test_doesnt_get_dirs(self, KW, dl):
+        compose = mock.Mock(conf={"koji_profile": "koji"})
+
+        with self.assertRaises(RuntimeError) as ctx:
+            scm.get_dir_from_scm(
+                {"scm": "koji", "repo": "my-build-1.0-2", "dir": "*"},
+                self.destdir,
+                compose=compose,
+            )
+        self.assertIn("Only files can be exported", str(ctx.exception))
+        self.assertEqual(KW.mock_calls, [mock.call("koji")])
+        self.assertEqual(dl.mock_calls, [])
+
+    def _setup_koji_wrapper(self, KW, build_id, files):
+        KW.return_value.koji_module.config.topdir = "/mnt/koji"
+        KW.return_value.koji_module.config.topurl = "http://koji.local/koji"
+        KW.return_value.koji_module.pathinfo.typedir.return_value = "/mnt/koji/images"
+        buildinfo = {"build_id": build_id}
+        KW.return_value.koji_proxy.getBuild.return_value = buildinfo
+        KW.return_value.koji_proxy.listArchives.return_value = [
+            {"filename": f, "btype": "image"} for f in files
+        ]
+        KW.return_value.koji_proxy.listTagged.return_value = [buildinfo]
+
+    def test_get_from_build(self, KW, dl):
+        compose = mock.Mock(conf={"koji_profile": "koji"})
+
+        def download(src, dst):
+            touch(dst)
+
+        dl.side_effect = download
+
+        self._setup_koji_wrapper(KW, 123, ["abc.out", "abc.tar"])
+
+        retval = scm.get_file_from_scm(
+            {"scm": "koji", "repo": "my-build-1.0-2", "file": "*.tar"},
+            self.destdir,
+            compose=compose,
+        )
+        self.assertStructure(retval, ["abc.tar"])
+        self.assertEqual(
+            KW.mock_calls,
+            [
+                mock.call("koji"),
+                mock.call().koji_proxy.getBuild("my-build-1.0-2"),
+                mock.call().koji_proxy.listArchives(123),
+                mock.call().koji_module.pathinfo.typedir({"build_id": 123}, "image"),
+            ],
+        )
+        self.assertEqual(
+            dl.call_args_list,
+            [mock.call("http://koji.local/koji/images/abc.tar", mock.ANY)],
+        )
+
+    def test_get_from_latest_build(self, KW, dl):
+        compose = mock.Mock(conf={"koji_profile": "koji"})
+
+        def download(src, dst):
+            touch(dst)
+
+        dl.side_effect = download
+
+        self._setup_koji_wrapper(KW, 123, ["abc.out", "abc.tar"])
+
+        retval = scm.get_file_from_scm(
+            {"scm": "koji", "repo": "my-build", "file": "*.tar", "branch": "images"},
+            self.destdir,
+            compose=compose,
+        )
+        self.assertStructure(retval, ["abc.tar"])
+        self.assertEqual(
+            KW.mock_calls,
+            [
+                mock.call("koji"),
+                mock.call().koji_proxy.listTagged(
+                    "images", package="my-build", latest=True
+                ),
+                mock.call().koji_proxy.listArchives(123),
+                mock.call().koji_module.pathinfo.typedir({"build_id": 123}, "image"),
+            ],
+        )
+        self.assertEqual(
+            dl.call_args_list,
+            [mock.call("http://koji.local/koji/images/abc.tar", mock.ANY)],
+        )