Add 'pkgset_koji_builds' option to include extra builds in a compose

This PR adds new pkgset_koji_builds configuration option.

This option allows setting list of extra Koji build NVRs which will be
included in a compose. This is useful in two cases:

a) It allows generating standard composes with few packages update to
certain version to test how the compose behaves when the package is
updated for real.

b) It allows generating compose consisting only from particular builds
when pkgset_koji_tag = '' or None. This is useful when one want to
regenerate the compose with packages which are not tagged in single Koji
tag. This is very useful for ODCS when reproducing old composes.

Merges: https://pagure.io/pungi/pull-request/1049
Signed-off-by: Jan Kaluza <jkaluza@redhat.com>

doc/configuration.rst

@@ -470,6 +470,9 @@ Options
     (*str|[str]*) -- tag(s) to read package set from. This option can be
     omitted for modular composes.
 
+**pkgset_koji_builds**
+    (*str|[str]*) -- extra build(s) to include in a package set defined as NVRs.
+
 **pkgset_koji_module_tag**
    (*str|[str]*) -- tags to read module from. This option works similarly to
    listing tags in variants XML. If tags are specified and variants XML

pungi/checks.py

@@ -773,6 +773,7 @@ def make_schema():
             "koji_profile": {"type": "string"},
 
             "pkgset_koji_tag": {"$ref": "#/definitions/strings"},
+            "pkgset_koji_builds": {"$ref": "#/definitions/strings"},
             "pkgset_koji_module_tag": {
                 "$ref": "#/definitions/strings",
                 "default": [],

pungi/phases/pkgset/pkgsets.py

@@ -275,7 +275,8 @@ class FilelistPackageSet(PackageSetBase):
 class KojiPackageSet(PackageSetBase):
     def __init__(self, koji_wrapper, sigkey_ordering, arches=None, logger=None,
                  packages=None, allow_invalid_sigkeys=False,
-                 populate_only_packages=False, cache_region=None):
+                 populate_only_packages=False, cache_region=None,
+                 extra_builds=None):
         """
         Creates new KojiPackageSet.
 
@@ -302,6 +303,8 @@ class KojiPackageSet(PackageSetBase):
             will be used to cache the list of RPMs per Koji tag, so next calls
             of the KojiPackageSet.populate(...) method won't try fetching it
             again.
+        :param list extra_builds: Extra builds NVRs to get from Koji and include
+            in the package set.
         """
         super(KojiPackageSet, self).__init__(sigkey_ordering=sigkey_ordering,
                                              arches=arches, logger=logger,
@@ -311,6 +314,7 @@ class KojiPackageSet(PackageSetBase):
         self.packages = set(packages or [])
         self.populate_only_packages = populate_only_packages
         self.cache_region = cache_region
+        self.extra_builds = extra_builds or []
 
     def __getstate__(self):
         result = self.__dict__.copy()
@@ -330,7 +334,27 @@ class KojiPackageSet(PackageSetBase):
     def koji_proxy(self):
         return self.koji_wrapper.koji_proxy
 
+    def get_extra_rpms(self):
+        if not self.extra_builds:
+            return [], []
+
+        rpms = []
+        builds = []
+
+        builds = self.koji_wrapper.retrying_multicall_map(
+            self.koji_proxy, self.koji_proxy.getBuild, list_of_args=self.extra_builds)
+        rpms_in_builds = self.koji_wrapper.retrying_multicall_map(
+            self.koji_proxy, self.koji_proxy.listBuildRPMs, list_of_args=self.extra_builds)
+
+        rpms = []
+        for rpms_in_build in rpms_in_builds:
+            rpms += rpms_in_build
+        return rpms, builds
+
     def get_latest_rpms(self, tag, event, inherit=True):
+        if not tag:
+            return [], []
+
         if self.cache_region:
             cache_key = "KojiPackageSet.get_latest_rpms_%s_%s_%s" % (
                 tag, str(event), str(inherit))
@@ -400,6 +424,9 @@ class KojiPackageSet(PackageSetBase):
         msg = "Getting latest RPMs (tag: %s, event: %s, inherit: %s)" % (tag, event, inherit)
         self.log_info("[BEGIN] %s" % msg)
         rpms, builds = self.get_latest_rpms(tag, event, inherit=inherit)
+        extra_rpms, extra_builds = self.get_extra_rpms()
+        rpms += extra_rpms
+        builds += extra_builds
 
         builds_by_id = {}
         for build_info in builds:
@@ -460,8 +487,12 @@ class KojiPackageSet(PackageSetBase):
             with open(logfile, 'w') as f:
                 for rpm in rpms:
                     build = builds_by_id[rpm['build_id']]
+                    if 'tag_name' in build and 'tag_id' in build:
                         f.write('{name}-{ep}:{version}-{release}.{arch}: {tag} [{tag_id}]\n'.format(
                             tag=build['tag_name'], tag_id=build['tag_id'], ep=rpm['epoch'] or 0, **rpm))
+                    else:
+                        f.write('{name}-{ep}:{version}-{release}.{arch}: [pkgset_koji_builds]\n'.format(
+                            ep=rpm['epoch'] or 0, **rpm))
 
         self.log_info("[DONE ] %s" % msg)
         return result

pungi/phases/pkgset/sources/source_koji.py

@@ -528,7 +528,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
     # List of compose_tags per variant
     variant_tags = {}
 
-    # In case we use "nodeps" gather_method, we might now the final list of
+    # In case we use "nodeps" gather_method, we might know the final list of
     # packages which will end up in the compose even now, so instead of reading
     # all the packages from Koji tag, we can just cherry-pick the ones which
     # are really needed to do the compose and safe lot of time and resources
@@ -652,7 +652,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
                 arches=all_arches, packages=packages_to_gather,
                 allow_invalid_sigkeys=allow_invalid_sigkeys,
                 populate_only_packages=populate_only_packages_to_gather,
-                cache_region=compose.cache_region)
+                cache_region=compose.cache_region,
+                extra_builds=force_list(compose.conf.get("pkgset_koji_builds", [])))
             if old_file_cache_path:
                 pkgset.load_old_file_cache(old_file_cache_path)
             # Create a filename for log with package-to-tag mapping. The tag

pungi/wrappers/kojiwrapper.py

@@ -24,6 +24,7 @@ import koji
 from kobo.shortcuts import run
 import six
 from six.moves import configparser, shlex_quote
+import six.moves.xmlrpc_client as xmlrpclib
 
 from .. import util
 from ..arch_utils import getBaseArch
@@ -510,6 +511,88 @@ class KojiWrapper(object):
         builds = self.koji_proxy.listBuilds(taskID=task_id)
         return [build.get("nvr") for build in builds if build.get("nvr")]
 
+    def multicall_map(self, koji_session, koji_session_fnc, list_of_args=None, list_of_kwargs=None):
+        """
+        Calls the `koji_session_fnc` using Koji multicall feature N times based on the list of
+        arguments passed in `list_of_args` and `list_of_kwargs`.
+        Returns list of responses sorted the same way as input args/kwargs. In case of error,
+        the error message is logged and None is returned.
+
+        For example to get the package ids of "httpd" and "apr" packages:
+            ids = multicall_map(session, session.getPackageID, ["httpd", "apr"])
+            # ids is now [280, 632]
+
+        :param KojiSessions koji_session: KojiSession to use for multicall.
+        :param object koji_session_fnc: Python object representing the KojiSession method to call.
+        :param list list_of_args: List of args which are passed to each call of koji_session_fnc.
+        :param list list_of_kwargs: List of kwargs which are passed to each call of koji_session_fnc.
+        """
+        if list_of_args is None and list_of_kwargs is None:
+            raise ValueError("One of list_of_args or list_of_kwargs must be set.")
+
+        if (type(list_of_args) not in [type(None), list] or
+                type(list_of_kwargs) not in [type(None), list]):
+            raise ValueError("list_of_args and list_of_kwargs must be list or None.")
+
+        if list_of_kwargs is None:
+            list_of_kwargs = [{}] * len(list_of_args)
+        if list_of_args is None:
+            list_of_args = [[]] * len(list_of_kwargs)
+
+        if len(list_of_args) != len(list_of_kwargs):
+            raise ValueError("Length of list_of_args and list_of_kwargs must be the same.")
+
+        koji_session.multicall = True
+        for args, kwargs in zip(list_of_args, list_of_kwargs):
+            if type(args) != list:
+                args = [args]
+            if type(kwargs) != dict:
+                raise ValueError("Every item in list_of_kwargs must be a dict")
+            koji_session_fnc(*args, **kwargs)
+
+        responses = koji_session.multiCall(strict=True)
+
+        if not responses:
+            return None
+        if type(responses) != list:
+            raise ValueError(
+                "Fault element was returned for multicall of method %r: %r" % (
+                    koji_session_fnc, responses))
+
+        results = []
+
+        # For the response specification, see
+        # https://web.archive.org/web/20060624230303/http://www.xmlrpc.com/discuss/msgReader$1208?mode=topic
+        # Relevant part of this:
+        # Multicall returns an array of responses. There will be one response for each call in
+        # the original array. The result will either be a one-item array containing the result value,
+        # or a struct of the form found inside the standard <fault> element.
+        for response, args, kwargs in zip(responses, list_of_args, list_of_kwargs):
+            if type(response) == list:
+                if not response:
+                    raise ValueError(
+                        "Empty list returned for multicall of method %r with args %r, %r" % (
+                        koji_session_fnc, args, kwargs))
+                results.append(response[0])
+            else:
+                raise ValueError(
+                    "Unexpected data returned for multicall of method %r with args %r, %r: %r" % (
+                        koji_session_fnc, args, kwargs, response))
+
+        return results
+
+
+    @util.retry(wait_on=(xmlrpclib.ProtocolError, koji.GenericError))
+    def retrying_multicall_map(self, *args, **kwargs):
+        """
+        Retrying version of multicall_map. This tries to retry the Koji call
+        in case of koji.GenericError or xmlrpclib.ProtocolError.
+
+        Please refer to koji_multicall_map for further specification of arguments.
+        """
+        return self.multicall_map(*args, **kwargs)
+
+
 
 def get_buildroot_rpms(compose, task_id):
     """Get build root RPMs - either from runroot or local"""

tests/test_koji_wrapper.py

@@ -320,6 +320,20 @@ class KojiWrapperTest(KojiWrapperBaseTestCase):
         self.assertItemsEqual(result.keys(), ['aarch64', 'armhfp', 'x86_64'])
         self.assertItemsEqual(failed, ['ppc64le', 's390x'])
 
+    def test_multicall_map(self):
+        self.koji.koji_proxy = mock.Mock()
+        self.koji.koji_proxy.multiCall.return_value = [[1], [2]]
+
+        ret = self.koji.multicall_map(
+            self.koji.koji_proxy, self.koji.koji_proxy.getBuild, ["foo", "bar"],
+            [{"x":1}, {"x":2}])
+
+        self.assertItemsEqual(
+            self.koji.koji_proxy.getBuild.mock_calls,
+            [mock.call("foo", x=1), mock.call("bar", x=2)])
+        self.koji.koji_proxy.multiCall.assert_called_with(strict=True)
+        self.assertEqual(ret, [1, 2])
+
 
 class LiveMediaTestCase(KojiWrapperBaseTestCase):
     def test_get_live_media_cmd_minimal(self):

tests/test_pkgset_pkgsets.py

@@ -370,6 +370,52 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
                 {'x86_64': ['rpms/bash-debuginfo@4.3.42@4.fc24@x86_64',
                             'rpms/bash@4.3.42@4.fc24@x86_64']})
 
+    def test_extra_builds_attribute(self):
+        self._touch_files([
+            'rpms/pungi@4.1.3@3.fc25@noarch',
+            'rpms/pungi@4.1.3@3.fc25@src',
+            'rpms/bash@4.3.42@4.fc24@i686',
+            'rpms/bash@4.3.42@4.fc24@x86_64',
+            'rpms/bash@4.3.42@4.fc24@src',
+            'rpms/bash-debuginfo@4.3.42@4.fc24@i686',
+            'rpms/bash-debuginfo@4.3.42@4.fc24@x86_64',
+        ])
+
+        # Return "pungi" RPMs and builds using "get_latest_rpms" which gets
+        # them from Koji multiCall.
+        extra_rpms = [rpm for rpm in self.tagged_rpms[0]
+                      if rpm["name"] == "pungi"]
+        extra_builds = [build for build in self.tagged_rpms[1]
+                        if build["package_name"] == "pungi"]
+        self.koji_wrapper.retrying_multicall_map.side_effect = [
+            extra_builds, [extra_rpms]]
+
+        # Do not return "pungi" RPMs and builds using the listTaggedRPMs, so
+        # we can be sure "pungi" gets into compose using the `extra_builds`.
+        self.koji_wrapper.koji_proxy.listTaggedRPMS.return_value = [
+            [rpm for rpm in self.tagged_rpms[0] if rpm["name"] != "pungi"],
+            [b for b in self.tagged_rpms[1] if b["package_name"] != "pungi"]]
+
+        pkgset = pkgsets.KojiPackageSet(
+            self.koji_wrapper, [None],
+            extra_builds=["pungi-4.1.3-3.fc25"])
+
+        result = pkgset.populate('f25', logfile=self.topdir + '/pkgset.log')
+
+        self.assertEqual(
+            self.koji_wrapper.koji_proxy.mock_calls,
+            [mock.call.listTaggedRPMS('f25', event=None, inherit=True, latest=True)])
+
+        self.assertPkgsetEqual(result,
+                               {'src': ['rpms/pungi@4.1.3@3.fc25@src',
+                                        'rpms/bash@4.3.42@4.fc24@src'],
+                                'noarch': ['rpms/pungi@4.1.3@3.fc25@noarch'],
+                                'i686': ['rpms/bash@4.3.42@4.fc24@i686',
+                                         'rpms/bash-debuginfo@4.3.42@4.fc24@i686'],
+                                'x86_64': ['rpms/bash@4.3.42@4.fc24@x86_64',
+                                           'rpms/bash-debuginfo@4.3.42@4.fc24@x86_64']})
+
+
 @mock.patch('kobo.pkgset.FileCache', new=MockFileCache)
 class TestMergePackageSets(PkgsetCompareMixin, unittest.TestCase):
     def test_merge_in_another_arch(self):