From 37c89dfde642accae3d41502e3e78b23c76fc347 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Fri, 14 Sep 2018 10:59:53 +0200 Subject: [PATCH] Add 'pkgset_koji_builds' option to include extra builds in a compose This PR adds new pkgset_koji_builds configuration option. This option allows setting list of extra Koji build NVRs which will be included in a compose. This is useful in two cases: a) It allows generating standard composes with few packages update to certain version to test how the compose behaves when the package is updated for real. b) It allows generating compose consisting only from particular builds when pkgset_koji_tag = '' or None. This is useful when one want to regenerate the compose with packages which are not tagged in single Koji tag. This is very useful for ODCS when reproducing old composes. Merges: https://pagure.io/pungi/pull-request/1049 Signed-off-by: Jan Kaluza --- doc/configuration.rst | 3 + pungi/checks.py | 1 + pungi/phases/pkgset/pkgsets.py | 37 +++++++++- pungi/phases/pkgset/sources/source_koji.py | 5 +- pungi/wrappers/kojiwrapper.py | 83 ++++++++++++++++++++++ tests/test_koji_wrapper.py | 14 ++++ tests/test_pkgset_pkgsets.py | 46 ++++++++++++ 7 files changed, 184 insertions(+), 5 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 9059d273..e084c6f8 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -470,6 +470,9 @@ Options (*str|[str]*) -- tag(s) to read package set from. This option can be omitted for modular composes. +**pkgset_koji_builds** + (*str|[str]*) -- extra build(s) to include in a package set defined as NVRs. + **pkgset_koji_module_tag** (*str|[str]*) -- tags to read module from. This option works similarly to listing tags in variants XML. If tags are specified and variants XML diff --git a/pungi/checks.py b/pungi/checks.py index 7a2f9a8d..ec1c2adc 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -773,6 +773,7 @@ def make_schema(): "koji_profile": {"type": "string"}, "pkgset_koji_tag": {"$ref": "#/definitions/strings"}, + "pkgset_koji_builds": {"$ref": "#/definitions/strings"}, "pkgset_koji_module_tag": { "$ref": "#/definitions/strings", "default": [], diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index 48b5c349..8786015e 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -275,7 +275,8 @@ class FilelistPackageSet(PackageSetBase): class KojiPackageSet(PackageSetBase): def __init__(self, koji_wrapper, sigkey_ordering, arches=None, logger=None, packages=None, allow_invalid_sigkeys=False, - populate_only_packages=False, cache_region=None): + populate_only_packages=False, cache_region=None, + extra_builds=None): """ Creates new KojiPackageSet. @@ -302,6 +303,8 @@ class KojiPackageSet(PackageSetBase): will be used to cache the list of RPMs per Koji tag, so next calls of the KojiPackageSet.populate(...) method won't try fetching it again. + :param list extra_builds: Extra builds NVRs to get from Koji and include + in the package set. """ super(KojiPackageSet, self).__init__(sigkey_ordering=sigkey_ordering, arches=arches, logger=logger, @@ -311,6 +314,7 @@ class KojiPackageSet(PackageSetBase): self.packages = set(packages or []) self.populate_only_packages = populate_only_packages self.cache_region = cache_region + self.extra_builds = extra_builds or [] def __getstate__(self): result = self.__dict__.copy() @@ -330,7 +334,27 @@ class KojiPackageSet(PackageSetBase): def koji_proxy(self): return self.koji_wrapper.koji_proxy + def get_extra_rpms(self): + if not self.extra_builds: + return [], [] + + rpms = [] + builds = [] + + builds = self.koji_wrapper.retrying_multicall_map( + self.koji_proxy, self.koji_proxy.getBuild, list_of_args=self.extra_builds) + rpms_in_builds = self.koji_wrapper.retrying_multicall_map( + self.koji_proxy, self.koji_proxy.listBuildRPMs, list_of_args=self.extra_builds) + + rpms = [] + for rpms_in_build in rpms_in_builds: + rpms += rpms_in_build + return rpms, builds + def get_latest_rpms(self, tag, event, inherit=True): + if not tag: + return [], [] + if self.cache_region: cache_key = "KojiPackageSet.get_latest_rpms_%s_%s_%s" % ( tag, str(event), str(inherit)) @@ -400,6 +424,9 @@ class KojiPackageSet(PackageSetBase): msg = "Getting latest RPMs (tag: %s, event: %s, inherit: %s)" % (tag, event, inherit) self.log_info("[BEGIN] %s" % msg) rpms, builds = self.get_latest_rpms(tag, event, inherit=inherit) + extra_rpms, extra_builds = self.get_extra_rpms() + rpms += extra_rpms + builds += extra_builds builds_by_id = {} for build_info in builds: @@ -460,8 +487,12 @@ class KojiPackageSet(PackageSetBase): with open(logfile, 'w') as f: for rpm in rpms: build = builds_by_id[rpm['build_id']] - f.write('{name}-{ep}:{version}-{release}.{arch}: {tag} [{tag_id}]\n'.format( - tag=build['tag_name'], tag_id=build['tag_id'], ep=rpm['epoch'] or 0, **rpm)) + if 'tag_name' in build and 'tag_id' in build: + f.write('{name}-{ep}:{version}-{release}.{arch}: {tag} [{tag_id}]\n'.format( + tag=build['tag_name'], tag_id=build['tag_id'], ep=rpm['epoch'] or 0, **rpm)) + else: + f.write('{name}-{ep}:{version}-{release}.{arch}: [pkgset_koji_builds]\n'.format( + ep=rpm['epoch'] or 0, **rpm)) self.log_info("[DONE ] %s" % msg) return result diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 3b4b5903..450d271e 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -528,7 +528,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): # List of compose_tags per variant variant_tags = {} - # In case we use "nodeps" gather_method, we might now the final list of + # In case we use "nodeps" gather_method, we might know the final list of # packages which will end up in the compose even now, so instead of reading # all the packages from Koji tag, we can just cherry-pick the ones which # are really needed to do the compose and safe lot of time and resources @@ -652,7 +652,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): arches=all_arches, packages=packages_to_gather, allow_invalid_sigkeys=allow_invalid_sigkeys, populate_only_packages=populate_only_packages_to_gather, - cache_region=compose.cache_region) + cache_region=compose.cache_region, + extra_builds=force_list(compose.conf.get("pkgset_koji_builds", []))) if old_file_cache_path: pkgset.load_old_file_cache(old_file_cache_path) # Create a filename for log with package-to-tag mapping. The tag diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py index abcdecb9..c69af800 100644 --- a/pungi/wrappers/kojiwrapper.py +++ b/pungi/wrappers/kojiwrapper.py @@ -24,6 +24,7 @@ import koji from kobo.shortcuts import run import six from six.moves import configparser, shlex_quote +import six.moves.xmlrpc_client as xmlrpclib from .. import util from ..arch_utils import getBaseArch @@ -510,6 +511,88 @@ class KojiWrapper(object): builds = self.koji_proxy.listBuilds(taskID=task_id) return [build.get("nvr") for build in builds if build.get("nvr")] + def multicall_map(self, koji_session, koji_session_fnc, list_of_args=None, list_of_kwargs=None): + """ + Calls the `koji_session_fnc` using Koji multicall feature N times based on the list of + arguments passed in `list_of_args` and `list_of_kwargs`. + Returns list of responses sorted the same way as input args/kwargs. In case of error, + the error message is logged and None is returned. + + For example to get the package ids of "httpd" and "apr" packages: + ids = multicall_map(session, session.getPackageID, ["httpd", "apr"]) + # ids is now [280, 632] + + :param KojiSessions koji_session: KojiSession to use for multicall. + :param object koji_session_fnc: Python object representing the KojiSession method to call. + :param list list_of_args: List of args which are passed to each call of koji_session_fnc. + :param list list_of_kwargs: List of kwargs which are passed to each call of koji_session_fnc. + """ + if list_of_args is None and list_of_kwargs is None: + raise ValueError("One of list_of_args or list_of_kwargs must be set.") + + if (type(list_of_args) not in [type(None), list] or + type(list_of_kwargs) not in [type(None), list]): + raise ValueError("list_of_args and list_of_kwargs must be list or None.") + + if list_of_kwargs is None: + list_of_kwargs = [{}] * len(list_of_args) + if list_of_args is None: + list_of_args = [[]] * len(list_of_kwargs) + + if len(list_of_args) != len(list_of_kwargs): + raise ValueError("Length of list_of_args and list_of_kwargs must be the same.") + + koji_session.multicall = True + for args, kwargs in zip(list_of_args, list_of_kwargs): + if type(args) != list: + args = [args] + if type(kwargs) != dict: + raise ValueError("Every item in list_of_kwargs must be a dict") + koji_session_fnc(*args, **kwargs) + + responses = koji_session.multiCall(strict=True) + + if not responses: + return None + if type(responses) != list: + raise ValueError( + "Fault element was returned for multicall of method %r: %r" % ( + koji_session_fnc, responses)) + + results = [] + + # For the response specification, see + # https://web.archive.org/web/20060624230303/http://www.xmlrpc.com/discuss/msgReader$1208?mode=topic + # Relevant part of this: + # Multicall returns an array of responses. There will be one response for each call in + # the original array. The result will either be a one-item array containing the result value, + # or a struct of the form found inside the standard element. + for response, args, kwargs in zip(responses, list_of_args, list_of_kwargs): + if type(response) == list: + if not response: + raise ValueError( + "Empty list returned for multicall of method %r with args %r, %r" % ( + koji_session_fnc, args, kwargs)) + results.append(response[0]) + else: + raise ValueError( + "Unexpected data returned for multicall of method %r with args %r, %r: %r" % ( + koji_session_fnc, args, kwargs, response)) + + return results + + + @util.retry(wait_on=(xmlrpclib.ProtocolError, koji.GenericError)) + def retrying_multicall_map(self, *args, **kwargs): + """ + Retrying version of multicall_map. This tries to retry the Koji call + in case of koji.GenericError or xmlrpclib.ProtocolError. + + Please refer to koji_multicall_map for further specification of arguments. + """ + return self.multicall_map(*args, **kwargs) + + def get_buildroot_rpms(compose, task_id): """Get build root RPMs - either from runroot or local""" diff --git a/tests/test_koji_wrapper.py b/tests/test_koji_wrapper.py index 0f20562a..e99cc137 100644 --- a/tests/test_koji_wrapper.py +++ b/tests/test_koji_wrapper.py @@ -320,6 +320,20 @@ class KojiWrapperTest(KojiWrapperBaseTestCase): self.assertItemsEqual(result.keys(), ['aarch64', 'armhfp', 'x86_64']) self.assertItemsEqual(failed, ['ppc64le', 's390x']) + def test_multicall_map(self): + self.koji.koji_proxy = mock.Mock() + self.koji.koji_proxy.multiCall.return_value = [[1], [2]] + + ret = self.koji.multicall_map( + self.koji.koji_proxy, self.koji.koji_proxy.getBuild, ["foo", "bar"], + [{"x":1}, {"x":2}]) + + self.assertItemsEqual( + self.koji.koji_proxy.getBuild.mock_calls, + [mock.call("foo", x=1), mock.call("bar", x=2)]) + self.koji.koji_proxy.multiCall.assert_called_with(strict=True) + self.assertEqual(ret, [1, 2]) + class LiveMediaTestCase(KojiWrapperBaseTestCase): def test_get_live_media_cmd_minimal(self): diff --git a/tests/test_pkgset_pkgsets.py b/tests/test_pkgset_pkgsets.py index f514b566..4c9a1bd9 100644 --- a/tests/test_pkgset_pkgsets.py +++ b/tests/test_pkgset_pkgsets.py @@ -370,6 +370,52 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase): {'x86_64': ['rpms/bash-debuginfo@4.3.42@4.fc24@x86_64', 'rpms/bash@4.3.42@4.fc24@x86_64']}) + def test_extra_builds_attribute(self): + self._touch_files([ + 'rpms/pungi@4.1.3@3.fc25@noarch', + 'rpms/pungi@4.1.3@3.fc25@src', + 'rpms/bash@4.3.42@4.fc24@i686', + 'rpms/bash@4.3.42@4.fc24@x86_64', + 'rpms/bash@4.3.42@4.fc24@src', + 'rpms/bash-debuginfo@4.3.42@4.fc24@i686', + 'rpms/bash-debuginfo@4.3.42@4.fc24@x86_64', + ]) + + # Return "pungi" RPMs and builds using "get_latest_rpms" which gets + # them from Koji multiCall. + extra_rpms = [rpm for rpm in self.tagged_rpms[0] + if rpm["name"] == "pungi"] + extra_builds = [build for build in self.tagged_rpms[1] + if build["package_name"] == "pungi"] + self.koji_wrapper.retrying_multicall_map.side_effect = [ + extra_builds, [extra_rpms]] + + # Do not return "pungi" RPMs and builds using the listTaggedRPMs, so + # we can be sure "pungi" gets into compose using the `extra_builds`. + self.koji_wrapper.koji_proxy.listTaggedRPMS.return_value = [ + [rpm for rpm in self.tagged_rpms[0] if rpm["name"] != "pungi"], + [b for b in self.tagged_rpms[1] if b["package_name"] != "pungi"]] + + pkgset = pkgsets.KojiPackageSet( + self.koji_wrapper, [None], + extra_builds=["pungi-4.1.3-3.fc25"]) + + result = pkgset.populate('f25', logfile=self.topdir + '/pkgset.log') + + self.assertEqual( + self.koji_wrapper.koji_proxy.mock_calls, + [mock.call.listTaggedRPMS('f25', event=None, inherit=True, latest=True)]) + + self.assertPkgsetEqual(result, + {'src': ['rpms/pungi@4.1.3@3.fc25@src', + 'rpms/bash@4.3.42@4.fc24@src'], + 'noarch': ['rpms/pungi@4.1.3@3.fc25@noarch'], + 'i686': ['rpms/bash@4.3.42@4.fc24@i686', + 'rpms/bash-debuginfo@4.3.42@4.fc24@i686'], + 'x86_64': ['rpms/bash@4.3.42@4.fc24@x86_64', + 'rpms/bash-debuginfo@4.3.42@4.fc24@x86_64']}) + + @mock.patch('kobo.pkgset.FileCache', new=MockFileCache) class TestMergePackageSets(PkgsetCompareMixin, unittest.TestCase): def test_merge_in_another_arch(self):