From 828557b4d673e6e492cba17419b6567b1b7a9851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 29 Aug 2018 13:02:29 +0200 Subject: [PATCH] pkgset: Respect koji event when searching for modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the search result, we should ignore any module build that finished after the event that we are working with. Fixes: https://pagure.io/pungi/issue/999 Signed-off-by: Lubomír Sedlář --- pungi/phases/pkgset/sources/source_koji.py | 36 +++++++++----- tests/test_pkgset_source_koji.py | 57 +++++++++++++++++++++- 2 files changed, 78 insertions(+), 15 deletions(-) diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index bc62c181..2bb652ae 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -103,9 +103,10 @@ def variant_dict_from_str(compose, module_str): @retry(wait_on=IOError) -def get_koji_modules(compose, koji_wrapper, module_info_str): +def get_koji_modules(compose, koji_wrapper, event, module_info_str): """ - :param koji_wrapper : koji wrapper instance + :param koji_wrapper: koji wrapper instance + :param event: event at which to perform the query :param module_info_str: str, mmd or module dict :return final list of module_info which pass repoclosure @@ -126,17 +127,19 @@ def get_koji_modules(compose, koji_wrapper, module_info_str): query_str = query_str.replace('*.*', '*') koji_builds = koji_proxy.search(query_str, "build", "glob") - # Error reporting - if not koji_builds: - raise ValueError( - "No module build found for %r (queried for %r)" - % (module_info_str, query_str) - ) modules = [] for build in koji_builds: md = koji_proxy.getBuild(build["id"]) + if md["completion_ts"] > event["ts"]: + # The build finished after the event at which we are limited to, + # ignore it. + compose.log_debug( + "Module build %s is too new, ignoring it." % build["name"] + ) + continue + if not md["extra"]: continue @@ -171,6 +174,12 @@ def get_koji_modules(compose, koji_wrapper, module_info_str): for rpm in rpms] modules.append(md) + if not modules: + raise ValueError( + "No module build found for %r (queried for %r)" + % (module_info_str, query_str) + ) + # If there is version provided, then all modules with that version will go # in. In case version is missing, we will find the latest version and # include all modules with that version. @@ -266,7 +275,7 @@ def _log_modulemd(compose, variant, mmd): def _get_modules_from_koji( - compose, koji_wrapper, variant, variant_tags, module_tag_rpm_filter + compose, koji_wrapper, event, variant, variant_tags, module_tag_rpm_filter ): """ Loads modules for given `variant` from koji `session`, adds them to @@ -282,7 +291,7 @@ def _get_modules_from_koji( # Find out all modules in every variant and add their Koji tags # to variant and variant_tags list. for module in variant.get_modules(): - koji_modules = get_koji_modules(compose, koji_wrapper, module["name"]) + koji_modules = get_koji_modules(compose, koji_wrapper, event, module["name"]) for koji_module in koji_modules: mmd = Modulemd.Module.new_from_string(koji_module["modulemd"]) mmd.upgrade() @@ -425,7 +434,7 @@ def _find_old_file_cache_path(compose): return old_file_cache_path -def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): +def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): all_arches = set(["src"]) for arch in compose.get_arches(): is_multilib = is_arch_multilib(compose.conf, arch) @@ -494,7 +503,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): _get_modules_from_koji_tags( compose, koji_wrapper, - event_id, + event, variant, variant_tags, module_tag_rpm_filter, @@ -506,6 +515,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): _get_modules_from_koji( compose, koji_wrapper, + event, variant, variant_tags, module_tag_rpm_filter, @@ -573,7 +583,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): should_inherit = inherit if is_traditional else inherit_modules pkgset.populate( compose_tag, - event_id, + event, inherit=should_inherit, logfile=logfile, exclude_packages=module_tag_rpm_filter.get(compose_tag), diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index 470a6e94..b0f11ea4 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -353,6 +353,7 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase): 'release': '20180406051653.2e6f5e0a', 'state': 1, 'version': 'master_dash', + 'completion_ts': 1433473124.0, } ] mock_archives = [ @@ -385,9 +386,12 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase): self.koji_wrapper.koji_proxy.getBuild.return_value = mock_build_md[0] self.koji_wrapper.koji_proxy.listArchives.return_value = mock_archives self.koji_wrapper.koji_proxy.listRPMs.return_value = mock_rpms + event = {"id": 12345, "ts": 1533473124.0} module_info_str = "testmodule2:master-dash:20180406051653:96c371af" - result = source_koji.get_koji_modules(self.compose, self.koji_wrapper, module_info_str) + result = source_koji.get_koji_modules( + self.compose, self.koji_wrapper, event, module_info_str + ) assert type(result) is list assert len(result) == 1 @@ -407,6 +411,49 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase): self.koji_wrapper.koji_proxy.listRPMs.assert_called_once_with( imageID=mock_archives[0]["id"]) + def test_get_koji_modules_filter_by_event(self): + mock_build_ids = [ + {"id": 1065873, "name": "testmodule2-master_dash-20180406051653.96c371af"} + ] + mock_extra = { + "typeinfo": { + "module": { + "content_koji_tag": "module-b62270b82443edde", + "modulemd_str": mock.Mock()} + } + } + mock_build_md = [ + { + "id": 1065873, + "epoch": None, + "extra": mock_extra, + "name": "testmodule2", + "nvr": "testmodule2-master_dash-20180406051653.2e6f5e0a", + "release": "20180406051653.2e6f5e0a", + "state": 1, + "version": "master_dash", + "completion_ts": 1633473124.0, + } + ] + + self.koji_wrapper.koji_proxy.search.return_value = mock_build_ids + self.koji_wrapper.koji_proxy.getBuild.return_value = mock_build_md[0] + event = {"id": 12345, "ts": 1533473124.0} + + with self.assertRaises(ValueError) as ctx: + source_koji.get_koji_modules( + self.compose, self.koji_wrapper, event, "testmodule2:master-dash" + ) + + self.assertIn("No module build found", str(ctx.exception)) + + self.koji_wrapper.koji_proxy.search.assert_called_once_with( + "testmodule2-master_dash-*", "build", "glob" + ) + self.koji_wrapper.koji_proxy.getBuild.assert_called_once_with(mock_build_ids[0]["id"]) + self.koji_wrapper.koji_proxy.listArchives.assert_not_called() + self.koji_wrapper.koji_proxy.listRPMs.assert_not_called() + def test_get_koji_modules_no_version(self): mock_build_ids = [ {'id': 1065873, 'name': 'testmodule2-master-20180406051653.2e6f5e0a'}, @@ -438,6 +485,7 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase): 'release': '20180406051653.2e6f5e0a', 'state': 1, 'version': 'master', + 'completion_ts': 1433473124.0, }, { 'id': 1065874, @@ -448,6 +496,7 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase): 'release': '20180406051653.96c371af', 'state': 1, 'version': 'master', + 'completion_ts': 1433473124.0, } ] mock_archives = [ @@ -499,8 +548,12 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase): self.koji_wrapper.koji_proxy.listArchives.side_effect = mock_archives self.koji_wrapper.koji_proxy.listRPMs.side_effect = mock_rpms + event = {"id": 12345, "ts": 1533473124.0} + module_info_str = "testmodule2:master" - result = source_koji.get_koji_modules(self.compose, self.koji_wrapper, module_info_str) + result = source_koji.get_koji_modules( + self.compose, self.koji_wrapper, event, module_info_str + ) assert type(result) is list assert len(result) == 2