diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index e04d954b..d79566a0 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -828,6 +828,8 @@ class KojiMockPackageSet(PackageSetBase): cache_region=None, extra_builds=None, extra_tasks=None, + signed_packages_retries=0, + signed_packages_wait=30, ): """ Creates new KojiPackageSet. @@ -876,10 +878,11 @@ class KojiMockPackageSet(PackageSetBase): self.extra_builds = extra_builds or [] self.extra_tasks = extra_tasks or [] self.reuse = None + self.signed_packages_retries = signed_packages_retries + self.signed_packages_wait = signed_packages_wait def __getstate__(self): result = self.__dict__.copy() - result["koji_profile"] = self.koji_wrapper.profile del result["koji_wrapper"] del result["_logger"] if "cache_region" in result: @@ -887,8 +890,6 @@ class KojiMockPackageSet(PackageSetBase): return result def __setstate__(self, data): - koji_profile = data.pop("koji_profile") - self.koji_wrapper = pungi.wrappers.kojiwrapper.KojiWrapper(koji_profile) self._logger = None self.__dict__.update(data) @@ -1202,20 +1203,26 @@ class KojiMockPackageSet(PackageSetBase): % (old_koji_event, koji_event) ) changed = self.koji_proxy.queryHistory( - tables=["tag_listing"], tag=tag, afterEvent=old_koji_event + tables=["tag_listing", "tag_inheritance"], + tag=tag, + afterEvent=min(koji_event, old_koji_event), + beforeEvent=max(koji_event, old_koji_event) + 1, ) if changed["tag_listing"]: self.log_debug("Builds under tag %s changed. Can't reuse." % tag) return False + if changed["tag_inheritance"]: + self.log_debug("Tag inheritance %s changed. Can't reuse." % tag) + return False if inherit: inherit_tags = self.koji_proxy.getFullInheritance(tag, koji_event) for t in inherit_tags: changed = self.koji_proxy.queryHistory( - tables=["tag_listing"], + tables=["tag_listing", "tag_inheritance"], tag=t["name"], - afterEvent=old_koji_event, - beforeEvent=koji_event + 1, + afterEvent=min(koji_event, old_koji_event), + beforeEvent=max(koji_event, old_koji_event) + 1, ) if changed["tag_listing"]: self.log_debug( @@ -1223,6 +1230,9 @@ class KojiMockPackageSet(PackageSetBase): % t["name"] ) return False + if changed["tag_inheritance"]: + self.log_debug("Tag inheritance %s changed. Can't reuse." % tag) + return False repo_dir = compose.paths.work.pkgset_repo(tag, create_dir=False) old_repo_dir = compose.paths.old_compose_path(repo_dir) diff --git a/pungi/phases/pkgset/sources/source_kojimock.py b/pungi/phases/pkgset/sources/source_kojimock.py index b3fcd7f0..5aaeb056 100644 --- a/pungi/phases/pkgset/sources/source_kojimock.py +++ b/pungi/phases/pkgset/sources/source_kojimock.py @@ -35,7 +35,13 @@ import pungi.wrappers.kojiwrapper from pungi.wrappers.comps import CompsWrapper from pungi.wrappers.mbs import MBSWrapper import pungi.phases.pkgset.pkgsets -from pungi.util import retry, get_arch_variant_data, get_variant_data +from pungi.util import ( + retry, + get_arch_variant_data, + get_variant_data, + read_single_module_stream_from_string, + read_single_module_stream_from_file, +) from pungi.module_util import Modulemd from pungi.phases.pkgset.common import MaterializedPackageSet, get_all_arches @@ -212,7 +218,12 @@ def get_pkgset_from_koji(compose, koji_wrapper, path_prefix): def _add_module_to_variant( - koji_wrapper, variant, build, add_to_variant_modules=False, compose=None + koji_wrapper, + variant, + build, + add_to_variant_modules=False, + compose=None, + exclude_module_ns=None, ): """ Adds module defined by Koji build info to variant. @@ -222,6 +233,7 @@ def _add_module_to_variant( :param bool add_to_variant_modules: Adds the modules also to variant.modules. :param compose: Compose object to get filters from + :param list exclude_module_ns: Module name:stream which will be excluded. """ mmds = {} archives = koji_wrapper.koji_proxy.listArchives(build["id"]) @@ -246,6 +258,10 @@ def _add_module_to_variant( info = build["extra"]["typeinfo"]["module"] nsvc = "%(name)s:%(stream)s:%(version)s:%(context)s" % info + ns = "%(name)s:%(stream)s" % info + + if exclude_module_ns and ns in exclude_module_ns: + return added = False @@ -253,12 +269,14 @@ def _add_module_to_variant( if _is_filtered_out(compose, variant, arch, info["name"], info["stream"]): compose.log_debug("Module %s is filtered from %s.%s", nsvc, variant, arch) continue - + filename = "modulemd.%s.txt" % arch try: - mmd = Modulemd.ModuleStream.read_file( - mmds["modulemd.%s.txt" % arch], strict=True + mod_stream = read_single_module_stream_from_file( + mmds[filename], compose, arch, build ) - variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd + if mod_stream: + added = True + variant.arch_mmds.setdefault(arch, {})[nsvc] = mod_stream added = True except KeyError: # There is no modulemd for this arch. This could mean an arch was @@ -344,8 +362,8 @@ def _add_scratch_modules_to_variant( tag_to_mmd.setdefault(tag, {}) for arch in variant.arches: try: - mmd = Modulemd.ModuleStream.read_string( - final_modulemd[arch], strict=True + mmd = read_single_module_stream_from_string( + final_modulemd[arch] ) variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd except KeyError: diff --git a/tests/fixtures/mmds/modules/fake_arch/module:master-20190318-abcdef b/tests/fixtures/mmds/modules/fake_arch/module:master-20190318-abcdef new file mode 100644 index 00000000..e03147d2 --- /dev/null +++ b/tests/fixtures/mmds/modules/fake_arch/module:master-20190318-abcdef @@ -0,0 +1,20 @@ +--- +document: modulemd +version: 2 +data: + name: module + stream: master + version: 20190318 + context: abcdef + arch: armhfp + summary: Dummy module + description: Dummy module + license: + module: + - Beerware + content: + - Beerware + artifacts: + rpms: + - foobar-0:1.0-1.noarch +... diff --git a/tests/test_pkgset_common.py b/tests/test_pkgset_common.py index 4e4ae9fe..9209e408 100755 --- a/tests/test_pkgset_common.py +++ b/tests/test_pkgset_common.py @@ -129,16 +129,16 @@ class TestMaterializedPkgsetCreate(helpers.PungiTestCase): amm.assert_called_once() self.assertEqual( - amm.mock_calls[0].args[1], os.path.join(self.topdir, "work/x86_64/repo/foo") + amm.mock_calls[0][1][1], os.path.join(self.topdir, "work/x86_64/repo/foo") ) - self.assertIsInstance(amm.mock_calls[0].args[2], Modulemd.ModuleIndex) - self.assertIsNotNone(amm.mock_calls[0].args[2].get_module("mod_name")) + self.assertIsInstance(amm.mock_calls[0][1][2], Modulemd.ModuleIndex) + self.assertIsNotNone(amm.mock_calls[0][1][2].get_module("mod_name")) # Check if proper Index is used by add_modular_metadata self.assertIsNotNone( - amm.mock_calls[0].args[2].get_module("mod_name").get_obsoletes() + amm.mock_calls[0][1][2].get_module("mod_name").get_obsoletes() ) self.assertEqual( - amm.mock_calls[0].args[3], + amm.mock_calls[0][1][3], os.path.join(self.topdir, "logs/x86_64/arch_repo_modulemd.foo.x86_64.log"), ) diff --git a/tests/test_pkgset_pkgsets.py b/tests/test_pkgset_pkgsets.py index 8513b956..8452d653 100644 --- a/tests/test_pkgset_pkgsets.py +++ b/tests/test_pkgset_pkgsets.py @@ -312,13 +312,9 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase): ) self.assertRegex(str(ctx.exception), figure) - @ddt.data( - pkgsets.KojiPackageSet, - pkgsets.KojiMockPackageSet, - ) @mock.patch("os.path.isfile") @mock.patch("time.sleep") - def test_find_signed_after_wait(self, sleep, isfile, package_set): + def test_find_signed_after_wait(self, sleep, isfile): checked_files = set() def check_file(path): @@ -331,7 +327,7 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase): isfile.side_effect = check_file fst_key, snd_key = ["cafebabe", "deadbeef"] - pkgset = package_set( + pkgset = pkgsets.KojiPackageSet( "pkgset", self.koji_wrapper, [fst_key, snd_key], @@ -411,13 +407,9 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase): r"^RPM\(s\) not found for sigs: .+Check log for details.+", ) - @ddt.data( - pkgsets.KojiPackageSet, - pkgsets.KojiMockPackageSet, - ) @mock.patch("time.sleep") - def test_can_not_find_signed_package_with_retries(self, time, package_set): - pkgset = package_set( + def test_can_not_find_signed_package_with_retries(self, time): + pkgset = pkgsets.KojiPackageSet( "pkgset", self.koji_wrapper, ["cafebabe"], @@ -767,8 +759,8 @@ class TestReuseKojiPkgset(helpers.PungiTestCase): mock_old_topdir.return_value = self.old_compose_dir self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1]) self.koji_wrapper.koji_proxy.queryHistory.side_effect = [ - {"tag_listing": []}, - {"tag_listing": [{}]}, + {"tag_listing": [], "tag_inheritance": []}, + {"tag_listing": [{}], "tag_inheritance": []}, ] self.koji_wrapper.koji_proxy.getFullInheritance.return_value = [ {"name": self.inherited_tag} @@ -795,7 +787,9 @@ class TestReuseKojiPkgset(helpers.PungiTestCase): def test_reuse_failed_load_reuse_file(self, mock_old_topdir, mock_exists): mock_old_topdir.return_value = self.old_compose_dir self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1]) - self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": []} + self.koji_wrapper.koji_proxy.queryHistory.return_value = { + "tag_listing": [], "tag_inheritance": [] + } self.koji_wrapper.koji_proxy.getFullInheritance.return_value = [] self.pkgset.load_old_file_cache = mock.Mock( side_effect=Exception("unknown error") @@ -827,7 +821,9 @@ class TestReuseKojiPkgset(helpers.PungiTestCase): def test_reuse_criteria_not_match(self, mock_old_topdir, mock_exists): mock_old_topdir.return_value = self.old_compose_dir self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1]) - self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": []} + self.koji_wrapper.koji_proxy.queryHistory.return_value = { + "tag_listing": [], "tag_inheritance": [] + } self.koji_wrapper.koji_proxy.getFullInheritance.return_value = [] self.pkgset.load_old_file_cache = mock.Mock( return_value={"allow_invalid_sigkeys": True} @@ -866,7 +862,9 @@ class TestReuseKojiPkgset(helpers.PungiTestCase): def test_reuse_pkgset(self, mock_old_topdir, mock_exists, mock_copy_all): mock_old_topdir.return_value = self.old_compose_dir self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1]) - self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": []} + self.koji_wrapper.koji_proxy.queryHistory.return_value = { + "tag_listing": [], "tag_inheritance": [] + } self.koji_wrapper.koji_proxy.getFullInheritance.return_value = [] self.pkgset.load_old_file_cache = mock.Mock( return_value={ diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index 61b45700..7fb0f86d 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -818,7 +818,7 @@ class TestAddModuleToVariantForKojiMock(helpers.PungiTestCase): super(TestAddModuleToVariantForKojiMock, self).setUp() self.koji = mock.Mock() self.koji.koji_module.pathinfo.typedir.return_value = "/koji" - self.koji.koji_module.pathinfo.topdir = "/mnt/koji" + self.koji.koji_module.pathinfo.topdir = MMDS_DIR files = ["modulemd.x86_64.txt", "modulemd.armv7hl.txt", "modulemd.txt"] self.koji.koji_proxy.listArchives.return_value = [ {"btype": "module", "filename": fname} for fname in files