From 6208dae869d74973453be6c04e16db832489488d Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Mon, 16 Oct 2017 14:20:37 +0200 Subject: [PATCH] pkgset: Cherry-pick packages from Koji when we know already what packages will end up in compose Merges: https://pagure.io/pungi/pull-request/763 Signed-off-by: Jan Kaluza --- pungi/phases/gather/__init__.py | 58 ++++++++++++++- pungi/phases/pkgset/pkgsets.py | 15 +++- pungi/phases/pkgset/sources/source_koji.py | 18 ++++- pungi/phases/pkgset/sources/source_repos.py | 21 +----- tests/test_gather_phase.py | 82 +++++++++++++++++++++ tests/test_pkgset_pkgsets.py | 25 +++++++ tests/test_pkgset_source_koji.py | 21 ++++++ 7 files changed, 216 insertions(+), 24 deletions(-) diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index c8551a79..9e84e6d4 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -335,12 +335,13 @@ def write_prepopulate_file(compose): shutil.rmtree(tmp_dir) -def get_prepopulate_packages(compose, arch, variant): +def get_prepopulate_packages(compose, arch, variant, include_arch=True): """Read prepopulate file and return list of packages for given tree. If ``variant`` is ``None``, all variants in the file are considered. The result of this function is a set of strings of format - ``package_name.arch``. + ``package_name.arch``. If ``include_arch`` is False, the ".arch" suffix + is not included in packages in returned list. """ result = set() @@ -361,7 +362,10 @@ def get_prepopulate_packages(compose, arch, variant): raise ValueError( "Incompatible package arch '%s' for tree arch '%s' in prepopulate package '%s'" % (pkg_arch, arch, pkg_name)) - result.add(i) + if include_arch: + result.add(i) + else: + result.add(pkg_name) return result @@ -491,3 +495,51 @@ def get_system_release_packages(compose, arch, variant, package_sets): filter_packages.add((pkg.name, None)) return packages, filter_packages + + +def get_packages_to_gather(compose, arch=None, variant=None, include_arch=True, + include_prepopulated=False): + """ + Returns the list of names of packages and list of names of groups which + would be included in a compose as GATHER phase result. + This works only for "comps" or "json" gather_source. For "module" + gather_source, this always return an empty list, because it is not clear + what packages will end up in a compose before the gather phase is run. + + :param str arch: Arch to return packages for. If not set, returns packages + for all arches. + :param Variant variant: Variant to return packages for, If not set, returns + packages for all variants of a compose. + :param include_arch: When True, the arch of package will be included in + returned list as ["pkg_name.arch", ...]. Otherwise only + ["pkg_name", ...] is returned. + :param include_prepopulated: When True, the prepopulated packages will + be included in a list of packages. + """ + if compose.conf["gather_source"] == "module": + return [] + + arches = [arch] if arch else compose.get_arches() + + GatherSource = get_gather_source(compose.conf["gather_source"]) + src = GatherSource(compose) + + packages = set([]) + groups = set([]) + for arch in arches: + pkgs, grps = src(arch, variant) + groups = groups.union(set(grps)) + + additional_packages = get_additional_packages(compose, arch, None) + for pkg_name, pkg_arch in pkgs | additional_packages: + if not include_arch or pkg_arch is None: + packages.add(pkg_name) + else: + packages.add("%s.%s" % (pkg_name, pkg_arch)) + + if include_prepopulated: + prepopulated = get_prepopulate_packages( + compose, arch, variant, include_arch) + packages = packages.union(prepopulated) + + return list(packages), list(groups) diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index c9ba6426..6363f549 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -208,10 +208,13 @@ class FilelistPackageSet(PackageSetBase): class KojiPackageSet(PackageSetBase): - def __init__(self, koji_wrapper, sigkey_ordering, arches=None, logger=None): + def __init__(self, koji_wrapper, sigkey_ordering, arches=None, logger=None, + packages=None): super(KojiPackageSet, self).__init__(sigkey_ordering=sigkey_ordering, arches=arches, logger=logger) self.koji_wrapper = koji_wrapper + # Names of packages to look for in the Koji tag. + self.packages = packages def __getstate__(self): result = self.__dict__.copy() @@ -281,6 +284,7 @@ class KojiPackageSet(PackageSetBase): builds_by_id.setdefault(build_info["build_id"], build_info) skipped_arches = [] + skipped_packages_count = 0 for rpm_info in rpms: if self.arches and rpm_info["arch"] not in self.arches: if rpm_info["arch"] not in skipped_arches: @@ -288,11 +292,20 @@ class KojiPackageSet(PackageSetBase): skipped_arches.append(rpm_info["arch"]) continue + if self.packages and rpm_info['name'] not in self.packages: + skipped_packages_count += 1 + continue + build_info = builds_by_id[rpm_info["build_id"]] if rpm_info["arch"] in ("src", "nosrc"): result_srpms.append((rpm_info, build_info)) else: result_rpms.append((rpm_info, build_info)) + + if skipped_packages_count: + self.log_debug("Skipped %d packages, not marked as to be " + "included in a compose." % skipped_packages_count) + result = self.read_packages(result_rpms, result_srpms) # Create a log with package NEVRAs and the tag they are coming from diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index ab08fe5b..dd6060cb 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -21,11 +21,13 @@ import re from kobo.shortcuts import force_list import pungi.wrappers.kojiwrapper +from pungi.wrappers.comps import CompsWrapper import pungi.phases.pkgset.pkgsets from pungi.arch import get_valid_arches from pungi.util import is_arch_multilib, retry from pungi.phases.pkgset.common import create_arch_repos, create_global_repo, populate_arch_pkgsets +from pungi.phases.gather import get_packages_to_gather import pungi.phases.pkgset.source @@ -163,6 +165,20 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): # List of compose_tags per variant variant_tags = {} + # In case we use "nodeps" gather_method, we might now the final list of + # packages which will end up in the compose even now, so instead of + # reading all the packages from Koji tag, we can just cherry-pick the ones + # which are really needed to do the compose and safe lot of time and + # resources here. + packages_to_gather = [] + if compose.conf["gather_method"] == "nodeps": + packages_to_gather, groups = get_packages_to_gather( + compose, include_arch=False, include_prepopulated=True) + if groups: + comps = CompsWrapper(compose.paths.work.comps()) + for group in groups: + packages_to_gather += comps.get_packages(group) + session = get_pdc_client_session(compose) for variant in compose.all_variants.values(): variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( @@ -219,7 +235,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): "'%s'" % compose_tag) pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( koji_wrapper, compose.conf["sigkeys"], logger=compose._logger, - arches=all_arches) + arches=all_arches, packages=packages_to_gather) # Create a filename for log with package-to-tag mapping. The tag # name is included in filename, so any slashes in it are replaced # with underscores just to be safe. diff --git a/pungi/phases/pkgset/sources/source_repos.py b/pungi/phases/pkgset/sources/source_repos.py index 83748a96..77955e3b 100644 --- a/pungi/phases/pkgset/sources/source_repos.py +++ b/pungi/phases/pkgset/sources/source_repos.py @@ -25,7 +25,7 @@ from pungi.util import makedirs, is_arch_multilib from pungi.wrappers.pungi import PungiWrapper from pungi.phases.pkgset.common import create_global_repo, create_arch_repos, populate_arch_pkgsets -from pungi.phases.gather import get_prepopulate_packages, get_additional_packages +from pungi.phases.gather import get_prepopulate_packages, get_packages_to_gather from pungi.linker import LinkerThread, LinkerPool @@ -160,24 +160,7 @@ def write_pungi_config(compose, arch, variant, repos=None, comps_repo=None, pack return compose.log_info(msg) - - # TODO move to a function - gather_source = "GatherSource%s" % compose.conf["gather_source"] - from pungi.phases.gather.source import GatherSourceContainer - import pungi.phases.gather.sources - GatherSourceContainer.register_module(pungi.phases.gather.sources) - container = GatherSourceContainer() - SourceClass = container[gather_source] - src = SourceClass(compose) - - packages = [] - pkgs, grps = src(arch, variant) - additional_packages = get_additional_packages(compose, arch, None) - for pkg_name, pkg_arch in pkgs | additional_packages: - if pkg_arch is None: - packages.append(pkg_name) - else: - packages.append("%s.%s" % (pkg_name, pkg_arch)) + packages, grps = get_packages_to_gather(compose, arch, variant) # include *all* packages providing system-release if "system-release" not in packages: diff --git a/tests/test_gather_phase.py b/tests/test_gather_phase.py index e953149e..14bde326 100644 --- a/tests/test_gather_phase.py +++ b/tests/test_gather_phase.py @@ -755,6 +755,17 @@ class TestGetPrepopulate(helpers.PungiTestCase): "bar.x86_64"] ) + def test_for_all_variants_include_arch_set_to_false(self): + helpers.copy_fixture('prepopulate.json', + os.path.join(self.topdir, 'work', 'global', 'prepopulate.json')) + self.assertItemsEqual( + gather.get_prepopulate_packages(self.compose, 'x86_64', None, + include_arch=False), + ["foo-common", + "foo", + "bar"] + ) + class TestGatherPhase(helpers.PungiTestCase): @mock.patch('pungi.phases.gather.link_files') @@ -797,3 +808,74 @@ class TestGatherPhase(helpers.PungiTestCase): self.assertEqual(gather_wrapper.call_args_list, []) self.assertTrue(os.path.isfile(os.path.join(self.topdir, 'compose', 'metadata', 'rpms.json'))) + + +class TestGetPackagesToGather(helpers.PungiTestCase): + def setUp(self): + super(TestGetPackagesToGather, self).setUp() + self.compose = helpers.DummyCompose(self.topdir, { + 'additional_packages': [ + ('.*', {'*': ['pkg', 'foo2.x86_64']}), + ] + }) + helpers.copy_fixture('prepopulate.json', + os.path.join(self.topdir, 'work', 'global', 'prepopulate.json')) + + @mock.patch('pungi.phases.gather.get_gather_source') + def test_all_arches(self, get_gather_source): + get_gather_source.return_value = mock.Mock( + return_value=mock.Mock(return_value=(set([('foo', None)]), set(['core']))) + ) + + packages, groups = gather.get_packages_to_gather(self.compose) + + self.assertItemsEqual(packages, ["foo", "foo2.x86_64", "pkg"]) + self.assertItemsEqual(groups, ["core"]) + + @mock.patch('pungi.phases.gather.get_gather_source') + def test_all_include_arch_set_to_false(self, get_gather_source): + get_gather_source.return_value = mock.Mock( + return_value=mock.Mock(return_value=(set([('foo', None)]), set(['core']))) + ) + + packages, groups = gather.get_packages_to_gather(self.compose, include_arch=False) + + self.assertItemsEqual(packages, ["foo", "foo2", "pkg"]) + self.assertItemsEqual(groups, ["core"]) + + @mock.patch('pungi.phases.gather.get_gather_source') + def test_all_include_prepopulated(self, get_gather_source): + get_gather_source.return_value = mock.Mock( + return_value=mock.Mock(return_value=(set([('foo', None)]), set(['core']))) + ) + + packages, groups = gather.get_packages_to_gather(self.compose, include_prepopulated=True) + + self.assertItemsEqual(packages, ["foo", "pkg", "foo-common.noarch", + "foo.x86_64", "foo.i686", "foo2.x86_64", + "bar.x86_64"]) + self.assertItemsEqual(groups, ["core"]) + + @mock.patch('pungi.phases.gather.get_gather_source') + def test_all_include_prepopulated_no_include_arch(self, get_gather_source): + get_gather_source.return_value = mock.Mock( + return_value=mock.Mock(return_value=(set([('foo', None)]), set(['core']))) + ) + + packages, groups = gather.get_packages_to_gather(self.compose, include_prepopulated=True, + include_arch=False) + + self.assertItemsEqual(packages, ["foo", "pkg", "foo-common", + "foo2", "bar"]) + self.assertItemsEqual(groups, ["core"]) + + @mock.patch('pungi.phases.gather.get_gather_source') + def test_all_one_arch(self, get_gather_source): + get_gather_source.return_value = mock.Mock( + return_value=mock.Mock(return_value=(set([('foo', None)]), set(['core']))) + ) + + packages, groups = gather.get_packages_to_gather(self.compose, "x86_64") + + self.assertItemsEqual(packages, ["foo", "pkg", "foo2.x86_64"]) + self.assertItemsEqual(groups, ["core"]) diff --git a/tests/test_pkgset_pkgsets.py b/tests/test_pkgset_pkgsets.py index 893e481d..926dd09f 100644 --- a/tests/test_pkgset_pkgsets.py +++ b/tests/test_pkgset_pkgsets.py @@ -275,6 +275,31 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase): self.assertRegexpMatches(str(ctx.exception), r'^RPM\(s\) not found for sigs: .+Check log for details.+') + def test_packages_attribute(self): + self._touch_files([ + 'rpms/pungi@4.1.3@3.fc25@noarch', + 'rpms/pungi@4.1.3@3.fc25@src', + 'rpms/bash@4.3.42@4.fc24@i686', + 'rpms/bash@4.3.42@4.fc24@x86_64', + 'rpms/bash@4.3.42@4.fc24@src', + 'rpms/bash-debuginfo@4.3.42@4.fc24@i686', + 'rpms/bash-debuginfo@4.3.42@4.fc24@x86_64', + ]) + + pkgset = pkgsets.KojiPackageSet(self.koji_wrapper, [None], + packages=["bash"]) + + result = pkgset.populate('f25', logfile=self.topdir + '/pkgset.log') + + self.assertEqual( + self.koji_wrapper.koji_proxy.mock_calls, + [mock.call.listTaggedRPMS('f25', event=None, inherit=True, latest=True)]) + + self.assertPkgsetEqual(result, + {'src': ['rpms/bash@4.3.42@4.fc24@src'], + 'i686': ['rpms/bash@4.3.42@4.fc24@i686'], + 'x86_64': ['rpms/bash@4.3.42@4.fc24@x86_64']}) + @mock.patch('kobo.pkgset.FileCache', new=MockFileCache) class TestMergePackageSets(PkgsetCompareMixin, unittest.TestCase): diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index 892aac2b..21112a45 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -167,6 +167,27 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase): [mock.call.save_file_list(self.topdir + '/work/global/package_list/global.conf', remove_path_prefix='/prefix')]) + @mock.patch('cPickle.dumps') + @mock.patch('pungi.phases.pkgset.pkgsets.KojiPackageSet.populate') + @mock.patch('pungi.phases.pkgset.pkgsets.KojiPackageSet.save_file_list') + def test_populate_packages_to_gather(self, save_file_list, popuplate, + pickle_dumps): + self.compose = helpers.DummyCompose(self.topdir, { + 'gather_method': 'nodeps', + 'gather_source': 'none', + 'pkgset_koji_tag': 'f25', + 'sigkeys': mock.Mock(), + 'additional_packages': [ + ('.*', {'*': ['pkg', 'foo.x86_64']}), + ] + }) + self.compose.DEBUG = False + pickle_dumps.return_value = 'DATA' + + pkgset = source_koji.populate_global_pkgset( + self.compose, self.koji_wrapper, '/prefix', 123456) + self.assertItemsEqual(pkgset.packages, ["pkg", "foo"]) + class TestGetPackageSetFromKoji(helpers.PungiTestCase): def setUp(self):