From 63a8b7b6c9882227f2ee9f24666b97cbdcf732d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 1 Aug 2019 17:01:16 +0200 Subject: [PATCH] Remove package whitelist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This was a workaround to make some packages from the global repo invisible for depsolving. This is now handled by packages being in different repos. We can select which repos are enabled at which point. This achieves the same result, but much faster. Signed-off-by: Lubomír Sedlář --- bin/pungi-gather | 1 - pungi/gather_dnf.py | 17 ------ pungi/ks.py | 15 ----- pungi/phases/gather/methods/method_deps.py | 2 +- pungi/wrappers/pungi.py | 3 +- tests/test_gather.py | 58 ------------------ tests/test_gather_method_deps.py | 68 ++++++++-------------- 7 files changed, 26 insertions(+), 138 deletions(-) diff --git a/bin/pungi-gather b/bin/pungi-gather index 2e5c114f..fe99612e 100755 --- a/bin/pungi-gather +++ b/bin/pungi-gather @@ -129,7 +129,6 @@ def main(persistdir, cachedir): gather_opts.multilib_whitelist = ksparser.handler.multilib_whitelist gather_opts.prepopulate = ksparser.handler.prepopulate gather_opts.fulltree_excludes = ksparser.handler.fulltree_excludes - gather_opts.package_whitelist = ksparser.handler.package_whitelist g = Gather(dnf_obj, gather_opts) diff --git a/pungi/gather_dnf.py b/pungi/gather_dnf.py index 1d6c510a..9b06a232 100644 --- a/pungi/gather_dnf.py +++ b/pungi/gather_dnf.py @@ -69,8 +69,6 @@ class GatherOptions(pungi.common.OptionsBase): # lookaside repos; packages will be flagged accordingly self.lookaside_repos = [] - self.package_whitelist = set() - self.merge_options(**kwargs) def __str__(self): @@ -85,7 +83,6 @@ class GatherOptions(pungi.common.OptionsBase): 'multilib_blacklist=%d items' % len(self.multilib_blacklist), 'multilib_whitelist=%d items' % len(self.multilib_whitelist), 'lookaside_repos=%s' % self.lookaside_repos, - 'package_whitelist=%d items' % len(self.package_whitelist), 'prepopulate=%d items' % len(self.prepopulate) ] return '[\n%s\n]' % '\n'.join(' ' + l for l in lines) @@ -404,20 +401,6 @@ class Gather(GatherBase): 'q_source_packages', 'q_native_debug_packages', 'q_multilib_debug_packages'] - if self.opts.package_whitelist: - with Profiler("Gather._apply_excludes():apply-package-whitelist'"): - to_keep = [] - for pattern in self.opts.package_whitelist: - nvr = parse_nvr(pattern) - try: - nvr['epoch'] = int(nvr.pop('epoch')) - except ValueError: - pass - to_keep.extend(self._query.filter(**nvr).run()) - - for queue in all_queues: - setattr(self, queue, getattr(self, queue).filter(pkg=to_keep).latest().apply()) - with Profiler("Gather._apply_excludes():exclude-queries"): for queue in all_queues: self._filter_queue(queue, exclude) diff --git a/pungi/ks.py b/pungi/ks.py index 517242aa..ecb8821f 100644 --- a/pungi/ks.py +++ b/pungi/ks.py @@ -127,19 +127,6 @@ class PrepopulateSection(pykickstart.sections.Section): self.handler.prepopulate.add(line) -class PackageWhitelistSection(pykickstart.sections.Section): - sectionOpen = "%package-whitelist" - - def handleLine(self, line): - if not self.handler: - return - - (h, s, t) = line.partition('#') - line = h.rstrip() - - self.handler.package_whitelist.add(line) - - class KickstartParser(pykickstart.parser.KickstartParser): def setupSections(self): pykickstart.parser.KickstartParser.setupSections(self) @@ -147,7 +134,6 @@ class KickstartParser(pykickstart.parser.KickstartParser): self.registerSection(MultilibBlacklistSection(self.handler)) self.registerSection(MultilibWhitelistSection(self.handler)) self.registerSection(PrepopulateSection(self.handler)) - self.registerSection(PackageWhitelistSection(self.handler)) def get_packages(self, dnf_obj): packages = set() @@ -208,7 +194,6 @@ class PungiHandler(HandlerClass): self.multilib_blacklist = set() self.multilib_whitelist = set() self.prepopulate = set() - self.package_whitelist = set() def get_ksparser(ks_path=None): diff --git a/pungi/phases/gather/methods/method_deps.py b/pungi/phases/gather/methods/method_deps.py index 7cfb0104..d6575c01 100644 --- a/pungi/phases/gather/methods/method_deps.py +++ b/pungi/phases/gather/methods/method_deps.py @@ -23,7 +23,7 @@ from kobo.rpmlib import parse_nvra from pungi.util import get_arch_variant_data, temp_dir from pungi.wrappers.pungi import PungiWrapper -from pungi.arch import tree_arch_to_yum_arch, get_valid_arches +from pungi.arch import tree_arch_to_yum_arch import pungi.phases.gather import pungi.phases.gather.method diff --git a/pungi/wrappers/pungi.py b/pungi/wrappers/pungi.py index 04110f47..106a0846 100644 --- a/pungi/wrappers/pungi.py +++ b/pungi/wrappers/pungi.py @@ -47,7 +47,7 @@ class PungiWrapper(object): exclude_packages=None, comps_repo=None, lookaside_repos=None, fulltree_excludes=None, multilib_blacklist=None, multilib_whitelist=None, - prepopulate=None, package_whitelist=None): + prepopulate=None): groups = groups or [] exclude_packages = exclude_packages or {} lookaside_repos = lookaside_repos or {} @@ -92,7 +92,6 @@ class PungiWrapper(object): _write_ks_section(kickstart, "multilib-blacklist", multilib_blacklist) _write_ks_section(kickstart, "multilib-whitelist", multilib_whitelist) _write_ks_section(kickstart, "prepopulate", prepopulate) - _write_ks_section(kickstart, "package-whitelist", package_whitelist) kickstart.close() diff --git a/tests/test_gather.py b/tests/test_gather.py index 0f6814f1..75bb21ef 100644 --- a/tests/test_gather.py +++ b/tests/test_gather.py @@ -1854,66 +1854,8 @@ class DNFDepsolvingTestCase(DepsolvingBase, unittest.TestCase): def test_bash_older(self): pass - def test_whitelist_old_version(self): - # There are two version of dummy-bash in the package set; let's - # whitelist only the older one and its dependencies. - packages = [ - "dummy-bash", - ] - package_whitelist = [ - "dummy-basesystem-10.0-6", - "dummy-bash-debuginfo-4.2.37-5", - "dummy-bash-4.2.37-5", - "dummy-filesystem-4.2.37-6", - "dummy-glibc-common-2.14-5", - "dummy-glibc-debuginfo-common-2.14-5", - "dummy-glibc-debuginfo-2.14-5", - "dummy-glibc-2.14-5", - ] - pkg_map = self.go(packages, None, greedy="none", package_whitelist=package_whitelist) - - self.assertNotIn("dummy-bash-4.2.37-5.i686.rpm", pkg_map["rpm"]) - self.assertNotIn("dummy-bash-4.2.37-6.i686.rpm", pkg_map["rpm"]) - self.assertNotIn("dummy-bash-4.2.37-6.x86_64.rpm", pkg_map["rpm"]) - - self.assertItemsEqual(pkg_map["rpm"], [ - "dummy-basesystem-10.0-6.noarch.rpm", - "dummy-bash-4.2.37-5.x86_64.rpm", - "dummy-filesystem-4.2.37-6.x86_64.rpm", - "dummy-glibc-2.14-5.x86_64.rpm", - "dummy-glibc-common-2.14-5.x86_64.rpm", - ]) - self.assertItemsEqual(pkg_map["srpm"], [ - "dummy-basesystem-10.0-6.src.rpm", - "dummy-bash-4.2.37-5.src.rpm", - "dummy-filesystem-4.2.37-6.src.rpm", - "dummy-glibc-2.14-5.src.rpm", - ]) - self.assertItemsEqual(pkg_map["debuginfo"], [ - "dummy-bash-debuginfo-4.2.37-5.x86_64.rpm", - "dummy-glibc-debuginfo-2.14-5.x86_64.rpm", - "dummy-glibc-debuginfo-common-2.14-5.x86_64.rpm", - ]) - def test_firefox_selfhosting_with_krb5_lookaside(self): super(DNFDepsolvingTestCase, self).test_firefox_selfhosting_with_krb5_lookaside() self.assertFlags("dummy-krb5-devel-1.10-5.x86_64", [PkgFlag.lookaside]) self.assertFlags("dummy-krb5-1.10-5.src", [PkgFlag.lookaside]) self.assertFlags("dummy-krb5-debuginfo-1.10-5.x86_64", [PkgFlag.lookaside]) - - def test_package_whitelist(self): - packages = ['*'] - whitelist = [ - 'dummy-bash-4.2.37-6', - ] - - pkg_map = self.go(packages, None, package_whitelist=whitelist) - - self.assertItemsEqual(pkg_map["rpm"], [ - 'dummy-bash-4.2.37-6.x86_64.rpm', - ]) - self.assertItemsEqual(pkg_map["srpm"], [ - 'dummy-bash-4.2.37-6.src.rpm', - ]) - self.assertItemsEqual(pkg_map["debuginfo"], [ - ]) diff --git a/tests/test_gather_method_deps.py b/tests/test_gather_method_deps.py index 79a3d373..1225c9ba 100644 --- a/tests/test_gather_method_deps.py +++ b/tests/test_gather_method_deps.py @@ -14,13 +14,13 @@ class TestWritePungiConfig(helpers.PungiTestCase): def setUp(self): super(TestWritePungiConfig, self).setUp() self.compose = helpers.DummyCompose(self.topdir, {}) + self.package_sets = self._make_pkgset_phase(["p1"]).package_sets def assertWritten(self, PungiWrapper, **kwargs): wrapper = PungiWrapper.return_value self.assertEqual(wrapper.mock_calls, [mock.call.write_kickstart(**kwargs)]) - @helpers.unittest.skip("temporarily broken") @mock.patch('pungi.phases.gather.methods.method_deps.PungiWrapper') def test_correct(self, PungiWrapper): pkgs = [('pkg1', None), ('pkg2', 'x86_64')] @@ -30,19 +30,21 @@ class TestWritePungiConfig(helpers.PungiTestCase): black = mock.Mock() prepopulate = mock.Mock() fulltree = mock.Mock() - deps.write_pungi_config(self.compose, 'x86_64', self.compose.variants['Server'], - pkgs, grps, filter, white, black, - prepopulate=prepopulate, fulltree_excludes=fulltree) + deps.write_pungi_config( + self.compose, 'x86_64', self.compose.variants['Server'], + pkgs, grps, filter, white, black, + prepopulate=prepopulate, fulltree_excludes=fulltree, + package_sets=self.package_sets, + ) self.assertWritten(PungiWrapper, packages=['pkg1', 'pkg2.x86_64'], ks_path=self.topdir + '/work/x86_64/pungi/Server.x86_64.conf', lookaside_repos={}, multilib_whitelist=white, multilib_blacklist=black, groups=['grp1'], prepopulate=prepopulate, - repos={'pungi-repo': self.topdir + '/work/x86_64/repo', + repos={"pungi-repo-0": self.topdir + "/work/x86_64/repo/p1", 'comps-repo': self.topdir + '/work/x86_64/comps_repo_Server'}, exclude_packages=['pkg3', 'pkg4.x86_64'], - fulltree_excludes=fulltree, package_whitelist=set()) + fulltree_excludes=fulltree) - @helpers.unittest.skip("temporarily broken") @mock.patch("pungi.phases.gather.methods.method_deps.PungiWrapper") def test_duplicated_package_name(self, PungiWrapper): pkgs = [("pkg1", None), ("pkg1", "x86_64")] @@ -52,64 +54,42 @@ class TestWritePungiConfig(helpers.PungiTestCase): black = mock.Mock() prepopulate = mock.Mock() fulltree = mock.Mock() - deps.write_pungi_config(self.compose, "x86_64", self.compose.variants["Server"], - pkgs, grps, filter, white, black, - prepopulate=prepopulate, fulltree_excludes=fulltree) + deps.write_pungi_config( + self.compose, "x86_64", self.compose.variants["Server"], + pkgs, grps, filter, white, black, + prepopulate=prepopulate, fulltree_excludes=fulltree, + package_sets=self.package_sets, + ) self.assertWritten(PungiWrapper, packages=["pkg1", "pkg1.x86_64"], ks_path=self.topdir + "/work/x86_64/pungi/Server.x86_64.conf", lookaside_repos={}, multilib_whitelist=white, multilib_blacklist=black, groups=[], prepopulate=prepopulate, - repos={"pungi-repo": self.topdir + "/work/x86_64/repo", + repos={"pungi-repo-0": self.topdir + "/work/x86_64/repo/p1", "comps-repo": self.topdir + "/work/x86_64/comps_repo_Server"}, exclude_packages=["pkg2", "pkg2.x86_64"], - fulltree_excludes=fulltree, package_whitelist=set()) + fulltree_excludes=fulltree) - @helpers.unittest.skip("temporarily broken") @mock.patch('pungi.phases.gather.get_lookaside_repos') @mock.patch('pungi.phases.gather.methods.method_deps.PungiWrapper') def test_with_lookaside(self, PungiWrapper, glr): glr.return_value = ['http://example.com/repo'] pkgs = [('pkg1', None)] - deps.write_pungi_config(self.compose, 'x86_64', self.compose.variants['Server'], - pkgs, [], [], [], []) + deps.write_pungi_config( + self.compose, 'x86_64', self.compose.variants['Server'], + pkgs, [], [], [], [], + package_sets=self.package_sets, + ) self.assertWritten(PungiWrapper, packages=['pkg1'], ks_path=self.topdir + '/work/x86_64/pungi/Server.x86_64.conf', lookaside_repos={'lookaside-repo-0': 'http://example.com/repo'}, multilib_whitelist=[], multilib_blacklist=[], groups=[], prepopulate=None, - repos={'pungi-repo': self.topdir + '/work/x86_64/repo', + repos={"pungi-repo-0": self.topdir + "/work/x86_64/repo/p1", 'comps-repo': self.topdir + '/work/x86_64/comps_repo_Server'}, - exclude_packages=[], fulltree_excludes=None, - package_whitelist=set()) + exclude_packages=[], fulltree_excludes=None) self.assertEqual(glr.call_args_list, [mock.call(self.compose, 'x86_64', self.compose.variants['Server'])]) - @helpers.unittest.skip("temporarily broken") - @mock.patch('pungi.phases.gather.methods.method_deps.PungiWrapper') - def test_with_whitelist(self, PungiWrapper): - pkgs = [('pkg1', None), ('pkg2', 'x86_64')] - grps = ['grp1'] - filter = [('pkg3', None), ('pkg4', 'x86_64')] - mock_rpm = mock.Mock(version='1.0.0', release='1', epoch=0) - mock_rpm.name = 'pkg' - self.compose.variants['Server'].pkgset.rpms_by_arch['x86_64'] = [mock_rpm] - white = mock.Mock() - black = mock.Mock() - prepopulate = mock.Mock() - fulltree = mock.Mock() - deps.write_pungi_config(self.compose, 'x86_64', self.compose.variants['Server'], - pkgs, grps, filter, white, black, - prepopulate=prepopulate, fulltree_excludes=fulltree) - self.assertWritten(PungiWrapper, packages=['pkg1', 'pkg2.x86_64'], - ks_path=self.topdir + '/work/x86_64/pungi/Server.x86_64.conf', - lookaside_repos={}, multilib_whitelist=white, multilib_blacklist=black, - groups=['grp1'], prepopulate=prepopulate, - repos={'pungi-repo': self.topdir + '/work/x86_64/repo', - 'comps-repo': self.topdir + '/work/x86_64/comps_repo_Server'}, - exclude_packages=['pkg3', 'pkg4.x86_64'], - fulltree_excludes=fulltree, - package_whitelist=set(['pkg-0:1.0.0-1'])) - @mock.patch('pungi.phases.gather.methods.method_deps.PungiWrapper') def test_without_input(self, PungiWrapper): with self.assertRaises(RuntimeError) as ctx: