Remove package whitelist

This was a workaround to make some packages from the global repo
invisible for depsolving. This is now handled by packages being in
different repos. We can select which repos are enabled at which point.
This achieves the same result, but much faster.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

bin/pungi-gather

@@ -129,7 +129,6 @@ def main(persistdir, cachedir):
     gather_opts.multilib_whitelist = ksparser.handler.multilib_whitelist
     gather_opts.prepopulate = ksparser.handler.prepopulate
     gather_opts.fulltree_excludes = ksparser.handler.fulltree_excludes
-    gather_opts.package_whitelist = ksparser.handler.package_whitelist
 
     g = Gather(dnf_obj, gather_opts)
 

pungi/gather_dnf.py

@@ -69,8 +69,6 @@ class GatherOptions(pungi.common.OptionsBase):
         # lookaside repos; packages will be flagged accordingly
         self.lookaside_repos = []
 
-        self.package_whitelist = set()
-
         self.merge_options(**kwargs)
 
     def __str__(self):
@@ -85,7 +83,6 @@ class GatherOptions(pungi.common.OptionsBase):
             'multilib_blacklist=%d items' % len(self.multilib_blacklist),
             'multilib_whitelist=%d items' % len(self.multilib_whitelist),
             'lookaside_repos=%s' % self.lookaside_repos,
-            'package_whitelist=%d items' % len(self.package_whitelist),
             'prepopulate=%d items' % len(self.prepopulate)
         ]
         return '[\n%s\n]' % '\n'.join('    ' + l for l in lines)
@@ -404,20 +401,6 @@ class Gather(GatherBase):
                       'q_source_packages', 'q_native_debug_packages',
                       'q_multilib_debug_packages']
 
-        if self.opts.package_whitelist:
-            with Profiler("Gather._apply_excludes():apply-package-whitelist'"):
-                to_keep = []
-                for pattern in self.opts.package_whitelist:
-                    nvr = parse_nvr(pattern)
-                    try:
-                        nvr['epoch'] = int(nvr.pop('epoch'))
-                    except ValueError:
-                        pass
-                    to_keep.extend(self._query.filter(**nvr).run())
-
-                for queue in all_queues:
-                    setattr(self, queue, getattr(self, queue).filter(pkg=to_keep).latest().apply())
-
         with Profiler("Gather._apply_excludes():exclude-queries"):
             for queue in all_queues:
                 self._filter_queue(queue, exclude)

pungi/ks.py

@@ -127,19 +127,6 @@ class PrepopulateSection(pykickstart.sections.Section):
         self.handler.prepopulate.add(line)
 
 
-class PackageWhitelistSection(pykickstart.sections.Section):
-    sectionOpen = "%package-whitelist"
-
-    def handleLine(self, line):
-        if not self.handler:
-            return
-
-        (h, s, t) = line.partition('#')
-        line = h.rstrip()
-
-        self.handler.package_whitelist.add(line)
-
-
 class KickstartParser(pykickstart.parser.KickstartParser):
     def setupSections(self):
         pykickstart.parser.KickstartParser.setupSections(self)
@@ -147,7 +134,6 @@ class KickstartParser(pykickstart.parser.KickstartParser):
         self.registerSection(MultilibBlacklistSection(self.handler))
         self.registerSection(MultilibWhitelistSection(self.handler))
         self.registerSection(PrepopulateSection(self.handler))
-        self.registerSection(PackageWhitelistSection(self.handler))
 
     def get_packages(self, dnf_obj):
         packages = set()
@@ -208,7 +194,6 @@ class PungiHandler(HandlerClass):
         self.multilib_blacklist = set()
         self.multilib_whitelist = set()
         self.prepopulate = set()
-        self.package_whitelist = set()
 
 
 def get_ksparser(ks_path=None):

pungi/phases/gather/methods/method_deps.py

@@ -23,7 +23,7 @@ from kobo.rpmlib import parse_nvra
 from pungi.util import get_arch_variant_data, temp_dir
 from pungi.wrappers.pungi import PungiWrapper
 
-from pungi.arch import tree_arch_to_yum_arch, get_valid_arches
+from pungi.arch import tree_arch_to_yum_arch
 import pungi.phases.gather
 
 import pungi.phases.gather.method

pungi/wrappers/pungi.py

@@ -47,7 +47,7 @@ class PungiWrapper(object):
                         exclude_packages=None, comps_repo=None,
                         lookaside_repos=None, fulltree_excludes=None,
                         multilib_blacklist=None, multilib_whitelist=None,
-                        prepopulate=None, package_whitelist=None):
+                        prepopulate=None):
         groups = groups or []
         exclude_packages = exclude_packages or {}
         lookaside_repos = lookaside_repos or {}
@@ -92,7 +92,6 @@ class PungiWrapper(object):
         _write_ks_section(kickstart, "multilib-blacklist", multilib_blacklist)
         _write_ks_section(kickstart, "multilib-whitelist", multilib_whitelist)
         _write_ks_section(kickstart, "prepopulate", prepopulate)
-        _write_ks_section(kickstart, "package-whitelist", package_whitelist)
 
         kickstart.close()
 

tests/test_gather.py

@@ -1854,66 +1854,8 @@ class DNFDepsolvingTestCase(DepsolvingBase, unittest.TestCase):
     def test_bash_older(self):
         pass
 
-    def test_whitelist_old_version(self):
-        # There are two version of dummy-bash in the package set; let's
-        # whitelist only the older one and its dependencies.
-        packages = [
-            "dummy-bash",
-        ]
-        package_whitelist = [
-            "dummy-basesystem-10.0-6",
-            "dummy-bash-debuginfo-4.2.37-5",
-            "dummy-bash-4.2.37-5",
-            "dummy-filesystem-4.2.37-6",
-            "dummy-glibc-common-2.14-5",
-            "dummy-glibc-debuginfo-common-2.14-5",
-            "dummy-glibc-debuginfo-2.14-5",
-            "dummy-glibc-2.14-5",
-        ]
-        pkg_map = self.go(packages, None, greedy="none", package_whitelist=package_whitelist)
-
-        self.assertNotIn("dummy-bash-4.2.37-5.i686.rpm", pkg_map["rpm"])
-        self.assertNotIn("dummy-bash-4.2.37-6.i686.rpm", pkg_map["rpm"])
-        self.assertNotIn("dummy-bash-4.2.37-6.x86_64.rpm", pkg_map["rpm"])
-
-        self.assertItemsEqual(pkg_map["rpm"], [
-            "dummy-basesystem-10.0-6.noarch.rpm",
-            "dummy-bash-4.2.37-5.x86_64.rpm",
-            "dummy-filesystem-4.2.37-6.x86_64.rpm",
-            "dummy-glibc-2.14-5.x86_64.rpm",
-            "dummy-glibc-common-2.14-5.x86_64.rpm",
-        ])
-        self.assertItemsEqual(pkg_map["srpm"], [
-            "dummy-basesystem-10.0-6.src.rpm",
-            "dummy-bash-4.2.37-5.src.rpm",
-            "dummy-filesystem-4.2.37-6.src.rpm",
-            "dummy-glibc-2.14-5.src.rpm",
-        ])
-        self.assertItemsEqual(pkg_map["debuginfo"], [
-            "dummy-bash-debuginfo-4.2.37-5.x86_64.rpm",
-            "dummy-glibc-debuginfo-2.14-5.x86_64.rpm",
-            "dummy-glibc-debuginfo-common-2.14-5.x86_64.rpm",
-        ])
-
     def test_firefox_selfhosting_with_krb5_lookaside(self):
         super(DNFDepsolvingTestCase, self).test_firefox_selfhosting_with_krb5_lookaside()
         self.assertFlags("dummy-krb5-devel-1.10-5.x86_64", [PkgFlag.lookaside])
         self.assertFlags("dummy-krb5-1.10-5.src", [PkgFlag.lookaside])
         self.assertFlags("dummy-krb5-debuginfo-1.10-5.x86_64", [PkgFlag.lookaside])
-
-    def test_package_whitelist(self):
-        packages = ['*']
-        whitelist = [
-            'dummy-bash-4.2.37-6',
-        ]
-
-        pkg_map = self.go(packages, None, package_whitelist=whitelist)
-
-        self.assertItemsEqual(pkg_map["rpm"], [
-            'dummy-bash-4.2.37-6.x86_64.rpm',
-        ])
-        self.assertItemsEqual(pkg_map["srpm"], [
-            'dummy-bash-4.2.37-6.src.rpm',
-        ])
-        self.assertItemsEqual(pkg_map["debuginfo"], [
-        ])

tests/test_gather_method_deps.py

@@ -14,13 +14,13 @@ class TestWritePungiConfig(helpers.PungiTestCase):
     def setUp(self):
         super(TestWritePungiConfig, self).setUp()
         self.compose = helpers.DummyCompose(self.topdir, {})
+        self.package_sets = self._make_pkgset_phase(["p1"]).package_sets
 
     def assertWritten(self, PungiWrapper, **kwargs):
         wrapper = PungiWrapper.return_value
         self.assertEqual(wrapper.mock_calls,
                          [mock.call.write_kickstart(**kwargs)])
 
-    @helpers.unittest.skip("temporarily broken")
     @mock.patch('pungi.phases.gather.methods.method_deps.PungiWrapper')
     def test_correct(self, PungiWrapper):
         pkgs = [('pkg1', None), ('pkg2', 'x86_64')]
@@ -30,19 +30,21 @@ class TestWritePungiConfig(helpers.PungiTestCase):
         black = mock.Mock()
         prepopulate = mock.Mock()
         fulltree = mock.Mock()
-        deps.write_pungi_config(self.compose, 'x86_64', self.compose.variants['Server'],
+        deps.write_pungi_config(
+            self.compose, 'x86_64', self.compose.variants['Server'],
             pkgs, grps, filter, white, black,
-                                prepopulate=prepopulate, fulltree_excludes=fulltree)
+            prepopulate=prepopulate, fulltree_excludes=fulltree,
+            package_sets=self.package_sets,
+        )
         self.assertWritten(PungiWrapper, packages=['pkg1', 'pkg2.x86_64'],
                            ks_path=self.topdir + '/work/x86_64/pungi/Server.x86_64.conf',
                            lookaside_repos={}, multilib_whitelist=white, multilib_blacklist=black,
                            groups=['grp1'], prepopulate=prepopulate,
-                           repos={'pungi-repo': self.topdir + '/work/x86_64/repo',
+                           repos={"pungi-repo-0": self.topdir + "/work/x86_64/repo/p1",
                                   'comps-repo': self.topdir + '/work/x86_64/comps_repo_Server'},
                            exclude_packages=['pkg3', 'pkg4.x86_64'],
-                           fulltree_excludes=fulltree, package_whitelist=set())
+                           fulltree_excludes=fulltree)
 
-    @helpers.unittest.skip("temporarily broken")
     @mock.patch("pungi.phases.gather.methods.method_deps.PungiWrapper")
     def test_duplicated_package_name(self, PungiWrapper):
         pkgs = [("pkg1", None), ("pkg1", "x86_64")]
@@ -52,64 +54,42 @@ class TestWritePungiConfig(helpers.PungiTestCase):
         black = mock.Mock()
         prepopulate = mock.Mock()
         fulltree = mock.Mock()
-        deps.write_pungi_config(self.compose, "x86_64", self.compose.variants["Server"],
+        deps.write_pungi_config(
+            self.compose, "x86_64", self.compose.variants["Server"],
             pkgs, grps, filter, white, black,
-                                prepopulate=prepopulate, fulltree_excludes=fulltree)
+            prepopulate=prepopulate, fulltree_excludes=fulltree,
+            package_sets=self.package_sets,
+        )
         self.assertWritten(PungiWrapper, packages=["pkg1", "pkg1.x86_64"],
                            ks_path=self.topdir + "/work/x86_64/pungi/Server.x86_64.conf",
                            lookaside_repos={}, multilib_whitelist=white, multilib_blacklist=black,
                            groups=[], prepopulate=prepopulate,
-                           repos={"pungi-repo": self.topdir + "/work/x86_64/repo",
+                           repos={"pungi-repo-0": self.topdir + "/work/x86_64/repo/p1",
                                   "comps-repo": self.topdir + "/work/x86_64/comps_repo_Server"},
                            exclude_packages=["pkg2", "pkg2.x86_64"],
-                           fulltree_excludes=fulltree, package_whitelist=set())
+                           fulltree_excludes=fulltree)
 
-    @helpers.unittest.skip("temporarily broken")
     @mock.patch('pungi.phases.gather.get_lookaside_repos')
     @mock.patch('pungi.phases.gather.methods.method_deps.PungiWrapper')
     def test_with_lookaside(self, PungiWrapper, glr):
         glr.return_value = ['http://example.com/repo']
         pkgs = [('pkg1', None)]
-        deps.write_pungi_config(self.compose, 'x86_64', self.compose.variants['Server'],
+        deps.write_pungi_config(
-                                pkgs, [], [], [], [])
+            self.compose, 'x86_64', self.compose.variants['Server'],
+            pkgs, [], [], [], [],
+            package_sets=self.package_sets,
+        )
         self.assertWritten(PungiWrapper, packages=['pkg1'],
                            ks_path=self.topdir + '/work/x86_64/pungi/Server.x86_64.conf',
                            lookaside_repos={'lookaside-repo-0': 'http://example.com/repo'},
                            multilib_whitelist=[], multilib_blacklist=[],
                            groups=[], prepopulate=None,
-                           repos={'pungi-repo': self.topdir + '/work/x86_64/repo',
+                           repos={"pungi-repo-0": self.topdir + "/work/x86_64/repo/p1",
                                   'comps-repo': self.topdir + '/work/x86_64/comps_repo_Server'},
-                           exclude_packages=[], fulltree_excludes=None,
+                           exclude_packages=[], fulltree_excludes=None)
-                           package_whitelist=set())
         self.assertEqual(glr.call_args_list,
                          [mock.call(self.compose, 'x86_64', self.compose.variants['Server'])])
 
-    @helpers.unittest.skip("temporarily broken")
-    @mock.patch('pungi.phases.gather.methods.method_deps.PungiWrapper')
-    def test_with_whitelist(self, PungiWrapper):
-        pkgs = [('pkg1', None), ('pkg2', 'x86_64')]
-        grps = ['grp1']
-        filter = [('pkg3', None), ('pkg4', 'x86_64')]
-        mock_rpm = mock.Mock(version='1.0.0', release='1', epoch=0)
-        mock_rpm.name = 'pkg'
-        self.compose.variants['Server'].pkgset.rpms_by_arch['x86_64'] = [mock_rpm]
-        white = mock.Mock()
-        black = mock.Mock()
-        prepopulate = mock.Mock()
-        fulltree = mock.Mock()
-        deps.write_pungi_config(self.compose, 'x86_64', self.compose.variants['Server'],
-                                pkgs, grps, filter, white, black,
-                                prepopulate=prepopulate, fulltree_excludes=fulltree)
-        self.assertWritten(PungiWrapper, packages=['pkg1', 'pkg2.x86_64'],
-                           ks_path=self.topdir + '/work/x86_64/pungi/Server.x86_64.conf',
-                           lookaside_repos={}, multilib_whitelist=white, multilib_blacklist=black,
-                           groups=['grp1'], prepopulate=prepopulate,
-                           repos={'pungi-repo': self.topdir + '/work/x86_64/repo',
-                                  'comps-repo': self.topdir + '/work/x86_64/comps_repo_Server'},
-                           exclude_packages=['pkg3', 'pkg4.x86_64'],
-                           fulltree_excludes=fulltree,
-                           package_whitelist=set(['pkg-0:1.0.0-1']))
-
     @mock.patch('pungi.phases.gather.methods.method_deps.PungiWrapper')
     def test_without_input(self, PungiWrapper):
         with self.assertRaises(RuntimeError) as ctx: