almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Remove package whitelist

Browse Source 
This was a workaround to make some packages from the global repo
invisible for depsolving. This is now handled by packages being in
different repos. We can select which repos are enabled at which point.
This achieves the same result, but much faster.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2019-08-01 17:01:16 +02:00
parent cec3efed51
commit 63a8b7b6c9
7 changed files with 26 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
bin/pungi-gather
View File

@ -129,7 +129,6 @@ def main(persistdir, cachedir):
gather_opts.multilib_whitelist = ksparser.handler.multilib_whitelist
gather_opts.prepopulate = ksparser.handler.prepopulate
gather_opts.fulltree_excludes = ksparser.handler.fulltree_excludes
gather_opts.package_whitelist = ksparser.handler.package_whitelist
g = Gather(dnf_obj, gather_opts)

17
pungi/gather_dnf.py
View File

@ -69,8 +69,6 @@ class GatherOptions(pungi.common.OptionsBase):
# lookaside repos; packages will be flagged accordingly
self.lookaside_repos = []
self.package_whitelist = set()
self.merge_options(**kwargs)
def __str__(self):
@ -85,7 +83,6 @@ class GatherOptions(pungi.common.OptionsBase):
'multilib_blacklist=%d items' % len(self.multilib_blacklist),
'multilib_whitelist=%d items' % len(self.multilib_whitelist),
'lookaside_repos=%s' % self.lookaside_repos,
'package_whitelist=%d items' % len(self.package_whitelist),
'prepopulate=%d items' % len(self.prepopulate)
]
return '[\n%s\n]' % '\n'.join(' ' + l for l in lines)
@ -404,20 +401,6 @@ class Gather(GatherBase):
'q_source_packages', 'q_native_debug_packages',
'q_multilib_debug_packages']
if self.opts.package_whitelist:
with Profiler("Gather._apply_excludes():apply-package-whitelist'"):
to_keep = []
for pattern in self.opts.package_whitelist:
nvr = parse_nvr(pattern)
try:
nvr['epoch'] = int(nvr.pop('epoch'))
except ValueError:
pass
to_keep.extend(self._query.filter(**nvr).run())
for queue in all_queues:
setattr(self, queue, getattr(self, queue).filter(pkg=to_keep).latest().apply())
with Profiler("Gather._apply_excludes():exclude-queries"):
for queue in all_queues:
self._filter_queue(queue, exclude)

15
pungi/ks.py
View File

@ -127,19 +127,6 @@ class PrepopulateSection(pykickstart.sections.Section):
self.handler.prepopulate.add(line)
class PackageWhitelistSection(pykickstart.sections.Section):
sectionOpen = "%package-whitelist"
def handleLine(self, line):
if not self.handler:
return
(h, s, t) = line.partition('#')
line = h.rstrip()
self.handler.package_whitelist.add(line)
class KickstartParser(pykickstart.parser.KickstartParser):
def setupSections(self):
pykickstart.parser.KickstartParser.setupSections(self)
@ -147,7 +134,6 @@ class KickstartParser(pykickstart.parser.KickstartParser):
self.registerSection(MultilibBlacklistSection(self.handler))
self.registerSection(MultilibWhitelistSection(self.handler))
self.registerSection(PrepopulateSection(self.handler))
self.registerSection(PackageWhitelistSection(self.handler))
def get_packages(self, dnf_obj):
packages = set()
@ -208,7 +194,6 @@ class PungiHandler(HandlerClass):
self.multilib_blacklist = set()
self.multilib_whitelist = set()
self.prepopulate = set()
self.package_whitelist = set()
def get_ksparser(ks_path=None):

2
pungi/phases/gather/methods/method_deps.py
View File

@ -23,7 +23,7 @@ from kobo.rpmlib import parse_nvra
from pungi.util import get_arch_variant_data, temp_dir
from pungi.wrappers.pungi import PungiWrapper
from pungi.arch import tree_arch_to_yum_arch, get_valid_arches
from pungi.arch import tree_arch_to_yum_arch
import pungi.phases.gather
import pungi.phases.gather.method

3
pungi/wrappers/pungi.py
View File

@ -47,7 +47,7 @@ class PungiWrapper(object):
exclude_packages=None, comps_repo=None,
lookaside_repos=None, fulltree_excludes=None,
multilib_blacklist=None, multilib_whitelist=None,
prepopulate=None, package_whitelist=None):
prepopulate=None):
groups = groups or []
exclude_packages = exclude_packages or {}
lookaside_repos = lookaside_repos or {}
@ -92,7 +92,6 @@ class PungiWrapper(object):
_write_ks_section(kickstart, "multilib-blacklist", multilib_blacklist)
_write_ks_section(kickstart, "multilib-whitelist", multilib_whitelist)
_write_ks_section(kickstart, "prepopulate", prepopulate)
_write_ks_section(kickstart, "package-whitelist", package_whitelist)
kickstart.close()

58
tests/test_gather.py
View File

@ -1854,66 +1854,8 @@ class DNFDepsolvingTestCase(DepsolvingBase, unittest.TestCase):
def test_bash_older(self):
pass
def test_whitelist_old_version(self):
# There are two version of dummy-bash in the package set; let's
# whitelist only the older one and its dependencies.
packages = [
"dummy-bash",
]
package_whitelist = [
"dummy-basesystem-10.0-6",
"dummy-bash-debuginfo-4.2.37-5",
"dummy-bash-4.2.37-5",
"dummy-filesystem-4.2.37-6",
"dummy-glibc-common-2.14-5",
"dummy-glibc-debuginfo-common-2.14-5",
"dummy-glibc-debuginfo-2.14-5",
"dummy-glibc-2.14-5",
]
pkg_map = self.go(packages, None, greedy="none", package_whitelist=package_whitelist)
self.assertNotIn("dummy-bash-4.2.37-5.i686.rpm", pkg_map["rpm"])
self.assertNotIn("dummy-bash-4.2.37-6.i686.rpm", pkg_map["rpm"])
self.assertNotIn("dummy-bash-4.2.37-6.x86_64.rpm", pkg_map["rpm"])
self.assertItemsEqual(pkg_map["rpm"], [
"dummy-basesystem-10.0-6.noarch.rpm",
"dummy-bash-4.2.37-5.x86_64.rpm",
"dummy-filesystem-4.2.37-6.x86_64.rpm",
"dummy-glibc-2.14-5.x86_64.rpm",
"dummy-glibc-common-2.14-5.x86_64.rpm",
])
self.assertItemsEqual(pkg_map["srpm"], [
"dummy-basesystem-10.0-6.src.rpm",
"dummy-bash-4.2.37-5.src.rpm",
"dummy-filesystem-4.2.37-6.src.rpm",
"dummy-glibc-2.14-5.src.rpm",
])
self.assertItemsEqual(pkg_map["debuginfo"], [
"dummy-bash-debuginfo-4.2.37-5.x86_64.rpm",
"dummy-glibc-debuginfo-2.14-5.x86_64.rpm",
"dummy-glibc-debuginfo-common-2.14-5.x86_64.rpm",
])
def test_firefox_selfhosting_with_krb5_lookaside(self):
super(DNFDepsolvingTestCase, self).test_firefox_selfhosting_with_krb5_lookaside()
self.assertFlags("dummy-krb5-devel-1.10-5.x86_64", [PkgFlag.lookaside])
self.assertFlags("dummy-krb5-1.10-5.src", [PkgFlag.lookaside])
self.assertFlags("dummy-krb5-debuginfo-1.10-5.x86_64", [PkgFlag.lookaside])
def test_package_whitelist(self):
packages = ['*']
whitelist = [
'dummy-bash-4.2.37-6',
]
pkg_map = self.go(packages, None, package_whitelist=whitelist)
self.assertItemsEqual(pkg_map["rpm"], [
'dummy-bash-4.2.37-6.x86_64.rpm',
])
self.assertItemsEqual(pkg_map["srpm"], [
'dummy-bash-4.2.37-6.src.rpm',
])
self.assertItemsEqual(pkg_map["debuginfo"], [
])

68
tests/test_gather_method_deps.py
View File

@ -14,13 +14,13 @@ class TestWritePungiConfig(helpers.PungiTestCase):
def setUp(self):
super(TestWritePungiConfig, self).setUp()
self.compose = helpers.DummyCompose(self.topdir, {})
self.package_sets = self._make_pkgset_phase(["p1"]).package_sets
def assertWritten(self, PungiWrapper, **kwargs):
wrapper = PungiWrapper.return_value
self.assertEqual(wrapper.mock_calls,
[mock.call.write_kickstart(**kwargs)])
@helpers.unittest.skip("temporarily broken")
@mock.patch('pungi.phases.gather.methods.method_deps.PungiWrapper')
def test_correct(self, PungiWrapper):
pkgs = [('pkg1', None), ('pkg2', 'x86_64')]
@ -30,19 +30,21 @@ class TestWritePungiConfig(helpers.PungiTestCase):
black = mock.Mock()
prepopulate = mock.Mock()
fulltree = mock.Mock()
deps.write_pungi_config(self.compose, 'x86_64', self.compose.variants['Server'],
pkgs, grps, filter, white, black,
prepopulate=prepopulate, fulltree_excludes=fulltree)
deps.write_pungi_config(
self.compose, 'x86_64', self.compose.variants['Server'],
pkgs, grps, filter, white, black,
prepopulate=prepopulate, fulltree_excludes=fulltree,
package_sets=self.package_sets,
)
self.assertWritten(PungiWrapper, packages=['pkg1', 'pkg2.x86_64'],
ks_path=self.topdir + '/work/x86_64/pungi/Server.x86_64.conf',
lookaside_repos={}, multilib_whitelist=white, multilib_blacklist=black,
groups=['grp1'], prepopulate=prepopulate,
repos={'pungi-repo': self.topdir + '/work/x86_64/repo',
repos={"pungi-repo-0": self.topdir + "/work/x86_64/repo/p1",
'comps-repo': self.topdir + '/work/x86_64/comps_repo_Server'},
exclude_packages=['pkg3', 'pkg4.x86_64'],
fulltree_excludes=fulltree, package_whitelist=set())
fulltree_excludes=fulltree)
@helpers.unittest.skip("temporarily broken")
@mock.patch("pungi.phases.gather.methods.method_deps.PungiWrapper")
def test_duplicated_package_name(self, PungiWrapper):
pkgs = [("pkg1", None), ("pkg1", "x86_64")]
@ -52,64 +54,42 @@ class TestWritePungiConfig(helpers.PungiTestCase):
black = mock.Mock()
prepopulate = mock.Mock()
fulltree = mock.Mock()
deps.write_pungi_config(self.compose, "x86_64", self.compose.variants["Server"],
pkgs, grps, filter, white, black,
prepopulate=prepopulate, fulltree_excludes=fulltree)
deps.write_pungi_config(
self.compose, "x86_64", self.compose.variants["Server"],
pkgs, grps, filter, white, black,
prepopulate=prepopulate, fulltree_excludes=fulltree,
package_sets=self.package_sets,
)
self.assertWritten(PungiWrapper, packages=["pkg1", "pkg1.x86_64"],
ks_path=self.topdir + "/work/x86_64/pungi/Server.x86_64.conf",
lookaside_repos={}, multilib_whitelist=white, multilib_blacklist=black,
groups=[], prepopulate=prepopulate,
repos={"pungi-repo": self.topdir + "/work/x86_64/repo",
repos={"pungi-repo-0": self.topdir + "/work/x86_64/repo/p1",
"comps-repo": self.topdir + "/work/x86_64/comps_repo_Server"},
exclude_packages=["pkg2", "pkg2.x86_64"],
fulltree_excludes=fulltree, package_whitelist=set())
fulltree_excludes=fulltree)
@helpers.unittest.skip("temporarily broken")
@mock.patch('pungi.phases.gather.get_lookaside_repos')
@mock.patch('pungi.phases.gather.methods.method_deps.PungiWrapper')
def test_with_lookaside(self, PungiWrapper, glr):
glr.return_value = ['http://example.com/repo']
pkgs = [('pkg1', None)]
deps.write_pungi_config(self.compose, 'x86_64', self.compose.variants['Server'],
pkgs, [], [], [], [])
deps.write_pungi_config(
self.compose, 'x86_64', self.compose.variants['Server'],
pkgs, [], [], [], [],
package_sets=self.package_sets,
)
self.assertWritten(PungiWrapper, packages=['pkg1'],
ks_path=self.topdir + '/work/x86_64/pungi/Server.x86_64.conf',
lookaside_repos={'lookaside-repo-0': 'http://example.com/repo'},
multilib_whitelist=[], multilib_blacklist=[],
groups=[], prepopulate=None,
repos={'pungi-repo': self.topdir + '/work/x86_64/repo',
repos={"pungi-repo-0": self.topdir + "/work/x86_64/repo/p1",
'comps-repo': self.topdir + '/work/x86_64/comps_repo_Server'},
exclude_packages=[], fulltree_excludes=None,
package_whitelist=set())
exclude_packages=[], fulltree_excludes=None)
self.assertEqual(glr.call_args_list,
[mock.call(self.compose, 'x86_64', self.compose.variants['Server'])])
@helpers.unittest.skip("temporarily broken")
@mock.patch('pungi.phases.gather.methods.method_deps.PungiWrapper')
def test_with_whitelist(self, PungiWrapper):
pkgs = [('pkg1', None), ('pkg2', 'x86_64')]
grps = ['grp1']
filter = [('pkg3', None), ('pkg4', 'x86_64')]
mock_rpm = mock.Mock(version='1.0.0', release='1', epoch=0)
mock_rpm.name = 'pkg'
self.compose.variants['Server'].pkgset.rpms_by_arch['x86_64'] = [mock_rpm]
white = mock.Mock()
black = mock.Mock()
prepopulate = mock.Mock()
fulltree = mock.Mock()
deps.write_pungi_config(self.compose, 'x86_64', self.compose.variants['Server'],
pkgs, grps, filter, white, black,
prepopulate=prepopulate, fulltree_excludes=fulltree)
self.assertWritten(PungiWrapper, packages=['pkg1', 'pkg2.x86_64'],
ks_path=self.topdir + '/work/x86_64/pungi/Server.x86_64.conf',
lookaside_repos={}, multilib_whitelist=white, multilib_blacklist=black,
groups=['grp1'], prepopulate=prepopulate,
repos={'pungi-repo': self.topdir + '/work/x86_64/repo',
'comps-repo': self.topdir + '/work/x86_64/comps_repo_Server'},
exclude_packages=['pkg3', 'pkg4.x86_64'],
fulltree_excludes=fulltree,
package_whitelist=set(['pkg-0:1.0.0-1']))
@mock.patch('pungi.phases.gather.methods.method_deps.PungiWrapper')
def test_without_input(self, PungiWrapper):
with self.assertRaises(RuntimeError) as ctx: