almalinux/pungi
13
1
Fork 0
Code Issues Pull Requests Releases Activity

gather: Get modular packages from fus

Browse Source 
Fus returns also RPMs in modules, but until latest version it only
worked if the package was in the same repo as the metadata. This changed
in latest version and now Pungi does not need to expand the list
anymore.

JIRA: COMPOSE-2779
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2018-07-31 14:01:19 +02:00
parent 5926858b58
commit b12deab153
4 changed files with 99 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
pungi/phases/gather/methods/method_hybrid.py
View File

@ -150,23 +150,22 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
self.multilib_methods, multilib_blacklist, multilib_whitelist self.multilib_methods, multilib_blacklist, multilib_whitelist
) )
platform, modular_rpms = create_module_repo(self.compose, variant, arch) platform = create_module_repo(self.compose, variant, arch)
packages.update( packages.update(
expand_groups(self.compose, arch, variant, groups, set_pkg_arch=False) expand_groups(self.compose, arch, variant, groups, set_pkg_arch=False)
) )
nvrs, modules = self.run_solver(variant, arch, packages, platform, modular_rpms) nvrs = self.run_solver(variant, arch, packages, platform)
return expand_packages( return expand_packages(
self._get_pkg_map(arch), self._get_pkg_map(arch),
variant.arch_mmds.get(arch, {}), variant.arch_mmds.get(arch, {}),
pungi.phases.gather.get_lookaside_repos(self.compose, arch, variant), pungi.phases.gather.get_lookaside_repos(self.compose, arch, variant),
nvrs, nvrs,
modules,
) )
# maybe check invalid sigkeys # maybe check invalid sigkeys
def run_solver(self, variant, arch, packages, platform, modular_rpms): def run_solver(self, variant, arch, packages, platform):
repos = [self.compose.paths.work.arch_repo(arch=arch)] repos = [self.compose.paths.work.arch_repo(arch=arch)]
modules = [] modules = []
@ -195,8 +194,8 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
arch, "hybrid-depsolver-%s-iter-%d" % (variant, step) arch, "hybrid-depsolver-%s-iter-%d" % (variant, step)
) )
run(cmd, logfile=logfile, show_cmd=True) run(cmd, logfile=logfile, show_cmd=True)
output, output_modules = fus.parse_output(logfile) output = fus.parse_output(logfile)
new_multilib = self.add_multilib(variant, arch, output, modular_rpms) new_multilib = self.add_multilib(variant, arch, output)
if new_multilib: if new_multilib:
input_packages.extend( input_packages.extend(
_fmt_pkg(pkg_name, pkg_arch) for pkg_name, pkg_arch in new_multilib _fmt_pkg(pkg_name, pkg_arch) for pkg_name, pkg_arch in new_multilib
@ -211,14 +210,17 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
# Nothing new was added, we can stop now. # Nothing new was added, we can stop now.
break break
return output, output_modules return output
def add_multilib(self, variant, arch, nvrs, modular_rpms): def add_multilib(self, variant, arch, nvrs):
added = set() added = set()
if not self.multilib_methods: if not self.multilib_methods:
return [] return []
for nvr, pkg_arch in nvrs: for nvr, pkg_arch, flags in nvrs:
if "modular" in flags:
continue
if pkg_arch != arch: if pkg_arch != arch:
# Not a native package, not checking to add multilib # Not a native package, not checking to add multilib
continue continue
@ -227,10 +229,6 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
nevr_copy = nevr.copy() nevr_copy = nevr.copy()
nevr_copy["arch"] = pkg_arch nevr_copy["arch"] = pkg_arch
if kobo.rpmlib.make_nvra(nevr_copy, force_epoch=True) in modular_rpms:
# Skip modular package
continue
if self.multilib.is_multilib(self._get_package("%s.%s" % (nvr, pkg_arch))): if self.multilib.is_multilib(self._get_package("%s.%s" % (nvr, pkg_arch))):
for add_arch in self.valid_arches: for add_arch in self.valid_arches:
if add_arch == arch: if add_arch == arch:
@ -239,7 +237,7 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
added.add((nevr["name"], add_arch)) added.add((nevr["name"], add_arch))
# Remove packages that are already present # Remove packages that are already present
for nvr, pkg_arch in nvrs: for nvr, pkg_arch, flags in nvrs:
existing = (nvr.rsplit("-", 2)[0], pkg_arch) existing = (nvr.rsplit("-", 2)[0], pkg_arch)
if existing in added: if existing in added:
added.remove(existing) added.remove(existing)
@ -250,7 +248,9 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
if not self.langpacks: if not self.langpacks:
return set() return set()
added = set() added = set()
for nvr, pkg_arch in nvrs: for nvr, pkg_arch, flags in nvrs:
if "modular" in flags:
continue
name = nvr.rsplit("-", 2)[0] name = nvr.rsplit("-", 2)[0]
if name in self.added_langpacks: if name in self.added_langpacks:
# This package is already processed. # This package is already processed.
@ -274,7 +274,6 @@ def create_module_repo(compose, variant, arch):
compose.log_debug("[BEGIN] %s" % msg) compose.log_debug("[BEGIN] %s" % msg)
platforms = set() platforms = set()
modular_rpms = set()
repo_path = compose.paths.work.module_repo(arch, variant) repo_path = compose.paths.work.module_repo(arch, variant)
@ -291,11 +290,6 @@ def create_module_repo(compose, variant, arch):
if streams: if streams:
platforms.update(streams.dup()) platforms.update(streams.dup())
# Collect all modular NEVRAs
artifacts = repo_mmd.get_rpm_artifacts()
if artifacts:
modular_rpms.update(artifacts.dup())
modules.append(repo_mmd) modules.append(repo_mmd)
if len(platforms) > 1: if len(platforms) > 1:
@ -331,7 +325,7 @@ def create_module_repo(compose, variant, arch):
run(cmd, logfile=log_file, show_cmd=True) run(cmd, logfile=log_file, show_cmd=True)
compose.log_debug("[DONE ] %s" % msg) compose.log_debug("[DONE ] %s" % msg)
return list(platforms)[0] if platforms else None, modular_rpms return list(platforms)[0] if platforms else None
def _fmt_pkg(pkg_name, arch): def _fmt_pkg(pkg_name, arch):
@ -366,7 +360,7 @@ def _make_result(paths):
return [{"path": path, "flags": []} for path in sorted(paths)] return [{"path": path, "flags": []} for path in sorted(paths)]
def expand_packages(nevra_to_pkg, variant_modules, lookasides, nvrs, modules): def expand_packages(nevra_to_pkg, variant_modules, lookasides, nvrs):
"""For each package add source RPM and possibly also debuginfo.""" """For each package add source RPM and possibly also debuginfo."""
# This will server as the final result. We collect sets of paths to the # This will server as the final result. We collect sets of paths to the
# packages. # packages.
@ -399,46 +393,29 @@ def expand_packages(nevra_to_pkg, variant_modules, lookasides, nvrs, modules):
) )
variant_mmd[nsvc] = mmd variant_mmd[nsvc] = mmd
for module in modules:
mmd = variant_mmd.get(module)
if not mmd:
continue
artifacts = mmd.get_rpm_artifacts()
if not artifacts:
continue
for rpm in artifacts.dup():
pkg = nevra_to_pkg[_nevra(**kobo.rpmlib.parse_nvra(rpm))]
if pkg_is_debug(pkg):
debuginfo.add(pkg.file_path)
else:
rpms.add(pkg.file_path)
# Add source package. We don't need modular packages, those are
# listed in modulemd.
try:
srpm_nevra = _get_srpm_nevra(pkg)
srpm = nevra_to_pkg[srpm_nevra]
if srpm.file_path not in lookaside_packages:
srpms.add(srpm.file_path)
except KeyError:
# Didn't find source RPM.. this should be logged
pass
# This is used to figure out which debuginfo packages to include. We keep # This is used to figure out which debuginfo packages to include. We keep
# track of package architectures from each SRPM. # track of package architectures from each SRPM.
srpm_arches = defaultdict(set) srpm_arches = defaultdict(set)
for nvr, arch in nvrs: for nvr, arch, flags in nvrs:
pkg = nevra_to_pkg["%s.%s" % (nvr, arch)] pkg = nevra_to_pkg["%s.%s" % (nvr, arch)]
if pkg.file_path in lookaside_packages: if pkg.file_path in lookaside_packages:
# Package is in lookaside, don't add it and ignore sources and # Package is in lookaside, don't add it and ignore sources and
# debuginfo too. # debuginfo too.
continue continue
rpms.add(pkg.file_path) if pkg_is_debug(pkg):
debuginfo.add(pkg.file_path)
else:
rpms.add(pkg.file_path)
try: try:
srpm_nevra = _get_srpm_nevra(pkg) srpm_nevra = _get_srpm_nevra(pkg)
srpm = nevra_to_pkg[srpm_nevra] srpm = nevra_to_pkg[srpm_nevra]
srpm_arches[srpm_nevra].add(arch) if "modular" not in flags:
# Only mark the arch for sources of non-modular packages. The
# debuginfo is explicitly listed in the output, and we don't
# want anything more.
srpm_arches[srpm_nevra].add(arch)
if srpm.file_path not in lookaside_packages: if srpm.file_path not in lookaside_packages:
srpms.add(srpm.file_path) srpms.add(srpm.file_path)
except KeyError: except KeyError:

16
pungi/wrappers/fus.py
View File

@ -54,17 +54,19 @@ def get_cmd(
def parse_output(output): def parse_output(output):
"""Read output of fus from the given filepath, and return a set of tuples """Read output of fus from the given filepath, and return a set of tuples
(NVR, arch) and a set of module NSVCs. (NVR, arch, flags) and a set of module NSVCs.
""" """
packages = set() packages = set()
modules = set()
with open(output) as f: with open(output) as f:
for line in f: for line in f:
if " " in line or "@" not in line: if " " in line or "@" not in line:
continue continue
nevra, _ = line.strip().rsplit("@", 1) nevra, _ = line.strip().rsplit("@", 1)
if nevra.startswith("module:"): if not nevra.startswith("module:"):
modules.add(nevra[7:].rsplit(".", 1)[0]) flags = set()
else: name, arch = nevra.rsplit(".", 1)
packages.add(tuple(nevra.rsplit(".", 1))) if name.startswith("*"):
return packages, modules flags.add("modular")
name = name[1:]
packages.add((name, arch, frozenset(flags)))
return packages

20
tests/test_fus_wrapper.py
View File

@ -57,21 +57,21 @@ class TestParseOutput(unittest.TestCase):
def test_skips_debug_line(self): def test_skips_debug_line(self):
touch(self.file, "debug line\n") touch(self.file, "debug line\n")
packages, modules = fus.parse_output(self.file) packages = fus.parse_output(self.file)
self.assertItemsEqual(packages, []) self.assertItemsEqual(packages, [])
self.assertItemsEqual(modules, [])
def test_separates_arch(self): def test_separates_arch(self):
touch(self.file, "pkg-1.0-1.x86_64@repo-0\npkg-1.0-1.i686@repo-0\n") touch(self.file, "pkg-1.0-1.x86_64@repo-0\npkg-1.0-1.i686@repo-0\n")
packages, modules = fus.parse_output(self.file) packages = fus.parse_output(self.file)
self.assertItemsEqual( self.assertItemsEqual(
packages, packages,
[("pkg-1.0-1", "x86_64"), ("pkg-1.0-1", "i686")], [("pkg-1.0-1", "x86_64", frozenset()), ("pkg-1.0-1", "i686", frozenset())],
) )
self.assertItemsEqual(modules, [])
def test_returns_modules(self): def test_marks_modular(self):
touch(self.file, "module:foo:1:201807131350:deadcafe.x86_64@repo-0\n") touch(self.file, "*pkg-1.0-1.x86_64@repo-0\n")
packages, modules = fus.parse_output(self.file) packages = fus.parse_output(self.file)
self.assertItemsEqual(packages, []) self.assertItemsEqual(
self.assertItemsEqual(modules, ["foo:1:201807131350:deadcafe"]) packages,
[("pkg-1.0-1", "x86_64", frozenset(["modular"]))],
)

107
tests/test_gather_method_hybrid.py
View File

@ -31,9 +31,8 @@ class TestMethodHybrid(helpers.PungiTestCase):
@mock.patch("pungi.phases.gather.methods.method_hybrid.create_module_repo") @mock.patch("pungi.phases.gather.methods.method_hybrid.create_module_repo")
def test_call_method(self, cmr, ep, eg, glr, CW): def test_call_method(self, cmr, ep, eg, glr, CW):
compose = helpers.DummyCompose(self.topdir, {}) compose = helpers.DummyCompose(self.topdir, {})
cmr.return_value = (mock.Mock(), mock.Mock())
m = hybrid.GatherMethodHybrid(compose) m = hybrid.GatherMethodHybrid(compose)
m.run_solver = mock.Mock(return_value=(mock.Mock(), mock.Mock())) m.run_solver = mock.Mock()
pkg = MockPkg( pkg = MockPkg(
name="pkg", name="pkg",
version="1", version="1",
@ -55,7 +54,7 @@ class TestMethodHybrid(helpers.PungiTestCase):
self.assertEqual(cmr.call_args_list, [mock.call(compose, variant, arch)]) self.assertEqual(cmr.call_args_list, [mock.call(compose, variant, arch)])
self.assertEqual( self.assertEqual(
m.run_solver.call_args_list, m.run_solver.call_args_list,
[mock.call(variant, arch, set(["pkg", "foo", "bar"]), *cmr.return_value)], [mock.call(variant, arch, set(["pkg", "foo", "bar"]), cmr.return_value)],
) )
self.assertEqual( self.assertEqual(
ep.call_args_list, ep.call_args_list,
@ -64,8 +63,7 @@ class TestMethodHybrid(helpers.PungiTestCase):
{"pkg-3:1-2.x86_64": pkg}, {"pkg-3:1-2.x86_64": pkg},
{}, {},
glr.return_value, glr.return_value,
m.run_solver.return_value[0], m.run_solver.return_value,
m.run_solver.return_value[1],
) )
], ],
) )
@ -221,10 +219,9 @@ class TestCreateModuleRepo(HelperMixin, helpers.PungiTestCase):
default = mock.Mock(peek_module_name=mock.Mock(return_value="mod")) default = mock.Mock(peek_module_name=mock.Mock(return_value="mod"))
imd.return_value = [default] imd.return_value = [default]
plat, pkgs = hybrid.create_module_repo(self.compose, self.variant, "x86_64") plat = hybrid.create_module_repo(self.compose, self.variant, "x86_64")
self.assertEqual(plat, "f29") self.assertEqual(plat, "f29")
self.assertItemsEqual(pkgs, ["pkg-1.0-1.x86_64"])
self.assertEqual( self.assertEqual(
Modulemd.mock_calls, [mock.call.dump([mod, default], mock.ANY)] Modulemd.mock_calls, [mock.call.dump([mod, default], mock.ANY)]
@ -284,7 +281,6 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
"x86_64", "x86_64",
[], [],
platform="pl", platform="pl",
modular_rpms=[],
) )
self.assertEqual(res, po.return_value) self.assertEqual(res, po.return_value)
@ -314,7 +310,6 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
"x86_64", "x86_64",
[("pkg", None)], [("pkg", None)],
platform=None, platform=None,
modular_rpms=[],
) )
self.assertEqual(res, po.return_value) self.assertEqual(res, po.return_value)
@ -330,15 +325,14 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
def test_with_langpacks(self, run, gc, po): def test_with_langpacks(self, run, gc, po):
self.phase.langpacks = {"pkg": set(["pkg-en"])} self.phase.langpacks = {"pkg": set(["pkg-en"])}
final = ([("pkg-1.0-1", "x86_64"), ("pkg-en-1.0-1", "noarch")], set()) final = [("pkg-1.0-1", "x86_64", []), ("pkg-en-1.0-1", "noarch", [])]
po.side_effect = [([("pkg-1.0-1", "x86_64")], set()), final] po.side_effect = [[("pkg-1.0-1", "x86_64", [])], final]
res = self.phase.run_solver( res = self.phase.run_solver(
self.compose.variants["Server"], self.compose.variants["Server"],
"x86_64", "x86_64",
[("pkg", None)], [("pkg", None)],
platform=None, platform=None,
modular_rpms=[],
) )
self.assertEqual(res, final) self.assertEqual(res, final)
@ -388,13 +382,12 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
} }
self.phase.packages = self.phase.package_maps["x86_64"] self.phase.packages = self.phase.package_maps["x86_64"]
final = [ final = [
("pkg-devel-1.0-1", "x86_64"), ("pkg-devel-1.0-1", "x86_64", []),
("foo-1.0-1", "x86_64"), ("foo-1.0-1", "x86_64", []),
("pkg-devel-1.0-1", "i686"), ("pkg-devel-1.0-1", "i686", []),
] ]
po.side_effect = [ po.side_effect = [
[[("pkg-devel-1.0-1", "x86_64"), ("foo-1.0-1", "x86_64")], set()], [("pkg-devel-1.0-1", "x86_64", []), ("foo-1.0-1", "x86_64", [])], final
[final, set()],
] ]
res = self.phase.run_solver( res = self.phase.run_solver(
@ -402,10 +395,9 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
"x86_64", "x86_64",
[("pkg-devel", None), ("foo", None)], [("pkg-devel", None), ("foo", None)],
platform=None, platform=None,
modular_rpms=[],
) )
self.assertEqual(res, (final, set())) self.assertEqual(res, final)
self.assertEqual( self.assertEqual(
po.call_args_list, [mock.call(self.logfile1), mock.call(self.logfile2)] po.call_args_list, [mock.call(self.logfile1), mock.call(self.logfile2)]
) )
@ -483,13 +475,12 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
} }
} }
final = [ final = [
("pkg-devel-1.0-1", "x86_64"), ("pkg-devel-1.0-1", "x86_64", []),
("foo-1.0-1", "x86_64"), ("foo-1.0-1", "x86_64", []),
("foo-1.0-1", "i686"), ("foo-1.0-1", "i686", []),
] ]
po.side_effect = [ po.side_effect = [
([("pkg-devel-1.0-1", "x86_64"), ("foo-1.0-1", "x86_64")], set()), [("pkg-devel-1.0-1", "x86_64", []), ("foo-1.0-1", "x86_64", [])], final
(final, set()),
] ]
res = self.phase.run_solver( res = self.phase.run_solver(
@ -497,10 +488,9 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
"x86_64", "x86_64",
[("pkg-devel", None), ("foo", None)], [("pkg-devel", None), ("foo", None)],
platform=None, platform=None,
modular_rpms=[],
) )
self.assertEqual(res, (final, set())) self.assertEqual(res, final)
self.assertEqual( self.assertEqual(
po.call_args_list, [mock.call(self.logfile1), mock.call(self.logfile2)] po.call_args_list, [mock.call(self.logfile1), mock.call(self.logfile2)]
) )
@ -560,7 +550,7 @@ class TestExpandPackages(helpers.PungiTestCase):
nevra_to_pkg = self._mk_packages() nevra_to_pkg = self._mk_packages()
res = hybrid.expand_packages( res = hybrid.expand_packages(
nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64")], [] nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64", [])]
) )
self.assertEqual( self.assertEqual(
@ -576,7 +566,7 @@ class TestExpandPackages(helpers.PungiTestCase):
nevra_to_pkg = self._mk_packages(debug_arch="x86_64") nevra_to_pkg = self._mk_packages(debug_arch="x86_64")
res = hybrid.expand_packages( res = hybrid.expand_packages(
nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64")], [] nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64", [])]
) )
self.assertEqual( self.assertEqual(
@ -588,11 +578,43 @@ class TestExpandPackages(helpers.PungiTestCase):
}, },
) )
def test_modular_include_src_but_not_debuginfo(self):
nevra_to_pkg = self._mk_packages(debug_arch="x86_64")
res = hybrid.expand_packages(
nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64", ["modular"])]
)
self.assertEqual(
res,
{
"rpm": [{"path": "/tmp/pkg.rpm", "flags": []}],
"srpm": [{"path": "/tmp/spkg.rpm", "flags": []}],
"debuginfo": [],
},
)
def test_modular_debug_in_correct_place(self):
nevra_to_pkg = self._mk_packages(debug_arch="x86_64")
res = hybrid.expand_packages(
nevra_to_pkg, {}, [], [("pkg-debuginfo-3:1-2", "x86_64", ["modular"])]
)
self.assertEqual(
res,
{
"rpm": [],
"srpm": [{"path": "/tmp/spkg.rpm", "flags": []}],
"debuginfo": [{"path": "/tmp/d1.rpm", "flags": []}],
},
)
def test_skip_debuginfo_for_different_arch(self): def test_skip_debuginfo_for_different_arch(self):
nevra_to_pkg = self._mk_packages(debug_arch="i686") nevra_to_pkg = self._mk_packages(debug_arch="i686")
res = hybrid.expand_packages( res = hybrid.expand_packages(
nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64")], [] nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64", [])]
) )
self.assertEqual( self.assertEqual(
@ -626,7 +648,7 @@ class TestExpandPackages(helpers.PungiTestCase):
cr.Metadata.return_value.get.side_effect = lambda key: repo[key] cr.Metadata.return_value.get.side_effect = lambda key: repo[key]
res = hybrid.expand_packages( res = hybrid.expand_packages(
nevra_to_pkg, {}, lookasides, [("pkg-3:1-2", "x86_64")], [] nevra_to_pkg, {}, lookasides, [("pkg-3:1-2", "x86_64", [])]
) )
self.assertEqual( self.assertEqual(
@ -654,30 +676,7 @@ class TestExpandPackages(helpers.PungiTestCase):
cr.Metadata.return_value.get.side_effect = lambda key: repo[key] cr.Metadata.return_value.get.side_effect = lambda key: repo[key]
res = hybrid.expand_packages( res = hybrid.expand_packages(
nevra_to_pkg, {}, lookasides, [("pkg-3:1-2", "x86_64")], [] nevra_to_pkg, {}, lookasides, [("pkg-3:1-2", "x86_64", [])]
) )
self.assertEqual(res, {"rpm": [], "srpm": [], "debuginfo": []}) self.assertEqual(res, {"rpm": [], "srpm": [], "debuginfo": []})
def test_expand_module_packages(self):
nevra_to_pkg = self._mk_packages(src=True)
mod = MockModule(
"foo",
stream="1.0",
version="201807131350",
context="deadcafe",
rpms=["pkg-3:1-2.x86_64"],
)
res = hybrid.expand_packages(
nevra_to_pkg, {"foo-1.0": mod}, [], [], ["foo:1.0:201807131350:deadcafe"]
)
self.assertEqual(
res,
{
"rpm": [{"flags": [], "path": "/tmp/pkg.rpm"}],
"srpm": [{"flags": [], "path": "/tmp/spkg.rpm"}],
"debuginfo": [],
},
)