gather: Get modular packages from fus
Fus returns also RPMs in modules, but until latest version it only worked if the package was in the same repo as the metadata. This changed in latest version and now Pungi does not need to expand the list anymore. JIRA: COMPOSE-2779 Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář
parent
5926858b58
commit
b12deab153
|
|
@ -150,23 +150,22 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
|
|||
self.multilib_methods, multilib_blacklist, multilib_whitelist
|
||||
)
|
||||
|
||||
platform, modular_rpms = create_module_repo(self.compose, variant, arch)
|
||||
platform = create_module_repo(self.compose, variant, arch)
|
||||
|
||||
packages.update(
|
||||
expand_groups(self.compose, arch, variant, groups, set_pkg_arch=False)
|
||||
)
|
||||
|
||||
nvrs, modules = self.run_solver(variant, arch, packages, platform, modular_rpms)
|
||||
nvrs = self.run_solver(variant, arch, packages, platform)
|
||||
return expand_packages(
|
||||
self._get_pkg_map(arch),
|
||||
variant.arch_mmds.get(arch, {}),
|
||||
pungi.phases.gather.get_lookaside_repos(self.compose, arch, variant),
|
||||
nvrs,
|
||||
modules,
|
||||
)
|
||||
# maybe check invalid sigkeys
|
||||
|
||||
def run_solver(self, variant, arch, packages, platform, modular_rpms):
|
||||
def run_solver(self, variant, arch, packages, platform):
|
||||
repos = [self.compose.paths.work.arch_repo(arch=arch)]
|
||||
|
||||
modules = []
|
||||
|
|
@ -195,8 +194,8 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
|
|||
arch, "hybrid-depsolver-%s-iter-%d" % (variant, step)
|
||||
)
|
||||
run(cmd, logfile=logfile, show_cmd=True)
|
||||
output, output_modules = fus.parse_output(logfile)
|
||||
new_multilib = self.add_multilib(variant, arch, output, modular_rpms)
|
||||
output = fus.parse_output(logfile)
|
||||
new_multilib = self.add_multilib(variant, arch, output)
|
||||
if new_multilib:
|
||||
input_packages.extend(
|
||||
_fmt_pkg(pkg_name, pkg_arch) for pkg_name, pkg_arch in new_multilib
|
||||
|
|
@ -211,14 +210,17 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
|
|||
# Nothing new was added, we can stop now.
|
||||
break
|
||||
|
||||
return output, output_modules
|
||||
return output
|
||||
|
||||
def add_multilib(self, variant, arch, nvrs, modular_rpms):
|
||||
def add_multilib(self, variant, arch, nvrs):
|
||||
added = set()
|
||||
if not self.multilib_methods:
|
||||
return []
|
||||
|
||||
for nvr, pkg_arch in nvrs:
|
||||
for nvr, pkg_arch, flags in nvrs:
|
||||
if "modular" in flags:
|
||||
continue
|
||||
|
||||
if pkg_arch != arch:
|
||||
# Not a native package, not checking to add multilib
|
||||
continue
|
||||
|
|
@ -227,10 +229,6 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
|
|||
nevr_copy = nevr.copy()
|
||||
nevr_copy["arch"] = pkg_arch
|
||||
|
||||
if kobo.rpmlib.make_nvra(nevr_copy, force_epoch=True) in modular_rpms:
|
||||
# Skip modular package
|
||||
continue
|
||||
|
||||
if self.multilib.is_multilib(self._get_package("%s.%s" % (nvr, pkg_arch))):
|
||||
for add_arch in self.valid_arches:
|
||||
if add_arch == arch:
|
||||
|
|
@ -239,7 +237,7 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
|
|||
added.add((nevr["name"], add_arch))
|
||||
|
||||
# Remove packages that are already present
|
||||
for nvr, pkg_arch in nvrs:
|
||||
for nvr, pkg_arch, flags in nvrs:
|
||||
existing = (nvr.rsplit("-", 2)[0], pkg_arch)
|
||||
if existing in added:
|
||||
added.remove(existing)
|
||||
|
|
@ -250,7 +248,9 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
|
|||
if not self.langpacks:
|
||||
return set()
|
||||
added = set()
|
||||
for nvr, pkg_arch in nvrs:
|
||||
for nvr, pkg_arch, flags in nvrs:
|
||||
if "modular" in flags:
|
||||
continue
|
||||
name = nvr.rsplit("-", 2)[0]
|
||||
if name in self.added_langpacks:
|
||||
# This package is already processed.
|
||||
|
|
@ -274,7 +274,6 @@ def create_module_repo(compose, variant, arch):
|
|||
compose.log_debug("[BEGIN] %s" % msg)
|
||||
|
||||
platforms = set()
|
||||
modular_rpms = set()
|
||||
|
||||
repo_path = compose.paths.work.module_repo(arch, variant)
|
||||
|
||||
|
|
@ -291,11 +290,6 @@ def create_module_repo(compose, variant, arch):
|
|||
if streams:
|
||||
platforms.update(streams.dup())
|
||||
|
||||
# Collect all modular NEVRAs
|
||||
artifacts = repo_mmd.get_rpm_artifacts()
|
||||
if artifacts:
|
||||
modular_rpms.update(artifacts.dup())
|
||||
|
||||
modules.append(repo_mmd)
|
||||
|
||||
if len(platforms) > 1:
|
||||
|
|
@ -331,7 +325,7 @@ def create_module_repo(compose, variant, arch):
|
|||
run(cmd, logfile=log_file, show_cmd=True)
|
||||
|
||||
compose.log_debug("[DONE ] %s" % msg)
|
||||
return list(platforms)[0] if platforms else None, modular_rpms
|
||||
return list(platforms)[0] if platforms else None
|
||||
|
||||
|
||||
def _fmt_pkg(pkg_name, arch):
|
||||
|
|
@ -366,7 +360,7 @@ def _make_result(paths):
|
|||
return [{"path": path, "flags": []} for path in sorted(paths)]
|
||||
|
||||
|
||||
def expand_packages(nevra_to_pkg, variant_modules, lookasides, nvrs, modules):
|
||||
def expand_packages(nevra_to_pkg, variant_modules, lookasides, nvrs):
|
||||
"""For each package add source RPM and possibly also debuginfo."""
|
||||
# This will server as the final result. We collect sets of paths to the
|
||||
# packages.
|
||||
|
|
@ -399,46 +393,29 @@ def expand_packages(nevra_to_pkg, variant_modules, lookasides, nvrs, modules):
|
|||
)
|
||||
variant_mmd[nsvc] = mmd
|
||||
|
||||
for module in modules:
|
||||
mmd = variant_mmd.get(module)
|
||||
if not mmd:
|
||||
continue
|
||||
artifacts = mmd.get_rpm_artifacts()
|
||||
if not artifacts:
|
||||
continue
|
||||
for rpm in artifacts.dup():
|
||||
pkg = nevra_to_pkg[_nevra(**kobo.rpmlib.parse_nvra(rpm))]
|
||||
if pkg_is_debug(pkg):
|
||||
debuginfo.add(pkg.file_path)
|
||||
else:
|
||||
rpms.add(pkg.file_path)
|
||||
# Add source package. We don't need modular packages, those are
|
||||
# listed in modulemd.
|
||||
try:
|
||||
srpm_nevra = _get_srpm_nevra(pkg)
|
||||
srpm = nevra_to_pkg[srpm_nevra]
|
||||
if srpm.file_path not in lookaside_packages:
|
||||
srpms.add(srpm.file_path)
|
||||
except KeyError:
|
||||
# Didn't find source RPM.. this should be logged
|
||||
pass
|
||||
|
||||
# This is used to figure out which debuginfo packages to include. We keep
|
||||
# track of package architectures from each SRPM.
|
||||
srpm_arches = defaultdict(set)
|
||||
|
||||
for nvr, arch in nvrs:
|
||||
for nvr, arch, flags in nvrs:
|
||||
pkg = nevra_to_pkg["%s.%s" % (nvr, arch)]
|
||||
if pkg.file_path in lookaside_packages:
|
||||
# Package is in lookaside, don't add it and ignore sources and
|
||||
# debuginfo too.
|
||||
continue
|
||||
rpms.add(pkg.file_path)
|
||||
if pkg_is_debug(pkg):
|
||||
debuginfo.add(pkg.file_path)
|
||||
else:
|
||||
rpms.add(pkg.file_path)
|
||||
|
||||
try:
|
||||
srpm_nevra = _get_srpm_nevra(pkg)
|
||||
srpm = nevra_to_pkg[srpm_nevra]
|
||||
srpm_arches[srpm_nevra].add(arch)
|
||||
if "modular" not in flags:
|
||||
# Only mark the arch for sources of non-modular packages. The
|
||||
# debuginfo is explicitly listed in the output, and we don't
|
||||
# want anything more.
|
||||
srpm_arches[srpm_nevra].add(arch)
|
||||
if srpm.file_path not in lookaside_packages:
|
||||
srpms.add(srpm.file_path)
|
||||
except KeyError:
|
||||
|
|
|
|||
|
|
@ -54,17 +54,19 @@ def get_cmd(
|
|||
|
||||
def parse_output(output):
|
||||
"""Read output of fus from the given filepath, and return a set of tuples
|
||||
(NVR, arch) and a set of module NSVCs.
|
||||
(NVR, arch, flags) and a set of module NSVCs.
|
||||
"""
|
||||
packages = set()
|
||||
modules = set()
|
||||
with open(output) as f:
|
||||
for line in f:
|
||||
if " " in line or "@" not in line:
|
||||
continue
|
||||
nevra, _ = line.strip().rsplit("@", 1)
|
||||
if nevra.startswith("module:"):
|
||||
modules.add(nevra[7:].rsplit(".", 1)[0])
|
||||
else:
|
||||
packages.add(tuple(nevra.rsplit(".", 1)))
|
||||
return packages, modules
|
||||
if not nevra.startswith("module:"):
|
||||
flags = set()
|
||||
name, arch = nevra.rsplit(".", 1)
|
||||
if name.startswith("*"):
|
||||
flags.add("modular")
|
||||
name = name[1:]
|
||||
packages.add((name, arch, frozenset(flags)))
|
||||
return packages
|
||||
|
|
|
|||
|
|
@ -57,21 +57,21 @@ class TestParseOutput(unittest.TestCase):
|
|||
|
||||
def test_skips_debug_line(self):
|
||||
touch(self.file, "debug line\n")
|
||||
packages, modules = fus.parse_output(self.file)
|
||||
packages = fus.parse_output(self.file)
|
||||
self.assertItemsEqual(packages, [])
|
||||
self.assertItemsEqual(modules, [])
|
||||
|
||||
def test_separates_arch(self):
|
||||
touch(self.file, "pkg-1.0-1.x86_64@repo-0\npkg-1.0-1.i686@repo-0\n")
|
||||
packages, modules = fus.parse_output(self.file)
|
||||
packages = fus.parse_output(self.file)
|
||||
self.assertItemsEqual(
|
||||
packages,
|
||||
[("pkg-1.0-1", "x86_64"), ("pkg-1.0-1", "i686")],
|
||||
[("pkg-1.0-1", "x86_64", frozenset()), ("pkg-1.0-1", "i686", frozenset())],
|
||||
)
|
||||
self.assertItemsEqual(modules, [])
|
||||
|
||||
def test_returns_modules(self):
|
||||
touch(self.file, "module:foo:1:201807131350:deadcafe.x86_64@repo-0\n")
|
||||
packages, modules = fus.parse_output(self.file)
|
||||
self.assertItemsEqual(packages, [])
|
||||
self.assertItemsEqual(modules, ["foo:1:201807131350:deadcafe"])
|
||||
def test_marks_modular(self):
|
||||
touch(self.file, "*pkg-1.0-1.x86_64@repo-0\n")
|
||||
packages = fus.parse_output(self.file)
|
||||
self.assertItemsEqual(
|
||||
packages,
|
||||
[("pkg-1.0-1", "x86_64", frozenset(["modular"]))],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -31,9 +31,8 @@ class TestMethodHybrid(helpers.PungiTestCase):
|
|||
@mock.patch("pungi.phases.gather.methods.method_hybrid.create_module_repo")
|
||||
def test_call_method(self, cmr, ep, eg, glr, CW):
|
||||
compose = helpers.DummyCompose(self.topdir, {})
|
||||
cmr.return_value = (mock.Mock(), mock.Mock())
|
||||
m = hybrid.GatherMethodHybrid(compose)
|
||||
m.run_solver = mock.Mock(return_value=(mock.Mock(), mock.Mock()))
|
||||
m.run_solver = mock.Mock()
|
||||
pkg = MockPkg(
|
||||
name="pkg",
|
||||
version="1",
|
||||
|
|
@ -55,7 +54,7 @@ class TestMethodHybrid(helpers.PungiTestCase):
|
|||
self.assertEqual(cmr.call_args_list, [mock.call(compose, variant, arch)])
|
||||
self.assertEqual(
|
||||
m.run_solver.call_args_list,
|
||||
[mock.call(variant, arch, set(["pkg", "foo", "bar"]), *cmr.return_value)],
|
||||
[mock.call(variant, arch, set(["pkg", "foo", "bar"]), cmr.return_value)],
|
||||
)
|
||||
self.assertEqual(
|
||||
ep.call_args_list,
|
||||
|
|
@ -64,8 +63,7 @@ class TestMethodHybrid(helpers.PungiTestCase):
|
|||
{"pkg-3:1-2.x86_64": pkg},
|
||||
{},
|
||||
glr.return_value,
|
||||
m.run_solver.return_value[0],
|
||||
m.run_solver.return_value[1],
|
||||
m.run_solver.return_value,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
|
@ -221,10 +219,9 @@ class TestCreateModuleRepo(HelperMixin, helpers.PungiTestCase):
|
|||
default = mock.Mock(peek_module_name=mock.Mock(return_value="mod"))
|
||||
imd.return_value = [default]
|
||||
|
||||
plat, pkgs = hybrid.create_module_repo(self.compose, self.variant, "x86_64")
|
||||
plat = hybrid.create_module_repo(self.compose, self.variant, "x86_64")
|
||||
|
||||
self.assertEqual(plat, "f29")
|
||||
self.assertItemsEqual(pkgs, ["pkg-1.0-1.x86_64"])
|
||||
|
||||
self.assertEqual(
|
||||
Modulemd.mock_calls, [mock.call.dump([mod, default], mock.ANY)]
|
||||
|
|
@ -284,7 +281,6 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
|
|||
"x86_64",
|
||||
[],
|
||||
platform="pl",
|
||||
modular_rpms=[],
|
||||
)
|
||||
|
||||
self.assertEqual(res, po.return_value)
|
||||
|
|
@ -314,7 +310,6 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
|
|||
"x86_64",
|
||||
[("pkg", None)],
|
||||
platform=None,
|
||||
modular_rpms=[],
|
||||
)
|
||||
|
||||
self.assertEqual(res, po.return_value)
|
||||
|
|
@ -330,15 +325,14 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
|
|||
|
||||
def test_with_langpacks(self, run, gc, po):
|
||||
self.phase.langpacks = {"pkg": set(["pkg-en"])}
|
||||
final = ([("pkg-1.0-1", "x86_64"), ("pkg-en-1.0-1", "noarch")], set())
|
||||
po.side_effect = [([("pkg-1.0-1", "x86_64")], set()), final]
|
||||
final = [("pkg-1.0-1", "x86_64", []), ("pkg-en-1.0-1", "noarch", [])]
|
||||
po.side_effect = [[("pkg-1.0-1", "x86_64", [])], final]
|
||||
|
||||
res = self.phase.run_solver(
|
||||
self.compose.variants["Server"],
|
||||
"x86_64",
|
||||
[("pkg", None)],
|
||||
platform=None,
|
||||
modular_rpms=[],
|
||||
)
|
||||
|
||||
self.assertEqual(res, final)
|
||||
|
|
@ -388,13 +382,12 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
|
|||
}
|
||||
self.phase.packages = self.phase.package_maps["x86_64"]
|
||||
final = [
|
||||
("pkg-devel-1.0-1", "x86_64"),
|
||||
("foo-1.0-1", "x86_64"),
|
||||
("pkg-devel-1.0-1", "i686"),
|
||||
("pkg-devel-1.0-1", "x86_64", []),
|
||||
("foo-1.0-1", "x86_64", []),
|
||||
("pkg-devel-1.0-1", "i686", []),
|
||||
]
|
||||
po.side_effect = [
|
||||
[[("pkg-devel-1.0-1", "x86_64"), ("foo-1.0-1", "x86_64")], set()],
|
||||
[final, set()],
|
||||
[("pkg-devel-1.0-1", "x86_64", []), ("foo-1.0-1", "x86_64", [])], final
|
||||
]
|
||||
|
||||
res = self.phase.run_solver(
|
||||
|
|
@ -402,10 +395,9 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
|
|||
"x86_64",
|
||||
[("pkg-devel", None), ("foo", None)],
|
||||
platform=None,
|
||||
modular_rpms=[],
|
||||
)
|
||||
|
||||
self.assertEqual(res, (final, set()))
|
||||
self.assertEqual(res, final)
|
||||
self.assertEqual(
|
||||
po.call_args_list, [mock.call(self.logfile1), mock.call(self.logfile2)]
|
||||
)
|
||||
|
|
@ -483,13 +475,12 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
|
|||
}
|
||||
}
|
||||
final = [
|
||||
("pkg-devel-1.0-1", "x86_64"),
|
||||
("foo-1.0-1", "x86_64"),
|
||||
("foo-1.0-1", "i686"),
|
||||
("pkg-devel-1.0-1", "x86_64", []),
|
||||
("foo-1.0-1", "x86_64", []),
|
||||
("foo-1.0-1", "i686", []),
|
||||
]
|
||||
po.side_effect = [
|
||||
([("pkg-devel-1.0-1", "x86_64"), ("foo-1.0-1", "x86_64")], set()),
|
||||
(final, set()),
|
||||
[("pkg-devel-1.0-1", "x86_64", []), ("foo-1.0-1", "x86_64", [])], final
|
||||
]
|
||||
|
||||
res = self.phase.run_solver(
|
||||
|
|
@ -497,10 +488,9 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
|
|||
"x86_64",
|
||||
[("pkg-devel", None), ("foo", None)],
|
||||
platform=None,
|
||||
modular_rpms=[],
|
||||
)
|
||||
|
||||
self.assertEqual(res, (final, set()))
|
||||
self.assertEqual(res, final)
|
||||
self.assertEqual(
|
||||
po.call_args_list, [mock.call(self.logfile1), mock.call(self.logfile2)]
|
||||
)
|
||||
|
|
@ -560,7 +550,7 @@ class TestExpandPackages(helpers.PungiTestCase):
|
|||
nevra_to_pkg = self._mk_packages()
|
||||
|
||||
res = hybrid.expand_packages(
|
||||
nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64")], []
|
||||
nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64", [])]
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
|
|
@ -576,7 +566,7 @@ class TestExpandPackages(helpers.PungiTestCase):
|
|||
nevra_to_pkg = self._mk_packages(debug_arch="x86_64")
|
||||
|
||||
res = hybrid.expand_packages(
|
||||
nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64")], []
|
||||
nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64", [])]
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
|
|
@ -588,11 +578,43 @@ class TestExpandPackages(helpers.PungiTestCase):
|
|||
},
|
||||
)
|
||||
|
||||
def test_modular_include_src_but_not_debuginfo(self):
|
||||
nevra_to_pkg = self._mk_packages(debug_arch="x86_64")
|
||||
|
||||
res = hybrid.expand_packages(
|
||||
nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64", ["modular"])]
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
res,
|
||||
{
|
||||
"rpm": [{"path": "/tmp/pkg.rpm", "flags": []}],
|
||||
"srpm": [{"path": "/tmp/spkg.rpm", "flags": []}],
|
||||
"debuginfo": [],
|
||||
},
|
||||
)
|
||||
|
||||
def test_modular_debug_in_correct_place(self):
|
||||
nevra_to_pkg = self._mk_packages(debug_arch="x86_64")
|
||||
|
||||
res = hybrid.expand_packages(
|
||||
nevra_to_pkg, {}, [], [("pkg-debuginfo-3:1-2", "x86_64", ["modular"])]
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
res,
|
||||
{
|
||||
"rpm": [],
|
||||
"srpm": [{"path": "/tmp/spkg.rpm", "flags": []}],
|
||||
"debuginfo": [{"path": "/tmp/d1.rpm", "flags": []}],
|
||||
},
|
||||
)
|
||||
|
||||
def test_skip_debuginfo_for_different_arch(self):
|
||||
nevra_to_pkg = self._mk_packages(debug_arch="i686")
|
||||
|
||||
res = hybrid.expand_packages(
|
||||
nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64")], []
|
||||
nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64", [])]
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
|
|
@ -626,7 +648,7 @@ class TestExpandPackages(helpers.PungiTestCase):
|
|||
cr.Metadata.return_value.get.side_effect = lambda key: repo[key]
|
||||
|
||||
res = hybrid.expand_packages(
|
||||
nevra_to_pkg, {}, lookasides, [("pkg-3:1-2", "x86_64")], []
|
||||
nevra_to_pkg, {}, lookasides, [("pkg-3:1-2", "x86_64", [])]
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
|
|
@ -654,30 +676,7 @@ class TestExpandPackages(helpers.PungiTestCase):
|
|||
cr.Metadata.return_value.get.side_effect = lambda key: repo[key]
|
||||
|
||||
res = hybrid.expand_packages(
|
||||
nevra_to_pkg, {}, lookasides, [("pkg-3:1-2", "x86_64")], []
|
||||
nevra_to_pkg, {}, lookasides, [("pkg-3:1-2", "x86_64", [])]
|
||||
)
|
||||
|
||||
self.assertEqual(res, {"rpm": [], "srpm": [], "debuginfo": []})
|
||||
|
||||
def test_expand_module_packages(self):
|
||||
nevra_to_pkg = self._mk_packages(src=True)
|
||||
mod = MockModule(
|
||||
"foo",
|
||||
stream="1.0",
|
||||
version="201807131350",
|
||||
context="deadcafe",
|
||||
rpms=["pkg-3:1-2.x86_64"],
|
||||
)
|
||||
|
||||
res = hybrid.expand_packages(
|
||||
nevra_to_pkg, {"foo-1.0": mod}, [], [], ["foo:1.0:201807131350:deadcafe"]
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
res,
|
||||
{
|
||||
"rpm": [{"flags": [], "path": "/tmp/pkg.rpm"}],
|
||||
"srpm": [{"flags": [], "path": "/tmp/spkg.rpm"}],
|
||||
"debuginfo": [],
|
||||
},
|
||||
)
|
||||
|
|
|
|||
Loading…
Reference in New Issue