diff --git a/pungi/paths.py b/pungi/paths.py index 3d921596..062cd35a 100644 --- a/pungi/paths.py +++ b/pungi/paths.py @@ -181,14 +181,6 @@ class WorkPaths(object): """ return self._repo("comps", arch, variant, create_dir=create_dir) - def module_repo(self, arch=None, variant=None, create_dir=True): - """ - Examples: - work/x86_64/module_repo_Server - work/global/module_repo - """ - return self._repo("module", arch, variant, create_dir=create_dir) - def arch_repo(self, arch=None, create_dir=True): """ Examples: diff --git a/pungi/phases/gather/methods/method_hybrid.py b/pungi/phases/gather/methods/method_hybrid.py index 9fe43b32..44c355fb 100644 --- a/pungi/phases/gather/methods/method_hybrid.py +++ b/pungi/phases/gather/methods/method_hybrid.py @@ -13,7 +13,6 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, see . -import gzip import os from collections import defaultdict from fnmatch import fnmatch @@ -23,18 +22,15 @@ import kobo.rpmlib from kobo.shortcuts import run import pungi.phases.gather.method -from pungi import Modulemd, multilib_dnf +from pungi import multilib_dnf from pungi.arch import get_valid_arches, tree_arch_to_yum_arch from pungi.phases.gather import _mk_pkg_map -from pungi.phases.createrepo import add_modular_metadata from pungi.util import ( get_arch_variant_data, - iter_module_defaults, pkg_is_debug, ) from pungi.wrappers import fus from pungi.wrappers.comps import CompsWrapper -from pungi.wrappers.createrepo import CreaterepoWrapper from .method_nodeps import expand_groups @@ -198,7 +194,7 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): set(p.name for p in self.expand_list(multilib_whitelist)), ) - platform = create_module_repo(self.compose, variant, arch) + platform = get_platform(self.compose, variant, arch) packages.update( expand_groups(self.compose, arch, variant, groups, set_pkg_arch=False) @@ -227,10 +223,8 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): result_modules = set() modules = [] - if variant.arch_mmds.get(arch): - repos.append(self.compose.paths.work.module_repo(arch, variant)) - for mmd in variant.arch_mmds[arch].values(): - modules.append("%s:%s" % (mmd.peek_name(), mmd.peek_stream())) + for mmd in variant.arch_mmds.get(arch, {}).values(): + modules.append("%s:%s" % (mmd.peek_name(), mmd.peek_stream())) input_packages = [] for pkg_name, pkg_arch in packages: @@ -398,105 +392,22 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): return packages -def get_lookaside_modules(lookasides): - """Get list of NSVC of all modules in all lookaside repos.""" - modules = set() +def get_platform(compose, variant, arch): + """Find platform stream for modules. Raises RuntimeError if there are + conflicting requests. + """ platforms = set() - for repo in lookasides: - repo = fus._prep_path(repo) - repomd = cr.Repomd(os.path.join(repo, "repodata/repomd.xml")) - for rec in repomd.records: - if rec.type != "modules": - continue - # No with statement on Python 2.6 for GzipFile... - gzipped_file = gzip.GzipFile(os.path.join(repo, rec.location_href), "r") - # This can't use _from_stream, since gobject-introspection - # refuses to pass a file object. - mmds = Modulemd.objects_from_string(gzipped_file.read()) - gzipped_file.close() - for mmd in mmds: - if isinstance(mmd, Modulemd.Module): - modules.add( - "%s:%s:%s:%s" - % ( - mmd.peek_name(), - mmd.peek_stream(), - mmd.peek_version(), - mmd.peek_context(), - ) - ) - for dep in mmd.peek_dependencies(): - streams = dep.peek_requires().get("platform") - if streams: - platforms.update(streams.dup()) - return modules, platforms - -def create_module_repo(compose, variant, arch): - """Create repository with module metadata. There are no packages otherwise.""" - createrepo_c = compose.conf["createrepo_c"] - createrepo_checksum = compose.conf["createrepo_checksum"] - msg = "Creating repo with modular metadata for %s.%s" % (variant, arch) - - repo_path = compose.paths.work.module_repo(arch, variant) - - compose.log_debug("[BEGIN] %s" % msg) - - lookaside_modules, platforms = get_lookaside_modules( - pungi.phases.gather.get_lookaside_repos(compose, arch, variant) - ) - - # Add modular metadata to it - included = set() - modules = [] - - # We need to include metadata for all variants. The packages are in the - # set, so we need their metadata. for var in compose.all_variants.values(): for mmd in var.arch_mmds.get(arch, {}).values(): - # Set the arch field, but no other changes are needed. - repo_mmd = mmd.copy() - repo_mmd.set_arch(tree_arch_to_yum_arch(arch)) - - for dep in repo_mmd.peek_dependencies(): + for dep in mmd.peek_dependencies(): streams = dep.peek_requires().get("platform") if streams: platforms.update(streams.dup()) - nsvc = "%s:%s:%s:%s" % ( - repo_mmd.peek_name(), - repo_mmd.peek_stream(), - repo_mmd.peek_version(), - repo_mmd.peek_context(), - ) - if nsvc not in lookaside_modules and nsvc not in included: - modules.append(repo_mmd) - included.add(nsvc) - if len(platforms) > 1: raise RuntimeError("There are conflicting requests for platform.") - module_names = set([x.get_name() for x in modules]) - defaults_dir = compose.paths.work.module_defaults_dir() - for mmddef in iter_module_defaults(defaults_dir): - if mmddef.peek_module_name() in module_names: - modules.append(mmddef) - - if modules: - # Initialize empty repo - repo = CreaterepoWrapper(createrepo_c=createrepo_c) - cmd = repo.get_createrepo_cmd( - repo_path, database=False, outputdir=repo_path, checksum=createrepo_checksum - ) - logfile = "module_repo-%s" % variant - run(cmd, logfile=compose.paths.log.log_file(arch, logfile), show_cmd=True) - - log_file = compose.paths.log.log_file( - arch, "gather-modifyrepo-modules-%s" % variant - ) - add_modular_metadata(repo, repo_path, modules, log_file) - - compose.log_debug("[DONE ] %s" % msg) return list(platforms)[0] if platforms else None diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py index 0d888e2c..60e7a76c 100644 --- a/pungi/phases/pkgset/common.py +++ b/pungi/phases/pkgset/common.py @@ -22,7 +22,8 @@ from kobo.threads import run_in_threads import pungi.phases.pkgset.pkgsets from pungi.arch import get_valid_arches from pungi.wrappers.createrepo import CreaterepoWrapper -from pungi.util import is_arch_multilib, find_old_compose +from pungi.util import is_arch_multilib, find_old_compose, iter_module_defaults +from pungi.phases.createrepo import add_modular_metadata # TODO: per arch? @@ -116,4 +117,13 @@ def _create_arch_repo(worker_thread, args, task_num): baseurl="file://%s" % path_prefix, workers=compose.conf["createrepo_num_workers"], update_md_path=repo_dir_global, checksum=createrepo_checksum) run(cmd, logfile=compose.paths.log.log_file(arch, "arch_repo"), show_cmd=True) + # Add modulemd to the repo for all modules in all variants on this architecture. + mmds = list(iter_module_defaults(compose.paths.work.module_defaults_dir())) + for variant in compose.get_variants(arch=arch): + mmds.extend(variant.arch_mmds.get(arch, {}).values()) + if mmds: + add_modular_metadata( + repo, repo_dir, mmds, compose.paths.log.log_file(arch, "arch_repo_modulemd") + ) + compose.log_info("[DONE ] %s" % msg) diff --git a/tests/test_gather_method_hybrid.py b/tests/test_gather_method_hybrid.py index d973e99f..fa1409f6 100644 --- a/tests/test_gather_method_hybrid.py +++ b/tests/test_gather_method_hybrid.py @@ -28,8 +28,8 @@ class TestMethodHybrid(helpers.PungiTestCase): @mock.patch("pungi.phases.gather.get_lookaside_repos") @mock.patch("pungi.phases.gather.methods.method_hybrid.expand_groups") @mock.patch("pungi.phases.gather.methods.method_hybrid.expand_packages") - @mock.patch("pungi.phases.gather.methods.method_hybrid.create_module_repo") - def test_call_method(self, cmr, ep, eg, glr, CW): + @mock.patch("pungi.phases.gather.methods.method_hybrid.get_platform") + def test_call_method(self, gp, ep, eg, glr, CW): compose = helpers.DummyCompose(self.topdir, {}) m = hybrid.GatherMethodHybrid(compose) m.run_solver = mock.Mock(return_value=(mock.Mock(), mock.Mock())) @@ -58,7 +58,7 @@ class TestMethodHybrid(helpers.PungiTestCase): ) self.assertEqual(res, ep.return_value) - self.assertEqual(cmr.call_args_list, [mock.call(compose, variant, arch)]) + self.assertEqual(gp.call_args_list, [mock.call(compose, variant, arch)]) self.assertEqual( m.run_solver.call_args_list, [ @@ -66,7 +66,7 @@ class TestMethodHybrid(helpers.PungiTestCase): variant, arch, set(["pkg", "foo", "bar", ("prep", "noarch")]), - cmr.return_value, + gp.return_value, [], ) ], @@ -250,62 +250,26 @@ class HelperMixin(object): return os.path.join(self.compose.topdir, "work/x86_64/%s" % name) -@mock.patch("pungi.phases.gather.methods.method_hybrid.add_modular_metadata") -@mock.patch("pungi.phases.gather.methods.method_hybrid.run") -class TestCreateModuleRepo(HelperMixin, helpers.PungiTestCase): +class TestGetPlatform(HelperMixin, helpers.PungiTestCase): def setUp(self): - super(TestCreateModuleRepo, self).setUp() + super(TestGetPlatform, self).setUp() self.compose = helpers.DummyCompose(self.topdir, {}) self.variant = self.compose.variants["Server"] - def test_no_modules(self, run, Modulemd): - plat = hybrid.create_module_repo(self.compose, self.variant, "x86_64") - + def test_no_modules(self): + plat = hybrid.get_platform(self.compose, self.variant, "x86_64") self.assertIsNone(plat) - self.assertEqual(run.call_args_list, []) - self.assertEqual(Modulemd.mock_calls, []) - def test_more_than_one_platform(self, run, add_modular_metadata): + def test_more_than_one_platform(self): self.variant.arch_mmds["x86_64"] = { "mod:1": MockModule("mod", platform="f29"), "mod:2": MockModule("mod", platform="f30"), } with self.assertRaises(RuntimeError) as ctx: - hybrid.create_module_repo(self.compose, self.variant, "x86_64") + hybrid.get_platform(self.compose, self.variant, "x86_64") self.assertIn("conflicting requests for platform", str(ctx.exception)) - self.assertEqual(run.call_args_list, []) - self.assertEqual(add_modular_metadata.mock_calls, []) - - @mock.patch("pungi.phases.gather.methods.method_hybrid.iter_module_defaults") - def test_creating_repo_with_module_and_default(self, imd, run, add_modular_metadata): - mod = MockModule("mod", platform="f29") - self.variant.arch_mmds["x86_64"] = {"mod:1": mod} - default = mock.Mock(peek_module_name=mock.Mock(return_value="mod")) - imd.return_value = [default] - - plat = hybrid.create_module_repo(self.compose, self.variant, "x86_64") - - self.assertEqual(plat, "f29") - - self.assertEqual( - add_modular_metadata.call_args_list, - [ - mock.call( - mock.ANY, - self._repo("module_repo_Server"), - [mod, default], - mock.ANY, - ), - ], - ) - self.assertEqual( - # Get first positional argument of the first call, and since it's - # an array, take first two elements. - run.call_args_list[0][0][0][:2], - ["createrepo_c", self._repo("module_repo_Server")] - ) class ModifiedMagicMock(mock.MagicMock): @@ -389,7 +353,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): mock.call( self.config1, "x86_64", - [self._repo("repo"), self._repo("module_repo_Server")], + [self._repo("repo")], [], platform="pl", filter_packages=[("foo", None)], @@ -445,7 +409,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): mock.call( self.config1, "x86_64", - [self._repo("repo"), self._repo("module_repo_Server")], + [self._repo("repo")], [], platform="pl", filter_packages=["foo"],