From a73099d44618bbe1ea7da777cb892491c6de0b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 19 Oct 2018 10:27:04 +0200 Subject: [PATCH] Support more specific config for devel modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The initial implementation is very coarse. It enables it for all variants and all modules. That is not always wanted. With this patch, the config file has to explicitly list the devel modules for each variant that should have it. The variant must be configured also to include the non-devel module (but the module may be in lookaside so it won't be included). We now include module metadata in the internal lookaside repo, so that this whole thing works if one variant is built on top of another. JIRA: COMPOSE-3034 Signed-off-by: Lubomír Sedlář --- pungi/checks.py | 4 +- pungi/phases/createrepo.py | 43 ++++++++++++------ pungi/phases/gather/__init__.py | 26 ++++++++++- pungi/phases/gather/methods/method_hybrid.py | 48 ++++++++++---------- pungi/phases/gather/sources/source_module.py | 27 +++++++++-- pungi/wrappers/variants.py | 1 + tests/helpers.py | 1 + tests/test_gather_method_hybrid.py | 33 +++++++------- 8 files changed, 121 insertions(+), 62 deletions(-) diff --git a/pungi/checks.py b/pungi/checks.py index 8bb2ae15..614be512 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -609,8 +609,8 @@ def make_schema(): "default": False, }, "include_devel_modules": { - "type": "boolean", - "default": False, + "type": "object", + "default": {}, }, "pkgset_source": { diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index f33f7d4a..042233bc 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -254,24 +254,41 @@ def create_variant_repo(compose, arch, variant, pkg_type, modules_metadata=None) if mmddef.peek_module_name() in module_names: modules.append(mmddef) - with temp_dir() as tmp_dir: - modules_path = os.path.join(tmp_dir, "modules.yaml") - Modulemd.dump(modules, modules_path) + log_file = compose.paths.log.log_file(arch, "modifyrepo-modules-%s" % variant) + add_modular_metadata(repo, repo_dir, modules, log_file) - cmd = repo.get_modifyrepo_cmd(os.path.join(repo_dir, "repodata"), - modules_path, mdtype="modules", - compress_type="gz") - log_file = compose.paths.log.log_file( - arch, "modifyrepo-modules-%s" % variant) - run(cmd, logfile=log_file, show_cmd=True) - - for module_id, module_rpms in metadata: - modulemd_path = os.path.join(types[pkg_type][1](relative=True), find_file_in_repodata(repo_dir, 'modules')) - modules_metadata.prepare_module_metadata(variant, arch, module_id, modulemd_path, types[pkg_type][0], list(module_rpms)) + for module_id, module_rpms in metadata: + modulemd_path = os.path.join( + types[pkg_type][1](relative=True), + find_file_in_repodata(repo_dir, 'modules'), + ) + modules_metadata.prepare_module_metadata( + variant, + arch, + module_id, + modulemd_path, + types[pkg_type][0], + list(module_rpms), + ) compose.log_info("[DONE ] %s" % msg) +def add_modular_metadata(repo, repo_path, mmd, log_file): + """Add modular metadata into a repository.""" + with temp_dir() as tmp_dir: + modules_path = os.path.join(tmp_dir, "modules.yaml") + Modulemd.dump(mmd, modules_path) + + cmd = repo.get_modifyrepo_cmd( + os.path.join(repo_path, "repodata"), + modules_path, + mdtype="modules", + compress_type="gz" + ) + run(cmd, logfile=log_file, show_cmd=True) + + def find_file_in_repodata(repo_path, type_): dom = xml.dom.minidom.parse(os.path.join(repo_path, 'repodata', 'repomd.xml')) for entry in dom.getElementsByTagName('data'): diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 745bd57c..f22920be 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -28,11 +28,12 @@ from ...wrappers.createrepo import CreaterepoWrapper import pungi.wrappers.kojiwrapper from pungi import Modulemd -from pungi.arch import get_compatible_arches, split_name_arch +from pungi.arch import get_compatible_arches, split_name_arch, tree_arch_to_yum_arch from pungi.graph import SimpleAcyclicOrientedGraph from pungi.phases.base import PhaseBase from pungi.util import (get_arch_data, get_arch_variant_data, get_variant_data, - makedirs) + makedirs, iter_module_defaults) +from pungi.phases.createrepo import add_modular_metadata def get_gather_source(name): @@ -401,6 +402,27 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map): run(cmd, logfile=compose.paths.log.log_file(arch, "lookaside_repo_%s" % (variant.uid)), show_cmd=True) + + # Add modular metadata into the repo + if variant.arch_mmds: + mmds = [] + for mmd in variant.arch_mmds[arch].values(): + # Set the arch field, but no other changes are needed. + repo_mmd = mmd.copy() + repo_mmd.set_arch(tree_arch_to_yum_arch(arch)) + mmds.append(repo_mmd) + + module_names = set([x.get_name() for x in mmds]) + defaults_dir = compose.paths.work.module_defaults_dir() + for mmddef in iter_module_defaults(defaults_dir): + if mmddef.peek_module_name() in module_names: + mmds.append(mmddef) + + log_file = compose.paths.log.log_file( + arch, "lookaside_repo_modules_%s" % (variant.uid) + ) + add_modular_metadata(cr, repo, mmds, log_file) + compose.log_info('[DONE ] %s', msg) return repo diff --git a/pungi/phases/gather/methods/method_hybrid.py b/pungi/phases/gather/methods/method_hybrid.py index bda6c01f..0dcde6f6 100644 --- a/pungi/phases/gather/methods/method_hybrid.py +++ b/pungi/phases/gather/methods/method_hybrid.py @@ -13,22 +13,25 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, see . -from collections import defaultdict -import os -from kobo.shortcuts import run -import kobo.rpmlib -from fnmatch import fnmatch import gzip +import os +from collections import defaultdict +from fnmatch import fnmatch +from itertools import chain + +import createrepo_c as cr +import kobo.rpmlib +from kobo.shortcuts import run import pungi.phases.gather.method from pungi import Modulemd, multilib_dnf from pungi.arch import get_valid_arches, tree_arch_to_yum_arch from pungi.phases.gather import _mk_pkg_map +from pungi.phases.createrepo import add_modular_metadata from pungi.util import ( get_arch_variant_data, iter_module_defaults, pkg_is_debug, - temp_dir, ) from pungi.wrappers import fus from pungi.wrappers.comps import CompsWrapper @@ -36,8 +39,6 @@ from pungi.wrappers.createrepo import CreaterepoWrapper from .method_nodeps import expand_groups -import createrepo_c as cr - class FakePackage(object): """This imitates a DNF package object and can be passed to python-multilib @@ -127,6 +128,8 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): for var in self.compose.all_variants.values(): for mmd in var.arch_mmds.get(self.arch, {}).values(): self.modular_packages.update(mmd.get_rpm_artifacts().dup()) + for mmd in var.dev_mmds.get(self.arch, {}).values(): + self.modular_packages.update(mmd.get_rpm_artifacts().dup()) def prepare_langpacks(self, arch, variant): if not self.compose.has_comps: @@ -236,8 +239,10 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): # useless for us anyway). env = os.environ.copy() env["G_MESSAGES_PREFIXED"] = "" + self.compose.log_debug("[BEGIN] Running fus") run(cmd, logfile=logfile, show_cmd=True, env=env) output, out_modules = fus.parse_output(logfile) + self.compose.log_debug("[DONE ] Running fus") new_multilib = self.add_multilib(variant, arch, output, old_multilib) old_multilib = new_multilib if new_multilib: @@ -287,6 +292,7 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): def add_langpacks(self, nvrs): if not self.langpacks: return set() + added = set() for nvr, pkg_arch, flags in nvrs: if "modular" in flags: @@ -376,12 +382,15 @@ def create_module_repo(compose, variant, arch): ) # Add modular metadata to it + included = set() modules = [] # We need to include metadata for all variants. The packages are in the # set, so we need their metadata. for var in compose.all_variants.values(): - for mmd in var.arch_mmds.get(arch, {}).values(): + for mmd in chain( + var.arch_mmds.get(arch, {}).values(), var.dev_mmds.get(arch, {}).values() + ): # Set the arch field, but no other changes are needed. repo_mmd = mmd.copy() repo_mmd.set_arch(tree_arch_to_yum_arch(arch)) @@ -397,8 +406,9 @@ def create_module_repo(compose, variant, arch): repo_mmd.peek_version(), repo_mmd.peek_context(), ) - if nsvc not in lookaside_modules: + if nsvc not in lookaside_modules and nsvc not in included: modules.append(repo_mmd) + included.add(nsvc) if len(platforms) > 1: raise RuntimeError("There are conflicting requests for platform.") @@ -418,20 +428,10 @@ def create_module_repo(compose, variant, arch): logfile = "module_repo-%s" % variant run(cmd, logfile=compose.paths.log.log_file(arch, logfile), show_cmd=True) - with temp_dir() as tmp_dir: - modules_path = os.path.join(tmp_dir, "modules.yaml") - Modulemd.dump(modules, modules_path) - - cmd = repo.get_modifyrepo_cmd( - os.path.join(repo_path, "repodata"), - modules_path, - mdtype="modules", - compress_type="gz", - ) - log_file = compose.paths.log.log_file( - arch, "gather-modifyrepo-modules-%s" % variant - ) - run(cmd, logfile=log_file, show_cmd=True) + log_file = compose.paths.log.log_file( + arch, "gather-modifyrepo-modules-%s" % variant + ) + add_modular_metadata(repo, repo_path, modules, log_file) compose.log_debug("[DONE ] %s" % msg) return list(platforms)[0] if platforms else None diff --git a/pungi/phases/gather/sources/source_module.py b/pungi/phases/gather/sources/source_module.py index 4b678892..782eb8db 100644 --- a/pungi/phases/gather/sources/source_module.py +++ b/pungi/phases/gather/sources/source_module.py @@ -59,6 +59,8 @@ class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase): # Generate architecture specific modulemd metadata, so we can # store per-architecture artifacts there later. variant.arch_mmds.setdefault(arch, {}) + variant.dev_mmds.setdefault(arch, {}) + include_devel = self.compose.conf.get("include_devel_modules", {}).get(variant.uid, []) for mmd in variant.mmds: nsvc = "%s:%s:%s:%s" % ( mmd.peek_name(), @@ -70,14 +72,18 @@ class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase): arch_mmd = mmd.copy() variant.arch_mmds[arch][nsvc] = arch_mmd - if self.compose.conf["include_devel_modules"]: + if self.compose.conf.get("include_devel_modules"): + # Devel modules are enabled, we need to create it. devel_nsvc = "%s-devel:%s:%s:%s" % ( mmd.peek_name(), mmd.peek_stream(), mmd.peek_version(), mmd.peek_context(), ) - if devel_nsvc not in variant.arch_mmds[arch]: + if ( + devel_nsvc not in variant.arch_mmds[arch] + and devel_nsvc not in variant.dev_mmds[arch] + ): arch_mmd = mmd.copy() arch_mmd.set_name(arch_mmd.peek_name() + "-devel") # Depend on the actual module @@ -86,9 +92,15 @@ class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase): # Delete API and profiles arch_mmd.set_rpm_api(Modulemd.SimpleSet()) arch_mmd.clear_profiles() + + ns = "%s:%s" % (arch_mmd.peek_name(), arch_mmd.peek_stream()) + # Store the new modulemd - variant.arch_mmds[arch][devel_nsvc] = arch_mmd variant.module_uid_to_koji_tag[devel_nsvc] = variant.module_uid_to_koji_tag.get(nsvc) + if ns in include_devel: + variant.arch_mmds[arch][devel_nsvc] = arch_mmd + else: + variant.dev_mmds[arch][devel_nsvc] = arch_mmd # Contains per-module RPMs added to variant. added_rpms = {} @@ -121,7 +133,7 @@ class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase): added_rpms[nsvc].append(str(rpm_obj.nevra)) log.write('Adding %s because it is in %s\n' % (rpm_obj, nsvc)) - elif self.compose.conf["include_devel_modules"]: + elif self.compose.conf.get("include_devel_modules"): nsvc_devel = "%s-devel:%s:%s:%s" % ( mmd.peek_name(), mmd.peek_stream(), @@ -141,7 +153,12 @@ class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase): # have not been added to variant from the `arch_mmd`. This package # list is later used in createrepo phase to generated modules.yaml. for nsvc, rpm_nevras in added_rpms.items(): - arch_mmd = variant.arch_mmds[arch][nsvc] + # If we added some RPMs from a module, that module must exist in + # exactly one of the dicts. We need to find the metadata object for + # it. + arch_mmd = variant.arch_mmds[arch].get(nsvc) or variant.dev_mmds[arch].get( + nsvc + ) added_artifacts = Modulemd.SimpleSet() for rpm_nevra in rpm_nevras: diff --git a/pungi/wrappers/variants.py b/pungi/wrappers/variants.py index 5689d346..d0447e06 100755 --- a/pungi/wrappers/variants.py +++ b/pungi/wrappers/variants.py @@ -266,6 +266,7 @@ class Variant(object): self.pkgset = None self.mmds = [] self.arch_mmds = {} + self.dev_mmds = {} self.module_uid_to_koji_tag = {} self.nsvc_to_pkgset = {} diff --git a/tests/helpers.py b/tests/helpers.py index 5537ce97..878b7528 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -62,6 +62,7 @@ class MockVariant(mock.Mock): self.parent = kwargs.get('parent', None) self.mmds = [] self.arch_mmds = {} + self.dev_mmds = {} self.module_uid_to_koji_tag = {} self.variants = {} self.pkgset = mock.Mock(rpms_by_arch={}) diff --git a/tests/test_gather_method_hybrid.py b/tests/test_gather_method_hybrid.py index 66e5492f..2e355e30 100644 --- a/tests/test_gather_method_hybrid.py +++ b/tests/test_gather_method_hybrid.py @@ -235,7 +235,7 @@ class HelperMixin(object): return os.path.join(self.compose.topdir, "work/x86_64/%s" % name) -@mock.patch("pungi.phases.gather.methods.method_hybrid.Modulemd") +@mock.patch("pungi.phases.gather.methods.method_hybrid.add_modular_metadata") @mock.patch("pungi.phases.gather.methods.method_hybrid.run") class TestCreateModuleRepo(HelperMixin, helpers.PungiTestCase): def setUp(self): @@ -250,7 +250,7 @@ class TestCreateModuleRepo(HelperMixin, helpers.PungiTestCase): self.assertEqual(run.call_args_list, []) self.assertEqual(Modulemd.mock_calls, []) - def test_more_than_one_platform(self, run, Modulemd): + def test_more_than_one_platform(self, run, add_modular_metadata): self.variant.arch_mmds["x86_64"] = { "mod:1": MockModule("mod", platform="f29"), "mod:2": MockModule("mod", platform="f30"), @@ -261,10 +261,10 @@ class TestCreateModuleRepo(HelperMixin, helpers.PungiTestCase): self.assertIn("conflicting requests for platform", str(ctx.exception)) self.assertEqual(run.call_args_list, []) - self.assertEqual(Modulemd.mock_calls, []) + self.assertEqual(add_modular_metadata.mock_calls, []) @mock.patch("pungi.phases.gather.methods.method_hybrid.iter_module_defaults") - def test_creating_repo_with_module_and_default(self, imd, run, Modulemd): + def test_creating_repo_with_module_and_default(self, imd, run, add_modular_metadata): mod = MockModule("mod", platform="f29") self.variant.arch_mmds["x86_64"] = {"mod:1": mod} default = mock.Mock(peek_module_name=mock.Mock(return_value="mod")) @@ -275,21 +275,22 @@ class TestCreateModuleRepo(HelperMixin, helpers.PungiTestCase): self.assertEqual(plat, "f29") self.assertEqual( - Modulemd.mock_calls, [mock.call.dump([mod, default], mock.ANY)] - ) - create, modify = run.call_args_list - self.assertEqual( - create[0][0][:2], ["createrepo_c", self._repo("module_repo_Server")] - ) - self.assertEqual( - modify[0][0][:4], + add_modular_metadata.call_args_list, [ - "modifyrepo_c", - Modulemd.mock_calls[0][1][1], - self._repo("module_repo_Server/repodata"), - "--mdtype=modules", + mock.call( + mock.ANY, + self._repo("module_repo_Server"), + [mod, default], + mock.ANY, + ), ], ) + self.assertEqual( + # Get first positional argument of the first call, and since it's + # an array, take first two elements. + run.call_args_list[0][0][0][:2], + ["createrepo_c", self._repo("module_repo_Server")] + ) class ModifiedMagicMock(mock.MagicMock):