From cc4d99441cd2c20802a3114484463afd45965fc6 Mon Sep 17 00:00:00 2001 From: stepan_oksanichenko Date: Wed, 12 May 2021 18:14:49 +0300 Subject: [PATCH] LNX-108: Add multiarch support to pungi @BS-NOBUILD @BS-TARGET-CL8 Change-Id: Ibfd540454941922d790ae4e56cc0992c0c85635d --- pungi/phases/pkgset/sources/source_koji.py | 25 +++++++++----- pungi/scripts/create_extra_repo.py | 11 +++--- pungi/scripts/gather_modules.py | 19 ++++++---- pungi/wrappers/comps.py | 2 ++ pungi/wrappers/kojimock.py | 40 +++++++++++++++------- tests/test_gather_modules.py | 6 ++-- tests/test_koji_local_source.py | 8 +++-- tests/test_pkgset_source_koji.py | 17 ++++----- 8 files changed, 81 insertions(+), 47 deletions(-) diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 23cc09e1..bc02c498 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -228,9 +228,12 @@ def _add_module_to_variant( # Skip non module archives continue filename = archive["filename"] - # TODO: add support for muplitple arches - file_path = os.path.join(koji_wrapper.koji_module.pathinfo.topdir, 'modules', - build['extra']['typeinfo']['module']['content_koji_tag']) + file_path = os.path.join( + koji_wrapper.koji_module.pathinfo.topdir, + 'modules', + build['arch'], + build['extra']['typeinfo']['module']['content_koji_tag'] + ) mmds[filename] = file_path if len(mmds) <= 1: @@ -488,16 +491,15 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules) info.get("context"), ) nvr_patterns.add((pattern, spec["name"])) - modules_to_keep = [] - for mb in module_builds: + for mb in sorted(module_builds, key=lambda i: i['name']): # Split release from the build into version and context ver, ctx = mb["release"].split(".") # Values in `mb` are from Koji build. There's nvr and name, version and # release. The input pattern specifies modular name, stream, version # and context. - for (n, s, v, c), spec in nvr_patterns: + for (n, s, v, c), spec in sorted(nvr_patterns): if ( # We always have a name and stream... mb["name"] == n @@ -509,7 +511,6 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules) ): modules_to_keep.append(mb) expected_modules.discard(spec) - break return modules_to_keep @@ -543,7 +544,7 @@ def _filter_expected_modules( if cond1 and cond2: expected_modules = { expected_module for expected_module in expected_modules if - not all( + not any( re.findall( filtered_module, expected_module, @@ -573,7 +574,13 @@ def _get_modules_from_koji_tags( ] # Get set of configured module names for this variant. If nothing is # configured, the set is empty. - expected_modules = set(spec["name"] for spec in variant.get_modules()) + expected_modules = [] + for spec in variant.get_modules(): + name, stream = spec['name'].split(':') + expected_modules.append( + ':'.join((name, stream.replace('-', '_'))) + ) + expected_modules = set(expected_modules) # Find out all modules in every variant and add their Koji tags # to variant and variant_tags list. koji_proxy = koji_wrapper.koji_proxy diff --git a/pungi/scripts/create_extra_repo.py b/pungi/scripts/create_extra_repo.py index 9598ea41..fd5898ae 100644 --- a/pungi/scripts/create_extra_repo.py +++ b/pungi/scripts/create_extra_repo.py @@ -78,11 +78,12 @@ class CreateExtraRepo(PackagesGenerator): """ Dump merged modules data to an local repo """ - with open(self.default_modules_yaml_path, 'w') as yaml_file: - yaml.dump_all( - self.local_modules_data, - yaml_file, - ) + if self.local_modules_data: + with open(self.default_modules_yaml_path, 'w') as yaml_file: + yaml.dump_all( + self.local_modules_data, + yaml_file, + ) @staticmethod def get_repo_info_from_bs_repo( diff --git a/pungi/scripts/gather_modules.py b/pungi/scripts/gather_modules.py index 4885f8ce..de0649e9 100644 --- a/pungi/scripts/gather_modules.py +++ b/pungi/scripts/gather_modules.py @@ -51,19 +51,24 @@ def collect_modules(modules_paths: List[BinaryIO], target_dir: str): documents = yaml.load_all(data, Loader=yaml.BaseLoader) for doc in documents: if doc['document'] == 'modulemd-defaults': - name = doc['data']['module'] + '.yaml' + name = f"{doc['data']['module']}.yaml" path = os.path.join(module_defaults_path, name) logging.info('Found %s module defaults', name) else: # pungi.phases.pkgset.sources.source_koji.get_koji_modules stream = doc['data']['stream'].replace('-', '_') - name = '%s-%s-%s.%s' % ( - doc['data']['name'], - stream, - doc['data']['version'], - doc['data']['context'] + doc_data = doc['data'] + name = f"{doc_data['name']}-{stream}-" \ + f"{doc_data['version']}.{doc_data['context']}" + arch_dir = os.path.join( + modules_path, + doc_data['arch'] + ) + os.makedirs(arch_dir, exist_ok=True) + path = os.path.join( + arch_dir, + name, ) - path = os.path.join(modules_path, name) logging.info('Found module %s', name) if 'artifacts' not in doc['data']: diff --git a/pungi/wrappers/comps.py b/pungi/wrappers/comps.py index aa7685b4..5a8ee09c 100644 --- a/pungi/wrappers/comps.py +++ b/pungi/wrappers/comps.py @@ -299,6 +299,8 @@ class CompsWrapper(object): append_common_info(doc, group_node, group, force_description=True) append_bool(doc, group_node, "default", group.default) append_bool(doc, group_node, "uservisible", group.uservisible) + if group.display_order is not None: + append(doc, group_node, "display_order", str(group.display_order)) if group.lang_only: append(doc, group_node, "langonly", group.lang_only) diff --git a/pungi/wrappers/kojimock.py b/pungi/wrappers/kojimock.py index cac1449b..56813158 100644 --- a/pungi/wrappers/kojimock.py +++ b/pungi/wrappers/kojimock.py @@ -35,6 +35,7 @@ class Module: stream: str version: str context: str + arch: str class KojiMock: @@ -49,16 +50,22 @@ class KojiMock: def _gather_modules(self, modules_dir): modules = {} - for index, f in enumerate(os.listdir(modules_dir)): - parsed = parse_nvra(f) - modules[index] = Module( - name=parsed['name'], - nvr=f, - version=parsed['release'], - context=parsed['arch'], - stream=parsed['version'], - build_id=index + for arch in os.listdir(modules_dir): + arch_dir = os.path.join( + modules_dir, + arch, ) + for index, f in enumerate(os.listdir(arch_dir)): + parsed = parse_nvra(f) + modules[index] = Module( + name=parsed['name'], + nvr=f, + version=parsed['release'], + context=parsed['arch'], + stream=parsed['version'], + build_id=index, + arch=arch, + ) return modules def getLastEvent(self, *args, **kwargs): @@ -124,6 +131,7 @@ class KojiMock: 'release': '%s.%s' % (module.version, module.context), 'completion_ts': BUILD_TIME, 'state': 'COMPLETE', + 'arch': module.arch, 'extra': { 'typeinfo': { 'module': { @@ -152,7 +160,7 @@ class KojiMock: return [ { 'build_id': module.build_id, - 'filename': 'modulemd.x86_64.txt', + 'filename': f'modulemd.{module.arch}.txt', 'btype': 'module' }, # noone ever uses this file @@ -197,7 +205,11 @@ class KojiMock: # get nvras for modular packages nvras = set() for module in self._modules.values(): - path = os.path.join(self._modules_dir, module.nvr) + path = os.path.join( + self._modules_dir, + module.arch, + module.nvr, + ) info = Modulemd.ModuleStream.read_string(open(path).read(), strict=True) for package in info.get_rpm_artifacts(): @@ -232,8 +244,12 @@ class KojiMock: """ Get list of builds for module and given module tag name. """ - path = os.path.join(self._modules_dir, tag_name) module = self._get_module_by_name(tag_name) + path = os.path.join( + self._modules_dir, + module.arch, + tag_name, + ) builds = [ { diff --git a/tests/test_gather_modules.py b/tests/test_gather_modules.py index 4b683bfc..0930ef06 100644 --- a/tests/test_gather_modules.py +++ b/tests/test_gather_modules.py @@ -105,15 +105,15 @@ class TestModulesYamlParser(TestCase): self.assertEqual([MODULES_YAML_GZ, 'modules', 'module_defaults'], os.listdir(PATH_TO_KOJI)) self.assertEqual(['mariadb-devel-10.3_1-8010020200108182321.cdc1202b', 'javapackages-tools-201801-8000020190628172923.b07bea58'], - os.listdir(os.path.join(PATH_TO_KOJI, 'modules'))) + os.listdir(os.path.join(PATH_TO_KOJI, 'modules/x86_64'))) self.assertEqual(['ant.yaml'], os.listdir(os.path.join(PATH_TO_KOJI, 'module_defaults'))) # check that modules were exported self.assertEqual(MARIADB_MODULE, yaml.load( - open(os.path.join(PATH_TO_KOJI, 'modules', 'mariadb-devel-10.3_1-8010020200108182321.cdc1202b')))) + open(os.path.join(PATH_TO_KOJI, 'modules/x86_64', 'mariadb-devel-10.3_1-8010020200108182321.cdc1202b')))) self.assertEqual(JAVAPACKAGES_TOOLS_MODULE, yaml.load( - open(os.path.join(PATH_TO_KOJI, 'modules', 'javapackages-tools-201801-8000020190628172923.b07bea58')))) + open(os.path.join(PATH_TO_KOJI, 'modules/x86_64', 'javapackages-tools-201801-8000020190628172923.b07bea58')))) # check that defaults were copied self.assertEqual(ANT_DEFAULTS, yaml.load( diff --git a/tests/test_koji_local_source.py b/tests/test_koji_local_source.py index fc4c03cf..0c64032a 100644 --- a/tests/test_koji_local_source.py +++ b/tests/test_koji_local_source.py @@ -103,13 +103,13 @@ version: '1' self.setUpPyfakefs() os.makedirs(PATH_TO_REPOS) - os.makedirs(os.path.join(PATH_TO_REPOS, 'modules')) + os.makedirs(os.path.join(PATH_TO_REPOS, 'modules/x86_64')) - with open(os.path.join(PATH_TO_REPOS, 'modules', + with open(os.path.join(PATH_TO_REPOS, 'modules/x86_64', 'javapackages-tools-201801-8000020190628172923.b07bea58'), 'w') as f: f.write(self.JAVAPACKAGES_TOOLS_MODULE) - with open(os.path.join(PATH_TO_REPOS, 'modules', + with open(os.path.join(PATH_TO_REPOS, 'modules/x86_64', 'mariadb-devel-10.3-8010020200108182321.cdc1202b'), 'w') as f: f.write(self.MARIADB_MODULE) @@ -122,6 +122,7 @@ version: '1' @ddt.data( [0, { 'completion_ts': 0, + 'arch': 'x86_64', 'extra': { 'typeinfo': { 'module': { @@ -141,6 +142,7 @@ version: '1' }], [1, { 'completion_ts': 0, + 'arch': 'x86_64', 'extra': { 'typeinfo': { 'module': { diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index 9d844bc6..6c90ab54 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -703,6 +703,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase): ] + [{"btype": "foo"}] self.buildinfo = { "id": 1234, + "arch": "fake_arch", "extra": { "typeinfo": { "module": { @@ -734,7 +735,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase): # }, "x86_64": { "module:master:20190318:abcdef": MockModule( - "/mnt/koji/modules/module:master-20190318-abcdef" + "/mnt/koji/modules/fake_arch/module:master-20190318-abcdef" ), }, }, @@ -748,7 +749,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase): "x86_64" ], arch_mmds={ - "x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe")} + "x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe")} }, modules=[{"name": "m1:latest-20190101:cafe", "glob": False}], ) @@ -765,9 +766,9 @@ class TestAddModuleToVariant(helpers.PungiTestCase): # }, "x86_64": { "module:master:20190318:abcdef": MockModule( - "/mnt/koji/modules/module:master-20190318-abcdef" + "/mnt/koji/modules/fake_arch/module:master-20190318-abcdef" ), - "m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe"), + "m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe"), }, }, ) @@ -795,7 +796,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase): # }, "x86_64": { "module:master:20190318:abcdef": MockModule( - "/mnt/koji/modules/module:master-20190318-abcdef" + "/mnt/koji/modules/fake_arch/module:master-20190318-abcdef" ) }, }, @@ -811,7 +812,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase): "x86_64" ], arch_mmds={ - "x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe")} + "x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe")} }, modules=[{"name": "m1:latest-20190101:cafe", "glob": False}], ) @@ -830,9 +831,9 @@ class TestAddModuleToVariant(helpers.PungiTestCase): # }, "x86_64": { "module:master:20190318:abcdef": MockModule( - "/mnt/koji/modules/module:master-20190318-abcdef" + "/mnt/koji/modules/fake_arch/module:master-20190318-abcdef" ), - "m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe"), + "m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe"), }, }, )