LNX-108: Add multiarch support to pungi

@BS-NOBUILD @BS-TARGET-CL8 Change-Id: Ibfd540454941922d790ae4e56cc0992c0c85635d
2021-05-12 18:14:49 +03:00 · 2021-05-12 18:14:49 +03:00 · cc4d99441c
parent a435eeed06
commit cc4d99441c
8 changed files with 81 additions and 47 deletions
--- a/pungi/phases/pkgset/sources/source_koji.py
+++ b/pungi/phases/pkgset/sources/source_koji.py
@ -228,9 +228,12 @@ def _add_module_to_variant(
            # Skip non module archives
            continue
        filename = archive["filename"]
-        # TODO: add support for muplitple arches
-        file_path = os.path.join(koji_wrapper.koji_module.pathinfo.topdir, 'modules',
-                                 build['extra']['typeinfo']['module']['content_koji_tag'])
+        file_path = os.path.join(
+            koji_wrapper.koji_module.pathinfo.topdir,
+            'modules',
+            build['arch'],
+            build['extra']['typeinfo']['module']['content_koji_tag']
+        )
        mmds[filename] = file_path

    if len(mmds) <= 1:
@ -488,16 +491,15 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules)
            info.get("context"),
        )
        nvr_patterns.add((pattern, spec["name"]))
-
    modules_to_keep = []

-    for mb in module_builds:
+    for mb in sorted(module_builds, key=lambda i: i['name']):
        # Split release from the build into version and context
        ver, ctx = mb["release"].split(".")
        # Values in `mb` are from Koji build. There's nvr and name, version and
        # release. The input pattern specifies modular name, stream, version
        # and context.
-        for (n, s, v, c), spec in nvr_patterns:
+        for (n, s, v, c), spec in sorted(nvr_patterns):
            if (
                # We always have a name and stream...
                mb["name"] == n
@ -509,7 +511,6 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules)
            ):
                modules_to_keep.append(mb)
                expected_modules.discard(spec)
-                break

    return modules_to_keep

@ -543,7 +544,7 @@ def _filter_expected_modules(
            if cond1 and cond2:
                expected_modules = {
                    expected_module for expected_module in expected_modules if
-                    not all(
+                    not any(
                        re.findall(
                            filtered_module,
                            expected_module,
@ -573,7 +574,13 @@ def _get_modules_from_koji_tags(
    ]
    # Get set of configured module names for this variant. If nothing is
    # configured, the set is empty.
-    expected_modules = set(spec["name"] for spec in variant.get_modules())
+    expected_modules = []
+    for spec in variant.get_modules():
+        name, stream = spec['name'].split(':')
+        expected_modules.append(
+            ':'.join((name, stream.replace('-', '_')))
+        )
+    expected_modules = set(expected_modules)
    # Find out all modules in every variant and add their Koji tags
    # to variant and variant_tags list.
    koji_proxy = koji_wrapper.koji_proxy
--- a/pungi/scripts/create_extra_repo.py
+++ b/pungi/scripts/create_extra_repo.py
@ -78,11 +78,12 @@ class CreateExtraRepo(PackagesGenerator):
        """
        Dump merged modules data to an local repo
        """
-        with open(self.default_modules_yaml_path, 'w') as yaml_file:
-            yaml.dump_all(
-                self.local_modules_data,
-                yaml_file,
-            )
+        if self.local_modules_data:
+            with open(self.default_modules_yaml_path, 'w') as yaml_file:
+                yaml.dump_all(
+                    self.local_modules_data,
+                    yaml_file,
+                )

    @staticmethod
    def get_repo_info_from_bs_repo(
--- a/pungi/scripts/gather_modules.py
+++ b/pungi/scripts/gather_modules.py
@ -51,19 +51,24 @@ def collect_modules(modules_paths: List[BinaryIO], target_dir: str):
        documents = yaml.load_all(data, Loader=yaml.BaseLoader)
        for doc in documents:
            if doc['document'] == 'modulemd-defaults':
-                name = doc['data']['module'] + '.yaml'
+                name = f"{doc['data']['module']}.yaml"
                path = os.path.join(module_defaults_path, name)
                logging.info('Found %s module defaults', name)
            else:
                # pungi.phases.pkgset.sources.source_koji.get_koji_modules
                stream = doc['data']['stream'].replace('-', '_')
-                name = '%s-%s-%s.%s' % (
-                    doc['data']['name'],
-                    stream,
-                    doc['data']['version'],
-                    doc['data']['context']
+                doc_data = doc['data']
+                name = f"{doc_data['name']}-{stream}-" \
+                       f"{doc_data['version']}.{doc_data['context']}"
+                arch_dir = os.path.join(
+                    modules_path,
+                    doc_data['arch']
+                )
+                os.makedirs(arch_dir, exist_ok=True)
+                path = os.path.join(
+                    arch_dir,
+                    name,
                )
-                path = os.path.join(modules_path, name)
                logging.info('Found module %s', name)

                if 'artifacts' not in doc['data']:
--- a/pungi/wrappers/comps.py
+++ b/pungi/wrappers/comps.py
@ -299,6 +299,8 @@ class CompsWrapper(object):
            append_common_info(doc, group_node, group, force_description=True)
            append_bool(doc, group_node, "default", group.default)
            append_bool(doc, group_node, "uservisible", group.uservisible)
+            if group.display_order is not None:
+                append(doc, group_node, "display_order", str(group.display_order))

            if group.lang_only:
                append(doc, group_node, "langonly", group.lang_only)
--- a/pungi/wrappers/kojimock.py
+++ b/pungi/wrappers/kojimock.py
@ -35,6 +35,7 @@ class Module:
    stream: str
    version: str
    context: str
+    arch: str


 class KojiMock:
@ -49,16 +50,22 @@ class KojiMock:

    def _gather_modules(self, modules_dir):
        modules = {}
-        for index, f in enumerate(os.listdir(modules_dir)):
-            parsed = parse_nvra(f)
-            modules[index] = Module(
-                name=parsed['name'],
-                nvr=f,
-                version=parsed['release'],
-                context=parsed['arch'],
-                stream=parsed['version'],
-                build_id=index
+        for arch in os.listdir(modules_dir):
+            arch_dir = os.path.join(
+                modules_dir,
+                arch,
            )
+            for index, f in enumerate(os.listdir(arch_dir)):
+                parsed = parse_nvra(f)
+                modules[index] = Module(
+                    name=parsed['name'],
+                    nvr=f,
+                    version=parsed['release'],
+                    context=parsed['arch'],
+                    stream=parsed['version'],
+                    build_id=index,
+                    arch=arch,
+                )
        return modules

    def getLastEvent(self, *args, **kwargs):
@ -124,6 +131,7 @@ class KojiMock:
            'release': '%s.%s' % (module.version, module.context),
            'completion_ts': BUILD_TIME,
            'state': 'COMPLETE',
+            'arch': module.arch,
            'extra': {
                'typeinfo': {
                    'module': {
@ -152,7 +160,7 @@ class KojiMock:
        return [
            {
                'build_id': module.build_id,
-                'filename': 'modulemd.x86_64.txt',
+                'filename': f'modulemd.{module.arch}.txt',
                'btype': 'module'
            },
            # noone ever uses this file
@ -197,7 +205,11 @@ class KojiMock:
        # get nvras for modular packages
        nvras = set()
        for module in self._modules.values():
-            path = os.path.join(self._modules_dir, module.nvr)
+            path = os.path.join(
+                self._modules_dir,
+                module.arch,
+                module.nvr,
+            )
            info = Modulemd.ModuleStream.read_string(open(path).read(), strict=True)

            for package in info.get_rpm_artifacts():
@ -232,8 +244,12 @@ class KojiMock:
        """
        Get list of builds for module and given module tag name.
        """
-        path = os.path.join(self._modules_dir, tag_name)
        module = self._get_module_by_name(tag_name)
+        path = os.path.join(
+            self._modules_dir,
+            module.arch,
+            tag_name,
+        )

        builds = [
            {
--- a/tests/test_gather_modules.py
+++ b/tests/test_gather_modules.py
@ -105,15 +105,15 @@ class TestModulesYamlParser(TestCase):
        self.assertEqual([MODULES_YAML_GZ, 'modules', 'module_defaults'], os.listdir(PATH_TO_KOJI))
        self.assertEqual(['mariadb-devel-10.3_1-8010020200108182321.cdc1202b',
                          'javapackages-tools-201801-8000020190628172923.b07bea58'],
-                         os.listdir(os.path.join(PATH_TO_KOJI, 'modules')))
+                         os.listdir(os.path.join(PATH_TO_KOJI, 'modules/x86_64')))
        self.assertEqual(['ant.yaml'],
                         os.listdir(os.path.join(PATH_TO_KOJI, 'module_defaults')))

        # check that modules were exported
        self.assertEqual(MARIADB_MODULE, yaml.load(
-            open(os.path.join(PATH_TO_KOJI, 'modules', 'mariadb-devel-10.3_1-8010020200108182321.cdc1202b'))))
+            open(os.path.join(PATH_TO_KOJI, 'modules/x86_64', 'mariadb-devel-10.3_1-8010020200108182321.cdc1202b'))))
        self.assertEqual(JAVAPACKAGES_TOOLS_MODULE, yaml.load(
-            open(os.path.join(PATH_TO_KOJI, 'modules', 'javapackages-tools-201801-8000020190628172923.b07bea58'))))
+            open(os.path.join(PATH_TO_KOJI, 'modules/x86_64', 'javapackages-tools-201801-8000020190628172923.b07bea58'))))

        # check that defaults were copied
        self.assertEqual(ANT_DEFAULTS, yaml.load(
--- a/tests/test_koji_local_source.py
+++ b/tests/test_koji_local_source.py
@ -103,13 +103,13 @@ version: '1'
        self.setUpPyfakefs()

        os.makedirs(PATH_TO_REPOS)
-        os.makedirs(os.path.join(PATH_TO_REPOS, 'modules'))
+        os.makedirs(os.path.join(PATH_TO_REPOS, 'modules/x86_64'))

-        with open(os.path.join(PATH_TO_REPOS, 'modules',
+        with open(os.path.join(PATH_TO_REPOS, 'modules/x86_64',
                               'javapackages-tools-201801-8000020190628172923.b07bea58'), 'w') as f:
            f.write(self.JAVAPACKAGES_TOOLS_MODULE)

-        with open(os.path.join(PATH_TO_REPOS, 'modules',
+        with open(os.path.join(PATH_TO_REPOS, 'modules/x86_64',
                               'mariadb-devel-10.3-8010020200108182321.cdc1202b'), 'w') as f:
            f.write(self.MARIADB_MODULE)

@ -122,6 +122,7 @@ version: '1'
    @ddt.data(
        [0, {
            'completion_ts': 0,
+            'arch': 'x86_64',
            'extra': {
                'typeinfo': {
                    'module': {
@ -141,6 +142,7 @@ version: '1'
        }],
        [1, {
            'completion_ts': 0,
+            'arch': 'x86_64',
            'extra': {
                'typeinfo': {
                    'module': {
--- a/tests/test_pkgset_source_koji.py
+++ b/tests/test_pkgset_source_koji.py
@ -703,6 +703,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
        ] + [{"btype": "foo"}]
        self.buildinfo = {
            "id": 1234,
+            "arch": "fake_arch",
            "extra": {
                "typeinfo": {
                    "module": {
@ -734,7 +735,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
                # },
                "x86_64": {
                    "module:master:20190318:abcdef": MockModule(
-                        "/mnt/koji/modules/module:master-20190318-abcdef"
+                        "/mnt/koji/modules/fake_arch/module:master-20190318-abcdef"
                    ),
                },
            },
@ -748,7 +749,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
                "x86_64"
            ],
            arch_mmds={
-                "x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe")}
+                "x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe")}
            },
            modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
        )
@ -765,9 +766,9 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
                # },
                "x86_64": {
                    "module:master:20190318:abcdef": MockModule(
-                        "/mnt/koji/modules/module:master-20190318-abcdef"
+                        "/mnt/koji/modules/fake_arch/module:master-20190318-abcdef"
                    ),
-                    "m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe"),
+                    "m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe"),
                },
            },
        )
@ -795,7 +796,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
                # },
                "x86_64": {
                    "module:master:20190318:abcdef": MockModule(
-                        "/mnt/koji/modules/module:master-20190318-abcdef"
+                        "/mnt/koji/modules/fake_arch/module:master-20190318-abcdef"
                    )
                },
            },
@ -811,7 +812,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
                "x86_64"
            ],
            arch_mmds={
-                "x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe")}
+                "x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe")}
            },
            modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
        )
@ -830,9 +831,9 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
                # },
                "x86_64": {
                    "module:master:20190318:abcdef": MockModule(
-                        "/mnt/koji/modules/module:master-20190318-abcdef"
+                        "/mnt/koji/modules/fake_arch/module:master-20190318-abcdef"
                    ),
-                    "m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe"),
+                    "m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe"),
                },
            },
        )