LNX-108: Add multiarch support to pungi

@BS-NOBUILD
@BS-TARGET-CL8

Change-Id: Ibfd540454941922d790ae4e56cc0992c0c85635d

pungi/phases/pkgset/sources/source_koji.py

@@ -228,9 +228,12 @@ def _add_module_to_variant(
             # Skip non module archives
             continue
         filename = archive["filename"]
-        # TODO: add support for muplitple arches
+        file_path = os.path.join(
-        file_path = os.path.join(koji_wrapper.koji_module.pathinfo.topdir, 'modules',
+            koji_wrapper.koji_module.pathinfo.topdir,
-                                 build['extra']['typeinfo']['module']['content_koji_tag'])
+            'modules',
+            build['arch'],
+            build['extra']['typeinfo']['module']['content_koji_tag']
+        )
         mmds[filename] = file_path
 
     if len(mmds) <= 1:
@@ -488,16 +491,15 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules)
             info.get("context"),
         )
         nvr_patterns.add((pattern, spec["name"]))
-
     modules_to_keep = []
 
-    for mb in module_builds:
+    for mb in sorted(module_builds, key=lambda i: i['name']):
         # Split release from the build into version and context
         ver, ctx = mb["release"].split(".")
         # Values in `mb` are from Koji build. There's nvr and name, version and
         # release. The input pattern specifies modular name, stream, version
         # and context.
-        for (n, s, v, c), spec in nvr_patterns:
+        for (n, s, v, c), spec in sorted(nvr_patterns):
             if (
                 # We always have a name and stream...
                 mb["name"] == n
@@ -509,7 +511,6 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules)
             ):
                 modules_to_keep.append(mb)
                 expected_modules.discard(spec)
-                break
 
     return modules_to_keep
 
@@ -543,7 +544,7 @@ def _filter_expected_modules(
             if cond1 and cond2:
                 expected_modules = {
                     expected_module for expected_module in expected_modules if
-                    not all(
+                    not any(
                         re.findall(
                             filtered_module,
                             expected_module,
@@ -573,7 +574,13 @@ def _get_modules_from_koji_tags(
     ]
     # Get set of configured module names for this variant. If nothing is
     # configured, the set is empty.
-    expected_modules = set(spec["name"] for spec in variant.get_modules())
+    expected_modules = []
+    for spec in variant.get_modules():
+        name, stream = spec['name'].split(':')
+        expected_modules.append(
+            ':'.join((name, stream.replace('-', '_')))
+        )
+    expected_modules = set(expected_modules)
     # Find out all modules in every variant and add their Koji tags
     # to variant and variant_tags list.
     koji_proxy = koji_wrapper.koji_proxy

pungi/scripts/create_extra_repo.py

@@ -78,11 +78,12 @@ class CreateExtraRepo(PackagesGenerator):
         """
         Dump merged modules data to an local repo
         """
-        with open(self.default_modules_yaml_path, 'w') as yaml_file:
+        if self.local_modules_data:
-            yaml.dump_all(
+            with open(self.default_modules_yaml_path, 'w') as yaml_file:
-                self.local_modules_data,
+                yaml.dump_all(
-                yaml_file,
+                    self.local_modules_data,
-            )
+                    yaml_file,
+                )
 
     @staticmethod
     def get_repo_info_from_bs_repo(

pungi/scripts/gather_modules.py

@@ -51,19 +51,24 @@ def collect_modules(modules_paths: List[BinaryIO], target_dir: str):
         documents = yaml.load_all(data, Loader=yaml.BaseLoader)
         for doc in documents:
             if doc['document'] == 'modulemd-defaults':
-                name = doc['data']['module'] + '.yaml'
+                name = f"{doc['data']['module']}.yaml"
                 path = os.path.join(module_defaults_path, name)
                 logging.info('Found %s module defaults', name)
             else:
                 # pungi.phases.pkgset.sources.source_koji.get_koji_modules
                 stream = doc['data']['stream'].replace('-', '_')
-                name = '%s-%s-%s.%s' % (
+                doc_data = doc['data']
-                    doc['data']['name'],
+                name = f"{doc_data['name']}-{stream}-" \
-                    stream,
+                       f"{doc_data['version']}.{doc_data['context']}"
-                    doc['data']['version'],
+                arch_dir = os.path.join(
-                    doc['data']['context']
+                    modules_path,
+                    doc_data['arch']
+                )
+                os.makedirs(arch_dir, exist_ok=True)
+                path = os.path.join(
+                    arch_dir,
+                    name,
                 )
-                path = os.path.join(modules_path, name)
                 logging.info('Found module %s', name)
 
                 if 'artifacts' not in doc['data']:

pungi/wrappers/comps.py

@@ -299,6 +299,8 @@ class CompsWrapper(object):
             append_common_info(doc, group_node, group, force_description=True)
             append_bool(doc, group_node, "default", group.default)
             append_bool(doc, group_node, "uservisible", group.uservisible)
+            if group.display_order is not None:
+                append(doc, group_node, "display_order", str(group.display_order))
 
             if group.lang_only:
                 append(doc, group_node, "langonly", group.lang_only)

pungi/wrappers/kojimock.py

@@ -35,6 +35,7 @@ class Module:
     stream: str
     version: str
     context: str
+    arch: str
 
 
 class KojiMock:
@@ -49,16 +50,22 @@ class KojiMock:
 
     def _gather_modules(self, modules_dir):
         modules = {}
-        for index, f in enumerate(os.listdir(modules_dir)):
+        for arch in os.listdir(modules_dir):
-            parsed = parse_nvra(f)
+            arch_dir = os.path.join(
-            modules[index] = Module(
+                modules_dir,
-                name=parsed['name'],
+                arch,
-                nvr=f,
-                version=parsed['release'],
-                context=parsed['arch'],
-                stream=parsed['version'],
-                build_id=index
             )
+            for index, f in enumerate(os.listdir(arch_dir)):
+                parsed = parse_nvra(f)
+                modules[index] = Module(
+                    name=parsed['name'],
+                    nvr=f,
+                    version=parsed['release'],
+                    context=parsed['arch'],
+                    stream=parsed['version'],
+                    build_id=index,
+                    arch=arch,
+                )
         return modules
 
     def getLastEvent(self, *args, **kwargs):
@@ -124,6 +131,7 @@ class KojiMock:
             'release': '%s.%s' % (module.version, module.context),
             'completion_ts': BUILD_TIME,
             'state': 'COMPLETE',
+            'arch': module.arch,
             'extra': {
                 'typeinfo': {
                     'module': {
@@ -152,7 +160,7 @@ class KojiMock:
         return [
             {
                 'build_id': module.build_id,
-                'filename': 'modulemd.x86_64.txt',
+                'filename': f'modulemd.{module.arch}.txt',
                 'btype': 'module'
             },
             # noone ever uses this file
@@ -197,7 +205,11 @@ class KojiMock:
         # get nvras for modular packages
         nvras = set()
         for module in self._modules.values():
-            path = os.path.join(self._modules_dir, module.nvr)
+            path = os.path.join(
+                self._modules_dir,
+                module.arch,
+                module.nvr,
+            )
             info = Modulemd.ModuleStream.read_string(open(path).read(), strict=True)
 
             for package in info.get_rpm_artifacts():
@@ -232,8 +244,12 @@ class KojiMock:
         """
         Get list of builds for module and given module tag name.
         """
-        path = os.path.join(self._modules_dir, tag_name)
         module = self._get_module_by_name(tag_name)
+        path = os.path.join(
+            self._modules_dir,
+            module.arch,
+            tag_name,
+        )
 
         builds = [
             {

tests/test_gather_modules.py

@@ -105,15 +105,15 @@ class TestModulesYamlParser(TestCase):
         self.assertEqual([MODULES_YAML_GZ, 'modules', 'module_defaults'], os.listdir(PATH_TO_KOJI))
         self.assertEqual(['mariadb-devel-10.3_1-8010020200108182321.cdc1202b',
                           'javapackages-tools-201801-8000020190628172923.b07bea58'],
-                         os.listdir(os.path.join(PATH_TO_KOJI, 'modules')))
+                         os.listdir(os.path.join(PATH_TO_KOJI, 'modules/x86_64')))
         self.assertEqual(['ant.yaml'],
                          os.listdir(os.path.join(PATH_TO_KOJI, 'module_defaults')))
 
         # check that modules were exported
         self.assertEqual(MARIADB_MODULE, yaml.load(
-            open(os.path.join(PATH_TO_KOJI, 'modules', 'mariadb-devel-10.3_1-8010020200108182321.cdc1202b'))))
+            open(os.path.join(PATH_TO_KOJI, 'modules/x86_64', 'mariadb-devel-10.3_1-8010020200108182321.cdc1202b'))))
         self.assertEqual(JAVAPACKAGES_TOOLS_MODULE, yaml.load(
-            open(os.path.join(PATH_TO_KOJI, 'modules', 'javapackages-tools-201801-8000020190628172923.b07bea58'))))
+            open(os.path.join(PATH_TO_KOJI, 'modules/x86_64', 'javapackages-tools-201801-8000020190628172923.b07bea58'))))
 
         # check that defaults were copied
         self.assertEqual(ANT_DEFAULTS, yaml.load(

tests/test_koji_local_source.py

@@ -103,13 +103,13 @@ version: '1'
         self.setUpPyfakefs()
 
         os.makedirs(PATH_TO_REPOS)
-        os.makedirs(os.path.join(PATH_TO_REPOS, 'modules'))
+        os.makedirs(os.path.join(PATH_TO_REPOS, 'modules/x86_64'))
 
-        with open(os.path.join(PATH_TO_REPOS, 'modules',
+        with open(os.path.join(PATH_TO_REPOS, 'modules/x86_64',
                                'javapackages-tools-201801-8000020190628172923.b07bea58'), 'w') as f:
             f.write(self.JAVAPACKAGES_TOOLS_MODULE)
 
-        with open(os.path.join(PATH_TO_REPOS, 'modules',
+        with open(os.path.join(PATH_TO_REPOS, 'modules/x86_64',
                                'mariadb-devel-10.3-8010020200108182321.cdc1202b'), 'w') as f:
             f.write(self.MARIADB_MODULE)
 
@@ -122,6 +122,7 @@ version: '1'
     @ddt.data(
         [0, {
             'completion_ts': 0,
+            'arch': 'x86_64',
             'extra': {
                 'typeinfo': {
                     'module': {
@@ -141,6 +142,7 @@ version: '1'
         }],
         [1, {
             'completion_ts': 0,
+            'arch': 'x86_64',
             'extra': {
                 'typeinfo': {
                     'module': {

tests/test_pkgset_source_koji.py

@@ -703,6 +703,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
         ] + [{"btype": "foo"}]
         self.buildinfo = {
             "id": 1234,
+            "arch": "fake_arch",
             "extra": {
                 "typeinfo": {
                     "module": {
@@ -734,7 +735,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
                 # },
                 "x86_64": {
                     "module:master:20190318:abcdef": MockModule(
-                        "/mnt/koji/modules/module:master-20190318-abcdef"
+                        "/mnt/koji/modules/fake_arch/module:master-20190318-abcdef"
                     ),
                 },
             },
@@ -748,7 +749,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
                 "x86_64"
             ],
             arch_mmds={
-                "x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe")}
+                "x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe")}
             },
             modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
         )
@@ -765,9 +766,9 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
                 # },
                 "x86_64": {
                     "module:master:20190318:abcdef": MockModule(
-                        "/mnt/koji/modules/module:master-20190318-abcdef"
+                        "/mnt/koji/modules/fake_arch/module:master-20190318-abcdef"
                     ),
-                    "m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe"),
+                    "m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe"),
                 },
             },
         )
@@ -795,7 +796,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
                 # },
                 "x86_64": {
                     "module:master:20190318:abcdef": MockModule(
-                        "/mnt/koji/modules/module:master-20190318-abcdef"
+                        "/mnt/koji/modules/fake_arch/module:master-20190318-abcdef"
                     )
                 },
             },
@@ -811,7 +812,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
                 "x86_64"
             ],
             arch_mmds={
-                "x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe")}
+                "x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe")}
             },
             modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
         )
@@ -830,9 +831,9 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
                 # },
                 "x86_64": {
                     "module:master:20190318:abcdef": MockModule(
-                        "/mnt/koji/modules/module:master-20190318-abcdef"
+                        "/mnt/koji/modules/fake_arch/module:master-20190318-abcdef"
                     ),
-                    "m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe"),
+                    "m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe"),
                 },
             },
         )