LNX-102: Add tool that collects information about modules

Add special tool that gathers given modules.tar.gz files
and collects information about modules into two dirs:
 - module_defaults
 - modules

 First one is used by pungi during repocreate phase and
 the second one is used by koji mock to get list of
 available modules and their versions.

Change-Id: I50a095a5f3bafa7e7a1effc2c0d4a2fc52ba603b

pungi/scripts/gather_modules.py

@@ -0,0 +1,64 @@
+import argparse
+import gzip
+import os
+import typing
+from argparse import ArgumentParser
+from typing import List
+import logging
+import yaml
+
+
+def collect_modules(modules_paths: List[typing.BinaryIO], target_dir: str):
+    """
+    Read given modules.yaml.gz files and export modules
+    and modulemd files from it.
+    Returns:
+        object:
+    """
+    modules_path = os.path.join(target_dir, 'modules')
+    module_defaults_path = os.path.join(target_dir, 'module_defaults')
+    os.makedirs(modules_path, exist_ok=True)
+    os.makedirs(module_defaults_path, exist_ok=True)
+
+    for module_file in modules_paths:
+        data = gzip.decompress(module_file.read())
+        documents = yaml.load_all(data, Loader=yaml.BaseLoader)
+        for doc in documents:
+            if doc['document'] == 'modulemd-defaults':
+                name = doc['data']['module'] + '.yaml'
+                path = os.path.join(module_defaults_path, name)
+                logging.info('Found %s module defaults', name)
+            else:
+                name = '%s-%s-%s.%s' % (
+                    doc['data']['name'],
+                    doc['data']['stream'],
+                    doc['data']['version'],
+                    doc['data']['context']
+                )
+                path = os.path.join(modules_path, name)
+                logging.info('Found module %s', name)
+
+                if 'artifacts' not in doc['data']:
+                    logging.warning('RPM %s does not have explicit list of artifacts', name)
+
+            with open(path, 'w') as f:
+                yaml.dump(doc, f, default_flow_style=False)
+
+
+def cli_main():
+    parser = ArgumentParser()
+    parser.add_argument(
+        '-p', '--path', required=True,
+        type=argparse.FileType('rb'), nargs='+',
+        help='Path to modules.yaml.gz file. '
+             'You may pass multiple files by passing -p path1 path2'
+    )
+    parser.add_argument('-t', '--target', required=True)
+
+    namespace = parser.parse_args()
+
+    collect_modules(namespace.path, namespace.target)
+
+
+if __name__ == '__main__':
+    cli_main()

setup.py

@@ -47,6 +47,7 @@ setup(
             "pungi-gather = pungi.scripts.pungi_gather:cli_main",
             "pungi-config-dump = pungi.scripts.config_dump:cli_main",
             "pungi-config-validate = pungi.scripts.config_validate:cli_main",
+            "pungi-gather-modules = pungi.scripts.gather_modules:cli_main",
         ]
     },
     scripts=["contrib/yum-dnf-compare/pungi-compare-depsolving"],
@@ -66,5 +67,5 @@ setup(
         "dogpile.cache",
     ],
     extras_require={':python_version=="2.7"': ["enum34", "lockfile"]},
-    tests_require=["mock", "pytest", "pytest-cov"],
+    tests_require=["mock", "pytest", "pytest-cov", "pyfakefs"],
 )

tests/test_gather_modules.py

@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*-
+import gzip
+import os
+from io import StringIO
+import yaml
+from pungi.scripts.gather_modules import collect_modules
+import unittest
+from pyfakefs.fake_filesystem_unittest import TestCase
+
+MARIADB_MODULE = yaml.load("""
+---
+document: modulemd
+version: 2
+data:
+  name: mariadb-devel
+  stream: 10.3
+  version: 8010020200108182321
+  context: cdc1202b
+  arch: x86_64
+  summary: MariaDB Module
+  description: >-
+    MariaDB is a community developed branch of MySQL. 
+  components:
+    rpms:
+      Judy:
+        rationale: MariaDB dependency for OQgraph computation engine
+        ref: a3583b33f939e74a530f2a1dff0552dff2c8ea73
+        buildorder: 4
+        arches: [aarch64, i686, ppc64le, x86_64]
+  artifacts:
+    rpms:
+    - Judy-0:1.0.5-18.module_el8.1.0+217+4d875839.i686
+    - Judy-debuginfo-0:1.0.5-18.module_el8.1.0+217+4d875839.i686
+""", Loader=yaml.BaseLoader)
+
+JAVAPACKAGES_TOOLS_MODULE = yaml.load("""
+---
+document: modulemd
+version: 2
+data:
+  name: javapackages-tools
+  stream: 201801
+  version: 8000020190628172923
+  context: b07bea58
+  arch: x86_64
+  summary: Tools and macros for Java packaging support
+  description: >-
+    Java Packages Tools is a collection of tools that make it easier to build RPM
+    packages containing software running on Java platform.
+  components:
+    rpms:
+      ant:
+        rationale: "Runtime dependency of ant-contrib"
+        ref: 2eaf095676540e2805ee7e8c7f6f78285c428fdc
+        arches: [aarch64, i686, ppc64le, x86_64]
+  artifacts:
+    rpms:
+    - ant-0:1.10.5-1.module_el8.0.0+30+832da3a1.noarch
+    - ant-0:1.10.5-1.module_el8.0.0+30+832da3a1.src
+""", Loader=yaml.BaseLoader)
+
+ANT_DEFAULTS = yaml.load("""
+data:
+  module: ant
+  profiles:
+    '1.10':
+    - common
+  stream: '1.10'
+document: modulemd-defaults
+version: '1'
+""", Loader=yaml.BaseLoader)
+
+
+PATH_TO_KOJI = '/path/to/koji'
+MODULES_YAML_GZ = 'modules.yaml.gz'
+
+
+class TestModulesYamlParser(TestCase):
+
+    maxDiff = None
+
+    def setUp(self):
+        self.setUpPyfakefs()
+
+    def _prepare_test_data(self):
+        """
+        Create modules.yaml.gz with some test data
+        """
+        os.makedirs(PATH_TO_KOJI)
+        modules_gz_path = os.path.join(PATH_TO_KOJI, MODULES_YAML_GZ)
+        # dump modules into compressed file as in generic repos for rpm
+        io = StringIO()
+        yaml.dump_all([MARIADB_MODULE, JAVAPACKAGES_TOOLS_MODULE, ANT_DEFAULTS], io)
+        with open(os.path.join(PATH_TO_KOJI, MODULES_YAML_GZ), 'wb') as f:
+            f.write(gzip.compress(io.getvalue().encode()))
+        return modules_gz_path
+
+    def test_export_modules(self):
+        modules_gz_path = self._prepare_test_data()
+
+        paths = [open(modules_gz_path, 'rb')]
+        collect_modules(paths, PATH_TO_KOJI)
+
+        # check directory structure matches expected
+        self.assertEqual([MODULES_YAML_GZ, 'modules', 'module_defaults'], os.listdir(PATH_TO_KOJI))
+        self.assertEqual(['mariadb-devel-10.3-8010020200108182321.cdc1202b',
+                          'javapackages-tools-201801-8000020190628172923.b07bea58'],
+                         os.listdir(os.path.join(PATH_TO_KOJI, 'modules')))
+        self.assertEqual(['ant.yaml'],
+                         os.listdir(os.path.join(PATH_TO_KOJI, 'module_defaults')))
+
+        # check that modules were exported
+        self.assertEqual(MARIADB_MODULE, yaml.load(
+            open(os.path.join(PATH_TO_KOJI, 'modules', 'mariadb-devel-10.3-8010020200108182321.cdc1202b'))))
+        self.assertEqual(JAVAPACKAGES_TOOLS_MODULE, yaml.load(
+            open(os.path.join(PATH_TO_KOJI, 'modules', 'javapackages-tools-201801-8000020190628172923.b07bea58'))))
+
+        # check that defaults were copied
+        self.assertEqual(ANT_DEFAULTS, yaml.load(
+            open(os.path.join(PATH_TO_KOJI, 'module_defaults', 'ant.yaml'))))
+
+
+if __name__ == '__main__':
+    unittest.main()