From 58062170414affcdda11cd430dd685eacef3834c Mon Sep 17 00:00:00 2001 From: oshyshatskyi Date: Tue, 29 Dec 2020 11:49:33 +0200 Subject: [PATCH] LNX-102: Add tool that collects information about modules Add special tool that gathers given modules.tar.gz files and collects information about modules into two dirs: - module_defaults - modules First one is used by pungi during repocreate phase and the second one is used by koji mock to get list of available modules and their versions. Change-Id: I50a095a5f3bafa7e7a1effc2c0d4a2fc52ba603b --- pungi/scripts/gather_modules.py | 64 +++++++++++++++++ setup.py | 3 +- tests/test_gather_modules.py | 124 ++++++++++++++++++++++++++++++++ 3 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 pungi/scripts/gather_modules.py create mode 100644 tests/test_gather_modules.py diff --git a/pungi/scripts/gather_modules.py b/pungi/scripts/gather_modules.py new file mode 100644 index 00000000..1a21e51c --- /dev/null +++ b/pungi/scripts/gather_modules.py @@ -0,0 +1,64 @@ +import argparse +import gzip +import os +import typing +from argparse import ArgumentParser +from typing import List +import logging +import yaml + + +def collect_modules(modules_paths: List[typing.BinaryIO], target_dir: str): + """ + Read given modules.yaml.gz files and export modules + and modulemd files from it. + Returns: + object: + """ + modules_path = os.path.join(target_dir, 'modules') + module_defaults_path = os.path.join(target_dir, 'module_defaults') + os.makedirs(modules_path, exist_ok=True) + os.makedirs(module_defaults_path, exist_ok=True) + + for module_file in modules_paths: + data = gzip.decompress(module_file.read()) + documents = yaml.load_all(data, Loader=yaml.BaseLoader) + for doc in documents: + if doc['document'] == 'modulemd-defaults': + name = doc['data']['module'] + '.yaml' + path = os.path.join(module_defaults_path, name) + logging.info('Found %s module defaults', name) + else: + name = '%s-%s-%s.%s' % ( + doc['data']['name'], + doc['data']['stream'], + doc['data']['version'], + doc['data']['context'] + ) + path = os.path.join(modules_path, name) + logging.info('Found module %s', name) + + if 'artifacts' not in doc['data']: + logging.warning('RPM %s does not have explicit list of artifacts', name) + + with open(path, 'w') as f: + yaml.dump(doc, f, default_flow_style=False) + + +def cli_main(): + parser = ArgumentParser() + parser.add_argument( + '-p', '--path', required=True, + type=argparse.FileType('rb'), nargs='+', + help='Path to modules.yaml.gz file. ' + 'You may pass multiple files by passing -p path1 path2' + ) + parser.add_argument('-t', '--target', required=True) + + namespace = parser.parse_args() + + collect_modules(namespace.path, namespace.target) + + +if __name__ == '__main__': + cli_main() diff --git a/setup.py b/setup.py index 054102dd..404a359e 100755 --- a/setup.py +++ b/setup.py @@ -47,6 +47,7 @@ setup( "pungi-gather = pungi.scripts.pungi_gather:cli_main", "pungi-config-dump = pungi.scripts.config_dump:cli_main", "pungi-config-validate = pungi.scripts.config_validate:cli_main", + "pungi-gather-modules = pungi.scripts.gather_modules:cli_main", ] }, scripts=["contrib/yum-dnf-compare/pungi-compare-depsolving"], @@ -66,5 +67,5 @@ setup( "dogpile.cache", ], extras_require={':python_version=="2.7"': ["enum34", "lockfile"]}, - tests_require=["mock", "pytest", "pytest-cov"], + tests_require=["mock", "pytest", "pytest-cov", "pyfakefs"], ) diff --git a/tests/test_gather_modules.py b/tests/test_gather_modules.py new file mode 100644 index 00000000..0a1cb856 --- /dev/null +++ b/tests/test_gather_modules.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- +import gzip +import os +from io import StringIO +import yaml +from pungi.scripts.gather_modules import collect_modules +import unittest +from pyfakefs.fake_filesystem_unittest import TestCase + +MARIADB_MODULE = yaml.load(""" +--- +document: modulemd +version: 2 +data: + name: mariadb-devel + stream: 10.3 + version: 8010020200108182321 + context: cdc1202b + arch: x86_64 + summary: MariaDB Module + description: >- + MariaDB is a community developed branch of MySQL. + components: + rpms: + Judy: + rationale: MariaDB dependency for OQgraph computation engine + ref: a3583b33f939e74a530f2a1dff0552dff2c8ea73 + buildorder: 4 + arches: [aarch64, i686, ppc64le, x86_64] + artifacts: + rpms: + - Judy-0:1.0.5-18.module_el8.1.0+217+4d875839.i686 + - Judy-debuginfo-0:1.0.5-18.module_el8.1.0+217+4d875839.i686 +""", Loader=yaml.BaseLoader) + +JAVAPACKAGES_TOOLS_MODULE = yaml.load(""" +--- +document: modulemd +version: 2 +data: + name: javapackages-tools + stream: 201801 + version: 8000020190628172923 + context: b07bea58 + arch: x86_64 + summary: Tools and macros for Java packaging support + description: >- + Java Packages Tools is a collection of tools that make it easier to build RPM + packages containing software running on Java platform. + components: + rpms: + ant: + rationale: "Runtime dependency of ant-contrib" + ref: 2eaf095676540e2805ee7e8c7f6f78285c428fdc + arches: [aarch64, i686, ppc64le, x86_64] + artifacts: + rpms: + - ant-0:1.10.5-1.module_el8.0.0+30+832da3a1.noarch + - ant-0:1.10.5-1.module_el8.0.0+30+832da3a1.src +""", Loader=yaml.BaseLoader) + +ANT_DEFAULTS = yaml.load(""" +data: + module: ant + profiles: + '1.10': + - common + stream: '1.10' +document: modulemd-defaults +version: '1' +""", Loader=yaml.BaseLoader) + + +PATH_TO_KOJI = '/path/to/koji' +MODULES_YAML_GZ = 'modules.yaml.gz' + + +class TestModulesYamlParser(TestCase): + + maxDiff = None + + def setUp(self): + self.setUpPyfakefs() + + def _prepare_test_data(self): + """ + Create modules.yaml.gz with some test data + """ + os.makedirs(PATH_TO_KOJI) + modules_gz_path = os.path.join(PATH_TO_KOJI, MODULES_YAML_GZ) + # dump modules into compressed file as in generic repos for rpm + io = StringIO() + yaml.dump_all([MARIADB_MODULE, JAVAPACKAGES_TOOLS_MODULE, ANT_DEFAULTS], io) + with open(os.path.join(PATH_TO_KOJI, MODULES_YAML_GZ), 'wb') as f: + f.write(gzip.compress(io.getvalue().encode())) + return modules_gz_path + + def test_export_modules(self): + modules_gz_path = self._prepare_test_data() + + paths = [open(modules_gz_path, 'rb')] + collect_modules(paths, PATH_TO_KOJI) + + # check directory structure matches expected + self.assertEqual([MODULES_YAML_GZ, 'modules', 'module_defaults'], os.listdir(PATH_TO_KOJI)) + self.assertEqual(['mariadb-devel-10.3-8010020200108182321.cdc1202b', + 'javapackages-tools-201801-8000020190628172923.b07bea58'], + os.listdir(os.path.join(PATH_TO_KOJI, 'modules'))) + self.assertEqual(['ant.yaml'], + os.listdir(os.path.join(PATH_TO_KOJI, 'module_defaults'))) + + # check that modules were exported + self.assertEqual(MARIADB_MODULE, yaml.load( + open(os.path.join(PATH_TO_KOJI, 'modules', 'mariadb-devel-10.3-8010020200108182321.cdc1202b')))) + self.assertEqual(JAVAPACKAGES_TOOLS_MODULE, yaml.load( + open(os.path.join(PATH_TO_KOJI, 'modules', 'javapackages-tools-201801-8000020190628172923.b07bea58')))) + + # check that defaults were copied + self.assertEqual(ANT_DEFAULTS, yaml.load( + open(os.path.join(PATH_TO_KOJI, 'module_defaults', 'ant.yaml')))) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file