LNX-102: Add tool that collects information about modules
Add special tool that gathers given modules.tar.gz files and collects information about modules into two dirs: - module_defaults - modules First one is used by pungi during repocreate phase and the second one is used by koji mock to get list of available modules and their versions. Change-Id: I50a095a5f3bafa7e7a1effc2c0d4a2fc52ba603b
This commit is contained in:
oshyshatskyi
parent
67eacf8483
commit
5806217041
|
|
@ -0,0 +1,64 @@
|
|||
import argparse
|
||||
import gzip
|
||||
import os
|
||||
import typing
|
||||
from argparse import ArgumentParser
|
||||
from typing import List
|
||||
import logging
|
||||
import yaml
|
||||
|
||||
|
||||
def collect_modules(modules_paths: List[typing.BinaryIO], target_dir: str):
|
||||
"""
|
||||
Read given modules.yaml.gz files and export modules
|
||||
and modulemd files from it.
|
||||
Returns:
|
||||
object:
|
||||
"""
|
||||
modules_path = os.path.join(target_dir, 'modules')
|
||||
module_defaults_path = os.path.join(target_dir, 'module_defaults')
|
||||
os.makedirs(modules_path, exist_ok=True)
|
||||
os.makedirs(module_defaults_path, exist_ok=True)
|
||||
|
||||
for module_file in modules_paths:
|
||||
data = gzip.decompress(module_file.read())
|
||||
documents = yaml.load_all(data, Loader=yaml.BaseLoader)
|
||||
for doc in documents:
|
||||
if doc['document'] == 'modulemd-defaults':
|
||||
name = doc['data']['module'] + '.yaml'
|
||||
path = os.path.join(module_defaults_path, name)
|
||||
logging.info('Found %s module defaults', name)
|
||||
else:
|
||||
name = '%s-%s-%s.%s' % (
|
||||
doc['data']['name'],
|
||||
doc['data']['stream'],
|
||||
doc['data']['version'],
|
||||
doc['data']['context']
|
||||
)
|
||||
path = os.path.join(modules_path, name)
|
||||
logging.info('Found module %s', name)
|
||||
|
||||
if 'artifacts' not in doc['data']:
|
||||
logging.warning('RPM %s does not have explicit list of artifacts', name)
|
||||
|
||||
with open(path, 'w') as f:
|
||||
yaml.dump(doc, f, default_flow_style=False)
|
||||
|
||||
|
||||
def cli_main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument(
|
||||
'-p', '--path', required=True,
|
||||
type=argparse.FileType('rb'), nargs='+',
|
||||
help='Path to modules.yaml.gz file. '
|
||||
'You may pass multiple files by passing -p path1 path2'
|
||||
)
|
||||
parser.add_argument('-t', '--target', required=True)
|
||||
|
||||
namespace = parser.parse_args()
|
||||
|
||||
collect_modules(namespace.path, namespace.target)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
cli_main()
|
||||
3
setup.py
3
setup.py
|
|
@ -47,6 +47,7 @@ setup(
|
|||
"pungi-gather = pungi.scripts.pungi_gather:cli_main",
|
||||
"pungi-config-dump = pungi.scripts.config_dump:cli_main",
|
||||
"pungi-config-validate = pungi.scripts.config_validate:cli_main",
|
||||
"pungi-gather-modules = pungi.scripts.gather_modules:cli_main",
|
||||
]
|
||||
},
|
||||
scripts=["contrib/yum-dnf-compare/pungi-compare-depsolving"],
|
||||
|
|
@ -66,5 +67,5 @@ setup(
|
|||
"dogpile.cache",
|
||||
],
|
||||
extras_require={':python_version=="2.7"': ["enum34", "lockfile"]},
|
||||
tests_require=["mock", "pytest", "pytest-cov"],
|
||||
tests_require=["mock", "pytest", "pytest-cov", "pyfakefs"],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,124 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
import gzip
|
||||
import os
|
||||
from io import StringIO
|
||||
import yaml
|
||||
from pungi.scripts.gather_modules import collect_modules
|
||||
import unittest
|
||||
from pyfakefs.fake_filesystem_unittest import TestCase
|
||||
|
||||
MARIADB_MODULE = yaml.load("""
|
||||
---
|
||||
document: modulemd
|
||||
version: 2
|
||||
data:
|
||||
name: mariadb-devel
|
||||
stream: 10.3
|
||||
version: 8010020200108182321
|
||||
context: cdc1202b
|
||||
arch: x86_64
|
||||
summary: MariaDB Module
|
||||
description: >-
|
||||
MariaDB is a community developed branch of MySQL.
|
||||
components:
|
||||
rpms:
|
||||
Judy:
|
||||
rationale: MariaDB dependency for OQgraph computation engine
|
||||
ref: a3583b33f939e74a530f2a1dff0552dff2c8ea73
|
||||
buildorder: 4
|
||||
arches: [aarch64, i686, ppc64le, x86_64]
|
||||
artifacts:
|
||||
rpms:
|
||||
- Judy-0:1.0.5-18.module_el8.1.0+217+4d875839.i686
|
||||
- Judy-debuginfo-0:1.0.5-18.module_el8.1.0+217+4d875839.i686
|
||||
""", Loader=yaml.BaseLoader)
|
||||
|
||||
JAVAPACKAGES_TOOLS_MODULE = yaml.load("""
|
||||
---
|
||||
document: modulemd
|
||||
version: 2
|
||||
data:
|
||||
name: javapackages-tools
|
||||
stream: 201801
|
||||
version: 8000020190628172923
|
||||
context: b07bea58
|
||||
arch: x86_64
|
||||
summary: Tools and macros for Java packaging support
|
||||
description: >-
|
||||
Java Packages Tools is a collection of tools that make it easier to build RPM
|
||||
packages containing software running on Java platform.
|
||||
components:
|
||||
rpms:
|
||||
ant:
|
||||
rationale: "Runtime dependency of ant-contrib"
|
||||
ref: 2eaf095676540e2805ee7e8c7f6f78285c428fdc
|
||||
arches: [aarch64, i686, ppc64le, x86_64]
|
||||
artifacts:
|
||||
rpms:
|
||||
- ant-0:1.10.5-1.module_el8.0.0+30+832da3a1.noarch
|
||||
- ant-0:1.10.5-1.module_el8.0.0+30+832da3a1.src
|
||||
""", Loader=yaml.BaseLoader)
|
||||
|
||||
ANT_DEFAULTS = yaml.load("""
|
||||
data:
|
||||
module: ant
|
||||
profiles:
|
||||
'1.10':
|
||||
- common
|
||||
stream: '1.10'
|
||||
document: modulemd-defaults
|
||||
version: '1'
|
||||
""", Loader=yaml.BaseLoader)
|
||||
|
||||
|
||||
PATH_TO_KOJI = '/path/to/koji'
|
||||
MODULES_YAML_GZ = 'modules.yaml.gz'
|
||||
|
||||
|
||||
class TestModulesYamlParser(TestCase):
|
||||
|
||||
maxDiff = None
|
||||
|
||||
def setUp(self):
|
||||
self.setUpPyfakefs()
|
||||
|
||||
def _prepare_test_data(self):
|
||||
"""
|
||||
Create modules.yaml.gz with some test data
|
||||
"""
|
||||
os.makedirs(PATH_TO_KOJI)
|
||||
modules_gz_path = os.path.join(PATH_TO_KOJI, MODULES_YAML_GZ)
|
||||
# dump modules into compressed file as in generic repos for rpm
|
||||
io = StringIO()
|
||||
yaml.dump_all([MARIADB_MODULE, JAVAPACKAGES_TOOLS_MODULE, ANT_DEFAULTS], io)
|
||||
with open(os.path.join(PATH_TO_KOJI, MODULES_YAML_GZ), 'wb') as f:
|
||||
f.write(gzip.compress(io.getvalue().encode()))
|
||||
return modules_gz_path
|
||||
|
||||
def test_export_modules(self):
|
||||
modules_gz_path = self._prepare_test_data()
|
||||
|
||||
paths = [open(modules_gz_path, 'rb')]
|
||||
collect_modules(paths, PATH_TO_KOJI)
|
||||
|
||||
# check directory structure matches expected
|
||||
self.assertEqual([MODULES_YAML_GZ, 'modules', 'module_defaults'], os.listdir(PATH_TO_KOJI))
|
||||
self.assertEqual(['mariadb-devel-10.3-8010020200108182321.cdc1202b',
|
||||
'javapackages-tools-201801-8000020190628172923.b07bea58'],
|
||||
os.listdir(os.path.join(PATH_TO_KOJI, 'modules')))
|
||||
self.assertEqual(['ant.yaml'],
|
||||
os.listdir(os.path.join(PATH_TO_KOJI, 'module_defaults')))
|
||||
|
||||
# check that modules were exported
|
||||
self.assertEqual(MARIADB_MODULE, yaml.load(
|
||||
open(os.path.join(PATH_TO_KOJI, 'modules', 'mariadb-devel-10.3-8010020200108182321.cdc1202b'))))
|
||||
self.assertEqual(JAVAPACKAGES_TOOLS_MODULE, yaml.load(
|
||||
open(os.path.join(PATH_TO_KOJI, 'modules', 'javapackages-tools-201801-8000020190628172923.b07bea58'))))
|
||||
|
||||
# check that defaults were copied
|
||||
self.assertEqual(ANT_DEFAULTS, yaml.load(
|
||||
open(os.path.join(PATH_TO_KOJI, 'module_defaults', 'ant.yaml'))))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Loading…
Reference in New Issue