LNX-108: Add multiarch support to pungi

@BS-NOBUILD
@BS-TARGET-CL8

Change-Id: Ibfd540454941922d790ae4e56cc0992c0c85635d
This commit is contained in:
stepan_oksanichenko 2021-05-12 18:14:49 +03:00
parent a435eeed06
commit cc4d99441c
8 changed files with 81 additions and 47 deletions

View File

@ -228,9 +228,12 @@ def _add_module_to_variant(
# Skip non module archives
continue
filename = archive["filename"]
# TODO: add support for muplitple arches
file_path = os.path.join(koji_wrapper.koji_module.pathinfo.topdir, 'modules',
build['extra']['typeinfo']['module']['content_koji_tag'])
file_path = os.path.join(
koji_wrapper.koji_module.pathinfo.topdir,
'modules',
build['arch'],
build['extra']['typeinfo']['module']['content_koji_tag']
)
mmds[filename] = file_path
if len(mmds) <= 1:
@ -488,16 +491,15 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules)
info.get("context"),
)
nvr_patterns.add((pattern, spec["name"]))
modules_to_keep = []
for mb in module_builds:
for mb in sorted(module_builds, key=lambda i: i['name']):
# Split release from the build into version and context
ver, ctx = mb["release"].split(".")
# Values in `mb` are from Koji build. There's nvr and name, version and
# release. The input pattern specifies modular name, stream, version
# and context.
for (n, s, v, c), spec in nvr_patterns:
for (n, s, v, c), spec in sorted(nvr_patterns):
if (
# We always have a name and stream...
mb["name"] == n
@ -509,7 +511,6 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules)
):
modules_to_keep.append(mb)
expected_modules.discard(spec)
break
return modules_to_keep
@ -543,7 +544,7 @@ def _filter_expected_modules(
if cond1 and cond2:
expected_modules = {
expected_module for expected_module in expected_modules if
not all(
not any(
re.findall(
filtered_module,
expected_module,
@ -573,7 +574,13 @@ def _get_modules_from_koji_tags(
]
# Get set of configured module names for this variant. If nothing is
# configured, the set is empty.
expected_modules = set(spec["name"] for spec in variant.get_modules())
expected_modules = []
for spec in variant.get_modules():
name, stream = spec['name'].split(':')
expected_modules.append(
':'.join((name, stream.replace('-', '_')))
)
expected_modules = set(expected_modules)
# Find out all modules in every variant and add their Koji tags
# to variant and variant_tags list.
koji_proxy = koji_wrapper.koji_proxy

View File

@ -78,6 +78,7 @@ class CreateExtraRepo(PackagesGenerator):
"""
Dump merged modules data to an local repo
"""
if self.local_modules_data:
with open(self.default_modules_yaml_path, 'w') as yaml_file:
yaml.dump_all(
self.local_modules_data,

View File

@ -51,19 +51,24 @@ def collect_modules(modules_paths: List[BinaryIO], target_dir: str):
documents = yaml.load_all(data, Loader=yaml.BaseLoader)
for doc in documents:
if doc['document'] == 'modulemd-defaults':
name = doc['data']['module'] + '.yaml'
name = f"{doc['data']['module']}.yaml"
path = os.path.join(module_defaults_path, name)
logging.info('Found %s module defaults', name)
else:
# pungi.phases.pkgset.sources.source_koji.get_koji_modules
stream = doc['data']['stream'].replace('-', '_')
name = '%s-%s-%s.%s' % (
doc['data']['name'],
stream,
doc['data']['version'],
doc['data']['context']
doc_data = doc['data']
name = f"{doc_data['name']}-{stream}-" \
f"{doc_data['version']}.{doc_data['context']}"
arch_dir = os.path.join(
modules_path,
doc_data['arch']
)
os.makedirs(arch_dir, exist_ok=True)
path = os.path.join(
arch_dir,
name,
)
path = os.path.join(modules_path, name)
logging.info('Found module %s', name)
if 'artifacts' not in doc['data']:

View File

@ -299,6 +299,8 @@ class CompsWrapper(object):
append_common_info(doc, group_node, group, force_description=True)
append_bool(doc, group_node, "default", group.default)
append_bool(doc, group_node, "uservisible", group.uservisible)
if group.display_order is not None:
append(doc, group_node, "display_order", str(group.display_order))
if group.lang_only:
append(doc, group_node, "langonly", group.lang_only)

View File

@ -35,6 +35,7 @@ class Module:
stream: str
version: str
context: str
arch: str
class KojiMock:
@ -49,7 +50,12 @@ class KojiMock:
def _gather_modules(self, modules_dir):
modules = {}
for index, f in enumerate(os.listdir(modules_dir)):
for arch in os.listdir(modules_dir):
arch_dir = os.path.join(
modules_dir,
arch,
)
for index, f in enumerate(os.listdir(arch_dir)):
parsed = parse_nvra(f)
modules[index] = Module(
name=parsed['name'],
@ -57,7 +63,8 @@ class KojiMock:
version=parsed['release'],
context=parsed['arch'],
stream=parsed['version'],
build_id=index
build_id=index,
arch=arch,
)
return modules
@ -124,6 +131,7 @@ class KojiMock:
'release': '%s.%s' % (module.version, module.context),
'completion_ts': BUILD_TIME,
'state': 'COMPLETE',
'arch': module.arch,
'extra': {
'typeinfo': {
'module': {
@ -152,7 +160,7 @@ class KojiMock:
return [
{
'build_id': module.build_id,
'filename': 'modulemd.x86_64.txt',
'filename': f'modulemd.{module.arch}.txt',
'btype': 'module'
},
# noone ever uses this file
@ -197,7 +205,11 @@ class KojiMock:
# get nvras for modular packages
nvras = set()
for module in self._modules.values():
path = os.path.join(self._modules_dir, module.nvr)
path = os.path.join(
self._modules_dir,
module.arch,
module.nvr,
)
info = Modulemd.ModuleStream.read_string(open(path).read(), strict=True)
for package in info.get_rpm_artifacts():
@ -232,8 +244,12 @@ class KojiMock:
"""
Get list of builds for module and given module tag name.
"""
path = os.path.join(self._modules_dir, tag_name)
module = self._get_module_by_name(tag_name)
path = os.path.join(
self._modules_dir,
module.arch,
tag_name,
)
builds = [
{

View File

@ -105,15 +105,15 @@ class TestModulesYamlParser(TestCase):
self.assertEqual([MODULES_YAML_GZ, 'modules', 'module_defaults'], os.listdir(PATH_TO_KOJI))
self.assertEqual(['mariadb-devel-10.3_1-8010020200108182321.cdc1202b',
'javapackages-tools-201801-8000020190628172923.b07bea58'],
os.listdir(os.path.join(PATH_TO_KOJI, 'modules')))
os.listdir(os.path.join(PATH_TO_KOJI, 'modules/x86_64')))
self.assertEqual(['ant.yaml'],
os.listdir(os.path.join(PATH_TO_KOJI, 'module_defaults')))
# check that modules were exported
self.assertEqual(MARIADB_MODULE, yaml.load(
open(os.path.join(PATH_TO_KOJI, 'modules', 'mariadb-devel-10.3_1-8010020200108182321.cdc1202b'))))
open(os.path.join(PATH_TO_KOJI, 'modules/x86_64', 'mariadb-devel-10.3_1-8010020200108182321.cdc1202b'))))
self.assertEqual(JAVAPACKAGES_TOOLS_MODULE, yaml.load(
open(os.path.join(PATH_TO_KOJI, 'modules', 'javapackages-tools-201801-8000020190628172923.b07bea58'))))
open(os.path.join(PATH_TO_KOJI, 'modules/x86_64', 'javapackages-tools-201801-8000020190628172923.b07bea58'))))
# check that defaults were copied
self.assertEqual(ANT_DEFAULTS, yaml.load(

View File

@ -103,13 +103,13 @@ version: '1'
self.setUpPyfakefs()
os.makedirs(PATH_TO_REPOS)
os.makedirs(os.path.join(PATH_TO_REPOS, 'modules'))
os.makedirs(os.path.join(PATH_TO_REPOS, 'modules/x86_64'))
with open(os.path.join(PATH_TO_REPOS, 'modules',
with open(os.path.join(PATH_TO_REPOS, 'modules/x86_64',
'javapackages-tools-201801-8000020190628172923.b07bea58'), 'w') as f:
f.write(self.JAVAPACKAGES_TOOLS_MODULE)
with open(os.path.join(PATH_TO_REPOS, 'modules',
with open(os.path.join(PATH_TO_REPOS, 'modules/x86_64',
'mariadb-devel-10.3-8010020200108182321.cdc1202b'), 'w') as f:
f.write(self.MARIADB_MODULE)
@ -122,6 +122,7 @@ version: '1'
@ddt.data(
[0, {
'completion_ts': 0,
'arch': 'x86_64',
'extra': {
'typeinfo': {
'module': {
@ -141,6 +142,7 @@ version: '1'
}],
[1, {
'completion_ts': 0,
'arch': 'x86_64',
'extra': {
'typeinfo': {
'module': {

View File

@ -703,6 +703,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
] + [{"btype": "foo"}]
self.buildinfo = {
"id": 1234,
"arch": "fake_arch",
"extra": {
"typeinfo": {
"module": {
@ -734,7 +735,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
# },
"x86_64": {
"module:master:20190318:abcdef": MockModule(
"/mnt/koji/modules/module:master-20190318-abcdef"
"/mnt/koji/modules/fake_arch/module:master-20190318-abcdef"
),
},
},
@ -748,7 +749,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
"x86_64"
],
arch_mmds={
"x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe")}
"x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe")}
},
modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
)
@ -765,9 +766,9 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
# },
"x86_64": {
"module:master:20190318:abcdef": MockModule(
"/mnt/koji/modules/module:master-20190318-abcdef"
"/mnt/koji/modules/fake_arch/module:master-20190318-abcdef"
),
"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe"),
"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe"),
},
},
)
@ -795,7 +796,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
# },
"x86_64": {
"module:master:20190318:abcdef": MockModule(
"/mnt/koji/modules/module:master-20190318-abcdef"
"/mnt/koji/modules/fake_arch/module:master-20190318-abcdef"
)
},
},
@ -811,7 +812,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
"x86_64"
],
arch_mmds={
"x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe")}
"x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe")}
},
modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
)
@ -830,9 +831,9 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
# },
"x86_64": {
"module:master:20190318:abcdef": MockModule(
"/mnt/koji/modules/module:master-20190318-abcdef"
"/mnt/koji/modules/fake_arch/module:master-20190318-abcdef"
),
"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe"),
"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe"),
},
},
)