almalinux
/
pungi
15
1
Fork 0
Code Issues Pull Requests Releases Activity

LNX-108: Add multiarch support to pungi 

@BS-NOBUILD
@BS-TARGET-CL8

Change-Id: Ibfd540454941922d790ae4e56cc0992c0c85635d
This commit is contained in:
stepan_oksanichenko 2021-05-12 18:14:49 +03:00
parent a435eeed06
commit cc4d99441c
8 changed files with 81 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
pungi/phases/pkgset/sources/source_koji.py
View File

@ -228,9 +228,12 @@ def _add_module_to_variant(
# Skip non module archives # Skip non module archives
continue continue
filename = archive["filename"] filename = archive["filename"]
# TODO: add support for muplitple arches file_path = os.path.join(
file_path = os.path.join(koji_wrapper.koji_module.pathinfo.topdir, 'modules', koji_wrapper.koji_module.pathinfo.topdir,
build['extra']['typeinfo']['module']['content_koji_tag']) 'modules',
build['arch'],
build['extra']['typeinfo']['module']['content_koji_tag']
)
mmds[filename] = file_path mmds[filename] = file_path
if len(mmds) <= 1: if len(mmds) <= 1:
@ -488,16 +491,15 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules)
info.get("context"), info.get("context"),
) )
nvr_patterns.add((pattern, spec["name"])) nvr_patterns.add((pattern, spec["name"]))
modules_to_keep = [] modules_to_keep = []
for mb in module_builds: for mb in sorted(module_builds, key=lambda i: i['name']):
# Split release from the build into version and context # Split release from the build into version and context
ver, ctx = mb["release"].split(".") ver, ctx = mb["release"].split(".")
# Values in `mb` are from Koji build. There's nvr and name, version and # Values in `mb` are from Koji build. There's nvr and name, version and
# release. The input pattern specifies modular name, stream, version # release. The input pattern specifies modular name, stream, version
# and context. # and context.
for (n, s, v, c), spec in nvr_patterns: for (n, s, v, c), spec in sorted(nvr_patterns):
if ( if (
# We always have a name and stream... # We always have a name and stream...
mb["name"] == n mb["name"] == n
@ -509,7 +511,6 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules)
): ):
modules_to_keep.append(mb) modules_to_keep.append(mb)
expected_modules.discard(spec) expected_modules.discard(spec)
break
return modules_to_keep return modules_to_keep
@ -543,7 +544,7 @@ def _filter_expected_modules(
if cond1 and cond2: if cond1 and cond2:
expected_modules = { expected_modules = {
expected_module for expected_module in expected_modules if expected_module for expected_module in expected_modules if
not all( not any(
re.findall( re.findall(
filtered_module, filtered_module,
expected_module, expected_module,
@ -573,7 +574,13 @@ def _get_modules_from_koji_tags(
] ]
# Get set of configured module names for this variant. If nothing is # Get set of configured module names for this variant. If nothing is
# configured, the set is empty. # configured, the set is empty.
expected_modules = set(spec["name"] for spec in variant.get_modules()) expected_modules = []
for spec in variant.get_modules():
name, stream = spec['name'].split(':')
expected_modules.append(
':'.join((name, stream.replace('-', '_')))
)
expected_modules = set(expected_modules)
# Find out all modules in every variant and add their Koji tags # Find out all modules in every variant and add their Koji tags
# to variant and variant_tags list. # to variant and variant_tags list.
koji_proxy = koji_wrapper.koji_proxy koji_proxy = koji_wrapper.koji_proxy

11
pungi/scripts/create_extra_repo.py
View File

@ -78,11 +78,12 @@ class CreateExtraRepo(PackagesGenerator):
""" """
Dump merged modules data to an local repo Dump merged modules data to an local repo
""" """
with open(self.default_modules_yaml_path, 'w') as yaml_file: if self.local_modules_data:
yaml.dump_all( with open(self.default_modules_yaml_path, 'w') as yaml_file:
self.local_modules_data, yaml.dump_all(
yaml_file, self.local_modules_data,
) yaml_file,
)
@staticmethod @staticmethod
def get_repo_info_from_bs_repo( def get_repo_info_from_bs_repo(

19
pungi/scripts/gather_modules.py
View File

@ -51,19 +51,24 @@ def collect_modules(modules_paths: List[BinaryIO], target_dir: str):
documents = yaml.load_all(data, Loader=yaml.BaseLoader) documents = yaml.load_all(data, Loader=yaml.BaseLoader)
for doc in documents: for doc in documents:
if doc['document'] == 'modulemd-defaults': if doc['document'] == 'modulemd-defaults':
name = doc['data']['module'] + '.yaml' name = f"{doc['data']['module']}.yaml"
path = os.path.join(module_defaults_path, name) path = os.path.join(module_defaults_path, name)
logging.info('Found %s module defaults', name) logging.info('Found %s module defaults', name)
else: else:
# pungi.phases.pkgset.sources.source_koji.get_koji_modules # pungi.phases.pkgset.sources.source_koji.get_koji_modules
stream = doc['data']['stream'].replace('-', '_') stream = doc['data']['stream'].replace('-', '_')
name = '%s-%s-%s.%s' % ( doc_data = doc['data']
doc['data']['name'], name = f"{doc_data['name']}-{stream}-" \
stream, f"{doc_data['version']}.{doc_data['context']}"
doc['data']['version'], arch_dir = os.path.join(
doc['data']['context'] modules_path,
doc_data['arch']
)
os.makedirs(arch_dir, exist_ok=True)
path = os.path.join(
arch_dir,
name,
) )
path = os.path.join(modules_path, name)
logging.info('Found module %s', name) logging.info('Found module %s', name)
if 'artifacts' not in doc['data']: if 'artifacts' not in doc['data']:

2
pungi/wrappers/comps.py
View File

@ -299,6 +299,8 @@ class CompsWrapper(object):
append_common_info(doc, group_node, group, force_description=True) append_common_info(doc, group_node, group, force_description=True)
append_bool(doc, group_node, "default", group.default) append_bool(doc, group_node, "default", group.default)
append_bool(doc, group_node, "uservisible", group.uservisible) append_bool(doc, group_node, "uservisible", group.uservisible)
if group.display_order is not None:
append(doc, group_node, "display_order", str(group.display_order))
if group.lang_only: if group.lang_only:
append(doc, group_node, "langonly", group.lang_only) append(doc, group_node, "langonly", group.lang_only)

40
pungi/wrappers/kojimock.py
View File

@ -35,6 +35,7 @@ class Module:
stream: str stream: str
version: str version: str
context: str context: str
arch: str
class KojiMock: class KojiMock:
@ -49,16 +50,22 @@ class KojiMock:
def _gather_modules(self, modules_dir): def _gather_modules(self, modules_dir):
modules = {} modules = {}
for index, f in enumerate(os.listdir(modules_dir)): for arch in os.listdir(modules_dir):
parsed = parse_nvra(f) arch_dir = os.path.join(
modules[index] = Module( modules_dir,
name=parsed['name'], arch,
nvr=f,
version=parsed['release'],
context=parsed['arch'],
stream=parsed['version'],
build_id=index
) )
for index, f in enumerate(os.listdir(arch_dir)):
parsed = parse_nvra(f)
modules[index] = Module(
name=parsed['name'],
nvr=f,
version=parsed['release'],
context=parsed['arch'],
stream=parsed['version'],
build_id=index,
arch=arch,
)
return modules return modules
def getLastEvent(self, *args, **kwargs): def getLastEvent(self, *args, **kwargs):
@ -124,6 +131,7 @@ class KojiMock:
'release': '%s.%s' % (module.version, module.context), 'release': '%s.%s' % (module.version, module.context),
'completion_ts': BUILD_TIME, 'completion_ts': BUILD_TIME,
'state': 'COMPLETE', 'state': 'COMPLETE',
'arch': module.arch,
'extra': { 'extra': {
'typeinfo': { 'typeinfo': {
'module': { 'module': {
@ -152,7 +160,7 @@ class KojiMock:
return [ return [
{ {
'build_id': module.build_id, 'build_id': module.build_id,
'filename': 'modulemd.x86_64.txt', 'filename': f'modulemd.{module.arch}.txt',
'btype': 'module' 'btype': 'module'
}, },
# noone ever uses this file # noone ever uses this file
@ -197,7 +205,11 @@ class KojiMock:
# get nvras for modular packages # get nvras for modular packages
nvras = set() nvras = set()
for module in self._modules.values(): for module in self._modules.values():
path = os.path.join(self._modules_dir, module.nvr) path = os.path.join(
self._modules_dir,
module.arch,
module.nvr,
)
info = Modulemd.ModuleStream.read_string(open(path).read(), strict=True) info = Modulemd.ModuleStream.read_string(open(path).read(), strict=True)
for package in info.get_rpm_artifacts(): for package in info.get_rpm_artifacts():
@ -232,8 +244,12 @@ class KojiMock:
""" """
Get list of builds for module and given module tag name. Get list of builds for module and given module tag name.
""" """
path = os.path.join(self._modules_dir, tag_name)
module = self._get_module_by_name(tag_name) module = self._get_module_by_name(tag_name)
path = os.path.join(
self._modules_dir,
module.arch,
tag_name,
)
builds = [ builds = [
{ {

6
tests/test_gather_modules.py
View File

@ -105,15 +105,15 @@ class TestModulesYamlParser(TestCase):
self.assertEqual([MODULES_YAML_GZ, 'modules', 'module_defaults'], os.listdir(PATH_TO_KOJI)) self.assertEqual([MODULES_YAML_GZ, 'modules', 'module_defaults'], os.listdir(PATH_TO_KOJI))
self.assertEqual(['mariadb-devel-10.3_1-8010020200108182321.cdc1202b', self.assertEqual(['mariadb-devel-10.3_1-8010020200108182321.cdc1202b',
'javapackages-tools-201801-8000020190628172923.b07bea58'], 'javapackages-tools-201801-8000020190628172923.b07bea58'],
os.listdir(os.path.join(PATH_TO_KOJI, 'modules'))) os.listdir(os.path.join(PATH_TO_KOJI, 'modules/x86_64')))
self.assertEqual(['ant.yaml'], self.assertEqual(['ant.yaml'],
os.listdir(os.path.join(PATH_TO_KOJI, 'module_defaults'))) os.listdir(os.path.join(PATH_TO_KOJI, 'module_defaults')))
# check that modules were exported # check that modules were exported
self.assertEqual(MARIADB_MODULE, yaml.load( self.assertEqual(MARIADB_MODULE, yaml.load(
open(os.path.join(PATH_TO_KOJI, 'modules', 'mariadb-devel-10.3_1-8010020200108182321.cdc1202b')))) open(os.path.join(PATH_TO_KOJI, 'modules/x86_64', 'mariadb-devel-10.3_1-8010020200108182321.cdc1202b'))))
self.assertEqual(JAVAPACKAGES_TOOLS_MODULE, yaml.load( self.assertEqual(JAVAPACKAGES_TOOLS_MODULE, yaml.load(
open(os.path.join(PATH_TO_KOJI, 'modules', 'javapackages-tools-201801-8000020190628172923.b07bea58')))) open(os.path.join(PATH_TO_KOJI, 'modules/x86_64', 'javapackages-tools-201801-8000020190628172923.b07bea58'))))
# check that defaults were copied # check that defaults were copied
self.assertEqual(ANT_DEFAULTS, yaml.load( self.assertEqual(ANT_DEFAULTS, yaml.load(

8
tests/test_koji_local_source.py
View File

@ -103,13 +103,13 @@ version: '1'
self.setUpPyfakefs() self.setUpPyfakefs()
os.makedirs(PATH_TO_REPOS) os.makedirs(PATH_TO_REPOS)
os.makedirs(os.path.join(PATH_TO_REPOS, 'modules')) os.makedirs(os.path.join(PATH_TO_REPOS, 'modules/x86_64'))
with open(os.path.join(PATH_TO_REPOS, 'modules', with open(os.path.join(PATH_TO_REPOS, 'modules/x86_64',
'javapackages-tools-201801-8000020190628172923.b07bea58'), 'w') as f: 'javapackages-tools-201801-8000020190628172923.b07bea58'), 'w') as f:
f.write(self.JAVAPACKAGES_TOOLS_MODULE) f.write(self.JAVAPACKAGES_TOOLS_MODULE)
with open(os.path.join(PATH_TO_REPOS, 'modules', with open(os.path.join(PATH_TO_REPOS, 'modules/x86_64',
'mariadb-devel-10.3-8010020200108182321.cdc1202b'), 'w') as f: 'mariadb-devel-10.3-8010020200108182321.cdc1202b'), 'w') as f:
f.write(self.MARIADB_MODULE) f.write(self.MARIADB_MODULE)
@ -122,6 +122,7 @@ version: '1'
@ddt.data( @ddt.data(
[0, { [0, {
'completion_ts': 0, 'completion_ts': 0,
'arch': 'x86_64',
'extra': { 'extra': {
'typeinfo': { 'typeinfo': {
'module': { 'module': {
@ -141,6 +142,7 @@ version: '1'
}], }],
[1, { [1, {
'completion_ts': 0, 'completion_ts': 0,
'arch': 'x86_64',
'extra': { 'extra': {
'typeinfo': { 'typeinfo': {
'module': { 'module': {

17
tests/test_pkgset_source_koji.py
View File

@ -703,6 +703,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
] + [{"btype": "foo"}] ] + [{"btype": "foo"}]
self.buildinfo = { self.buildinfo = {
"id": 1234, "id": 1234,
"arch": "fake_arch",
"extra": { "extra": {
"typeinfo": { "typeinfo": {
"module": { "module": {
@ -734,7 +735,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
# }, # },
"x86_64": { "x86_64": {
"module:master:20190318:abcdef": MockModule( "module:master:20190318:abcdef": MockModule(
"/mnt/koji/modules/module:master-20190318-abcdef" "/mnt/koji/modules/fake_arch/module:master-20190318-abcdef"
), ),
}, },
}, },
@ -748,7 +749,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
"x86_64" "x86_64"
], ],
arch_mmds={ arch_mmds={
"x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe")} "x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe")}
}, },
modules=[{"name": "m1:latest-20190101:cafe", "glob": False}], modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
) )
@ -765,9 +766,9 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
# }, # },
"x86_64": { "x86_64": {
"module:master:20190318:abcdef": MockModule( "module:master:20190318:abcdef": MockModule(
"/mnt/koji/modules/module:master-20190318-abcdef" "/mnt/koji/modules/fake_arch/module:master-20190318-abcdef"
), ),
"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe"), "m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe"),
}, },
}, },
) )
@ -795,7 +796,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
# }, # },
"x86_64": { "x86_64": {
"module:master:20190318:abcdef": MockModule( "module:master:20190318:abcdef": MockModule(
"/mnt/koji/modules/module:master-20190318-abcdef" "/mnt/koji/modules/fake_arch/module:master-20190318-abcdef"
) )
}, },
}, },
@ -811,7 +812,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
"x86_64" "x86_64"
], ],
arch_mmds={ arch_mmds={
"x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe")} "x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe")}
}, },
modules=[{"name": "m1:latest-20190101:cafe", "glob": False}], modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
) )
@ -830,9 +831,9 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
# }, # },
"x86_64": { "x86_64": {
"module:master:20190318:abcdef": MockModule( "module:master:20190318:abcdef": MockModule(
"/mnt/koji/modules/module:master-20190318-abcdef" "/mnt/koji/modules/fake_arch/module:master-20190318-abcdef"
), ),
"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe"), "m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe"),
}, },
}, },
) )