LNX-108: Add multiarch support to pungi
@BS-NOBUILD @BS-TARGET-CL8 Change-Id: Ibfd540454941922d790ae4e56cc0992c0c85635d
This commit is contained in:
parent
a435eeed06
commit
cc4d99441c
@ -228,9 +228,12 @@ def _add_module_to_variant(
|
|||||||
# Skip non module archives
|
# Skip non module archives
|
||||||
continue
|
continue
|
||||||
filename = archive["filename"]
|
filename = archive["filename"]
|
||||||
# TODO: add support for muplitple arches
|
file_path = os.path.join(
|
||||||
file_path = os.path.join(koji_wrapper.koji_module.pathinfo.topdir, 'modules',
|
koji_wrapper.koji_module.pathinfo.topdir,
|
||||||
build['extra']['typeinfo']['module']['content_koji_tag'])
|
'modules',
|
||||||
|
build['arch'],
|
||||||
|
build['extra']['typeinfo']['module']['content_koji_tag']
|
||||||
|
)
|
||||||
mmds[filename] = file_path
|
mmds[filename] = file_path
|
||||||
|
|
||||||
if len(mmds) <= 1:
|
if len(mmds) <= 1:
|
||||||
@ -488,16 +491,15 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules)
|
|||||||
info.get("context"),
|
info.get("context"),
|
||||||
)
|
)
|
||||||
nvr_patterns.add((pattern, spec["name"]))
|
nvr_patterns.add((pattern, spec["name"]))
|
||||||
|
|
||||||
modules_to_keep = []
|
modules_to_keep = []
|
||||||
|
|
||||||
for mb in module_builds:
|
for mb in sorted(module_builds, key=lambda i: i['name']):
|
||||||
# Split release from the build into version and context
|
# Split release from the build into version and context
|
||||||
ver, ctx = mb["release"].split(".")
|
ver, ctx = mb["release"].split(".")
|
||||||
# Values in `mb` are from Koji build. There's nvr and name, version and
|
# Values in `mb` are from Koji build. There's nvr and name, version and
|
||||||
# release. The input pattern specifies modular name, stream, version
|
# release. The input pattern specifies modular name, stream, version
|
||||||
# and context.
|
# and context.
|
||||||
for (n, s, v, c), spec in nvr_patterns:
|
for (n, s, v, c), spec in sorted(nvr_patterns):
|
||||||
if (
|
if (
|
||||||
# We always have a name and stream...
|
# We always have a name and stream...
|
||||||
mb["name"] == n
|
mb["name"] == n
|
||||||
@ -509,7 +511,6 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules)
|
|||||||
):
|
):
|
||||||
modules_to_keep.append(mb)
|
modules_to_keep.append(mb)
|
||||||
expected_modules.discard(spec)
|
expected_modules.discard(spec)
|
||||||
break
|
|
||||||
|
|
||||||
return modules_to_keep
|
return modules_to_keep
|
||||||
|
|
||||||
@ -543,7 +544,7 @@ def _filter_expected_modules(
|
|||||||
if cond1 and cond2:
|
if cond1 and cond2:
|
||||||
expected_modules = {
|
expected_modules = {
|
||||||
expected_module for expected_module in expected_modules if
|
expected_module for expected_module in expected_modules if
|
||||||
not all(
|
not any(
|
||||||
re.findall(
|
re.findall(
|
||||||
filtered_module,
|
filtered_module,
|
||||||
expected_module,
|
expected_module,
|
||||||
@ -573,7 +574,13 @@ def _get_modules_from_koji_tags(
|
|||||||
]
|
]
|
||||||
# Get set of configured module names for this variant. If nothing is
|
# Get set of configured module names for this variant. If nothing is
|
||||||
# configured, the set is empty.
|
# configured, the set is empty.
|
||||||
expected_modules = set(spec["name"] for spec in variant.get_modules())
|
expected_modules = []
|
||||||
|
for spec in variant.get_modules():
|
||||||
|
name, stream = spec['name'].split(':')
|
||||||
|
expected_modules.append(
|
||||||
|
':'.join((name, stream.replace('-', '_')))
|
||||||
|
)
|
||||||
|
expected_modules = set(expected_modules)
|
||||||
# Find out all modules in every variant and add their Koji tags
|
# Find out all modules in every variant and add their Koji tags
|
||||||
# to variant and variant_tags list.
|
# to variant and variant_tags list.
|
||||||
koji_proxy = koji_wrapper.koji_proxy
|
koji_proxy = koji_wrapper.koji_proxy
|
||||||
|
@ -78,6 +78,7 @@ class CreateExtraRepo(PackagesGenerator):
|
|||||||
"""
|
"""
|
||||||
Dump merged modules data to an local repo
|
Dump merged modules data to an local repo
|
||||||
"""
|
"""
|
||||||
|
if self.local_modules_data:
|
||||||
with open(self.default_modules_yaml_path, 'w') as yaml_file:
|
with open(self.default_modules_yaml_path, 'w') as yaml_file:
|
||||||
yaml.dump_all(
|
yaml.dump_all(
|
||||||
self.local_modules_data,
|
self.local_modules_data,
|
||||||
|
@ -51,19 +51,24 @@ def collect_modules(modules_paths: List[BinaryIO], target_dir: str):
|
|||||||
documents = yaml.load_all(data, Loader=yaml.BaseLoader)
|
documents = yaml.load_all(data, Loader=yaml.BaseLoader)
|
||||||
for doc in documents:
|
for doc in documents:
|
||||||
if doc['document'] == 'modulemd-defaults':
|
if doc['document'] == 'modulemd-defaults':
|
||||||
name = doc['data']['module'] + '.yaml'
|
name = f"{doc['data']['module']}.yaml"
|
||||||
path = os.path.join(module_defaults_path, name)
|
path = os.path.join(module_defaults_path, name)
|
||||||
logging.info('Found %s module defaults', name)
|
logging.info('Found %s module defaults', name)
|
||||||
else:
|
else:
|
||||||
# pungi.phases.pkgset.sources.source_koji.get_koji_modules
|
# pungi.phases.pkgset.sources.source_koji.get_koji_modules
|
||||||
stream = doc['data']['stream'].replace('-', '_')
|
stream = doc['data']['stream'].replace('-', '_')
|
||||||
name = '%s-%s-%s.%s' % (
|
doc_data = doc['data']
|
||||||
doc['data']['name'],
|
name = f"{doc_data['name']}-{stream}-" \
|
||||||
stream,
|
f"{doc_data['version']}.{doc_data['context']}"
|
||||||
doc['data']['version'],
|
arch_dir = os.path.join(
|
||||||
doc['data']['context']
|
modules_path,
|
||||||
|
doc_data['arch']
|
||||||
|
)
|
||||||
|
os.makedirs(arch_dir, exist_ok=True)
|
||||||
|
path = os.path.join(
|
||||||
|
arch_dir,
|
||||||
|
name,
|
||||||
)
|
)
|
||||||
path = os.path.join(modules_path, name)
|
|
||||||
logging.info('Found module %s', name)
|
logging.info('Found module %s', name)
|
||||||
|
|
||||||
if 'artifacts' not in doc['data']:
|
if 'artifacts' not in doc['data']:
|
||||||
|
@ -299,6 +299,8 @@ class CompsWrapper(object):
|
|||||||
append_common_info(doc, group_node, group, force_description=True)
|
append_common_info(doc, group_node, group, force_description=True)
|
||||||
append_bool(doc, group_node, "default", group.default)
|
append_bool(doc, group_node, "default", group.default)
|
||||||
append_bool(doc, group_node, "uservisible", group.uservisible)
|
append_bool(doc, group_node, "uservisible", group.uservisible)
|
||||||
|
if group.display_order is not None:
|
||||||
|
append(doc, group_node, "display_order", str(group.display_order))
|
||||||
|
|
||||||
if group.lang_only:
|
if group.lang_only:
|
||||||
append(doc, group_node, "langonly", group.lang_only)
|
append(doc, group_node, "langonly", group.lang_only)
|
||||||
|
@ -35,6 +35,7 @@ class Module:
|
|||||||
stream: str
|
stream: str
|
||||||
version: str
|
version: str
|
||||||
context: str
|
context: str
|
||||||
|
arch: str
|
||||||
|
|
||||||
|
|
||||||
class KojiMock:
|
class KojiMock:
|
||||||
@ -49,7 +50,12 @@ class KojiMock:
|
|||||||
|
|
||||||
def _gather_modules(self, modules_dir):
|
def _gather_modules(self, modules_dir):
|
||||||
modules = {}
|
modules = {}
|
||||||
for index, f in enumerate(os.listdir(modules_dir)):
|
for arch in os.listdir(modules_dir):
|
||||||
|
arch_dir = os.path.join(
|
||||||
|
modules_dir,
|
||||||
|
arch,
|
||||||
|
)
|
||||||
|
for index, f in enumerate(os.listdir(arch_dir)):
|
||||||
parsed = parse_nvra(f)
|
parsed = parse_nvra(f)
|
||||||
modules[index] = Module(
|
modules[index] = Module(
|
||||||
name=parsed['name'],
|
name=parsed['name'],
|
||||||
@ -57,7 +63,8 @@ class KojiMock:
|
|||||||
version=parsed['release'],
|
version=parsed['release'],
|
||||||
context=parsed['arch'],
|
context=parsed['arch'],
|
||||||
stream=parsed['version'],
|
stream=parsed['version'],
|
||||||
build_id=index
|
build_id=index,
|
||||||
|
arch=arch,
|
||||||
)
|
)
|
||||||
return modules
|
return modules
|
||||||
|
|
||||||
@ -124,6 +131,7 @@ class KojiMock:
|
|||||||
'release': '%s.%s' % (module.version, module.context),
|
'release': '%s.%s' % (module.version, module.context),
|
||||||
'completion_ts': BUILD_TIME,
|
'completion_ts': BUILD_TIME,
|
||||||
'state': 'COMPLETE',
|
'state': 'COMPLETE',
|
||||||
|
'arch': module.arch,
|
||||||
'extra': {
|
'extra': {
|
||||||
'typeinfo': {
|
'typeinfo': {
|
||||||
'module': {
|
'module': {
|
||||||
@ -152,7 +160,7 @@ class KojiMock:
|
|||||||
return [
|
return [
|
||||||
{
|
{
|
||||||
'build_id': module.build_id,
|
'build_id': module.build_id,
|
||||||
'filename': 'modulemd.x86_64.txt',
|
'filename': f'modulemd.{module.arch}.txt',
|
||||||
'btype': 'module'
|
'btype': 'module'
|
||||||
},
|
},
|
||||||
# noone ever uses this file
|
# noone ever uses this file
|
||||||
@ -197,7 +205,11 @@ class KojiMock:
|
|||||||
# get nvras for modular packages
|
# get nvras for modular packages
|
||||||
nvras = set()
|
nvras = set()
|
||||||
for module in self._modules.values():
|
for module in self._modules.values():
|
||||||
path = os.path.join(self._modules_dir, module.nvr)
|
path = os.path.join(
|
||||||
|
self._modules_dir,
|
||||||
|
module.arch,
|
||||||
|
module.nvr,
|
||||||
|
)
|
||||||
info = Modulemd.ModuleStream.read_string(open(path).read(), strict=True)
|
info = Modulemd.ModuleStream.read_string(open(path).read(), strict=True)
|
||||||
|
|
||||||
for package in info.get_rpm_artifacts():
|
for package in info.get_rpm_artifacts():
|
||||||
@ -232,8 +244,12 @@ class KojiMock:
|
|||||||
"""
|
"""
|
||||||
Get list of builds for module and given module tag name.
|
Get list of builds for module and given module tag name.
|
||||||
"""
|
"""
|
||||||
path = os.path.join(self._modules_dir, tag_name)
|
|
||||||
module = self._get_module_by_name(tag_name)
|
module = self._get_module_by_name(tag_name)
|
||||||
|
path = os.path.join(
|
||||||
|
self._modules_dir,
|
||||||
|
module.arch,
|
||||||
|
tag_name,
|
||||||
|
)
|
||||||
|
|
||||||
builds = [
|
builds = [
|
||||||
{
|
{
|
||||||
|
@ -105,15 +105,15 @@ class TestModulesYamlParser(TestCase):
|
|||||||
self.assertEqual([MODULES_YAML_GZ, 'modules', 'module_defaults'], os.listdir(PATH_TO_KOJI))
|
self.assertEqual([MODULES_YAML_GZ, 'modules', 'module_defaults'], os.listdir(PATH_TO_KOJI))
|
||||||
self.assertEqual(['mariadb-devel-10.3_1-8010020200108182321.cdc1202b',
|
self.assertEqual(['mariadb-devel-10.3_1-8010020200108182321.cdc1202b',
|
||||||
'javapackages-tools-201801-8000020190628172923.b07bea58'],
|
'javapackages-tools-201801-8000020190628172923.b07bea58'],
|
||||||
os.listdir(os.path.join(PATH_TO_KOJI, 'modules')))
|
os.listdir(os.path.join(PATH_TO_KOJI, 'modules/x86_64')))
|
||||||
self.assertEqual(['ant.yaml'],
|
self.assertEqual(['ant.yaml'],
|
||||||
os.listdir(os.path.join(PATH_TO_KOJI, 'module_defaults')))
|
os.listdir(os.path.join(PATH_TO_KOJI, 'module_defaults')))
|
||||||
|
|
||||||
# check that modules were exported
|
# check that modules were exported
|
||||||
self.assertEqual(MARIADB_MODULE, yaml.load(
|
self.assertEqual(MARIADB_MODULE, yaml.load(
|
||||||
open(os.path.join(PATH_TO_KOJI, 'modules', 'mariadb-devel-10.3_1-8010020200108182321.cdc1202b'))))
|
open(os.path.join(PATH_TO_KOJI, 'modules/x86_64', 'mariadb-devel-10.3_1-8010020200108182321.cdc1202b'))))
|
||||||
self.assertEqual(JAVAPACKAGES_TOOLS_MODULE, yaml.load(
|
self.assertEqual(JAVAPACKAGES_TOOLS_MODULE, yaml.load(
|
||||||
open(os.path.join(PATH_TO_KOJI, 'modules', 'javapackages-tools-201801-8000020190628172923.b07bea58'))))
|
open(os.path.join(PATH_TO_KOJI, 'modules/x86_64', 'javapackages-tools-201801-8000020190628172923.b07bea58'))))
|
||||||
|
|
||||||
# check that defaults were copied
|
# check that defaults were copied
|
||||||
self.assertEqual(ANT_DEFAULTS, yaml.load(
|
self.assertEqual(ANT_DEFAULTS, yaml.load(
|
||||||
|
@ -103,13 +103,13 @@ version: '1'
|
|||||||
self.setUpPyfakefs()
|
self.setUpPyfakefs()
|
||||||
|
|
||||||
os.makedirs(PATH_TO_REPOS)
|
os.makedirs(PATH_TO_REPOS)
|
||||||
os.makedirs(os.path.join(PATH_TO_REPOS, 'modules'))
|
os.makedirs(os.path.join(PATH_TO_REPOS, 'modules/x86_64'))
|
||||||
|
|
||||||
with open(os.path.join(PATH_TO_REPOS, 'modules',
|
with open(os.path.join(PATH_TO_REPOS, 'modules/x86_64',
|
||||||
'javapackages-tools-201801-8000020190628172923.b07bea58'), 'w') as f:
|
'javapackages-tools-201801-8000020190628172923.b07bea58'), 'w') as f:
|
||||||
f.write(self.JAVAPACKAGES_TOOLS_MODULE)
|
f.write(self.JAVAPACKAGES_TOOLS_MODULE)
|
||||||
|
|
||||||
with open(os.path.join(PATH_TO_REPOS, 'modules',
|
with open(os.path.join(PATH_TO_REPOS, 'modules/x86_64',
|
||||||
'mariadb-devel-10.3-8010020200108182321.cdc1202b'), 'w') as f:
|
'mariadb-devel-10.3-8010020200108182321.cdc1202b'), 'w') as f:
|
||||||
f.write(self.MARIADB_MODULE)
|
f.write(self.MARIADB_MODULE)
|
||||||
|
|
||||||
@ -122,6 +122,7 @@ version: '1'
|
|||||||
@ddt.data(
|
@ddt.data(
|
||||||
[0, {
|
[0, {
|
||||||
'completion_ts': 0,
|
'completion_ts': 0,
|
||||||
|
'arch': 'x86_64',
|
||||||
'extra': {
|
'extra': {
|
||||||
'typeinfo': {
|
'typeinfo': {
|
||||||
'module': {
|
'module': {
|
||||||
@ -141,6 +142,7 @@ version: '1'
|
|||||||
}],
|
}],
|
||||||
[1, {
|
[1, {
|
||||||
'completion_ts': 0,
|
'completion_ts': 0,
|
||||||
|
'arch': 'x86_64',
|
||||||
'extra': {
|
'extra': {
|
||||||
'typeinfo': {
|
'typeinfo': {
|
||||||
'module': {
|
'module': {
|
||||||
|
@ -703,6 +703,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
|
|||||||
] + [{"btype": "foo"}]
|
] + [{"btype": "foo"}]
|
||||||
self.buildinfo = {
|
self.buildinfo = {
|
||||||
"id": 1234,
|
"id": 1234,
|
||||||
|
"arch": "fake_arch",
|
||||||
"extra": {
|
"extra": {
|
||||||
"typeinfo": {
|
"typeinfo": {
|
||||||
"module": {
|
"module": {
|
||||||
@ -734,7 +735,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
|
|||||||
# },
|
# },
|
||||||
"x86_64": {
|
"x86_64": {
|
||||||
"module:master:20190318:abcdef": MockModule(
|
"module:master:20190318:abcdef": MockModule(
|
||||||
"/mnt/koji/modules/module:master-20190318-abcdef"
|
"/mnt/koji/modules/fake_arch/module:master-20190318-abcdef"
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@ -748,7 +749,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
|
|||||||
"x86_64"
|
"x86_64"
|
||||||
],
|
],
|
||||||
arch_mmds={
|
arch_mmds={
|
||||||
"x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe")}
|
"x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe")}
|
||||||
},
|
},
|
||||||
modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
|
modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
|
||||||
)
|
)
|
||||||
@ -765,9 +766,9 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
|
|||||||
# },
|
# },
|
||||||
"x86_64": {
|
"x86_64": {
|
||||||
"module:master:20190318:abcdef": MockModule(
|
"module:master:20190318:abcdef": MockModule(
|
||||||
"/mnt/koji/modules/module:master-20190318-abcdef"
|
"/mnt/koji/modules/fake_arch/module:master-20190318-abcdef"
|
||||||
),
|
),
|
||||||
"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe"),
|
"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe"),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@ -795,7 +796,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
|
|||||||
# },
|
# },
|
||||||
"x86_64": {
|
"x86_64": {
|
||||||
"module:master:20190318:abcdef": MockModule(
|
"module:master:20190318:abcdef": MockModule(
|
||||||
"/mnt/koji/modules/module:master-20190318-abcdef"
|
"/mnt/koji/modules/fake_arch/module:master-20190318-abcdef"
|
||||||
)
|
)
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@ -811,7 +812,7 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
|
|||||||
"x86_64"
|
"x86_64"
|
||||||
],
|
],
|
||||||
arch_mmds={
|
arch_mmds={
|
||||||
"x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe")}
|
"x86_64": {"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe")}
|
||||||
},
|
},
|
||||||
modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
|
modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
|
||||||
)
|
)
|
||||||
@ -830,9 +831,9 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
|
|||||||
# },
|
# },
|
||||||
"x86_64": {
|
"x86_64": {
|
||||||
"module:master:20190318:abcdef": MockModule(
|
"module:master:20190318:abcdef": MockModule(
|
||||||
"/mnt/koji/modules/module:master-20190318-abcdef"
|
"/mnt/koji/modules/fake_arch/module:master-20190318-abcdef"
|
||||||
),
|
),
|
||||||
"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/m1:latest:20190101:cafe"),
|
"m1:latest:20190101:cafe": MockModule("/mnt/koji/modules/fake_arch/m1:latest:20190101:cafe"),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
Loading…
Reference in New Issue
Block a user