pkgset: Load final modulemd files from Koji
Instead of loading the "source" modulemd, always get the final file for each architecture from Koji. Logging the downloaded files locally is no longer necessary, when debugging a problem we can find the files in the respective builds in Koji. Tests are updated to work with new code, and an obsolete test is removed. JIRA: COMPOSE-3147 Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář
parent
358fdd50ce
commit
9229699078
|
|
@ -26,7 +26,7 @@ from kobo.shortcuts import force_list, relative_path
|
|||
import pungi.wrappers.kojiwrapper
|
||||
from pungi.wrappers.comps import CompsWrapper
|
||||
import pungi.phases.pkgset.pkgsets
|
||||
from pungi.arch import get_valid_arches
|
||||
from pungi.arch import get_valid_arches, getBaseArch
|
||||
from pungi.util import is_arch_multilib, retry, find_old_compose
|
||||
from pungi import Modulemd
|
||||
|
||||
|
|
@ -207,36 +207,46 @@ def get_pkgset_from_koji(compose, koji_wrapper, path_prefix):
|
|||
return package_sets
|
||||
|
||||
|
||||
def _add_module_to_variant(variant, mmd, add_to_variant_modules=False):
|
||||
def _add_module_to_variant(koji_wrapper, variant, build, add_to_variant_modules=False):
|
||||
"""
|
||||
Adds module defined by Modulemd.Module `mmd` to variant.
|
||||
Adds module defined by Koji build info to variant.
|
||||
|
||||
:param Variant variant: Variant to add the module to.
|
||||
:param Modulemd.Module: Modulemd instance defining the module.
|
||||
:param int: build id
|
||||
:param bool add_to_variant_modules: Adds the modules also to
|
||||
variant.modules.
|
||||
"""
|
||||
# Get the NSVC of module and handle the case where for some reason the
|
||||
# name/strea/version is not set.
|
||||
if not mmd.get_name() or not mmd.get_stream() or not mmd.get_version():
|
||||
raise ValueError(
|
||||
"Input module %s does not name or stream or version set."
|
||||
% mmd.dumps())
|
||||
nsvc_list = [mmd.get_name(), mmd.get_stream(), str(mmd.get_version())]
|
||||
if mmd.get_context():
|
||||
nsvc_list.append(mmd.get_context())
|
||||
nsvc = ":".join(nsvc_list)
|
||||
mmds = {}
|
||||
archives = koji_wrapper.koji_proxy.listArchives(build["id"])
|
||||
for archive in archives:
|
||||
if archive["btype"] != "module":
|
||||
# Skip non module archives
|
||||
continue
|
||||
typedir = koji_wrapper.koji_module.pathinfo.typedir(build, archive["btype"])
|
||||
filename = archive["filename"]
|
||||
file_path = os.path.join(typedir, filename)
|
||||
try:
|
||||
# If there are two dots, the arch is in the middle. MBS uploads
|
||||
# files with actual architecture in the filename, but Pungi deals
|
||||
# in basearch. This assumes that each arch in the build maps to a
|
||||
# unique basearch.
|
||||
_, arch, _ = filename.split(".")
|
||||
filename = "modulemd.%s.txt" % getBaseArch(arch)
|
||||
except ValueError:
|
||||
pass
|
||||
mmds[filename] = Modulemd.Module.new_from_file(file_path)
|
||||
|
||||
variant.mmds.append(mmd)
|
||||
source_mmd = mmds["modulemd.txt"]
|
||||
nsvc = source_mmd.dup_nsvc()
|
||||
|
||||
variant.mmds.append(source_mmd)
|
||||
for arch in variant.arches:
|
||||
variant.arch_mmds.setdefault(arch, {})[nsvc] = mmds["modulemd.%s.txt" % arch]
|
||||
|
||||
if add_to_variant_modules:
|
||||
variant.modules.append(nsvc)
|
||||
|
||||
|
||||
def _log_modulemd(compose, variant, mmd):
|
||||
"""Dump module metadata to a log file for easy inspection."""
|
||||
mmd.dump(compose.paths.log.log_file('global', 'modulemd-%s-%s'
|
||||
% (variant.uid, mmd.dup_nsvc())))
|
||||
return source_mmd
|
||||
|
||||
|
||||
def _get_modules_from_koji(
|
||||
|
|
@ -258,10 +268,7 @@ def _get_modules_from_koji(
|
|||
for module in variant.get_modules():
|
||||
koji_modules = get_koji_modules(compose, koji_wrapper, event, module["name"])
|
||||
for koji_module in koji_modules:
|
||||
mmd = Modulemd.Module.new_from_string(koji_module["modulemd"])
|
||||
mmd.upgrade()
|
||||
_add_module_to_variant(variant, mmd)
|
||||
_log_modulemd(compose, variant, mmd)
|
||||
mmd = _add_module_to_variant(koji_wrapper, variant, koji_module)
|
||||
|
||||
tag = koji_module["tag"]
|
||||
uid = ':'.join([koji_module['name'], koji_module['stream'],
|
||||
|
|
@ -422,17 +429,10 @@ def _get_modules_from_koji_tags(
|
|||
build = koji_proxy.getBuild(build["build_id"])
|
||||
module_tag = build.get("extra", {}).get("typeinfo", {}).get(
|
||||
"module", {}).get("content_koji_tag", "")
|
||||
modulemd = build.get("extra", {}).get("typeinfo", {}).get(
|
||||
"module", {}).get("modulemd_str", "")
|
||||
if not module_tag or not modulemd:
|
||||
continue
|
||||
|
||||
variant_tags[variant].append(module_tag)
|
||||
|
||||
mmd = Modulemd.Module.new_from_string(modulemd)
|
||||
mmd.upgrade()
|
||||
_add_module_to_variant(variant, mmd, True)
|
||||
_log_modulemd(compose, variant, mmd)
|
||||
mmd = _add_module_to_variant(koji_wrapper, variant, build, True)
|
||||
|
||||
# Store mapping module-uid --> koji_tag into variant.
|
||||
# This is needed in createrepo phase where metadata is exposed by producmd
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
#!/usr/bin/env python2
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import json
|
||||
|
|
@ -132,75 +131,6 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
|
|||
with open(self.pkgset_path) as f:
|
||||
self.assertEqual(f.read(), 'DATA')
|
||||
|
||||
@unittest.skipUnless(Modulemd is not None, 'Modulemd not available') # noqa
|
||||
@mock.patch('six.moves.cPickle.dumps')
|
||||
@mock.patch('pungi.phases.pkgset.pkgsets.KojiPackageSet')
|
||||
@mock.patch('pungi.phases.pkgset.sources.source_koji.get_koji_modules')
|
||||
def test_pdc_log(self, get_koji_modules, KojiPackageSet, pickle_dumps):
|
||||
|
||||
pickle_dumps.return_value = b'DATA'
|
||||
|
||||
modulemd1 = """
|
||||
document: modulemd
|
||||
version: 2
|
||||
data:
|
||||
name: foo
|
||||
stream: bar
|
||||
version: 1
|
||||
summary: foo
|
||||
description: foo
|
||||
license:
|
||||
module:
|
||||
- MIT
|
||||
"""
|
||||
|
||||
modulemd2 = """
|
||||
document: modulemd
|
||||
version: 2
|
||||
data:
|
||||
name: foo
|
||||
stream: bar
|
||||
version: 4
|
||||
summary: foo
|
||||
description: foo
|
||||
license:
|
||||
module:
|
||||
- MIT
|
||||
"""
|
||||
|
||||
get_koji_modules.return_value = [
|
||||
{
|
||||
'abc': 'def',
|
||||
'modulemd': modulemd1,
|
||||
'tag': 'taggg',
|
||||
'uid': 'modulenamefoo:rhel:1:00000000',
|
||||
'name': 'modulenamefoo',
|
||||
'stream': 'rhel',
|
||||
'version': '1',
|
||||
'context': '00000000'
|
||||
},
|
||||
{
|
||||
'abc': 'def',
|
||||
'modulemd': modulemd2,
|
||||
'tag': 'taggg',
|
||||
'uid': 'modulenamefoo:rhel:4:00000000',
|
||||
'name': 'modulenamefoo',
|
||||
'stream': 'rhel',
|
||||
'version': '4',
|
||||
'context': '00000000'
|
||||
},
|
||||
]
|
||||
for name, variant in self.compose.variants.items():
|
||||
variant.get_modules = mock.MagicMock()
|
||||
if name == 'Server':
|
||||
variant.modules = [{'name': 'modulenamefoo'}]
|
||||
variant.get_modules.return_value = variant.modules
|
||||
|
||||
source_koji.populate_global_pkgset(
|
||||
self.compose, self.koji_wrapper, '/prefix', 123456)
|
||||
mmds = Modulemd.Module.new_all_from_file(self.koji_module_path)
|
||||
self.assertEqual(mmds[0].get_name(), "foo")
|
||||
|
||||
@mock.patch('six.moves.cPickle.dumps')
|
||||
@mock.patch('pungi.phases.pkgset.pkgsets.KojiPackageSet')
|
||||
def test_populate_with_multiple_koji_tags(self, KojiPackageSet, pickle_dumps):
|
||||
|
|
@ -735,5 +665,138 @@ class TestFilterByWhitelist(unittest.TestCase):
|
|||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
class MockModule(object):
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
|
||||
def __repr__(self):
|
||||
return "MockModule(%r)" % self.path
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.path == other.path
|
||||
|
||||
def dup_nsvc(self):
|
||||
return "module:master:20190318.abcdef"
|
||||
|
||||
|
||||
@mock.patch("pungi.Modulemd.Module.new_from_file", new=MockModule)
|
||||
class TestAddModuleToVariant(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.koji = mock.Mock()
|
||||
self.koji.koji_module.pathinfo.typedir.return_value = "/koji"
|
||||
files = ["modulemd.x86_64.txt", "modulemd.armv7hl.txt", "modulemd.txt"]
|
||||
self.koji.koji_proxy.listArchives.return_value = [
|
||||
{"btype": "module", "filename": fname} for fname in files
|
||||
] + [{"btype": "foo"}]
|
||||
|
||||
def test_adding_module(self):
|
||||
build = {"id": 1234}
|
||||
variant = mock.Mock(
|
||||
arches=["armhfp", "x86_64"], mmds=[], arch_mmds={}, modules=[]
|
||||
)
|
||||
|
||||
source_koji._add_module_to_variant(self.koji, variant, build)
|
||||
|
||||
self.assertEqual(variant.mmds, [MockModule("/koji/modulemd.txt")])
|
||||
self.assertEqual(
|
||||
variant.arch_mmds,
|
||||
{
|
||||
"armhfp": {
|
||||
"module:master:20190318.abcdef": MockModule("/koji/modulemd.armv7hl.txt"),
|
||||
},
|
||||
"x86_64": {
|
||||
"module:master:20190318.abcdef": MockModule("/koji/modulemd.x86_64.txt"),
|
||||
},
|
||||
},
|
||||
)
|
||||
self.assertEqual(variant.modules, [])
|
||||
|
||||
def test_adding_module_to_existing(self):
|
||||
build = {"id": 1234}
|
||||
variant = mock.Mock(
|
||||
arches=["armhfp", "x86_64"],
|
||||
mmds=[MockModule("/koji/m1.txt")],
|
||||
arch_mmds={
|
||||
"x86_64": {"m1:latest:20190101.cafe": MockModule("/koji/m1.x86_64.txt")}
|
||||
},
|
||||
modules=["m1:latest-20190101.cafe"],
|
||||
)
|
||||
|
||||
source_koji._add_module_to_variant(self.koji, variant, build)
|
||||
|
||||
self.assertEqual(
|
||||
variant.mmds, [MockModule("/koji/m1.txt"), MockModule("/koji/modulemd.txt")]
|
||||
)
|
||||
self.assertEqual(
|
||||
variant.arch_mmds,
|
||||
{
|
||||
"armhfp": {
|
||||
"module:master:20190318.abcdef": MockModule("/koji/modulemd.armv7hl.txt"),
|
||||
},
|
||||
"x86_64": {
|
||||
"module:master:20190318.abcdef": MockModule("/koji/modulemd.x86_64.txt"),
|
||||
"m1:latest:20190101.cafe": MockModule("/koji/m1.x86_64.txt"),
|
||||
},
|
||||
},
|
||||
)
|
||||
self.assertEqual(variant.modules, ["m1:latest-20190101.cafe"])
|
||||
|
||||
def test_adding_module_with_add_module(self):
|
||||
build = {"id": 1234}
|
||||
variant = mock.Mock(
|
||||
arches=["armhfp", "x86_64"], mmds=[], arch_mmds={}, modules=[]
|
||||
)
|
||||
|
||||
source_koji._add_module_to_variant(
|
||||
self.koji, variant, build, add_to_variant_modules=True
|
||||
)
|
||||
|
||||
self.assertEqual(variant.mmds, [MockModule("/koji/modulemd.txt")])
|
||||
self.assertEqual(
|
||||
variant.arch_mmds,
|
||||
{
|
||||
"armhfp": {
|
||||
"module:master:20190318.abcdef": MockModule("/koji/modulemd.armv7hl.txt"),
|
||||
},
|
||||
"x86_64": {
|
||||
"module:master:20190318.abcdef": MockModule("/koji/modulemd.x86_64.txt"),
|
||||
},
|
||||
},
|
||||
)
|
||||
self.assertEqual(variant.modules, ["module:master:20190318.abcdef"])
|
||||
|
||||
def test_adding_module_to_existing_with_add_module(self):
|
||||
build = {"id": 1234}
|
||||
variant = mock.Mock(
|
||||
arches=["armhfp", "x86_64"],
|
||||
mmds=[MockModule("/koji/m1.txt")],
|
||||
arch_mmds={
|
||||
"x86_64": {"m1:latest:20190101.cafe": MockModule("/koji/m1.x86_64.txt")}
|
||||
},
|
||||
modules=["m1:latest-20190101.cafe"],
|
||||
)
|
||||
|
||||
source_koji._add_module_to_variant(
|
||||
self.koji, variant, build, add_to_variant_modules=True
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
variant.mmds, [MockModule("/koji/m1.txt"), MockModule("/koji/modulemd.txt")]
|
||||
)
|
||||
self.assertEqual(
|
||||
variant.arch_mmds,
|
||||
{
|
||||
"armhfp": {
|
||||
"module:master:20190318.abcdef": MockModule("/koji/modulemd.armv7hl.txt"),
|
||||
},
|
||||
"x86_64": {
|
||||
"module:master:20190318.abcdef": MockModule("/koji/modulemd.x86_64.txt"),
|
||||
"m1:latest:20190101.cafe": MockModule("/koji/m1.x86_64.txt"),
|
||||
},
|
||||
},
|
||||
)
|
||||
self.assertEqual(
|
||||
variant.modules,
|
||||
["m1:latest-20190101.cafe", "module:master:20190318.abcdef"],
|
||||
)
|
||||
|
|
|
|||
Loading…
Reference in New Issue