almalinux/pungi
15
2
Fork 0
Code Issues Pull Requests Releases Activity

pkgset: Load modulemd only when needed

Browse Source 
We can avoid parsing source modulemd information since we can get the
same information from the Koji build info.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2019-06-28 08:42:50 +02:00
parent 04baa2a4db
commit da78b99fc0
2 changed files with 43 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
pungi/phases/pkgset/sources/source_koji.py
View File

@ -235,7 +235,7 @@ def _add_module_to_variant(koji_wrapper, variant, build, add_to_variant_modules=
filename = "modulemd.%s.txt" % getBaseArch(arch) filename = "modulemd.%s.txt" % getBaseArch(arch)
except ValueError: except ValueError:
pass pass
mmds[filename] = Modulemd.Module.new_from_file(file_path) mmds[filename] = file_path
if len(mmds) <= 1: if len(mmds) <= 1:
# There was only one modulemd file. This means the build is rather old # There was only one modulemd file. This means the build is rather old
@ -243,15 +243,12 @@ def _add_module_to_variant(koji_wrapper, variant, build, add_to_variant_modules=
# longer supported and should be rebuilt. Let's skip it. # longer supported and should be rebuilt. Let's skip it.
return return
source_mmd = mmds["modulemd.txt"] nsvc = "%(name)s:%(stream)s:%(version)s:%(context)s" % build["extra"]["typeinfo"]["module"]
# Set name from build to modulemd. Generally it will match, but for devel
# modules the source mmd contains the original name.
source_mmd.set_name(build["extra"]["typeinfo"]["module"]["name"])
nsvc = source_mmd.dup_nsvc()
for arch in variant.arches: for arch in variant.arches:
try: try:
variant.arch_mmds.setdefault(arch, {})[nsvc] = mmds["modulemd.%s.txt" % arch] mmd = Modulemd.Module.new_from_file(mmds["modulemd.%s.txt" % arch])
variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd
except KeyError: except KeyError:
# There is no modulemd for this arch. This could mean an arch was # There is no modulemd for this arch. This could mean an arch was
# added to the compose after the module was built. We don't want to # added to the compose after the module was built. We don't want to
@ -261,7 +258,7 @@ def _add_module_to_variant(koji_wrapper, variant, build, add_to_variant_modules=
if add_to_variant_modules: if add_to_variant_modules:
variant.modules.append(nsvc) variant.modules.append(nsvc)
return source_mmd return nsvc
def _get_modules_from_koji(compose, koji_wrapper, event, variant, variant_tags): def _get_modules_from_koji(compose, koji_wrapper, event, variant, variant_tags):
@ -281,12 +278,11 @@ def _get_modules_from_koji(compose, koji_wrapper, event, variant, variant_tags):
for module in variant.get_modules(): for module in variant.get_modules():
koji_modules = get_koji_modules(compose, koji_wrapper, event, module["name"]) koji_modules = get_koji_modules(compose, koji_wrapper, event, module["name"])
for koji_module in koji_modules: for koji_module in koji_modules:
mmd = _add_module_to_variant(koji_wrapper, variant, koji_module) nsvc = _add_module_to_variant(koji_wrapper, variant, koji_module)
if not mmd: if not nsvc:
continue continue
tag = koji_module["tag"] tag = koji_module["tag"]
nsvc = mmd.dup_nsvc()
variant_tags[variant].append(tag) variant_tags[variant].append(tag)
# Store mapping NSVC --> koji_tag into variant. # Store mapping NSVC --> koji_tag into variant.
@ -461,13 +457,12 @@ def _get_modules_from_koji_tags(compose, koji_wrapper, event_id, variant, varian
variant_tags[variant].append(module_tag) variant_tags[variant].append(module_tag)
mmd = _add_module_to_variant(koji_wrapper, variant, build, True) nsvc = _add_module_to_variant(koji_wrapper, variant, build, True)
if not mmd: if not nsvc:
continue continue
# Store mapping module-uid --> koji_tag into variant. # Store mapping module-uid --> koji_tag into variant.
# This is needed in createrepo phase where metadata is exposed by producmd # This is needed in createrepo phase where metadata is exposed by producmd
nsvc = mmd.dup_nsvc()
variant.module_uid_to_koji_tag[nsvc] = module_tag variant.module_uid_to_koji_tag[nsvc] = module_tag
module_msg = "Module {module} in variant {variant} will use Koji tag {tag}.".format( module_msg = "Module {module} in variant {variant} will use Koji tag {tag}.".format(

65
tests/test_pkgset_source_koji.py
View File

@ -684,12 +684,6 @@ class MockModule(object):
def __eq__(self, other): def __eq__(self, other):
return self.path == other.path return self.path == other.path
def dup_nsvc(self):
return "module:master:20190318.abcdef"
def set_name(self, name):
pass
@mock.patch("pungi.Modulemd.Module.new_from_file", new=MockModule) @mock.patch("pungi.Modulemd.Module.new_from_file", new=MockModule)
@unittest.skipIf(Modulemd is None, "Skipping tests, no module support") @unittest.skipIf(Modulemd is None, "Skipping tests, no module support")
@ -702,104 +696,113 @@ class TestAddModuleToVariant(unittest.TestCase):
self.koji.koji_proxy.listArchives.return_value = [ self.koji.koji_proxy.listArchives.return_value = [
{"btype": "module", "filename": fname} for fname in files {"btype": "module", "filename": fname} for fname in files
] + [{"btype": "foo"}] ] + [{"btype": "foo"}]
self.buildinfo = {
"id": 1234,
"extra": {
"typeinfo": {
"module": {
"name": "module",
"stream": "master",
"version": "20190318",
"context": "abcdef",
},
},
},
}
def test_adding_module(self): def test_adding_module(self):
build = {"id": 1234, "extra": {"typeinfo": {"module": {"name": "module"}}}}
variant = mock.Mock( variant = mock.Mock(
arches=["armhfp", "x86_64"], arch_mmds={}, modules=[] arches=["armhfp", "x86_64"], arch_mmds={}, modules=[]
) )
source_koji._add_module_to_variant(self.koji, variant, build) source_koji._add_module_to_variant(self.koji, variant, self.buildinfo)
self.assertEqual( self.assertEqual(
variant.arch_mmds, variant.arch_mmds,
{ {
"armhfp": { "armhfp": {
"module:master:20190318.abcdef": MockModule("/koji/modulemd.armv7hl.txt"), "module:master:20190318:abcdef": MockModule("/koji/modulemd.armv7hl.txt"),
}, },
"x86_64": { "x86_64": {
"module:master:20190318.abcdef": MockModule("/koji/modulemd.x86_64.txt"), "module:master:20190318:abcdef": MockModule("/koji/modulemd.x86_64.txt"),
}, },
}, },
) )
self.assertEqual(variant.modules, []) self.assertEqual(variant.modules, [])
def test_adding_module_to_existing(self): def test_adding_module_to_existing(self):
build = {"id": 1234, "extra": {"typeinfo": {"module": {"name": "module"}}}}
variant = mock.Mock( variant = mock.Mock(
arches=["armhfp", "x86_64"], arches=["armhfp", "x86_64"],
arch_mmds={ arch_mmds={
"x86_64": {"m1:latest:20190101.cafe": MockModule("/koji/m1.x86_64.txt")} "x86_64": {"m1:latest:20190101:cafe": MockModule("/koji/m1.x86_64.txt")}
}, },
modules=["m1:latest-20190101.cafe"], modules=["m1:latest-20190101:cafe"],
) )
source_koji._add_module_to_variant(self.koji, variant, build) source_koji._add_module_to_variant(self.koji, variant, self.buildinfo)
self.assertEqual( self.assertEqual(
variant.arch_mmds, variant.arch_mmds,
{ {
"armhfp": { "armhfp": {
"module:master:20190318.abcdef": MockModule("/koji/modulemd.armv7hl.txt"), "module:master:20190318:abcdef": MockModule("/koji/modulemd.armv7hl.txt"),
}, },
"x86_64": { "x86_64": {
"module:master:20190318.abcdef": MockModule("/koji/modulemd.x86_64.txt"), "module:master:20190318:abcdef": MockModule("/koji/modulemd.x86_64.txt"),
"m1:latest:20190101.cafe": MockModule("/koji/m1.x86_64.txt"), "m1:latest:20190101:cafe": MockModule("/koji/m1.x86_64.txt"),
}, },
}, },
) )
self.assertEqual(variant.modules, ["m1:latest-20190101.cafe"]) self.assertEqual(variant.modules, ["m1:latest-20190101:cafe"])
def test_adding_module_with_add_module(self): def test_adding_module_with_add_module(self):
build = {"id": 1234, "extra": {"typeinfo": {"module": {"name": "module"}}}}
variant = mock.Mock( variant = mock.Mock(
arches=["armhfp", "x86_64"], arch_mmds={}, modules=[] arches=["armhfp", "x86_64"], arch_mmds={}, modules=[]
) )
source_koji._add_module_to_variant( source_koji._add_module_to_variant(
self.koji, variant, build, add_to_variant_modules=True self.koji, variant, self.buildinfo, add_to_variant_modules=True
) )
self.assertEqual( self.assertEqual(
variant.arch_mmds, variant.arch_mmds,
{ {
"armhfp": { "armhfp": {
"module:master:20190318.abcdef": MockModule("/koji/modulemd.armv7hl.txt"), "module:master:20190318:abcdef": MockModule("/koji/modulemd.armv7hl.txt"),
}, },
"x86_64": { "x86_64": {
"module:master:20190318.abcdef": MockModule("/koji/modulemd.x86_64.txt"), "module:master:20190318:abcdef": MockModule("/koji/modulemd.x86_64.txt"),
}, },
}, },
) )
self.assertEqual(variant.modules, ["module:master:20190318.abcdef"]) self.assertEqual(variant.modules, ["module:master:20190318:abcdef"])
def test_adding_module_to_existing_with_add_module(self): def test_adding_module_to_existing_with_add_module(self):
build = {"id": 1234, "extra": {"typeinfo": {"module": {"name": "module"}}}}
variant = mock.Mock( variant = mock.Mock(
arches=["armhfp", "x86_64"], arches=["armhfp", "x86_64"],
arch_mmds={ arch_mmds={
"x86_64": {"m1:latest:20190101.cafe": MockModule("/koji/m1.x86_64.txt")} "x86_64": {"m1:latest:20190101:cafe": MockModule("/koji/m1.x86_64.txt")}
}, },
modules=["m1:latest-20190101.cafe"], modules=["m1:latest-20190101:cafe"],
) )
source_koji._add_module_to_variant( source_koji._add_module_to_variant(
self.koji, variant, build, add_to_variant_modules=True self.koji, variant, self.buildinfo, add_to_variant_modules=True
) )
self.assertEqual( self.assertEqual(
variant.arch_mmds, variant.arch_mmds,
{ {
"armhfp": { "armhfp": {
"module:master:20190318.abcdef": MockModule("/koji/modulemd.armv7hl.txt"), "module:master:20190318:abcdef": MockModule("/koji/modulemd.armv7hl.txt"),
}, },
"x86_64": { "x86_64": {
"module:master:20190318.abcdef": MockModule("/koji/modulemd.x86_64.txt"), "module:master:20190318:abcdef": MockModule("/koji/modulemd.x86_64.txt"),
"m1:latest:20190101.cafe": MockModule("/koji/m1.x86_64.txt"), "m1:latest:20190101:cafe": MockModule("/koji/m1.x86_64.txt"),
}, },
}, },
) )
self.assertEqual( self.assertEqual(
variant.modules, variant.modules,
["m1:latest-20190101.cafe", "module:master:20190318.abcdef"], ["m1:latest-20190101:cafe", "module:master:20190318:abcdef"],
) )