almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Supersede ModuleStream loading with ModuleIndex

Browse Source 
- Use ModuleIndex's update_from_file/update_from_string instead of ModuleStream's
read_file/read_string which is deprecated.
- Extend tests to work with real module streams instead of mocks.

Signed-off-by: Filip Valder <fvalder@redhat.com>
This commit is contained in:
Filip Valder 2021-08-11 13:15:29 +02:00
parent 5831d4ae1e
commit a7c111643d
8 changed files with 183 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pungi/phases/createrepo.py
View File

@ -31,7 +31,7 @@ from kobo.shortcuts import run, relative_path
from ..wrappers.scm import get_dir_from_scm
from ..wrappers.createrepo import CreaterepoWrapper
from .base import PhaseBase
from ..util import get_arch_variant_data, temp_dir
from ..util import get_arch_variant_data, temp_dir, read_single_module_stream_from_file
from ..module_util import Modulemd, collect_module_defaults
import productmd.rpms
@ -268,7 +268,7 @@ def create_variant_repo(
compose.log_debug("Adding extra modulemd for %s.%s", variant.uid, arch)
dirname = compose.paths.work.tmp_dir(variant=variant, create_dir=False)
for filepath in glob.glob(os.path.join(dirname, arch) + "/*.yaml"):
module_stream = Modulemd.ModuleStream.read_file(filepath, strict=True)
module_stream = read_single_module_stream_from_file(filepath)
if not mod_index.add_module_stream(module_stream):
raise RuntimeError(
"Failed parsing modulemd data from %s" % filepath

27
pungi/phases/pkgset/sources/source_koji.py
View File

@ -29,7 +29,13 @@ from pungi.wrappers.comps import CompsWrapper
from pungi.wrappers.mbs import MBSWrapper
import pungi.phases.pkgset.pkgsets
from pungi.arch import getBaseArch
from pungi.util import retry, get_arch_variant_data, get_variant_data
from pungi.util import (
retry,
get_arch_variant_data,
get_variant_data,
read_single_module_stream_from_file,
read_single_module_stream_from_string,
)
from pungi.module_util import Modulemd
from pungi.phases.pkgset.common import MaterializedPackageSet, get_all_arches
@ -258,17 +264,12 @@ def _add_module_to_variant(
compose.log_debug("Module %s is filtered from %s.%s", nsvc, variant, arch)
continue
try:
mmd = Modulemd.ModuleStream.read_file(
mmds["modulemd.%s.txt" % arch], strict=True
)
variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd
mod_stream = read_single_module_stream_from_file(
mmds["modulemd.%s.txt" % arch], compose, arch, build
)
if mod_stream:
added = True
except KeyError:
# There is no modulemd for this arch. This could mean an arch was
# added to the compose after the module was built. We don't want to
# process this, let's skip this module.
pass
variant.arch_mmds.setdefault(arch, {})[nsvc] = mod_stream
if not added:
# The module is filtered on all arches of this variant.
@ -348,9 +349,7 @@ def _add_scratch_modules_to_variant(
tag_to_mmd.setdefault(tag, {})
for arch in variant.arches:
try:
mmd = Modulemd.ModuleStream.read_string(
final_modulemd[arch], strict=True
)
mmd = read_single_module_stream_from_string(final_modulemd[arch])
variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd
except KeyError:
continue

41
pungi/util.py
View File

@ -34,6 +34,7 @@ import kobo.conf
from kobo.shortcuts import run, force_list
from kobo.threads import WorkerThread, ThreadPool
from productmd.common import get_major_version
from pungi.module_util import Modulemd
# Patterns that match all names of debuginfo packages
DEBUG_PATTERNS = ["*-debuginfo", "*-debuginfo-*", "*-debugsource"]
@ -1034,6 +1035,46 @@ def load_config(file_path, defaults={}):
return conf
def _read_single_module_stream(
file_or_string, compose=None, arch=None, build=None, is_file=True
):
try:
mod_index = Modulemd.ModuleIndex.new()
if is_file:
mod_index.update_from_file(file_or_string, True)
else:
mod_index.update_from_string(file_or_string, True)
mod_names = mod_index.get_module_names()
emit_warning = False
if len(mod_names) > 1:
emit_warning = True
mod_streams = mod_index.get_module(mod_names[0]).get_all_streams()
if len(mod_streams) > 1:
emit_warning = True
if emit_warning and compose:
compose.log_warning(
"Multiple modules/streams for arch: %s. Build: %s. "
"Processing first module/stream only.",
arch,
build,
)
return mod_streams[0]
except (KeyError, IndexError):
# There is no modulemd for this arch. This could mean an arch was
# added to the compose after the module was built. We don't want to
# process this, let's skip this module.
if compose:
compose.log_info("Skipping arch: %s. Build: %s", arch, build)
def read_single_module_stream_from_file(*args, **kwargs):
return _read_single_module_stream(*args, is_file=True, **kwargs)
def read_single_module_stream_from_string(*args, **kwargs):
return _read_single_module_stream(*args, is_file=False, **kwargs)
@contextlib.contextmanager
def as_local_file(url):
"""If URL points to a file over HTTP, the file will be downloaded locally

20
tests/fixtures/mmds/m1.x86_64.txt vendored Normal file
View File

@ -0,0 +1,20 @@
---
document: modulemd
version: 2
data:
name: m1
stream: latest
version: 20190101
context: cafe
arch: x86_64
summary: Dummy module
description: Dummy module
license:
module:
- Beerware
content:
- Beerware
artifacts:
rpms:
- foobar-0:1.0-1.noarch
...

20
tests/fixtures/mmds/modulemd.armv7hl.txt vendored Normal file
View File

@ -0,0 +1,20 @@
---
document: modulemd
version: 2
data:
name: module
stream: master
version: 20190318
context: abcdef
arch: armhfp
summary: Dummy module
description: Dummy module
license:
module:
- Beerware
content:
- Beerware
artifacts:
rpms:
- foobar-0:1.0-1.noarch
...

20
tests/fixtures/mmds/modulemd.x86_64.txt vendored Normal file
View File

@ -0,0 +1,20 @@
---
document: modulemd
version: 2
data:
name: module
stream: master
version: 20190318
context: abcdef
arch: x86_64
summary: Dummy module
description: Dummy module
license:
module:
- Beerware
content:
- Beerware
artifacts:
rpms:
- foobar-0:1.0-1.noarch
...

20
tests/fixtures/mmds/scratch-module.x86_64.txt vendored Normal file
View File

@ -0,0 +1,20 @@
---
document: modulemd
version: 2
data:
name: scratch-module
stream: master
version: 20200710
context: abcdef
arch: x86_64
summary: Dummy module
description: Dummy module
license:
module:
- Beerware
content:
- Beerware
artifacts:
rpms:
- foobar-0:1.0-1.noarch
...

136
tests/test_pkgset_source_koji.py
View File

@ -14,7 +14,9 @@ except ImportError:
from pungi.phases.pkgset.sources import source_koji
from tests import helpers
from pungi.module_util import Modulemd
from pungi.util import read_single_module_stream_from_file
MMDS_DIR = os.path.join(helpers.FIXTURE_DIR, "mmds")
EVENT_INFO = {"id": 15681980, "ts": 1460956382.81936}
TAG_INFO = {
"maven_support": False,
@ -672,24 +674,12 @@ class TestFilterByWhitelist(unittest.TestCase):
self.assertEqual(expected, set())
class MockModule(object):
def __init__(self, path, strict=True):
self.path = path
def __repr__(self):
return "MockModule(%r)" % self.path
def __eq__(self, other):
return self.path == other.path
@mock.patch("pungi.module_util.Modulemd.ModuleStream.read_file", new=MockModule)
@unittest.skipIf(Modulemd is None, "Skipping tests, no module support")
class TestAddModuleToVariant(helpers.PungiTestCase):
def setUp(self):
super(TestAddModuleToVariant, self).setUp()
self.koji = mock.Mock()
self.koji.koji_module.pathinfo.typedir.return_value = "/koji"
self.koji.koji_module.pathinfo.typedir.return_value = MMDS_DIR
files = ["modulemd.x86_64.txt", "modulemd.armv7hl.txt", "modulemd.txt"]
self.koji.koji_proxy.listArchives.return_value = [
{"btype": "module", "filename": fname} for fname in files
@ -713,50 +703,35 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
source_koji._add_module_to_variant(self.koji, variant, self.buildinfo)
self.assertEqual(
variant.arch_mmds,
{
"armhfp": {
"module:master:20190318:abcdef": MockModule(
"/koji/modulemd.armv7hl.txt"
),
},
"x86_64": {
"module:master:20190318:abcdef": MockModule(
"/koji/modulemd.x86_64.txt"
),
},
},
)
mod1 = variant.arch_mmds["armhfp"]["module:master:20190318:abcdef"]
self.assertEqual(mod1.get_NSVCA(), "module:master:20190318:abcdef:armhfp")
mod2 = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"]
self.assertEqual(mod2.get_NSVCA(), "module:master:20190318:abcdef:x86_64")
self.assertEqual(len(variant.arch_mmds), 2)
self.assertEqual(variant.modules, [])
def test_adding_module_to_existing(self):
variant = mock.Mock(
arches=["armhfp", "x86_64"],
arch_mmds={
"x86_64": {"m1:latest:20190101:cafe": MockModule("/koji/m1.x86_64.txt")}
"x86_64": {
"m1:latest:20190101:cafe": read_single_module_stream_from_file(
os.path.join(MMDS_DIR, "m1.x86_64.txt")
)
}
},
modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
)
source_koji._add_module_to_variant(self.koji, variant, self.buildinfo)
self.assertEqual(
variant.arch_mmds,
{
"armhfp": {
"module:master:20190318:abcdef": MockModule(
"/koji/modulemd.armv7hl.txt"
),
},
"x86_64": {
"module:master:20190318:abcdef": MockModule(
"/koji/modulemd.x86_64.txt"
),
"m1:latest:20190101:cafe": MockModule("/koji/m1.x86_64.txt"),
},
},
)
mod1 = variant.arch_mmds["armhfp"]["module:master:20190318:abcdef"]
self.assertEqual(mod1.get_NSVCA(), "module:master:20190318:abcdef:armhfp")
mod2 = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"]
self.assertEqual(mod2.get_NSVCA(), "module:master:20190318:abcdef:x86_64")
mod3 = variant.arch_mmds["x86_64"]["m1:latest:20190101:cafe"]
self.assertEqual(mod3.get_NSVCA(), "m1:latest:20190101:cafe:x86_64")
self.assertEqual(
variant.modules, [{"name": "m1:latest-20190101:cafe", "glob": False}]
)
@ -768,21 +743,11 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
self.koji, variant, self.buildinfo, add_to_variant_modules=True
)
self.assertEqual(
variant.arch_mmds,
{
"armhfp": {
"module:master:20190318:abcdef": MockModule(
"/koji/modulemd.armv7hl.txt"
),
},
"x86_64": {
"module:master:20190318:abcdef": MockModule(
"/koji/modulemd.x86_64.txt"
),
},
},
)
mod1 = variant.arch_mmds["armhfp"]["module:master:20190318:abcdef"]
self.assertEqual(mod1.get_NSVCA(), "module:master:20190318:abcdef:armhfp")
mod2 = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"]
self.assertEqual(mod2.get_NSVCA(), "module:master:20190318:abcdef:x86_64")
self.assertEqual(
variant.modules, [{"name": "module:master:20190318:abcdef", "glob": False}]
)
@ -791,7 +756,11 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
variant = mock.Mock(
arches=["armhfp", "x86_64"],
arch_mmds={
"x86_64": {"m1:latest:20190101:cafe": MockModule("/koji/m1.x86_64.txt")}
"x86_64": {
"m1:latest:20190101:cafe": read_single_module_stream_from_file(
os.path.join(MMDS_DIR, "m1.x86_64.txt")
)
}
},
modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
)
@ -800,22 +769,13 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
self.koji, variant, self.buildinfo, add_to_variant_modules=True
)
self.assertEqual(
variant.arch_mmds,
{
"armhfp": {
"module:master:20190318:abcdef": MockModule(
"/koji/modulemd.armv7hl.txt"
),
},
"x86_64": {
"module:master:20190318:abcdef": MockModule(
"/koji/modulemd.x86_64.txt"
),
"m1:latest:20190101:cafe": MockModule("/koji/m1.x86_64.txt"),
},
},
)
mod1 = variant.arch_mmds["armhfp"]["module:master:20190318:abcdef"]
self.assertEqual(mod1.get_NSVCA(), "module:master:20190318:abcdef:armhfp")
mod2 = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"]
self.assertEqual(mod2.get_NSVCA(), "module:master:20190318:abcdef:x86_64")
mod3 = variant.arch_mmds["x86_64"]["m1:latest:20190101:cafe"]
self.assertEqual(mod3.get_NSVCA(), "m1:latest:20190101:cafe:x86_64")
self.assertEqual(
variant.modules,
[
@ -891,12 +851,8 @@ class MockMBS(object):
return {"id": 1, "koji_tag": "scratch-module-tag", "name": "scratch-module"}
def final_modulemd(self, module_build_id):
return {"x86_64": ""}
class MockMmd(object):
def __init__(self, mmd, strict=True):
pass
with open(os.path.join(MMDS_DIR, "scratch-module.x86_64.txt")) as f:
return {"x86_64": f.read()}
@mock.patch("pungi.phases.pkgset.sources.source_koji.MBSWrapper", new=MockMBS)
@ -909,10 +865,7 @@ class TestAddScratchModuleToVariant(helpers.PungiTestCase):
)
self.nsvc = "scratch-module:master:20200710:abcdef"
@mock.patch(
"pungi.phases.pkgset.sources.source_koji.Modulemd.ModuleStream.read_string"
)
def test_adding_scratch_module(self, mock_mmd):
def test_adding_scratch_module(self):
variant = mock.Mock(
arches=["armhfp", "x86_64"],
arch_mmds={},
@ -927,11 +880,16 @@ class TestAddScratchModuleToVariant(helpers.PungiTestCase):
self.compose, variant, scratch_modules, variant_tags, tag_to_mmd
)
self.assertEqual(variant_tags, {variant: ["scratch-module-tag"]})
self.assertEqual(
variant.arch_mmds, {"x86_64": {self.nsvc: mock_mmd.return_value}}
variant.arch_mmds["x86_64"][self.nsvc].get_NSVCA(),
"scratch-module:master:20200710:abcdef:x86_64",
)
self.assertTrue(isinstance(tag_to_mmd["scratch-module-tag"]["x86_64"], set))
self.assertEqual(
tag_to_mmd, {"scratch-module-tag": {"x86_64": set([mock_mmd.return_value])}}
list(tag_to_mmd["scratch-module-tag"]["x86_64"])[0].get_NSVCA(),
"scratch-module:master:20200710:abcdef:x86_64",
)
self.assertEqual(variant.modules, [])