almalinux/pungi
15
2
Fork 0
Code Issues Pull Requests Releases Activity

Add module obsoletes feature

Browse Source 
JIRA: MODULAR-113
Merges: https://pagure.io/pungi/pull-request/1578
Signed-off-by: Filip Valder <fvalder@redhat.com>
This commit is contained in:
Filip Valder 2021-12-15 10:13:23 +01:00 committed by Lubomír Sedlář
parent 42f668d969
commit fe986d68b9
11 changed files with 124 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -14,3 +14,5 @@ htmlcov/
.idea/
.tox
.venv
.kdev4/
pungi.kdev4

10
doc/examples.rst
View File

@ -30,9 +30,17 @@ This is a shortened configuration for Fedora Radhide compose as of 2019-10-14.
module_defaults_dir = {
'scm': 'git',
'repo': 'https://pagure.io/releng/fedora-module-defaults.git',
'branch': 'master',
'branch': 'main',
'dir': '.'
}
# Optional module obsoletes configuration which is merged
# into the module index and gets resolved
module_obsoletes_dir = {
'scm': 'git',
'repo': 'https://pagure.io/releng/fedora-module-defaults.git',
'branch': 'main',
'dir': 'obsoletes'
}
variants_file='variants-fedora.xml'
sigkeys = ['12C944D0']

1
pungi/checks.py
View File

@ -736,6 +736,7 @@ def make_schema():
"patternProperties": {".+": {"$ref": "#/definitions/strings"}},
"additionalProperties": False,
},
"module_obsoletes_dir": {"$ref": "#/definitions/str_or_scm_dict"},
"create_optional_isos": {"type": "boolean", "default": False},
"symlink_isos_to": {"type": "string"},
"dogpile_cache_backend": {"type": "string"},

4
pungi/compose.py
View File

@ -377,6 +377,10 @@ class Compose(kobo.log.LoggingBase):
def has_module_defaults(self):
return bool(self.conf.get("module_defaults_dir", False))
@property
def has_module_obsoletes(self):
return bool(self.conf.get("module_obsoletes_dir", False))
@property
def config_dir(self):
return os.path.dirname(self.conf._open_file or "")

26
pungi/module_util.py
View File

@ -25,9 +25,10 @@ except (ImportError, ValueError):
Modulemd = None
def iter_module_defaults(path):
def iter_module_defaults_or_obsoletes(path, obsoletes=False):
"""Given a path to a directory with yaml files, yield each module default
in there as a pair (module_name, ModuleDefaults instance).
The same happens for module obsoletes if the obsoletes switch is True.
"""
# It is really tempting to merge all the module indexes into a single one
# and work with it. However that does not allow for detecting conflicting
@ -41,7 +42,10 @@ def iter_module_defaults(path):
index = Modulemd.ModuleIndex()
index.update_from_file(file, strict=False)
for module_name in index.get_module_names():
yield module_name, index.get_module(module_name).get_defaults()
if obsoletes:
yield module_name, index.get_module(module_name).get_obsoletes()
else:
yield module_name, index.get_module(module_name).get_defaults()
def collect_module_defaults(
@ -69,3 +73,21 @@ def collect_module_defaults(
mod_index.add_defaults(defaults)
return mod_index
def collect_module_obsoletes(obsoletes_dir, modules_to_load, mod_index=None):
"""Load module obsoletes into index.
This works in a similar fashion as collect_module_defaults except the overrides_dir
feature.
"""
mod_index = mod_index or Modulemd.ModuleIndex()
for module_name, obsoletes in iter_module_defaults_or_obsoletes(
obsoletes_dir, obsoletes=True
):
for obsolete in obsoletes:
if not modules_to_load or module_name in modules_to_load:
mod_index.add_obsoletes(obsoletes)
return mod_index

10
pungi/paths.py
View File

@ -509,6 +509,16 @@ class WorkPaths(object):
makedirs(path)
return path
def module_obsoletes_dir(self, create_dir=True):
"""
Example:
work/global/module_obsoletes
"""
path = os.path.join(self.topdir(create_dir=create_dir), "module_obsoletes")
if create_dir:
makedirs(path)
return path
def pkgset_file_cache(self, pkgset_name):
"""
Returns the path to file in which the cached version of

5
pungi/phases/createrepo.py
View File

@ -29,7 +29,7 @@ import productmd.rpms
from kobo.shortcuts import relative_path, run
from kobo.threads import ThreadPool, WorkerThread
from ..module_util import Modulemd, collect_module_defaults
from ..module_util import Modulemd, collect_module_defaults, collect_module_obsoletes
from ..util import (
get_arch_variant_data,
read_single_module_stream_from_file,
@ -266,6 +266,9 @@ def create_variant_repo(
defaults_dir, module_names, mod_index, overrides_dir=overrides_dir
)
obsoletes_dir = compose.paths.work.module_obsoletes_dir()
collect_module_obsoletes(obsoletes_dir, module_names, mod_index)
# Add extra modulemd files
if variant.uid in compose.conf.get("createrepo_extra_modulemd", {}):
compose.log_debug("Adding extra modulemd for %s.%s", variant.uid, arch)

8
pungi/phases/gather/__init__.py
View File

@ -33,7 +33,11 @@ except ImportError:
import pungi.wrappers.kojiwrapper
from pungi.arch import get_compatible_arches, split_name_arch
from pungi.compose import get_ordered_variant_uids
from pungi.module_util import Modulemd, collect_module_defaults
from pungi.module_util import (
Modulemd,
collect_module_defaults,
collect_module_obsoletes,
)
from pungi.phases.base import PhaseBase
from pungi.phases.createrepo import add_modular_metadata
from pungi.util import get_arch_data, get_arch_variant_data, get_variant_data, makedirs
@ -698,6 +702,8 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None):
collect_module_defaults(
defaults_dir, module_names, mod_index, overrides_dir=overrides_dir
)
obsoletes_dir = compose.paths.work.module_obsoletes_dir()
collect_module_obsoletes(obsoletes_dir, module_names, mod_index)
log_file = compose.paths.log.log_file(
arch, "lookaside_repo_modules_%s" % (variant.uid)

59
pungi/phases/init.py
View File

@ -24,7 +24,7 @@ from kobo.threads import run_in_threads
from pungi.phases.base import PhaseBase
from pungi.phases.gather import write_prepopulate_file
from pungi.util import temp_dir
from pungi.module_util import iter_module_defaults
from pungi.module_util import iter_module_defaults_or_obsoletes
from pungi.wrappers.comps import CompsWrapper
from pungi.wrappers.createrepo import CreaterepoWrapper
from pungi.wrappers.scm import get_dir_from_scm, get_file_from_scm
@ -68,10 +68,18 @@ class InitPhase(PhaseBase):
# download module defaults
if self.compose.has_module_defaults:
write_module_defaults(self.compose)
validate_module_defaults(
validate_module_defaults_or_obsoletes(
self.compose.paths.work.module_defaults_dir(create_dir=False)
)
# download module obsoletes
if self.compose.has_module_obsoletes:
write_module_obsoletes(self.compose)
validate_module_defaults_or_obsoletes(
self.compose.paths.work.module_obsoletes_dir(create_dir=False),
obsoletes=True,
)
# write prepopulate file
write_prepopulate_file(self.compose)
@ -218,28 +226,53 @@ def write_module_defaults(compose):
)
def validate_module_defaults(path):
def write_module_obsoletes(compose):
scm_dict = compose.conf["module_obsoletes_dir"]
if isinstance(scm_dict, dict):
if scm_dict["scm"] == "file":
scm_dict["dir"] = os.path.join(compose.config_dir, scm_dict["dir"])
else:
scm_dict = os.path.join(compose.config_dir, scm_dict)
with temp_dir(prefix="moduleobsoletes_") as tmp_dir:
get_dir_from_scm(scm_dict, tmp_dir, compose=compose)
compose.log_debug("Writing module obsoletes")
shutil.copytree(
tmp_dir,
compose.paths.work.module_obsoletes_dir(create_dir=False),
ignore=shutil.ignore_patterns(".git"),
)
def validate_module_defaults_or_obsoletes(path, obsoletes=False):
"""Make sure there are no conflicting defaults. Each module name can only
have one default stream.
have one default stream or module obsolete.
:param str path: directory with cloned module defaults
:param str path: directory with cloned module defaults/obsoletes
"""
seen_defaults = collections.defaultdict(set)
seen = collections.defaultdict(set)
mmd_type = "obsoletes" if obsoletes else "defaults"
for module_name, defaults in iter_module_defaults(path):
seen_defaults[module_name].add(defaults.get_default_stream())
for module_name, defaults_or_obsoletes in iter_module_defaults_or_obsoletes(
path, obsoletes
):
if obsoletes:
for obsolete in defaults_or_obsoletes:
seen[obsolete.props.module_name].add(obsolete)
else:
seen[module_name].add(defaults_or_obsoletes.get_default_stream())
errors = []
for module_name, defaults in seen_defaults.items():
if len(defaults) > 1:
for module_name, defaults_or_obsoletes in seen.items():
if len(defaults_or_obsoletes) > 1:
errors.append(
"Module %s has multiple defaults: %s"
% (module_name, ", ".join(sorted(defaults)))
"Module %s has multiple %s: %s"
% (module_name, mmd_type, ", ".join(sorted(defaults_or_obsoletes)))
)
if errors:
raise RuntimeError(
"There are duplicated module defaults:\n%s" % "\n".join(errors)
"There are duplicated module %s:\n%s" % (mmd_type, "\n".join(errors))
)

9
pungi/phases/pkgset/common.py
View File

@ -28,7 +28,11 @@ from pungi.util import (
PartialFuncWorkerThread,
PartialFuncThreadPool,
)
from pungi.module_util import Modulemd, collect_module_defaults
from pungi.module_util import (
Modulemd,
collect_module_defaults,
collect_module_obsoletes,
)
from pungi.phases.createrepo import add_modular_metadata
@ -159,6 +163,9 @@ def _create_arch_repo(worker_thread, args, task_num):
mod_index = collect_module_defaults(
compose.paths.work.module_defaults_dir(), names, overrides_dir=overrides_dir
)
mod_index = collect_module_obsoletes(
compose.paths.work.module_obsoletes_dir(), names, mod_index
)
for x in mmd:
mod_index.add_module_stream(x)
add_modular_metadata(

14
tests/test_initphase.py
View File

@ -24,7 +24,7 @@ from tests.helpers import (
@mock.patch("pungi.phases.init.run_in_threads", new=fake_run_in_threads)
@mock.patch("pungi.phases.init.validate_comps")
@mock.patch("pungi.phases.init.validate_module_defaults")
@mock.patch("pungi.phases.init.validate_module_defaults_or_obsoletes")
@mock.patch("pungi.phases.init.write_module_defaults")
@mock.patch("pungi.phases.init.write_global_comps")
@mock.patch("pungi.phases.init.write_arch_comps")
@ -46,6 +46,7 @@ class TestInitPhase(PungiTestCase):
compose = DummyCompose(self.topdir, {})
compose.has_comps = True
compose.has_module_defaults = False
compose.has_module_obsoletes = False
compose.setup_optional()
phase = init.InitPhase(compose)
phase.run()
@ -100,6 +101,7 @@ class TestInitPhase(PungiTestCase):
compose = DummyCompose(self.topdir, {})
compose.has_comps = True
compose.has_module_defaults = False
compose.has_module_obsoletes = False
compose.variants["Everything"].groups = []
compose.variants["Everything"].modules = []
phase = init.InitPhase(compose)
@ -156,6 +158,7 @@ class TestInitPhase(PungiTestCase):
compose = DummyCompose(self.topdir, {})
compose.has_comps = False
compose.has_module_defaults = False
compose.has_module_obsoletes = False
phase = init.InitPhase(compose)
phase.run()
@ -182,6 +185,7 @@ class TestInitPhase(PungiTestCase):
compose = DummyCompose(self.topdir, {})
compose.has_comps = False
compose.has_module_defaults = True
compose.has_module_obsoletes = False
phase = init.InitPhase(compose)
phase.run()
@ -620,13 +624,13 @@ class TestValidateModuleDefaults(PungiTestCase):
def test_valid_files(self):
self._write_defaults({"httpd": ["1"], "python": ["3.6"]})
init.validate_module_defaults(self.topdir)
init.validate_module_defaults_or_obsoletes(self.topdir)
def test_duplicated_stream(self):
self._write_defaults({"httpd": ["1"], "python": ["3.6", "3.5"]})
with self.assertRaises(RuntimeError) as ctx:
init.validate_module_defaults(self.topdir)
init.validate_module_defaults_or_obsoletes(self.topdir)
self.assertIn(
"Module python has multiple defaults: 3.5, 3.6", str(ctx.exception)
@ -636,7 +640,7 @@ class TestValidateModuleDefaults(PungiTestCase):
self._write_defaults({"httpd": ["1", "2"], "python": ["3.6", "3.5"]})
with self.assertRaises(RuntimeError) as ctx:
init.validate_module_defaults(self.topdir)
init.validate_module_defaults_or_obsoletes(self.topdir)
self.assertIn("Module httpd has multiple defaults: 1, 2", str(ctx.exception))
self.assertIn(
@ -661,7 +665,7 @@ class TestValidateModuleDefaults(PungiTestCase):
),
)
init.validate_module_defaults(self.topdir)
init.validate_module_defaults_or_obsoletes(self.topdir)
@mock.patch("pungi.phases.init.CompsWrapper")