almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Update from upstream #11

Closed
soksanichenko wants to merge 158 commits from a8_updated into a8
pull from: a8_updated
merge into: almalinux:a8
Conversation 0 Commits 158 Files Changed 128 +124 -24
11 changed files with 124 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit fe986d68b9 - Show all commits

2
.gitignore vendored
View File

@ -14,3 +14,5 @@ htmlcov/
.idea/
.tox
.venv
.kdev4/
pungi.kdev4

10
doc/examples.rst
View File

@ -30,9 +30,17 @@ This is a shortened configuration for Fedora Radhide compose as of 2019-10-14.
module_defaults_dir = {
'scm': 'git',
'repo': 'https://pagure.io/releng/fedora-module-defaults.git',
'branch': 'master',
'branch': 'main',
'dir': '.'
}
# Optional module obsoletes configuration which is merged
# into the module index and gets resolved
module_obsoletes_dir = {
'scm': 'git',
'repo': 'https://pagure.io/releng/fedora-module-defaults.git',
'branch': 'main',
'dir': 'obsoletes'
}
variants_file='variants-fedora.xml'
sigkeys = ['12C944D0']

1
pungi/checks.py
View File

@ -736,6 +736,7 @@ def make_schema():
"patternProperties": {".+": {"$ref": "#/definitions/strings"}},
"additionalProperties": False,
},
"module_obsoletes_dir": {"$ref": "#/definitions/str_or_scm_dict"},
"create_optional_isos": {"type": "boolean", "default": False},
"symlink_isos_to": {"type": "string"},
"dogpile_cache_backend": {"type": "string"},

4
pungi/compose.py
View File

@ -377,6 +377,10 @@ class Compose(kobo.log.LoggingBase):
def has_module_defaults(self):
return bool(self.conf.get("module_defaults_dir", False))
@property
def has_module_obsoletes(self):
return bool(self.conf.get("module_obsoletes_dir", False))
@property
def config_dir(self):
return os.path.dirname(self.conf._open_file or "")

26
pungi/module_util.py
View File

@ -25,9 +25,10 @@ except (ImportError, ValueError):
Modulemd = None
def iter_module_defaults(path):
def iter_module_defaults_or_obsoletes(path, obsoletes=False):
"""Given a path to a directory with yaml files, yield each module default
in there as a pair (module_name, ModuleDefaults instance).
The same happens for module obsoletes if the obsoletes switch is True.
"""
# It is really tempting to merge all the module indexes into a single one
# and work with it. However that does not allow for detecting conflicting
@ -41,7 +42,10 @@ def iter_module_defaults(path):
index = Modulemd.ModuleIndex()
index.update_from_file(file, strict=False)
for module_name in index.get_module_names():
yield module_name, index.get_module(module_name).get_defaults()
if obsoletes:
yield module_name, index.get_module(module_name).get_obsoletes()
else:
yield module_name, index.get_module(module_name).get_defaults()
def collect_module_defaults(
@ -69,3 +73,21 @@ def collect_module_defaults(
mod_index.add_defaults(defaults)
return mod_index
def collect_module_obsoletes(obsoletes_dir, modules_to_load, mod_index=None):
"""Load module obsoletes into index.
This works in a similar fashion as collect_module_defaults except the overrides_dir
feature.
"""
mod_index = mod_index or Modulemd.ModuleIndex()
for module_name, obsoletes in iter_module_defaults_or_obsoletes(
obsoletes_dir, obsoletes=True
):
for obsolete in obsoletes:
if not modules_to_load or module_name in modules_to_load:
mod_index.add_obsoletes(obsoletes)
return mod_index

10
pungi/paths.py
View File

@ -509,6 +509,16 @@ class WorkPaths(object):
makedirs(path)
return path
def module_obsoletes_dir(self, create_dir=True):
"""
Example:
work/global/module_obsoletes
"""
path = os.path.join(self.topdir(create_dir=create_dir), "module_obsoletes")
if create_dir:
makedirs(path)
return path
def pkgset_file_cache(self, pkgset_name):
"""
Returns the path to file in which the cached version of

5
pungi/phases/createrepo.py
View File

@ -29,7 +29,7 @@ import productmd.rpms
from kobo.shortcuts import relative_path, run
from kobo.threads import ThreadPool, WorkerThread
from ..module_util import Modulemd, collect_module_defaults
from ..module_util import Modulemd, collect_module_defaults, collect_module_obsoletes
from ..util import (
get_arch_variant_data,
read_single_module_stream_from_file,
@ -266,6 +266,9 @@ def create_variant_repo(
defaults_dir, module_names, mod_index, overrides_dir=overrides_dir
)
obsoletes_dir = compose.paths.work.module_obsoletes_dir()
collect_module_obsoletes(obsoletes_dir, module_names, mod_index)
# Add extra modulemd files
if variant.uid in compose.conf.get("createrepo_extra_modulemd", {}):
compose.log_debug("Adding extra modulemd for %s.%s", variant.uid, arch)

8
pungi/phases/gather/__init__.py
View File

@ -33,7 +33,11 @@ except ImportError:
import pungi.wrappers.kojiwrapper
from pungi.arch import get_compatible_arches, split_name_arch
from pungi.compose import get_ordered_variant_uids
from pungi.module_util import Modulemd, collect_module_defaults
from pungi.module_util import (
Modulemd,
collect_module_defaults,
collect_module_obsoletes,
)
from pungi.phases.base import PhaseBase
from pungi.phases.createrepo import add_modular_metadata
from pungi.util import get_arch_data, get_arch_variant_data, get_variant_data, makedirs
@ -698,6 +702,8 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None):
collect_module_defaults(
defaults_dir, module_names, mod_index, overrides_dir=overrides_dir
)
obsoletes_dir = compose.paths.work.module_obsoletes_dir()
collect_module_obsoletes(obsoletes_dir, module_names, mod_index)
log_file = compose.paths.log.log_file(
arch, "lookaside_repo_modules_%s" % (variant.uid)

59
pungi/phases/init.py
View File

@ -24,7 +24,7 @@ from kobo.threads import run_in_threads
from pungi.phases.base import PhaseBase
from pungi.phases.gather import write_prepopulate_file
from pungi.util import temp_dir
from pungi.module_util import iter_module_defaults
from pungi.module_util import iter_module_defaults_or_obsoletes
from pungi.wrappers.comps import CompsWrapper
from pungi.wrappers.createrepo import CreaterepoWrapper
from pungi.wrappers.scm import get_dir_from_scm, get_file_from_scm
@ -68,10 +68,18 @@ class InitPhase(PhaseBase):
# download module defaults
if self.compose.has_module_defaults:
write_module_defaults(self.compose)
validate_module_defaults(
validate_module_defaults_or_obsoletes(
self.compose.paths.work.module_defaults_dir(create_dir=False)
)
# download module obsoletes
if self.compose.has_module_obsoletes:
write_module_obsoletes(self.compose)
validate_module_defaults_or_obsoletes(
self.compose.paths.work.module_obsoletes_dir(create_dir=False),
obsoletes=True,
)
# write prepopulate file
write_prepopulate_file(self.compose)
@ -218,28 +226,53 @@ def write_module_defaults(compose):
)
def validate_module_defaults(path):
def write_module_obsoletes(compose):
scm_dict = compose.conf["module_obsoletes_dir"]
if isinstance(scm_dict, dict):
if scm_dict["scm"] == "file":
scm_dict["dir"] = os.path.join(compose.config_dir, scm_dict["dir"])
else:
scm_dict = os.path.join(compose.config_dir, scm_dict)
with temp_dir(prefix="moduleobsoletes_") as tmp_dir:
get_dir_from_scm(scm_dict, tmp_dir, compose=compose)
compose.log_debug("Writing module obsoletes")
shutil.copytree(
tmp_dir,
compose.paths.work.module_obsoletes_dir(create_dir=False),
ignore=shutil.ignore_patterns(".git"),
)
def validate_module_defaults_or_obsoletes(path, obsoletes=False):
"""Make sure there are no conflicting defaults. Each module name can only
have one default stream.
have one default stream or module obsolete.
:param str path: directory with cloned module defaults
:param str path: directory with cloned module defaults/obsoletes
"""
seen_defaults = collections.defaultdict(set)
seen = collections.defaultdict(set)
mmd_type = "obsoletes" if obsoletes else "defaults"
for module_name, defaults in iter_module_defaults(path):
seen_defaults[module_name].add(defaults.get_default_stream())
for module_name, defaults_or_obsoletes in iter_module_defaults_or_obsoletes(
path, obsoletes
):
if obsoletes:
for obsolete in defaults_or_obsoletes:
seen[obsolete.props.module_name].add(obsolete)
else:
seen[module_name].add(defaults_or_obsoletes.get_default_stream())
errors = []
for module_name, defaults in seen_defaults.items():
if len(defaults) > 1:
for module_name, defaults_or_obsoletes in seen.items():
if len(defaults_or_obsoletes) > 1:
errors.append(
"Module %s has multiple defaults: %s"
% (module_name, ", ".join(sorted(defaults)))
"Module %s has multiple %s: %s"
% (module_name, mmd_type, ", ".join(sorted(defaults_or_obsoletes)))
)
if errors:
raise RuntimeError(
"There are duplicated module defaults:\n%s" % "\n".join(errors)
"There are duplicated module %s:\n%s" % (mmd_type, "\n".join(errors))
)

9
pungi/phases/pkgset/common.py
View File

@ -28,7 +28,11 @@ from pungi.util import (
PartialFuncWorkerThread,
PartialFuncThreadPool,
)
from pungi.module_util import Modulemd, collect_module_defaults
from pungi.module_util import (
Modulemd,
collect_module_defaults,
collect_module_obsoletes,
)
from pungi.phases.createrepo import add_modular_metadata
@ -159,6 +163,9 @@ def _create_arch_repo(worker_thread, args, task_num):
mod_index = collect_module_defaults(
compose.paths.work.module_defaults_dir(), names, overrides_dir=overrides_dir
)
mod_index = collect_module_obsoletes(
compose.paths.work.module_obsoletes_dir(), names, mod_index
)
for x in mmd:
mod_index.add_module_stream(x)
add_modular_metadata(

14
tests/test_initphase.py
View File

@ -24,7 +24,7 @@ from tests.helpers import (
@mock.patch("pungi.phases.init.run_in_threads", new=fake_run_in_threads)
@mock.patch("pungi.phases.init.validate_comps")
@mock.patch("pungi.phases.init.validate_module_defaults")
@mock.patch("pungi.phases.init.validate_module_defaults_or_obsoletes")
@mock.patch("pungi.phases.init.write_module_defaults")
@mock.patch("pungi.phases.init.write_global_comps")
@mock.patch("pungi.phases.init.write_arch_comps")
@ -46,6 +46,7 @@ class TestInitPhase(PungiTestCase):
compose = DummyCompose(self.topdir, {})
compose.has_comps = True
compose.has_module_defaults = False
compose.has_module_obsoletes = False
compose.setup_optional()
phase = init.InitPhase(compose)
phase.run()
@ -100,6 +101,7 @@ class TestInitPhase(PungiTestCase):
compose = DummyCompose(self.topdir, {})
compose.has_comps = True
compose.has_module_defaults = False
compose.has_module_obsoletes = False
compose.variants["Everything"].groups = []
compose.variants["Everything"].modules = []
phase = init.InitPhase(compose)
@ -156,6 +158,7 @@ class TestInitPhase(PungiTestCase):
compose = DummyCompose(self.topdir, {})
compose.has_comps = False
compose.has_module_defaults = False
compose.has_module_obsoletes = False
phase = init.InitPhase(compose)
phase.run()
@ -182,6 +185,7 @@ class TestInitPhase(PungiTestCase):
compose = DummyCompose(self.topdir, {})
compose.has_comps = False
compose.has_module_defaults = True
compose.has_module_obsoletes = False
phase = init.InitPhase(compose)
phase.run()
@ -620,13 +624,13 @@ class TestValidateModuleDefaults(PungiTestCase):
def test_valid_files(self):
self._write_defaults({"httpd": ["1"], "python": ["3.6"]})
init.validate_module_defaults(self.topdir)
init.validate_module_defaults_or_obsoletes(self.topdir)
def test_duplicated_stream(self):
self._write_defaults({"httpd": ["1"], "python": ["3.6", "3.5"]})
with self.assertRaises(RuntimeError) as ctx:
init.validate_module_defaults(self.topdir)
init.validate_module_defaults_or_obsoletes(self.topdir)
self.assertIn(
"Module python has multiple defaults: 3.5, 3.6", str(ctx.exception)
@ -636,7 +640,7 @@ class TestValidateModuleDefaults(PungiTestCase):
self._write_defaults({"httpd": ["1", "2"], "python": ["3.6", "3.5"]})
with self.assertRaises(RuntimeError) as ctx:
init.validate_module_defaults(self.topdir)
init.validate_module_defaults_or_obsoletes(self.topdir)
self.assertIn("Module httpd has multiple defaults: 1, 2", str(ctx.exception))
self.assertIn(
@ -661,7 +665,7 @@ class TestValidateModuleDefaults(PungiTestCase):
),
)
init.validate_module_defaults(self.topdir)
init.validate_module_defaults_or_obsoletes(self.topdir)
@mock.patch("pungi.phases.init.CompsWrapper")