From f470599f6c804c06980ca43f32d3412e10b488e8 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Mon, 11 Jan 2021 09:58:26 +0800 Subject: [PATCH 001/137] React to SIGINT signal ODCS sends SIGINT signal. JIRA: RHELCMP-3687 Signed-off-by: Haibo Lin --- pungi/scripts/pungi_koji.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index 6131f27f..8c905565 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -615,6 +615,7 @@ def sigterm_handler(signum, frame): def cli_main(): + signal.signal(signal.SIGINT, sigterm_handler) signal.signal(signal.SIGTERM, sigterm_handler) try: From f518c1bb7c15960d759e6e584cc26ee68d4b2bf4 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Wed, 13 Jan 2021 15:34:09 +0800 Subject: [PATCH 002/137] Stop copying .git directory with module defaults JIRA: RHELCMP-3016 Fixes: https://pagure.io/pungi/issue/1464 Signed-off-by: Haibo Lin --- pungi/phases/init.py | 4 +++- tests/test_initphase.py | 10 +++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/pungi/phases/init.py b/pungi/phases/init.py index 5f68f29a..f0590128 100644 --- a/pungi/phases/init.py +++ b/pungi/phases/init.py @@ -212,7 +212,9 @@ def write_module_defaults(compose): get_dir_from_scm(scm_dict, tmp_dir, compose=compose) compose.log_debug("Writing module defaults") shutil.copytree( - tmp_dir, compose.paths.work.module_defaults_dir(create_dir=False) + tmp_dir, + compose.paths.work.module_defaults_dir(create_dir=False), + ignore=shutil.ignore_patterns(".git"), ) diff --git a/tests/test_initphase.py b/tests/test_initphase.py index 5f06a08e..afd2601e 100644 --- a/tests/test_initphase.py +++ b/tests/test_initphase.py @@ -529,10 +529,11 @@ class TestGetLookasideGroups(PungiTestCase): ) +@mock.patch("shutil.ignore_patterns") @mock.patch("shutil.copytree") @mock.patch("pungi.phases.init.get_dir_from_scm") class TestWriteModuleDefaults(PungiTestCase): - def test_clone_git(self, gdfs, ct): + def test_clone_git(self, gdfs, ct, igp): conf = {"scm": "git", "repo": "https://pagure.io/pungi.git", "dir": "."} compose = DummyCompose(self.topdir, {"module_defaults_dir": conf}) @@ -547,11 +548,12 @@ class TestWriteModuleDefaults(PungiTestCase): mock.call( gdfs.call_args_list[0][0][1], os.path.join(self.topdir, "work/global/module_defaults"), + ignore=igp(".git"), ) ], ) - def test_clone_file_scm(self, gdfs, ct): + def test_clone_file_scm(self, gdfs, ct, igp): conf = {"scm": "file", "dir": "defaults"} compose = DummyCompose(self.topdir, {"module_defaults_dir": conf}) compose.config_dir = "/home/releng/configs" @@ -574,11 +576,12 @@ class TestWriteModuleDefaults(PungiTestCase): mock.call( gdfs.call_args_list[0][0][1], os.path.join(self.topdir, "work/global/module_defaults"), + ignore=igp(".git"), ) ], ) - def test_clone_file_str(self, gdfs, ct): + def test_clone_file_str(self, gdfs, ct, igp): conf = "defaults" compose = DummyCompose(self.topdir, {"module_defaults_dir": conf}) compose.config_dir = "/home/releng/configs" @@ -595,6 +598,7 @@ class TestWriteModuleDefaults(PungiTestCase): mock.call( gdfs.call_args_list[0][0][1], os.path.join(self.topdir, "work/global/module_defaults"), + ignore=igp(".git"), ) ], ) From 9ea1098eaee9a6c1195d8b0e8e3073c2d4308cb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 21 Jan 2021 16:38:42 +0100 Subject: [PATCH 003/137] comps: Preserve default arg on groupid MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the wrapper processes comps file, it wasn't emitting "default" argument for groupid element. The default is false and most entries are actually using the default, so let's only emit it if set to true. Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1882358 Signed-off-by: Lubomír Sedlář --- pungi/wrappers/comps.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/pungi/wrappers/comps.py b/pungi/wrappers/comps.py index 4eaddb0e..aa7685b4 100644 --- a/pungi/wrappers/comps.py +++ b/pungi/wrappers/comps.py @@ -325,9 +325,8 @@ class CompsWrapper(object): group_node.appendChild(packagelist) for category in self.comps.categories: - groups = set(x.name for x in category.group_ids) & set( - self.get_comps_groups() - ) + group_ids = set(self.get_comps_groups()) + groups = set(g for g in category.group_ids if g.name in group_ids) if not groups: continue cat_node = doc.createElement("category") @@ -341,7 +340,7 @@ class CompsWrapper(object): append_grouplist(doc, cat_node, groups) for environment in sorted(self.comps.environments, key=attrgetter("id")): - groups = set(x.name for x in environment.group_ids) + groups = set(environment.group_ids) if not groups: continue env_node = doc.createElement("environment") @@ -356,10 +355,7 @@ class CompsWrapper(object): if environment.option_ids: append_grouplist( - doc, - env_node, - (x.name for x in environment.option_ids), - "optionlist", + doc, env_node, set(environment.option_ids), "optionlist", ) if self.comps.langpacks: @@ -460,8 +456,11 @@ def append(doc, parent, elem, content=None, lang=None, **kwargs): def append_grouplist(doc, parent, groups, elem="grouplist"): grouplist_node = doc.createElement(elem) - for groupid in sorted(groups): - append(doc, grouplist_node, "groupid", groupid) + for groupid in sorted(groups, key=lambda x: x.name): + kwargs = {} + if groupid.default: + kwargs["default"] = "true" + append(doc, grouplist_node, "groupid", groupid.name, **kwargs) parent.appendChild(grouplist_node) From 39b847094a98080ae9b7f9422fcefa34cc17411d Mon Sep 17 00:00:00 2001 From: Ken Dreyer Date: Fri, 22 Jan 2021 15:25:48 -0700 Subject: [PATCH 004/137] doc: remove default createrepo_checksum value from example createrepo_checksum already defaults to sha256. Remove this setting from the documented Minimal Example configuration to make it easier to read. Signed-off-by: Ken Dreyer --- doc/configuration.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index d4abb0d6..d1175baa 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -31,9 +31,6 @@ Minimal Config Example pkgset_source = "koji" pkgset_koji_tag = "f23" - # CREATEREPO - createrepo_checksum = "sha256" - # GATHER gather_method = "deps" greedy_method = "build" From 83458f26c27d21845071946f38c3587ea924bfa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 27 Jan 2021 15:41:33 +0100 Subject: [PATCH 005/137] pkgset: Drop kobo.plugins usage from GatherSources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Relates: https://pagure.io/pungi/issue/1488 Signed-off-by: Lubomír Sedlář --- pungi/phases/gather/__init__.py | 5 +--- pungi/phases/gather/source.py | 11 +-------- pungi/phases/gather/sources/__init__.py | 26 ++++++++++++++++++++ pungi/phases/gather/sources/source_comps.py | 2 -- pungi/phases/gather/sources/source_json.py | 2 -- pungi/phases/gather/sources/source_module.py | 2 -- pungi/phases/gather/sources/source_none.py | 2 -- 7 files changed, 28 insertions(+), 22 deletions(-) diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index bf190335..96b6d9af 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -45,11 +45,8 @@ from pungi.phases.createrepo import add_modular_metadata def get_gather_source(name): import pungi.phases.gather.sources - from .source import GatherSourceContainer - GatherSourceContainer.register_module(pungi.phases.gather.sources) - container = GatherSourceContainer() - return container["GatherSource%s" % name] + return pungi.phases.gather.sources.ALL_SOURCES[name.lower()] def get_gather_method(name): diff --git a/pungi/phases/gather/source.py b/pungi/phases/gather/source.py index c1d7c9c5..92c15df1 100644 --- a/pungi/phases/gather/source.py +++ b/pungi/phases/gather/source.py @@ -14,15 +14,6 @@ # along with this program; if not, see . -import kobo.plugins - - -class GatherSourceBase(kobo.plugins.Plugin): +class GatherSourceBase(object): def __init__(self, compose): self.compose = compose - - -class GatherSourceContainer(kobo.plugins.PluginContainer): - @classmethod - def normalize_name(cls, name): - return name.lower() diff --git a/pungi/phases/gather/sources/__init__.py b/pungi/phases/gather/sources/__init__.py index e69de29b..00ff61e8 100644 --- a/pungi/phases/gather/sources/__init__.py +++ b/pungi/phases/gather/sources/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + +from .source_comps import GatherSourceComps +from .source_json import GatherSourceJson +from .source_module import GatherSourceModule +from .source_none import GatherSourceNone + +ALL_SOURCES = { + "comps": GatherSourceComps, + "json": GatherSourceJson, + "module": GatherSourceModule, + "none": GatherSourceNone, +} diff --git a/pungi/phases/gather/sources/source_comps.py b/pungi/phases/gather/sources/source_comps.py index e9987dfe..e1247770 100644 --- a/pungi/phases/gather/sources/source_comps.py +++ b/pungi/phases/gather/sources/source_comps.py @@ -30,8 +30,6 @@ import pungi.phases.gather.source class GatherSourceComps(pungi.phases.gather.source.GatherSourceBase): - enabled = True - def __call__(self, arch, variant): groups = set() if not self.compose.conf.get("comps_file"): diff --git a/pungi/phases/gather/sources/source_json.py b/pungi/phases/gather/sources/source_json.py index 073935d8..2be88eb0 100644 --- a/pungi/phases/gather/sources/source_json.py +++ b/pungi/phases/gather/sources/source_json.py @@ -37,8 +37,6 @@ import pungi.phases.gather.source class GatherSourceJson(pungi.phases.gather.source.GatherSourceBase): - enabled = True - def __call__(self, arch, variant): json_path = self.compose.conf.get("gather_source_mapping") if not json_path: diff --git a/pungi/phases/gather/sources/source_module.py b/pungi/phases/gather/sources/source_module.py index beb108d2..be636bf0 100644 --- a/pungi/phases/gather/sources/source_module.py +++ b/pungi/phases/gather/sources/source_module.py @@ -26,8 +26,6 @@ import pungi.phases.gather.source class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase): - enabled = True - def __call__(self, arch, variant): groups = set() packages = set() diff --git a/pungi/phases/gather/sources/source_none.py b/pungi/phases/gather/sources/source_none.py index 35801e9f..a78b198a 100644 --- a/pungi/phases/gather/sources/source_none.py +++ b/pungi/phases/gather/sources/source_none.py @@ -29,7 +29,5 @@ import pungi.phases.gather.source class GatherSourceNone(pungi.phases.gather.source.GatherSourceBase): - enabled = True - def __call__(self, arch, variant): return set(), set() From 0f4b0577f7a650f78aa8155edd6ab8b3b36f0272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 27 Jan 2021 15:42:12 +0100 Subject: [PATCH 006/137] gather: Drop kobo.plugins usage from GatherMethod MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Relates: https://pagure.io/pungi/issue/1488 Signed-off-by: Lubomír Sedlář --- pungi/phases/gather/__init__.py | 5 +--- pungi/phases/gather/method.py | 11 +-------- pungi/phases/gather/methods/__init__.py | 24 ++++++++++++++++++++ pungi/phases/gather/methods/method_deps.py | 2 -- pungi/phases/gather/methods/method_hybrid.py | 2 -- pungi/phases/gather/methods/method_nodeps.py | 2 -- 6 files changed, 26 insertions(+), 20 deletions(-) diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 96b6d9af..b559cfbc 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -51,11 +51,8 @@ def get_gather_source(name): def get_gather_method(name): import pungi.phases.gather.methods - from .method import GatherMethodContainer - GatherMethodContainer.register_module(pungi.phases.gather.methods) - container = GatherMethodContainer() - return container["GatherMethod%s" % name] + return pungi.phases.gather.methods.ALL_METHODS[name.lower()] class GatherPhase(PhaseBase): diff --git a/pungi/phases/gather/method.py b/pungi/phases/gather/method.py index 7feb835f..94e5460b 100644 --- a/pungi/phases/gather/method.py +++ b/pungi/phases/gather/method.py @@ -14,15 +14,6 @@ # along with this program; if not, see . -import kobo.plugins - - -class GatherMethodBase(kobo.plugins.Plugin): +class GatherMethodBase(object): def __init__(self, compose): self.compose = compose - - -class GatherMethodContainer(kobo.plugins.PluginContainer): - @classmethod - def normalize_name(cls, name): - return name.lower() diff --git a/pungi/phases/gather/methods/__init__.py b/pungi/phases/gather/methods/__init__.py index e69de29b..905edf70 100644 --- a/pungi/phases/gather/methods/__init__.py +++ b/pungi/phases/gather/methods/__init__.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + +from .method_deps import GatherMethodDeps +from .method_nodeps import GatherMethodNodeps +from .method_hybrid import GatherMethodHybrid + +ALL_METHODS = { + "deps": GatherMethodDeps, + "nodeps": GatherMethodNodeps, + "hybrid": GatherMethodHybrid, +} diff --git a/pungi/phases/gather/methods/method_deps.py b/pungi/phases/gather/methods/method_deps.py index a0e0bee6..1f3f9659 100644 --- a/pungi/phases/gather/methods/method_deps.py +++ b/pungi/phases/gather/methods/method_deps.py @@ -31,8 +31,6 @@ import pungi.phases.gather.method class GatherMethodDeps(pungi.phases.gather.method.GatherMethodBase): - enabled = True - def __call__( self, arch, diff --git a/pungi/phases/gather/methods/method_hybrid.py b/pungi/phases/gather/methods/method_hybrid.py index 5d143199..3c673c0f 100644 --- a/pungi/phases/gather/methods/method_hybrid.py +++ b/pungi/phases/gather/methods/method_hybrid.py @@ -60,8 +60,6 @@ class FakePackage(object): class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): - enabled = True - def __init__(self, *args, **kwargs): super(GatherMethodHybrid, self).__init__(*args, **kwargs) self.package_maps = {} diff --git a/pungi/phases/gather/methods/method_nodeps.py b/pungi/phases/gather/methods/method_nodeps.py index cd625047..062a386b 100644 --- a/pungi/phases/gather/methods/method_nodeps.py +++ b/pungi/phases/gather/methods/method_nodeps.py @@ -28,8 +28,6 @@ from kobo.pkgset import SimpleRpmWrapper, RpmWrapper class GatherMethodNodeps(pungi.phases.gather.method.GatherMethodBase): - enabled = True - def __call__(self, arch, variant, *args, **kwargs): fname = "gather-nodeps-%s" % variant.uid if self.source_name: From c87fce30ac61baf648092c707af70709fb352dca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 27 Jan 2021 15:46:28 +0100 Subject: [PATCH 007/137] pkgset: Drop kobo.plugin usage from PkgsetSource MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Relates: https://pagure.io/pungi/issue/1488 Signed-off-by: Lubomír Sedlář --- pungi/phases/pkgset/__init__.py | 7 ++----- pungi/phases/pkgset/source.py | 11 +---------- pungi/phases/pkgset/sources/__init__.py | 22 +++++++++++++++++++++ pungi/phases/pkgset/sources/source_koji.py | 2 -- pungi/phases/pkgset/sources/source_repos.py | 2 -- 5 files changed, 25 insertions(+), 19 deletions(-) diff --git a/pungi/phases/pkgset/__init__.py b/pungi/phases/pkgset/__init__.py index 66fa4952..684f6e95 100644 --- a/pungi/phases/pkgset/__init__.py +++ b/pungi/phases/pkgset/__init__.py @@ -29,13 +29,10 @@ class PkgsetPhase(PhaseBase): self.path_prefix = None def run(self): - pkgset_source = "PkgsetSource%s" % self.compose.conf["pkgset_source"] - from .source import PkgsetSourceContainer from . import sources - PkgsetSourceContainer.register_module(sources) - container = PkgsetSourceContainer() - SourceClass = container[pkgset_source] + SourceClass = sources.ALL_SOURCES[self.compose.conf["pkgset_source"].lower()] + self.package_sets, self.path_prefix = SourceClass(self.compose)() def validate(self): diff --git a/pungi/phases/pkgset/source.py b/pungi/phases/pkgset/source.py index 472b4400..297d499e 100644 --- a/pungi/phases/pkgset/source.py +++ b/pungi/phases/pkgset/source.py @@ -14,15 +14,6 @@ # along with this program; if not, see . -import kobo.plugins - - -class PkgsetSourceBase(kobo.plugins.Plugin): +class PkgsetSourceBase(object): def __init__(self, compose): self.compose = compose - - -class PkgsetSourceContainer(kobo.plugins.PluginContainer): - @classmethod - def normalize_name(cls, name): - return name.lower() diff --git a/pungi/phases/pkgset/sources/__init__.py b/pungi/phases/pkgset/sources/__init__.py index e69de29b..b3bcba8c 100644 --- a/pungi/phases/pkgset/sources/__init__.py +++ b/pungi/phases/pkgset/sources/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + +from .source_koji import PkgsetSourceKoji +from .source_repos import PkgsetSourceRepos + +ALL_SOURCES = { + "koji": PkgsetSourceKoji, + "repos": PkgsetSourceRepos, +} diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index a557f604..5e96a5be 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -184,8 +184,6 @@ def get_koji_modules(compose, koji_wrapper, event, module_info_str): class PkgsetSourceKoji(pungi.phases.pkgset.source.PkgsetSourceBase): - enabled = True - def __call__(self): compose = self.compose koji_profile = compose.conf["koji_profile"] diff --git a/pungi/phases/pkgset/sources/source_repos.py b/pungi/phases/pkgset/sources/source_repos.py index 3d3701bb..63c7c4ec 100644 --- a/pungi/phases/pkgset/sources/source_repos.py +++ b/pungi/phases/pkgset/sources/source_repos.py @@ -31,8 +31,6 @@ import pungi.phases.pkgset.source class PkgsetSourceRepos(pungi.phases.pkgset.source.PkgsetSourceBase): - enabled = True - def __call__(self): package_sets, path_prefix = get_pkgset_from_repos(self.compose) return (package_sets, path_prefix) From 49a566152187d6e18658f74688a7505ec4916b40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 8 Jan 2021 14:34:34 +0100 Subject: [PATCH 008/137] pkgset: Remove reuse file when packages are not signed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In such case we never want to reuse the pkgset, as it risks leaking unsigned packages. Safest option is to remove the file completely. Fixes: https://pagure.io/pungi/issue/1480 JIRA: RHELCMP-3720 Signed-off-by: Lubomír Sedlář --- pungi/phases/pkgset/pkgsets.py | 10 ++++++++-- pungi/scripts/pungi_koji.py | 21 +++++++++++++++------ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index e82a47cf..9fa0bcf5 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -35,6 +35,10 @@ from pungi.util import pkg_is_srpm, copy_all from pungi.arch import get_valid_arches, is_excluded +class UnsignedPackagesError(RuntimeError): + pass + + class ExtendedRpmWrapper(kobo.pkgset.SimpleRpmWrapper): """ ExtendedRpmWrapper extracts only certain RPM fields instead of @@ -144,7 +148,7 @@ class PackageSetBase(kobo.log.LoggingBase): def raise_invalid_sigkeys_exception(self, rpminfos): """ - Raises RuntimeError containing details of RPMs with invalid + Raises UnsignedPackagesError containing details of RPMs with invalid sigkeys defined in `rpminfos`. """ @@ -166,7 +170,9 @@ class PackageSetBase(kobo.log.LoggingBase): if not isinstance(rpminfos, dict): rpminfos = {self.sigkey_ordering: rpminfos} - raise RuntimeError("\n".join(get_error(k, v) for k, v in rpminfos.items())) + raise UnsignedPackagesError( + "\n".join(get_error(k, v) for k, v in rpminfos.items()) + ) def read_packages(self, rpms, srpms): srpm_pool = ReaderPool(self, self._logger) diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index 8c905565..2a08840f 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -5,6 +5,7 @@ from __future__ import print_function import argparse import getpass +import glob import json import locale import logging @@ -327,12 +328,20 @@ def main(): ) notifier.compose = compose COMPOSE = compose - run_compose( - compose, - create_latest_link=create_latest_link, - latest_link_status=latest_link_status, - latest_link_components=latest_link_components, - ) + try: + run_compose( + compose, + create_latest_link=create_latest_link, + latest_link_status=latest_link_status, + latest_link_components=latest_link_components, + ) + except pungi.phases.pkgset.pkgsets.UnsignedPackagesError: + # There was an unsigned package somewhere. It is not safe to reuse any + # package set from this compose (since we could leak the unsigned + # package). Let's make sure all reuse files are deleted. + for fp in glob.glob(compose.paths.work.pkgset_reuse_file("*")): + os.unlink(fp) + raise def run_compose( From d4ee42ec236ad92611459c872500edbb745204dd Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Fri, 29 Jan 2021 12:00:54 +0800 Subject: [PATCH 009/137] pkgset: Check tag inheritance change before reuse JIRA: RHELCMP-2453 Signed-off-by: Haibo Lin --- pungi/phases/pkgset/pkgsets.py | 12 ++++++++++-- tests/test_pkgset_pkgsets.py | 24 ++++++++++++++++++------ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index 9fa0bcf5..7d39d8c3 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -731,17 +731,22 @@ class KojiPackageSet(PackageSetBase): % (old_koji_event, koji_event) ) changed = self.koji_proxy.queryHistory( - tables=["tag_listing"], tag=tag, afterEvent=old_koji_event + tables=["tag_listing", "tag_inheritance"], + tag=tag, + afterEvent=old_koji_event, ) if changed["tag_listing"]: self.log_debug("Builds under tag %s changed. Can't reuse." % tag) return False + if changed["tag_inheritance"]: + self.log_debug("Tag inheritance %s changed. Can't reuse." % tag) + return False if inherit: inherit_tags = self.koji_proxy.getFullInheritance(tag, koji_event) for t in inherit_tags: changed = self.koji_proxy.queryHistory( - tables=["tag_listing"], + tables=["tag_listing", "tag_inheritance"], tag=t["name"], afterEvent=old_koji_event, beforeEvent=koji_event + 1, @@ -752,6 +757,9 @@ class KojiPackageSet(PackageSetBase): % t["name"] ) return False + if changed["tag_inheritance"]: + self.log_debug("Tag inheritance %s changed. Can't reuse." % tag) + return False repo_dir = compose.paths.work.pkgset_repo(tag, create_dir=False) old_repo_dir = compose.paths.old_compose_path(repo_dir) diff --git a/tests/test_pkgset_pkgsets.py b/tests/test_pkgset_pkgsets.py index 94fae923..555a313b 100644 --- a/tests/test_pkgset_pkgsets.py +++ b/tests/test_pkgset_pkgsets.py @@ -632,7 +632,10 @@ class TestReuseKojiPkgset(helpers.PungiTestCase): def test_reuse_build_under_tag_changed(self, mock_old_topdir): mock_old_topdir.return_value = self.old_compose_dir self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1]) - self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": [{}]} + self.koji_wrapper.koji_proxy.queryHistory.return_value = { + "tag_listing": [{}], + "tag_inheritance": [], + } self.pkgset.try_to_reuse(self.compose, self.tag) @@ -652,8 +655,8 @@ class TestReuseKojiPkgset(helpers.PungiTestCase): mock_old_topdir.return_value = self.old_compose_dir self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1]) self.koji_wrapper.koji_proxy.queryHistory.side_effect = [ - {"tag_listing": []}, - {"tag_listing": [{}]}, + {"tag_listing": [], "tag_inheritance": []}, + {"tag_listing": [{}], "tag_inheritance": []}, ] self.koji_wrapper.koji_proxy.getFullInheritance.return_value = [ {"name": self.inherited_tag} @@ -680,7 +683,10 @@ class TestReuseKojiPkgset(helpers.PungiTestCase): def test_reuse_failed_load_reuse_file(self, mock_old_topdir, mock_exists): mock_old_topdir.return_value = self.old_compose_dir self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1]) - self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": []} + self.koji_wrapper.koji_proxy.queryHistory.return_value = { + "tag_listing": [], + "tag_inheritance": [], + } self.koji_wrapper.koji_proxy.getFullInheritance.return_value = [] self.pkgset.load_old_file_cache = mock.Mock( side_effect=Exception("unknown error") @@ -712,7 +718,10 @@ class TestReuseKojiPkgset(helpers.PungiTestCase): def test_reuse_criteria_not_match(self, mock_old_topdir, mock_exists): mock_old_topdir.return_value = self.old_compose_dir self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1]) - self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": []} + self.koji_wrapper.koji_proxy.queryHistory.return_value = { + "tag_listing": [], + "tag_inheritance": [], + } self.koji_wrapper.koji_proxy.getFullInheritance.return_value = [] self.pkgset.load_old_file_cache = mock.Mock( return_value={"allow_invalid_sigkeys": True} @@ -751,7 +760,10 @@ class TestReuseKojiPkgset(helpers.PungiTestCase): def test_reuse_pkgset(self, mock_old_topdir, mock_exists, mock_copy_all): mock_old_topdir.return_value = self.old_compose_dir self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1]) - self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": []} + self.koji_wrapper.koji_proxy.queryHistory.return_value = { + "tag_listing": [], + "tag_inheritance": [], + } self.koji_wrapper.koji_proxy.getFullInheritance.return_value = [] self.pkgset.load_old_file_cache = mock.Mock( return_value={ From daa0ca6106e2198ef1d2977c5440b66104322f4c Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Tue, 2 Feb 2021 18:02:12 +0800 Subject: [PATCH 010/137] pkgset: Include just one version of module When adding extra modules via option *pkgset_koji_module_builds*, all other versions of the same stream potentially available in a Brew tag should be skipped. JIRA: RHELCMP-3689 Signed-off-by: Haibo Lin --- pungi/phases/pkgset/sources/source_koji.py | 68 ++++++++++++++++++---- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 5e96a5be..c14d4eb2 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -202,7 +202,12 @@ def get_pkgset_from_koji(compose, koji_wrapper, path_prefix): def _add_module_to_variant( - koji_wrapper, variant, build, add_to_variant_modules=False, compose=None + koji_wrapper, + variant, + build, + add_to_variant_modules=False, + compose=None, + exclude_module_ns=None, ): """ Adds module defined by Koji build info to variant. @@ -212,6 +217,7 @@ def _add_module_to_variant( :param bool add_to_variant_modules: Adds the modules also to variant.modules. :param compose: Compose object to get filters from + :param list exclude_module_ns: Module name:stream which will be excluded. """ mmds = {} archives = koji_wrapper.koji_proxy.listArchives(build["id"]) @@ -241,6 +247,10 @@ def _add_module_to_variant( info = build["extra"]["typeinfo"]["module"] nsvc = "%(name)s:%(stream)s:%(version)s:%(context)s" % info + ns = "%(name)s:%(stream)s" % info + + if exclude_module_ns and ns in exclude_module_ns: + return added = False @@ -381,7 +391,7 @@ def _is_filtered_out(compose, variant, arch, module_name, module_stream): def _get_modules_from_koji( - compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd + compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd, exclude_module_ns ): """ Loads modules for given `variant` from koji `session`, adds them to @@ -392,6 +402,7 @@ def _get_modules_from_koji( :param Variant variant: Variant with modules to find. :param dict variant_tags: Dict populated by this method. Key is `variant` and value is list of Koji tags to get the RPMs from. + :param list exclude_module_ns: Module name:stream which will be excluded. """ # Find out all modules in every variant and add their Koji tags @@ -400,7 +411,11 @@ def _get_modules_from_koji( koji_modules = get_koji_modules(compose, koji_wrapper, event, module["name"]) for koji_module in koji_modules: nsvc = _add_module_to_variant( - koji_wrapper, variant, koji_module, compose=compose + koji_wrapper, + variant, + koji_module, + compose=compose, + exclude_module_ns=exclude_module_ns, ) if not nsvc: continue @@ -515,7 +530,13 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules) def _get_modules_from_koji_tags( - compose, koji_wrapper, event_id, variant, variant_tags, tag_to_mmd + compose, + koji_wrapper, + event_id, + variant, + variant_tags, + tag_to_mmd, + exclude_module_ns, ): """ Loads modules for given `variant` from Koji, adds them to @@ -527,6 +548,7 @@ def _get_modules_from_koji_tags( :param Variant variant: Variant with modules to find. :param dict variant_tags: Dict populated by this method. Key is `variant` and value is list of Koji tags to get the RPMs from. + :param list exclude_module_ns: Module name:stream which will be excluded. """ # Compose tags from configuration compose_tags = [ @@ -603,7 +625,12 @@ def _get_modules_from_koji_tags( variant_tags[variant].append(module_tag) nsvc = _add_module_to_variant( - koji_wrapper, variant, build, True, compose=compose + koji_wrapper, + variant, + build, + True, + compose=compose, + exclude_module_ns=exclude_module_ns, ) if not nsvc: continue @@ -693,23 +720,44 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): "modules." ) + extra_modules = get_variant_data( + compose.conf, "pkgset_koji_module_builds", variant + ) + + # When adding extra modules, other modules of the same name:stream available + # in brew tag should be excluded. + exclude_module_ns = [] + if extra_modules: + exclude_module_ns = [ + ":".join(nsvc.split(":")[:2]) for nsvc in extra_modules + ] + if modular_koji_tags or ( compose.conf["pkgset_koji_module_tag"] and variant.modules ): # List modules tagged in particular tags. _get_modules_from_koji_tags( - compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd + compose, + koji_wrapper, + event, + variant, + variant_tags, + tag_to_mmd, + exclude_module_ns, ) elif variant.modules: # Search each module in Koji separately. Tagging does not come into # play here. _get_modules_from_koji( - compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd + compose, + koji_wrapper, + event, + variant, + variant_tags, + tag_to_mmd, + exclude_module_ns, ) - extra_modules = get_variant_data( - compose.conf, "pkgset_koji_module_builds", variant - ) if extra_modules: _add_extra_modules_to_variant( compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd From 44f7eff1b73eb331768ae5c6800bc76d890eb80c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Tue, 2 Feb 2021 10:59:08 +0100 Subject: [PATCH 011/137] Move UnsignedPackagesError to a separate file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This file can contain all Pungi specific exceptions. It should also fix an issue encountered on Python 2.7: AttributeError: 'module' object has no attribute 'pkgsets' Signed-off-by: Lubomír Sedlář --- pungi/errors.py | 20 ++++++++++++++++++++ pungi/phases/pkgset/pkgsets.py | 5 +---- pungi/scripts/pungi_koji.py | 3 ++- 3 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 pungi/errors.py diff --git a/pungi/errors.py b/pungi/errors.py new file mode 100644 index 00000000..093c2c83 --- /dev/null +++ b/pungi/errors.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + + +class UnsignedPackagesError(RuntimeError): + """Raised when package set fails to find a properly signed copy of an + RPM.""" + + pass diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index 7d39d8c3..d4320865 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -33,10 +33,7 @@ from kobo.threads import WorkerThread, ThreadPool import pungi.wrappers.kojiwrapper from pungi.util import pkg_is_srpm, copy_all from pungi.arch import get_valid_arches, is_excluded - - -class UnsignedPackagesError(RuntimeError): - pass +from pungi.errors import UnsignedPackagesError class ExtendedRpmWrapper(kobo.pkgset.SimpleRpmWrapper): diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index 2a08840f..1e631718 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -21,6 +21,7 @@ from six.moves import shlex_quote from pungi.phases import PHASES_NAMES from pungi import get_full_version, util +from pungi.errors import UnsignedPackagesError # force C locales @@ -335,7 +336,7 @@ def main(): latest_link_status=latest_link_status, latest_link_components=latest_link_components, ) - except pungi.phases.pkgset.pkgsets.UnsignedPackagesError: + except UnsignedPackagesError: # There was an unsigned package somewhere. It is not safe to reuse any # package set from this compose (since we could leak the unsigned # package). Let's make sure all reuse files are deleted. From 36373479db874ec663f6c4dbb835990253dc2759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Mon, 8 Feb 2021 11:42:07 +0100 Subject: [PATCH 012/137] Move container metadata into compose object MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rather than tracking this directly in OSBS phase, move this into Compose object, which will allow access to this from multiple phases. Signed-off-by: Lubomír Sedlář --- pungi/compose.py | 15 +++++++++++++++ pungi/phases/osbs.py | 14 ++------------ pungi/scripts/pungi_koji.py | 2 +- tests/helpers.py | 1 + tests/test_compose.py | 24 ++++++++++++++++++++++++ tests/test_osbs_phase.py | 32 ++------------------------------ 6 files changed, 45 insertions(+), 43 deletions(-) diff --git a/pungi/compose.py b/pungi/compose.py index 2121a69f..ec9c5640 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -284,6 +284,8 @@ class Compose(kobo.log.LoggingBase): self.im.compose.respin = self.compose_respin self.im.metadata_path = self.paths.compose.metadata() + self.containers_metadata = {} + # Stores list of deliverables that failed, but did not abort the # compose. # {deliverable: [(Variant.uid, arch, subvariant)]} @@ -575,6 +577,19 @@ class Compose(kobo.log.LoggingBase): path = os.path.join(self.paths.work.tmp_dir(arch=arch, variant=variant)) return tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=path) + def dump_containers_metadata(self): + """Create a file with container metadata if there are any containers.""" + if not self.containers_metadata: + return + with open(self.paths.compose.metadata("osbs.json"), "w") as f: + json.dump( + self.containers_metadata, + f, + indent=4, + sort_keys=True, + separators=(",", ": "), + ) + def get_ordered_variant_uids(compose): if not hasattr(compose, "_ordered_variant_uids"): diff --git a/pungi/phases/osbs.py b/pungi/phases/osbs.py index 148b3c3e..7ad2abc8 100644 --- a/pungi/phases/osbs.py +++ b/pungi/phases/osbs.py @@ -17,7 +17,6 @@ class OSBSPhase(PhaseLoggerMixin, ConfigGuardedPhase): def __init__(self, compose): super(OSBSPhase, self).__init__(compose) self.pool = ThreadPool(logger=self.logger) - self.pool.metadata = {} self.pool.registries = {} def run(self): @@ -28,15 +27,6 @@ class OSBSPhase(PhaseLoggerMixin, ConfigGuardedPhase): self.pool.start() - def dump_metadata(self): - """Create a file with image metadata if the phase actually ran.""" - if self._skipped: - return - with open(self.compose.paths.compose.metadata("osbs.json"), "w") as f: - json.dump( - self.pool.metadata, f, indent=4, sort_keys=True, separators=(",", ": ") - ) - def request_push(self): """Store configuration data about where to push the created images and then send the same data to message bus. @@ -146,7 +136,7 @@ class OSBSThread(WorkerThread): metadata.update({"repositories": result["repositories"]}) # add a fake arch of 'scratch', so we can construct the metadata # in same data structure as real builds. - self.pool.metadata.setdefault(variant.uid, {}).setdefault( + compose.containers_metadata.setdefault(variant.uid, {}).setdefault( "scratch", [] ).append(metadata) return None @@ -180,7 +170,7 @@ class OSBSThread(WorkerThread): "Created Docker base image %s-%s-%s.%s" % (metadata["name"], metadata["version"], metadata["release"], arch) ) - self.pool.metadata.setdefault(variant.uid, {}).setdefault( + compose.containers_metadata.setdefault(variant.uid, {}).setdefault( arch, [] ).append(data) return nvr diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index 1e631718..d0e53bb8 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -538,7 +538,7 @@ def run_compose( and osbuild_phase.skip() ): compose.im.dump(compose.paths.compose.metadata("images.json")) - osbs_phase.dump_metadata() + compose.dump_containers_metadata() test_phase.start() test_phase.stop() diff --git a/tests/helpers.py b/tests/helpers.py index a3969953..3fdd1eae 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -226,6 +226,7 @@ class DummyCompose(object): self.require_deliverable = mock.Mock() self.should_create_yum_database = True self.cache_region = None + self.containers_metadata = {} def setup_optional(self): self.all_variants["Server-optional"] = MockVariant( diff --git a/tests/test_compose.py b/tests/test_compose.py index 85590273..628973bd 100644 --- a/tests/test_compose.py +++ b/tests/test_compose.py @@ -753,3 +753,27 @@ class StatusTest(unittest.TestCase): self.compose.conf["gather_backend"] = "yum" self.compose.conf["createrepo_database"] = False self.assertFalse(self.compose.should_create_yum_database) + + +class DumpContainerMetadataTest(unittest.TestCase): + def setUp(self): + self.tmp_dir = tempfile.mkdtemp() + with mock.patch("pungi.compose.ComposeInfo"): + self.compose = Compose({}, self.tmp_dir) + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + + def test_dump_metadata(self): + metadata = {"Server": {"x86_64": "Metadata"}} + self.compose.containers_metadata = metadata + self.compose.dump_containers_metadata() + + with open(self.tmp_dir + "/compose/metadata/osbs.json") as f: + data = json.load(f) + self.assertEqual(data, metadata) + + @mock.patch("pungi.phases.osbs.ThreadPool") + def test_dump_empty_metadata(self, ThreadPool): + self.compose.dump_containers_metadata() + self.assertFalse(os.path.isfile(self.tmp_dir + "/compose/metadata/osbs.json")) diff --git a/tests/test_osbs_phase.py b/tests/test_osbs_phase.py index 059ce828..03bd6740 100644 --- a/tests/test_osbs_phase.py +++ b/tests/test_osbs_phase.py @@ -36,34 +36,6 @@ class OSBSPhaseTest(helpers.PungiTestCase): phase = osbs.OSBSPhase(compose) self.assertTrue(phase.skip()) - @mock.patch("pungi.phases.osbs.ThreadPool") - def test_dump_metadata(self, ThreadPool): - compose = helpers.DummyCompose(self.topdir, {"osbs": {"^Everything$": {}}}) - compose.just_phases = None - compose.skip_phases = [] - compose.notifier = mock.Mock() - phase = osbs.OSBSPhase(compose) - phase.start() - phase.stop() - phase.pool.metadata = METADATA - phase.dump_metadata() - - with open(self.topdir + "/compose/metadata/osbs.json") as f: - data = json.load(f) - self.assertEqual(data, METADATA) - - @mock.patch("pungi.phases.osbs.ThreadPool") - def test_dump_metadata_after_skip(self, ThreadPool): - compose = helpers.DummyCompose(self.topdir, {}) - compose.just_phases = None - compose.skip_phases = [] - phase = osbs.OSBSPhase(compose) - phase.start() - phase.stop() - phase.dump_metadata() - - self.assertFalse(os.path.isfile(self.topdir + "/compose/metadata/osbs.json")) - @mock.patch("pungi.phases.osbs.ThreadPool") def test_request_push(self, ThreadPool): compose = helpers.DummyCompose(self.topdir, {"osbs": {"^Everything$": {}}}) @@ -190,7 +162,7 @@ SCRATCH_METADATA = { class OSBSThreadTest(helpers.PungiTestCase): def setUp(self): super(OSBSThreadTest, self).setUp() - self.pool = mock.Mock(metadata={}, registries={}) + self.pool = mock.Mock(registries={}) self.t = osbs.OSBSThread(self.pool) self.compose = helpers.DummyCompose( self.topdir, @@ -226,7 +198,7 @@ class OSBSThreadTest(helpers.PungiTestCase): metadata = copy.deepcopy(METADATA) metadata["Server"]["x86_64"][0]["compose_id"] = self.compose.compose_id metadata["Server"]["x86_64"][0]["koji_task"] = 12345 - self.assertEqual(self.pool.metadata, metadata) + self.assertEqual(self.compose.containers_metadata, metadata) def _assertCorrectCalls(self, opts, setupCalls=None, scratch=False): setupCalls = setupCalls or [] From 61e90fd7e060f14bee74312c694fd82a90941a65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Mon, 8 Feb 2021 13:23:00 +0100 Subject: [PATCH 013/137] osbs: Move metadata processing to standalone function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lubomír Sedlář --- pungi/phases/osbs.py | 117 ++++++++++++++++++++++--------------------- 1 file changed, 60 insertions(+), 57 deletions(-) diff --git a/pungi/phases/osbs.py b/pungi/phases/osbs.py index 7ad2abc8..a1e0db67 100644 --- a/pungi/phases/osbs.py +++ b/pungi/phases/osbs.py @@ -111,7 +111,7 @@ class OSBSThread(WorkerThread): ) scratch = config.get("scratch", False) - nvr = self._add_metadata(variant, task_id, compose, scratch) + nvr = add_metadata(variant, task_id, compose, scratch) if nvr: registry = get_registry(compose, nvr, registry) if registry: @@ -119,62 +119,6 @@ class OSBSThread(WorkerThread): self.pool.log_info("[DONE ] %s" % msg) - def _add_metadata(self, variant, task_id, compose, is_scratch): - # Create new Koji session. The task could take so long to finish that - # our session will expire. This second session does not need to be - # authenticated since it will only do reading operations. - koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"]) - - # Create metadata - metadata = { - "compose_id": compose.compose_id, - "koji_task": task_id, - } - - result = koji.koji_proxy.getTaskResult(task_id) - if is_scratch: - metadata.update({"repositories": result["repositories"]}) - # add a fake arch of 'scratch', so we can construct the metadata - # in same data structure as real builds. - compose.containers_metadata.setdefault(variant.uid, {}).setdefault( - "scratch", [] - ).append(metadata) - return None - - else: - build_id = int(result["koji_builds"][0]) - buildinfo = koji.koji_proxy.getBuild(build_id) - archives = koji.koji_proxy.listArchives(build_id) - - nvr = "%(name)s-%(version)s-%(release)s" % buildinfo - - metadata.update( - { - "name": buildinfo["name"], - "version": buildinfo["version"], - "release": buildinfo["release"], - "nvr": nvr, - "creation_time": buildinfo["creation_time"], - } - ) - for archive in archives: - data = { - "filename": archive["filename"], - "size": archive["size"], - "checksum": archive["checksum"], - } - data.update(archive["extra"]) - data.update(metadata) - arch = archive["extra"]["image"]["arch"] - self.pool.log_debug( - "Created Docker base image %s-%s-%s.%s" - % (metadata["name"], metadata["version"], metadata["release"], arch) - ) - compose.containers_metadata.setdefault(variant.uid, {}).setdefault( - arch, [] - ).append(data) - return nvr - def _get_repo(self, compose, repo, gpgkey=None): """ Return repo file URL of repo, if repo contains "://", it's already a @@ -221,3 +165,62 @@ class OSBSThread(WorkerThread): f.write("gpgkey=%s\n" % gpgkey) return util.translate_path(compose, repo_file) + + +def add_metadata(variant, task_id, compose, is_scratch): + """Given a task ID, find details about the container and add it to global + metadata.""" + # Create new Koji session. The task could take so long to finish that + # our session will expire. This second session does not need to be + # authenticated since it will only do reading operations. + koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"]) + + # Create metadata + metadata = { + "compose_id": compose.compose_id, + "koji_task": task_id, + } + + result = koji.koji_proxy.getTaskResult(task_id) + if is_scratch: + metadata.update({"repositories": result["repositories"]}) + # add a fake arch of 'scratch', so we can construct the metadata + # in same data structure as real builds. + compose.containers_metadata.setdefault(variant.uid, {}).setdefault( + "scratch", [] + ).append(metadata) + return None + + else: + build_id = int(result["koji_builds"][0]) + buildinfo = koji.koji_proxy.getBuild(build_id) + archives = koji.koji_proxy.listArchives(build_id) + + nvr = "%(name)s-%(version)s-%(release)s" % buildinfo + + metadata.update( + { + "name": buildinfo["name"], + "version": buildinfo["version"], + "release": buildinfo["release"], + "nvr": nvr, + "creation_time": buildinfo["creation_time"], + } + ) + for archive in archives: + data = { + "filename": archive["filename"], + "size": archive["size"], + "checksum": archive["checksum"], + } + data.update(archive["extra"]) + data.update(metadata) + arch = archive["extra"]["image"]["arch"] + compose.log_debug( + "Created Docker base image %s-%s-%s.%s" + % (metadata["name"], metadata["version"], metadata["release"], arch) + ) + compose.containers_metadata.setdefault(variant.uid, {}).setdefault( + arch, [] + ).append(data) + return nvr From 40133074b3123dc45ac8f638d8be937594c1073b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 5 Feb 2021 10:19:10 +0100 Subject: [PATCH 014/137] Add image-container phase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This phase runs after image-build and osbuild and can embed an image into a container. JIRA: RHELCMP-3820 Signed-off-by: Lubomír Sedlář --- doc/_static/phases.svg | 37 +++- doc/configuration.rst | 50 ++++++ doc/phases.rst | 16 +- pungi/checks.py | 20 +++ pungi/phases/__init__.py | 1 + pungi/phases/image_container.py | 120 +++++++++++++ pungi/scripts/pungi_koji.py | 7 +- tests/test_image_container_phase.py | 262 ++++++++++++++++++++++++++++ 8 files changed, 502 insertions(+), 11 deletions(-) create mode 100644 pungi/phases/image_container.py create mode 100644 tests/test_image_container_phase.py diff --git a/doc/_static/phases.svg b/doc/_static/phases.svg index 5083e3b7..b973798a 100644 --- a/doc/_static/phases.svg +++ b/doc/_static/phases.svg @@ -12,7 +12,7 @@ viewBox="0 0 610.46457 301.1662" id="svg2" version="1.1" - inkscape:version="1.0.1 (3bc2e813f5, 2020-09-07)" + inkscape:version="1.0.2 (e86c870879, 2021-01-15)" sodipodi:docname="phases.svg" inkscape:export-filename="/home/lsedlar/repos/pungi/doc/_static/phases.png" inkscape:export-xdpi="90" @@ -24,9 +24,9 @@ borderopacity="1.0" inkscape:pageopacity="1" inkscape:pageshadow="2" - inkscape:zoom="2.1213203" - inkscape:cx="276.65806" - inkscape:cy="189.24198" + inkscape:zoom="1.5" + inkscape:cx="9.4746397" + inkscape:cy="58.833855" inkscape:document-units="px" inkscape:current-layer="layer1" showgrid="false" @@ -70,7 +70,7 @@ image/svg+xml - + @@ -303,15 +303,15 @@ ImageChecksum + y="921.73846">ImageChecksum @@ -518,5 +518,24 @@ id="tspan301-5" style="font-size:12px;line-height:0">OSBuild + + ImageContainer diff --git a/doc/configuration.rst b/doc/configuration.rst index d1175baa..810af973 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1555,6 +1555,56 @@ OSBuild Composer for building images arch. +Image container +=============== + +This phase supports building containers in OSBS that embed an image created in +the same compose. This can be useful for delivering the image to users running +in containerized environments. + +Pungi will start a ``buildContainer`` task in Koji with configured source +repository. The ``Dockerfile`` can expect that a repo file will be injected +into the container that defines a repo named ``image-to-include``, and its +``baseurl`` will point to the image to include. It is possible to extract the +URL with a command like ``dnf config-manager --dump image-to-include | awk +'/baseurl =/{print $3}'``` + +**image_container** + (*dict*) -- configuration for building containers embedding an image. + + Format: ``{variant_uid_regex: [{...}]}``. + + The inner object will define a single container. These keys are required: + + * ``url``, ``target``, ``git_branch``. See OSBS section for definition of + these. + * ``image_spec`` -- (*object*) A string mapping of filters used to select + the image to embed. All images listed in metadata for the variant will be + processed. The keys of this filter are used to select metadata fields for + the image, and values are regular expression that need to match the + metadata value. + + The filter should match exactly one image. + + +Example config +-------------- +:: + + image_container = { + "^Server$": [{ + "url": "git://example.com/dockerfiles.git?#HEAD", + "target": "f24-container-candidate", + "git_branch": "f24", + "image_spec": { + "format": "qcow2", + "arch": "x86_64", + "path": ".*/guest-image-.*$", + } + }] + } + + OSTree Settings =============== diff --git a/doc/phases.rst b/doc/phases.rst index 2cb810a8..7ae5bcdc 100644 --- a/doc/phases.rst +++ b/doc/phases.rst @@ -115,16 +115,30 @@ ImageBuild This phase wraps up ``koji image-build``. It also updates the metadata ultimately responsible for ``images.json`` manifest. +OSBuild +------- + +Similarly to image build, this phases creates a koji `osbuild` task. In the +background it uses OSBuild Composer to create images. + OSBS ---- -This phase builds docker base images in `OSBS +This phase builds container base images in `OSBS `_. The finished images are available in registry provided by OSBS, but not downloaded directly into the compose. The is metadata about the created image in ``compose/metadata/osbs.json``. +ImageContainer +-------------- + +This phase builds a container image in OSBS, and stores the metadata in the +same file as OSBS phase. The container produced here wraps a different image, +created it ImageBuild or OSBuild phase. It can be useful to deliver a VM image +to containerized environments. + OSTreeInstaller --------------- diff --git a/pungi/checks.py b/pungi/checks.py index 94a40b77..cfefdfb2 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -1213,6 +1213,26 @@ def make_schema(): }, "additionalProperties": False, }, + "image_container": { + "type": "object", + "patternProperties": { + ".+": _one_or_list( + { + "type": "object", + "properties": { + "url": {"type": "url"}, + "target": {"type": "string"}, + "priority": {"type": "number"}, + "failable": {"type": "boolean"}, + "git_branch": {"type": "string"}, + "image_spec": {"type": "object"}, + }, + "required": ["url", "target", "git_branch", "image_spec"], + } + ), + }, + "additionalProperties": False, + }, "extra_files": _variant_arch_mapping( { "type": "array", diff --git a/pungi/phases/__init__.py b/pungi/phases/__init__.py index 7b28e4e5..3e124548 100644 --- a/pungi/phases/__init__.py +++ b/pungi/phases/__init__.py @@ -27,6 +27,7 @@ from .createiso import CreateisoPhase # noqa from .extra_isos import ExtraIsosPhase # noqa from .live_images import LiveImagesPhase # noqa from .image_build import ImageBuildPhase # noqa +from .image_container import ImageContainerPhase # noqa from .osbuild import OSBuildPhase # noqa from .repoclosure import RepoclosurePhase # noqa from .test import TestPhase # noqa diff --git a/pungi/phases/image_container.py b/pungi/phases/image_container.py new file mode 100644 index 00000000..cb72161f --- /dev/null +++ b/pungi/phases/image_container.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- + +import os +import re +from kobo.threads import ThreadPool, WorkerThread + +from .base import ConfigGuardedPhase, PhaseLoggerMixin +from .. import util +from ..wrappers import kojiwrapper +from ..phases.osbs import add_metadata + + +class ImageContainerPhase(PhaseLoggerMixin, ConfigGuardedPhase): + name = "image_container" + + def __init__(self, compose): + super(ImageContainerPhase, self).__init__(compose) + self.pool = ThreadPool(logger=self.logger) + self.pool.metadata = {} + + def run(self): + for variant in self.compose.get_variants(): + for conf in self.get_config_block(variant): + self.pool.add(ImageContainerThread(self.pool)) + self.pool.queue_put((self.compose, variant, conf)) + + self.pool.start() + + +class ImageContainerThread(WorkerThread): + def process(self, item, num): + compose, variant, config = item + self.num = num + with util.failable( + compose, + bool(config.pop("failable", None)), + variant, + "*", + "osbs", + logger=self.pool._logger, + ): + self.worker(compose, variant, config) + + def worker(self, compose, variant, config): + msg = "Image container task for variant %s" % variant.uid + self.pool.log_info("[BEGIN] %s" % msg) + + source = config.pop("url") + target = config.pop("target") + priority = config.pop("priority", None) + + config["yum_repourls"] = [ + self._get_repo( + compose, + variant, + config.get("arch_override", "").split(" "), + config.pop("image_spec"), + ) + ] + + # Start task + koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"]) + koji.login() + task_id = koji.koji_proxy.buildContainer( + source, target, config, priority=priority + ) + + # Wait for it to finish and capture the output into log file (even + # though there is not much there). + log_dir = os.path.join(compose.paths.log.topdir(), "image_container") + util.makedirs(log_dir) + log_file = os.path.join( + log_dir, "%s-%s-watch-task.log" % (variant.uid, self.num) + ) + if koji.watch_task(task_id, log_file) != 0: + raise RuntimeError( + "ImageContainer: task %s failed: see %s for details" + % (task_id, log_file) + ) + + add_metadata(variant, task_id, compose, config.get("scratch", False)) + + self.pool.log_info("[DONE ] %s" % msg) + + def _get_repo(self, compose, variant, arches, image_spec): + """ + Return a repo file that points baseurl to the image specified by + image_spec. + """ + image_paths = set() + + for arch in arches or compose.im.images[variant.uid].keys(): + for image in compose.im.images[variant.uid].get(arch, []): + for key, value in image_spec.items(): + if not re.match(value, getattr(image, key)): + break + else: + image_paths.add(image.path) + + if len(image_paths) != 1: + raise RuntimeError( + "%d images matched specification. Only one was expected." + % len(image_paths) + ) + + image_path = image_paths.pop() + absolute_path = os.path.join(compose.paths.compose.topdir(), image_path) + + repo_file = os.path.join( + compose.paths.work.tmp_dir(None, variant), + "image-container-%s-%s.repo" % (variant, self.num), + ) + with open(repo_file, "w") as f: + f.write("[image-to-include]\n") + f.write("name=Location of image to embed\n") + f.write("baseurl=%s\n" % util.translate_path(compose, absolute_path)) + f.write("enabled=0\n") + f.write("gpgcheck=0\n") + + return util.translate_path(compose, repo_file) diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index d0e53bb8..54763e3b 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -394,6 +394,7 @@ def run_compose( image_build_phase = pungi.phases.ImageBuildPhase(compose) osbuild_phase = pungi.phases.OSBuildPhase(compose) osbs_phase = pungi.phases.OSBSPhase(compose) + image_container_phase = pungi.phases.ImageContainerPhase(compose) image_checksum_phase = pungi.phases.ImageChecksumPhase(compose) repoclosure_phase = pungi.phases.RepoclosurePhase(compose) test_phase = pungi.phases.TestPhase(compose) @@ -417,6 +418,7 @@ def run_compose( extra_isos_phase, osbs_phase, osbuild_phase, + image_container_phase, ): if phase.skip(): continue @@ -516,9 +518,12 @@ def run_compose( livemedia_phase, osbuild_phase, ) + post_image_phase = pungi.phases.WeaverPhase( + compose, (image_checksum_phase, image_container_phase) + ) compose_images_phase = pungi.phases.WeaverPhase(compose, compose_images_schema) extra_phase_schema = ( - (compose_images_phase, image_checksum_phase), + (compose_images_phase, post_image_phase), osbs_phase, repoclosure_phase, ) diff --git a/tests/test_image_container_phase.py b/tests/test_image_container_phase.py new file mode 100644 index 00000000..65745b2d --- /dev/null +++ b/tests/test_image_container_phase.py @@ -0,0 +1,262 @@ +# -*- coding: utf-8 -*- + +import mock + +import os + +from tests import helpers +from pungi import checks +from pungi.phases import image_container + + +class ImageContainerPhaseTest(helpers.PungiTestCase): + @mock.patch("pungi.phases.image_container.ThreadPool") + def test_run(self, ThreadPool): + cfg = helpers.IterableMock() + compose = helpers.DummyCompose( + self.topdir, {"image_container": {"^Everything$": cfg}} + ) + + pool = ThreadPool.return_value + + phase = image_container.ImageContainerPhase(compose) + phase.run() + + self.assertEqual(len(pool.add.call_args_list), 1) + self.assertEqual( + pool.queue_put.call_args_list, + [mock.call((compose, compose.variants["Everything"], cfg))], + ) + + @mock.patch("pungi.phases.image_container.ThreadPool") + def test_skip_without_config(self, ThreadPool): + compose = helpers.DummyCompose(self.topdir, {}) + compose.just_phases = None + compose.skip_phases = [] + phase = image_container.ImageContainerPhase(compose) + self.assertTrue(phase.skip()) + + +class ImageContainerConfigTest(helpers.PungiTestCase): + def assertConfigMissing(self, cfg, key): + conf = helpers.load_config( + helpers.PKGSET_REPOS, **{"image_container": {"^Server$": cfg}} + ) + errors, warnings = checks.validate(conf, offline=True) + self.assertIn( + "Failed validation in image_container.^Server$: %r is not valid under any of the given schemas" # noqa: E501 + % cfg, + errors, + ) + self.assertIn(" Possible reason: %r is a required property" % key, errors) + self.assertEqual([], warnings) + + def test_correct(self): + conf = helpers.load_config( + helpers.PKGSET_REPOS, + **{ + "image_container": { + "^Server$": [ + { + "url": "http://example.com/repo.git#HEAD", + "target": "container-candidate", + "git_branch": "main", + "image_spec": {"type": "qcow2"}, + } + ] + } + } + ) + errors, warnings = checks.validate(conf, offline=True) + self.assertEqual([], errors) + self.assertEqual([], warnings) + + def test_missing_url(self): + self.assertConfigMissing( + { + "target": "container-candidate", + "git_branch": "main", + "image_spec": {"type": "qcow2"}, + }, + "url", + ) + + def test_missing_target(self): + self.assertConfigMissing( + { + "url": "http://example.com/repo.git#HEAD", + "git_branch": "main", + "image_spec": {"type": "qcow2"}, + }, + "target", + ) + + def test_missing_git_branch(self): + self.assertConfigMissing( + { + "url": "http://example.com/repo.git#HEAD", + "target": "container-candidate", + "image_spec": {"type": "qcow2"}, + }, + "git_branch", + ) + + def test_missing_image_spec(self): + self.assertConfigMissing( + { + "url": "http://example.com/repo.git#HEAD", + "target": "container-candidate", + "git_branch": "main", + }, + "image_spec", + ) + + +class ImageContainerThreadTest(helpers.PungiTestCase): + def setUp(self): + super(ImageContainerThreadTest, self).setUp() + self.pool = mock.Mock() + self.repofile_path = "work/global/tmp-Server/image-container-Server-1.repo" + self.t = image_container.ImageContainerThread(self.pool) + self.compose = helpers.DummyCompose( + self.topdir, + { + "koji_profile": "koji", + "translate_paths": [(self.topdir, "http://root")], + }, + ) + self.cfg = { + "url": "git://example.com/repo?#BEEFCAFE", + "target": "f24-docker-candidate", + "git_branch": "f24-docker", + "image_spec": {"type": "qcow2"}, + "arch_override": "x86_64", + } + self.compose.im.images["Server"] = { + "x86_64": [ + mock.Mock(path="Server/x86_64/iso/image.iso", type="iso"), + mock.Mock(path="Server/x86_64/images/image.qcow2", type="qcow2"), + ] + } + + def _setupMock(self, KojiWrapper): + self.wrapper = KojiWrapper.return_value + self.wrapper.koji_proxy.buildContainer.return_value = 12345 + self.wrapper.watch_task.return_value = 0 + + def assertRepoFile(self): + repofile = os.path.join(self.topdir, self.repofile_path) + with open(repofile) as f: + repo_content = list(f) + self.assertIn("[image-to-include]\n", repo_content) + self.assertIn( + "baseurl=http://root/compose/Server/x86_64/images/image.qcow2\n", + repo_content, + ) + self.assertIn("enabled=0\n", repo_content) + + def assertKojiCalls(self, cfg, scratch=False): + opts = { + "git_branch": cfg["git_branch"], + "arch_override": cfg["arch_override"], + "yum_repourls": ["http://root/" + self.repofile_path], + } + if scratch: + opts["scratch"] = True + self.assertEqual( + self.wrapper.mock_calls, + [ + mock.call.login(), + mock.call.koji_proxy.buildContainer( + cfg["url"], cfg["target"], opts, priority=None, + ), + mock.call.watch_task( + 12345, + os.path.join( + self.topdir, + "logs/global/image_container/Server-1-watch-task.log", + ), + ), + ], + ) + + @mock.patch("pungi.phases.image_container.add_metadata") + @mock.patch("pungi.phases.image_container.kojiwrapper.KojiWrapper") + def test_success(self, KojiWrapper, add_metadata): + self._setupMock(KojiWrapper) + + self.t.process( + (self.compose, self.compose.variants["Server"], self.cfg.copy()), 1 + ) + + self.assertRepoFile() + self.assertKojiCalls(self.cfg) + self.assertEqual( + add_metadata.call_args_list, + [mock.call(self.compose.variants["Server"], 12345, self.compose, False)], + ) + + @mock.patch("pungi.phases.image_container.add_metadata") + @mock.patch("pungi.phases.image_container.kojiwrapper.KojiWrapper") + def test_scratch_build(self, KojiWrapper, add_metadata): + self.cfg["scratch"] = True + self._setupMock(KojiWrapper) + + self.t.process( + (self.compose, self.compose.variants["Server"], self.cfg.copy()), 1 + ) + + self.assertRepoFile() + self.assertKojiCalls(self.cfg, scratch=True) + self.assertEqual( + add_metadata.call_args_list, + [mock.call(self.compose.variants["Server"], 12345, self.compose, True)], + ) + + @mock.patch("pungi.phases.image_container.add_metadata") + @mock.patch("pungi.phases.image_container.kojiwrapper.KojiWrapper") + def test_task_fail(self, KojiWrapper, add_metadata): + self._setupMock(KojiWrapper) + self.wrapper.watch_task.return_value = 1 + + with self.assertRaises(RuntimeError) as ctx: + self.t.process( + (self.compose, self.compose.variants["Server"], self.cfg.copy()), 1 + ) + + self.assertRegex(str(ctx.exception), r"task 12345 failed: see .+ for details") + self.assertRepoFile() + self.assertKojiCalls(self.cfg) + self.assertEqual(add_metadata.call_args_list, []) + + @mock.patch("pungi.phases.image_container.add_metadata") + @mock.patch("pungi.phases.image_container.kojiwrapper.KojiWrapper") + def test_task_fail_failable(self, KojiWrapper, add_metadata): + self.cfg["failable"] = "*" + self._setupMock(KojiWrapper) + self.wrapper.watch_task.return_value = 1 + + self.t.process( + (self.compose, self.compose.variants["Server"], self.cfg.copy()), 1 + ) + + self.assertRepoFile() + self.assertKojiCalls(self.cfg) + self.assertEqual(add_metadata.call_args_list, []) + + @mock.patch("pungi.phases.image_container.add_metadata") + @mock.patch("pungi.phases.image_container.kojiwrapper.KojiWrapper") + def test_non_unique_spec(self, KojiWrapper, add_metadata): + self.cfg["image_spec"] = {"path": ".*/image\\..*"} + self._setupMock(KojiWrapper) + + with self.assertRaises(RuntimeError) as ctx: + self.t.process( + (self.compose, self.compose.variants["Server"], self.cfg.copy()), 1 + ) + + self.assertRegex( + str(ctx.exception), "2 images matched specification. Only one was expected." + ) + self.assertEqual(self.wrapper.mock_calls, []) + self.assertEqual(add_metadata.call_args_list, []) From 64897d7d4840d9289aac44906bf1b198d69aa33f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 11 Feb 2021 15:22:18 +0100 Subject: [PATCH 015/137] pkgset: Add ability to wait for signed packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If packages are appearing quickly in Koji, and signing them is triggered by automation, there may be a delay between the package being signed and compose running. In such case it may be preferable to wait for the signed copy rather than fail the compose. JIRA: RHELCMP-3932 Signed-off-by: Lubomír Sedlář --- doc/configuration.rst | 11 +++ pungi/checks.py | 2 + pungi/phases/pkgset/pkgsets.py | 41 +++++++++--- pungi/phases/pkgset/sources/source_koji.py | 2 + tests/test_pkgset_pkgsets.py | 78 ++++++++++++++++++++++ 5 files changed, 123 insertions(+), 11 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 810af973..528e51f6 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -581,6 +581,17 @@ Options (for example) between composes, then Pungi may not respect those changes in your new compose. +**signed_packages_retries** = 1 + (*int*) -- In automated workflows a compose may start before signed + packages are written to disk. In such case it may make sense to wait for + the package to appear on storage. This option controls how many times to + try to look for the signed copy. + +**signed_packages_wait** = 30 + (*int*) -- Interval in seconds for how long to wait between attemts to find + signed packages. This option only makes sense when + ``signed_packages_retries`` is set higher than to 1. + Example ------- diff --git a/pungi/checks.py b/pungi/checks.py index cfefdfb2..c8b73421 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -722,6 +722,8 @@ def make_schema(): "minItems": 1, "default": [None], }, + "signed_packages_retries": {"type": "number", "default": 1}, + "signed_packages_wait": {"type": "number", "default": 30}, "variants_file": {"$ref": "#/definitions/str_or_scm_dict"}, "comps_file": {"$ref": "#/definitions/str_or_scm_dict"}, "comps_filter_environments": {"type": "boolean", "default": True}, diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index d4320865..e10a79bf 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -22,6 +22,7 @@ It automatically finds a signed copies according to *sigkey_ordering*. import itertools import json import os +import time from six.moves import cPickle as pickle import kobo.log @@ -332,6 +333,8 @@ class KojiPackageSet(PackageSetBase): cache_region=None, extra_builds=None, extra_tasks=None, + signed_packages_retries=1, + signed_packages_wait=30, ): """ Creates new KojiPackageSet. @@ -364,6 +367,9 @@ class KojiPackageSet(PackageSetBase): :param list extra_tasks: Extra RPMs defined as Koji task IDs to get from Koji and include in the package set. Useful when building testing compose with RPM scratch builds. + :param int signed_packages_retries: How many times should a search for + signed package be repeated. + :param int signed_packages_wait: How long to wait between search attemts. """ super(KojiPackageSet, self).__init__( name, @@ -380,6 +386,8 @@ class KojiPackageSet(PackageSetBase): self.extra_builds = extra_builds or [] self.extra_tasks = extra_tasks or [] self.reuse = None + self.signed_packages_retries = signed_packages_retries + self.signed_packages_wait = signed_packages_wait def __getstate__(self): result = self.__dict__.copy() @@ -506,17 +514,28 @@ class KojiPackageSet(PackageSetBase): pathinfo = self.koji_wrapper.koji_module.pathinfo paths = [] - for sigkey in self.sigkey_ordering: - if not sigkey: - # we're looking for *signed* copies here - continue - sigkey = sigkey.lower() - rpm_path = os.path.join( - pathinfo.build(build_info), pathinfo.signed(rpm_info, sigkey) - ) - paths.append(rpm_path) - if os.path.isfile(rpm_path): - return rpm_path + + retries = self.signed_packages_retries + while retries > 0: + for sigkey in self.sigkey_ordering: + if not sigkey: + # we're looking for *signed* copies here + continue + sigkey = sigkey.lower() + rpm_path = os.path.join( + pathinfo.build(build_info), pathinfo.signed(rpm_info, sigkey) + ) + if rpm_path not in paths: + paths.append(rpm_path) + if os.path.isfile(rpm_path): + return rpm_path + + # No signed copy was found, wait a little and try again. + retries -= 1 + if retries > 0: + nvr = "%(name)s-%(version)s-%(release)s" % rpm_info + self.log_debug("Waiting for signed package to appear for %s", nvr) + time.sleep(self.signed_packages_wait) if None in self.sigkey_ordering or "" in self.sigkey_ordering: # use an unsigned copy (if allowed) diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index c14d4eb2..bb574064 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -811,6 +811,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): cache_region=compose.cache_region, extra_builds=extra_builds, extra_tasks=extra_tasks, + signed_packages_retries=compose.conf["signed_packages_retries"], + signed_packages_wait=compose.conf["signed_packages_wait"], ) # Check if we have cache for this tag from previous compose. If so, use diff --git a/tests/test_pkgset_pkgsets.py b/tests/test_pkgset_pkgsets.py index 555a313b..a329d373 100644 --- a/tests/test_pkgset_pkgsets.py +++ b/tests/test_pkgset_pkgsets.py @@ -303,6 +303,58 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase): ) self.assertRegex(str(ctx.exception), figure) + @mock.patch("os.path.isfile") + @mock.patch("time.sleep") + def test_find_signed_after_wait(self, sleep, isfile): + checked_files = set() + + def check_file(path): + """First check for any path will fail, second and further will succeed.""" + if path in checked_files: + return True + checked_files.add(path) + return False + + isfile.side_effect = check_file + + fst_key, snd_key = ["cafebabe", "deadbeef"] + pkgset = pkgsets.KojiPackageSet( + "pkgset", + self.koji_wrapper, + [fst_key, snd_key], + arches=["x86_64"], + signed_packages_retries=3, + signed_packages_wait=5, + ) + + result = pkgset.populate("f25") + + self.assertEqual( + self.koji_wrapper.koji_proxy.mock_calls, + [mock.call.listTaggedRPMS("f25", event=None, inherit=True, latest=True)], + ) + + fst_pkg = "signed/%s/bash-debuginfo@4.3.42@4.fc24@x86_64" + snd_pkg = "signed/%s/bash@4.3.42@4.fc24@x86_64" + + self.assertPkgsetEqual( + result, {"x86_64": [fst_pkg % "cafebabe", snd_pkg % "cafebabe"]} + ) + # Wait once for each of the two packages + self.assertEqual(sleep.call_args_list, [mock.call(5)] * 2) + # Each file will be checked three times + self.assertEqual( + isfile.call_args_list, + [ + mock.call(os.path.join(self.topdir, fst_pkg % fst_key)), + mock.call(os.path.join(self.topdir, fst_pkg % snd_key)), + mock.call(os.path.join(self.topdir, fst_pkg % fst_key)), + mock.call(os.path.join(self.topdir, snd_pkg % fst_key)), + mock.call(os.path.join(self.topdir, snd_pkg % snd_key)), + mock.call(os.path.join(self.topdir, snd_pkg % fst_key)), + ], + ) + def test_can_not_find_signed_package_allow_invalid_sigkeys(self): pkgset = pkgsets.KojiPackageSet( "pkgset", @@ -346,6 +398,32 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase): r"^RPM\(s\) not found for sigs: .+Check log for details.+", ) + @mock.patch("time.sleep") + def test_can_not_find_signed_package_with_retries(self, time): + pkgset = pkgsets.KojiPackageSet( + "pkgset", + self.koji_wrapper, + ["cafebabe"], + arches=["x86_64"], + signed_packages_retries=3, + signed_packages_wait=5, + ) + + with self.assertRaises(RuntimeError) as ctx: + pkgset.populate("f25") + + self.assertEqual( + self.koji_wrapper.koji_proxy.mock_calls, + [mock.call.listTaggedRPMS("f25", event=None, inherit=True, latest=True)], + ) + + self.assertRegex( + str(ctx.exception), + r"^RPM\(s\) not found for sigs: .+Check log for details.+", + ) + # Two packages making three attempts each, so two waits per package. + self.assertEqual(time.call_args_list, [mock.call(5)] * 4) + def test_packages_attribute(self): self._touch_files( [ From 98359654cfdc61ca7166dc348ea111d7fd7d0a3f Mon Sep 17 00:00:00 2001 From: Ondrej Nosek Date: Fri, 12 Feb 2021 15:20:40 +0100 Subject: [PATCH 016/137] 4.2.8 release Signed-off-by: Ondrej Nosek --- doc/conf.py | 2 +- pungi.spec | 20 +++++++++++++++++++- setup.py | 2 +- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 6aa7e512..845600fc 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -53,7 +53,7 @@ copyright = u'2016, Red Hat, Inc.' # The short X.Y version. version = '4.2' # The full version, including alpha/beta/rc tags. -release = '4.2.7' +release = '4.2.8' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pungi.spec b/pungi.spec index 6d0e58d5..22b53512 100644 --- a/pungi.spec +++ b/pungi.spec @@ -1,5 +1,5 @@ Name: pungi -Version: 4.2.7 +Version: 4.2.8 Release: 1%{?dist} Summary: Distribution compose tool @@ -111,6 +111,24 @@ pytest cd tests && ./test_compose.sh %changelog +* Fri Feb 12 2021 Ondrej Nosek +- pkgset: Add ability to wait for signed packages (lsedlar) +- Add image-container phase (lsedlar) +- osbs: Move metadata processing to standalone function (lsedlar) +- Move container metadata into compose object (lsedlar) +- Move UnsignedPackagesError to a separate file (lsedlar) +- pkgset: Include just one version of module (hlin) +- pkgset: Check tag inheritance change before reuse (hlin) +- pkgset: Remove reuse file when packages are not signed (lsedlar) +- pkgset: Drop kobo.plugin usage from PkgsetSource (lsedlar) +- gather: Drop kobo.plugins usage from GatherMethod (lsedlar) +- pkgset: Drop kobo.plugins usage from GatherSources (lsedlar) +- doc: remove default createrepo_checksum value from example (kdreyer) +- comps: Preserve default arg on groupid (lsedlar) +- Stop copying .git directory with module defaults (hlin) +- React to SIGINT signal (hlin) +- scm: Only copy debugging data if we have a compose (lsedlar) + * Thu Dec 03 2020 Lubomír Sedlář 4.2.7-1 - osbuild: Fix not failing on failable tasks (lsedlar) - kojiwrapper: Use gssapi_login (lsedlar) diff --git a/setup.py b/setup.py index 054102dd..072811bf 100755 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ packages = sorted(packages) setup( name="pungi", - version="4.2.7", + version="4.2.8", description="Distribution compose tool", url="https://pagure.io/pungi", author="Dennis Gilmore", From 477dcf37d9c23da91514df66cd51670d112fd69c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 17 Feb 2021 10:52:08 +0100 Subject: [PATCH 017/137] Store extended traceback for gather errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a gathering thread raises an exception, it gets forwarded to the main thread and re-raised there. However, during this transition it loses details about exact location of the problem. This patch creates an extended traceback in the worker, which should make it easier to track the problem down later. JIRA: RHELCMP-4259 Signed-off-by: Lubomír Sedlář --- pungi/compose.py | 15 +++++++++++++++ pungi/phases/gather/__init__.py | 4 ++++ pungi/scripts/pungi_koji.py | 7 +------ tests/test_compose.py | 30 ++++++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 6 deletions(-) diff --git a/pungi/compose.py b/pungi/compose.py index ec9c5640..f1229439 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -26,6 +26,7 @@ import shutil import json import kobo.log +import kobo.tback from productmd.composeinfo import ComposeInfo from productmd.images import Images from dogpile.cache import make_region @@ -590,6 +591,20 @@ class Compose(kobo.log.LoggingBase): separators=(",", ": "), ) + def traceback(self, detail=None): + """Store an extended traceback. This method should only be called when + handling an exception. + + :param str detail: Extra information appended to the filename + """ + basename = "traceback" + if detail: + basename += "-" + detail + tb_path = self.paths.log.log_file("global", basename) + self.log_error("Extended traceback in: %s", tb_path) + with open(tb_path, "wb") as f: + f.write(kobo.tback.Traceback().get_traceback()) + def get_ordered_variant_uids(compose): if not hasattr(compose, "_ordered_variant_uids"): diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index b559cfbc..9338d5da 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -730,6 +730,10 @@ def _gather_variants( try: que.put((arch, gather_packages(*args, **kwargs))) except Exception as exc: + compose.log_error( + "Error in gathering for %s.%s: %s", variant, arch, exc + ) + compose.traceback("gather-%s-%s" % (variant, arch)) errors.put(exc) # Run gather_packages() in parallel with multi threads and store diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index 54763e3b..dd568128 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -637,15 +637,10 @@ def cli_main(): main() except (Exception, KeyboardInterrupt) as ex: if COMPOSE: - tb_path = COMPOSE.paths.log.log_file("global", "traceback") COMPOSE.log_error("Compose run failed: %s" % ex) - COMPOSE.log_error("Extended traceback in: %s" % tb_path) + COMPOSE.traceback() COMPOSE.log_critical("Compose failed: %s" % COMPOSE.topdir) COMPOSE.write_status("DOOMED") - import kobo.tback - - with open(tb_path, "wb") as f: - f.write(kobo.tback.Traceback().get_traceback()) else: print("Exception: %s" % ex) raise diff --git a/tests/test_compose.py b/tests/test_compose.py index 628973bd..94940909 100644 --- a/tests/test_compose.py +++ b/tests/test_compose.py @@ -777,3 +777,33 @@ class DumpContainerMetadataTest(unittest.TestCase): def test_dump_empty_metadata(self, ThreadPool): self.compose.dump_containers_metadata() self.assertFalse(os.path.isfile(self.tmp_dir + "/compose/metadata/osbs.json")) + + +class TracebackTest(unittest.TestCase): + def setUp(self): + self.tmp_dir = tempfile.mkdtemp() + with mock.patch("pungi.compose.ComposeInfo"): + self.compose = Compose({}, self.tmp_dir) + self.patcher = mock.patch("kobo.tback.Traceback") + self.Traceback = self.patcher.start() + self.Traceback.return_value.get_traceback.return_value = b"traceback" + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + self.patcher.stop() + + def assertTraceback(self, filename): + self.assertTrue( + os.path.isfile("%s/logs/global/%s.global.log" % (self.tmp_dir, filename)) + ) + self.assertEqual( + self.Traceback.mock_calls, [mock.call(), mock.call().get_traceback()] + ) + + def test_traceback_default(self): + self.compose.traceback() + self.assertTraceback("traceback") + + def test_with_detail(self): + self.compose.traceback("extra-info") + self.assertTraceback("traceback-extra-info") From 5b5069175d7f1dc1f613a958108f575f1f7a8211 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Mon, 22 Feb 2021 10:21:56 +0100 Subject: [PATCH 018/137] pkgset: Store module tag only if module is used MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a module is skipped from the compose, we should not add it to a mapping of module tags. If it's there, we then spend time building a repo for the module, and it get's passed to buildinstall, despite the packages not being supposed to be included in the compose. If the packages are not included in any variant, they shouldn't be available to buildinstall either. Signed-off-by: Lubomír Sedlář --- pungi/phases/pkgset/sources/source_koji.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index bb574064..f683b4ae 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -615,14 +615,6 @@ def _get_modules_from_koji_tags( for build in latest_builds: # Get the Build from Koji to get modulemd and module_tag. build = koji_proxy.getBuild(build["build_id"]) - module_tag = ( - build.get("extra", {}) - .get("typeinfo", {}) - .get("module", {}) - .get("content_koji_tag", "") - ) - - variant_tags[variant].append(module_tag) nsvc = _add_module_to_variant( koji_wrapper, @@ -635,6 +627,14 @@ def _get_modules_from_koji_tags( if not nsvc: continue + module_tag = ( + build.get("extra", {}) + .get("typeinfo", {}) + .get("module", {}) + .get("content_koji_tag", "") + ) + variant_tags[variant].append(module_tag) + tag_to_mmd.setdefault(module_tag, {}) for arch in variant.arch_mmds: try: From 735bfaa0d6338128a4b0f95acf333aa63a706fa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Tue, 23 Feb 2021 13:52:15 +0100 Subject: [PATCH 019/137] pkgset: Fix meaning of retries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The name brings a different expectation than how it actually worked. This patch makes the code work similarly to the expectation. Signed-off-by: Lubomír Sedlář --- doc/configuration.rst | 6 +++--- pungi/checks.py | 2 +- pungi/phases/pkgset/pkgsets.py | 10 +++++----- tests/test_pkgset_pkgsets.py | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 528e51f6..2fd50c63 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -581,16 +581,16 @@ Options (for example) between composes, then Pungi may not respect those changes in your new compose. -**signed_packages_retries** = 1 +**signed_packages_retries** = 0 (*int*) -- In automated workflows a compose may start before signed packages are written to disk. In such case it may make sense to wait for the package to appear on storage. This option controls how many times to - try to look for the signed copy. + retry looking for the signed copy. **signed_packages_wait** = 30 (*int*) -- Interval in seconds for how long to wait between attemts to find signed packages. This option only makes sense when - ``signed_packages_retries`` is set higher than to 1. + ``signed_packages_retries`` is set higher than to 0. Example diff --git a/pungi/checks.py b/pungi/checks.py index c8b73421..336a3378 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -722,7 +722,7 @@ def make_schema(): "minItems": 1, "default": [None], }, - "signed_packages_retries": {"type": "number", "default": 1}, + "signed_packages_retries": {"type": "number", "default": 0}, "signed_packages_wait": {"type": "number", "default": 30}, "variants_file": {"$ref": "#/definitions/str_or_scm_dict"}, "comps_file": {"$ref": "#/definitions/str_or_scm_dict"}, diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index e10a79bf..4b9c3fe8 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -333,7 +333,7 @@ class KojiPackageSet(PackageSetBase): cache_region=None, extra_builds=None, extra_tasks=None, - signed_packages_retries=1, + signed_packages_retries=0, signed_packages_wait=30, ): """ @@ -515,8 +515,8 @@ class KojiPackageSet(PackageSetBase): pathinfo = self.koji_wrapper.koji_module.pathinfo paths = [] - retries = self.signed_packages_retries - while retries > 0: + attempts_left = self.signed_packages_retries + 1 + while attempts_left > 0: for sigkey in self.sigkey_ordering: if not sigkey: # we're looking for *signed* copies here @@ -531,8 +531,8 @@ class KojiPackageSet(PackageSetBase): return rpm_path # No signed copy was found, wait a little and try again. - retries -= 1 - if retries > 0: + attempts_left -= 1 + if attempts_left > 0: nvr = "%(name)s-%(version)s-%(release)s" % rpm_info self.log_debug("Waiting for signed package to appear for %s", nvr) time.sleep(self.signed_packages_wait) diff --git a/tests/test_pkgset_pkgsets.py b/tests/test_pkgset_pkgsets.py index a329d373..4436d23a 100644 --- a/tests/test_pkgset_pkgsets.py +++ b/tests/test_pkgset_pkgsets.py @@ -323,7 +323,7 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase): self.koji_wrapper, [fst_key, snd_key], arches=["x86_64"], - signed_packages_retries=3, + signed_packages_retries=2, signed_packages_wait=5, ) @@ -405,7 +405,7 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase): self.koji_wrapper, ["cafebabe"], arches=["x86_64"], - signed_packages_retries=3, + signed_packages_retries=2, signed_packages_wait=5, ) From b217470464bf11b76010396f8f2164a8cf99259f Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Tue, 2 Mar 2021 18:19:05 +0800 Subject: [PATCH 020/137] Format code Code didn't get well formatted when jenkins unusable. Signed-off-by: Haibo Lin --- pungi/arch_utils.py | 12 +-- pungi/checks.py | 3 +- pungi/gather.py | 34 ++++----- pungi/graph.py | 3 +- pungi/phases/osbuild.py | 7 +- pungi/scripts/create_unified_isos.py | 5 +- pungi/scripts/pungi_gather.py | 19 +++-- pungi/util.py | 6 +- pungi/wrappers/comps.py | 5 +- pungi/wrappers/fus.py | 7 +- pungi/wrappers/jigdo.py | 2 +- pungi/wrappers/kojiwrapper.py | 17 ++++- pungi_utils/orchestrator.py | 3 +- tests/helpers.py | 10 ++- tests/test_buildinstall.py | 13 +++- tests/test_config.py | 109 +++++++++++++++++++++------ tests/test_createrepophase.py | 4 +- tests/test_extra_isos_phase.py | 7 +- tests/test_fus_wrapper.py | 3 +- tests/test_gather.py | 4 +- tests/test_gather_method_hybrid.py | 6 +- tests/test_image_container_phase.py | 5 +- tests/test_koji_wrapper.py | 4 +- tests/test_osbs_phase.py | 3 +- tests/test_runroot.py | 3 +- tests/test_test_phase.py | 3 +- tests/test_util.py | 6 +- 27 files changed, 214 insertions(+), 89 deletions(-) diff --git a/pungi/arch_utils.py b/pungi/arch_utils.py index c78082f0..d01eccd2 100644 --- a/pungi/arch_utils.py +++ b/pungi/arch_utils.py @@ -131,8 +131,8 @@ def getArchList(thisarch=None): # pragma: no cover def _try_read_cpuinfo(): # pragma: no cover - """ Try to read /proc/cpuinfo ... if we can't ignore errors (ie. proc not - mounted). """ + """Try to read /proc/cpuinfo ... if we can't ignore errors (ie. proc not + mounted).""" try: with open("/proc/cpuinfo", "r") as f: return f.readlines() @@ -141,8 +141,8 @@ def _try_read_cpuinfo(): # pragma: no cover def _parse_auxv(): # pragma: no cover - """ Read /proc/self/auxv and parse it into global dict for easier access - later on, very similar to what rpm does. """ + """Read /proc/self/auxv and parse it into global dict for easier access + later on, very similar to what rpm does.""" # In case we can't open and read /proc/self/auxv, just return try: with open("/proc/self/auxv", "rb") as f: @@ -326,8 +326,8 @@ def getMultiArchInfo(arch=canonArch): # pragma: no cover def getBaseArch(myarch=None): # pragma: no cover """returns 'base' arch for myarch, if specified, or canonArch if not. - base arch is the arch before noarch in the arches dict if myarch is not - a key in the multilibArches.""" + base arch is the arch before noarch in the arches dict if myarch is not + a key in the multilibArches.""" if not myarch: myarch = canonArch diff --git a/pungi/checks.py b/pungi/checks.py index 336a3378..d4a8ed26 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -75,8 +75,7 @@ def is_isohybrid_needed(conf): def is_genisoimage_needed(conf): - """This is only needed locally for createiso without runroot. - """ + """This is only needed locally for createiso without runroot.""" runroot_tag = conf.get("runroot_tag", "") if runroot_tag or conf.get("createiso_use_xorrisofs"): return False diff --git a/pungi/gather.py b/pungi/gather.py index 2c4f7a80..3411e0c3 100644 --- a/pungi/gather.py +++ b/pungi/gather.py @@ -519,7 +519,7 @@ class Pungi(PungiBase): def verifyCachePkg(self, po, path): # Stolen from yum """check the package checksum vs the cache - return True if pkg is good, False if not""" + return True if pkg is good, False if not""" (csum_type, csum) = po.returnIdSum() @@ -682,7 +682,7 @@ class Pungi(PungiBase): def get_package_deps(self, po): """Add the dependencies for a given package to the - transaction info""" + transaction info""" added = set() if po.repoid in self.lookaside_repos: # Don't resolve deps for stuff in lookaside. @@ -911,7 +911,7 @@ class Pungi(PungiBase): def getPackagesFromGroup(self, group): """Get a list of package names from a ksparser group object - Returns a list of package names""" + Returns a list of package names""" packages = [] @@ -951,7 +951,7 @@ class Pungi(PungiBase): def _addDefaultGroups(self, excludeGroups=None): """Cycle through the groups and return at list of the ones that ara - default.""" + default.""" excludeGroups = excludeGroups or [] # This is mostly stolen from anaconda. @@ -1217,8 +1217,8 @@ class Pungi(PungiBase): def createSourceHashes(self): """Create two dicts - one that maps binary POs to source POs, and - one that maps a single source PO to all binary POs it produces. - Requires yum still configured.""" + one that maps a single source PO to all binary POs it produces. + Requires yum still configured.""" self.src_by_bin = {} self.bin_by_src = {} self.logger.info("Generating source <-> binary package mappings") @@ -1232,8 +1232,8 @@ class Pungi(PungiBase): def add_srpms(self, po_list=None): """Cycle through the list of package objects and - find the sourcerpm for them. Requires yum still - configured and a list of package objects""" + find the sourcerpm for them. Requires yum still + configured and a list of package objects""" srpms = set() po_list = po_list or self.po_list @@ -1275,9 +1275,9 @@ class Pungi(PungiBase): def add_fulltree(self, srpm_po_list=None): """Cycle through all package objects, and add any - that correspond to a source rpm that we are including. - Requires yum still configured and a list of package - objects.""" + that correspond to a source rpm that we are including. + Requires yum still configured and a list of package + objects.""" self.logger.info("Completing package set") @@ -1357,8 +1357,8 @@ class Pungi(PungiBase): def getDebuginfoList(self): """Cycle through the list of package objects and find - debuginfo rpms for them. Requires yum still - configured and a list of package objects""" + debuginfo rpms for them. Requires yum still + configured and a list of package objects""" added = set() for po in self.all_pkgs: @@ -1398,7 +1398,7 @@ class Pungi(PungiBase): def _downloadPackageList(self, polist, relpkgdir): """Cycle through the list of package objects and - download them from their respective repos.""" + download them from their respective repos.""" for pkg in sorted(polist): repo = self.ayum.repos.getRepo(pkg.repoid) @@ -1533,7 +1533,7 @@ class Pungi(PungiBase): @yumlocked def downloadSRPMs(self): """Cycle through the list of srpms and - find the package objects for them, Then download them.""" + find the package objects for them, Then download them.""" # do the downloads self._downloadPackageList(self.srpm_po_list, os.path.join("source", "SRPMS")) @@ -1541,7 +1541,7 @@ class Pungi(PungiBase): @yumlocked def downloadDebuginfo(self): """Cycle through the list of debuginfo rpms and - download them.""" + download them.""" # do the downloads self._downloadPackageList( @@ -1980,7 +1980,7 @@ class Pungi(PungiBase): def doGetRelnotes(self): """Get extra files from packages in the tree to put in the topdir of - the tree.""" + the tree.""" docsdir = os.path.join(self.workdir, "docs") relnoterpms = self.config.get("pungi", "relnotepkgs").split() diff --git a/pungi/graph.py b/pungi/graph.py index 4e946f1b..03112951 100755 --- a/pungi/graph.py +++ b/pungi/graph.py @@ -54,8 +54,7 @@ class SimpleAcyclicOrientedGraph(object): return False if node in self._graph else True def remove_final_endpoint(self, node): - """ - """ + """""" remove_start_points = [] for start, ends in self._graph.items(): if node in ends: diff --git a/pungi/phases/osbuild.py b/pungi/phases/osbuild.py index b81afc9e..0d188e8b 100644 --- a/pungi/phases/osbuild.py +++ b/pungi/phases/osbuild.py @@ -96,7 +96,12 @@ class RunOSBuildThread(WorkerThread): self.can_fail = can_fail self.num = num with util.failable( - compose, can_fail, variant, "*", "osbuild", logger=self.pool._logger, + compose, + can_fail, + variant, + "*", + "osbuild", + logger=self.pool._logger, ): self.worker( compose, variant, config, arches, version, release, target, repo diff --git a/pungi/scripts/create_unified_isos.py b/pungi/scripts/create_unified_isos.py index 645debaf..81f47aed 100644 --- a/pungi/scripts/create_unified_isos.py +++ b/pungi/scripts/create_unified_isos.py @@ -16,7 +16,10 @@ def parse_args(): parser = argparse.ArgumentParser(add_help=True) parser.add_argument( - "compose", metavar="", nargs=1, help="path to compose", + "compose", + metavar="", + nargs=1, + help="path to compose", ) parser.add_argument( "--arch", diff --git a/pungi/scripts/pungi_gather.py b/pungi/scripts/pungi_gather.py index 22cebe13..232d54b7 100644 --- a/pungi/scripts/pungi_gather.py +++ b/pungi/scripts/pungi_gather.py @@ -18,13 +18,18 @@ from pungi.util import temp_dir def get_parser(): parser = argparse.ArgumentParser() parser.add_argument( - "--profiler", action="store_true", + "--profiler", + action="store_true", ) parser.add_argument( - "--arch", required=True, + "--arch", + required=True, ) parser.add_argument( - "--config", metavar="PATH", required=True, help="path to kickstart config file", + "--config", + metavar="PATH", + required=True, + help="path to kickstart config file", ) parser.add_argument( "--download-to", @@ -42,7 +47,9 @@ def get_parser(): group = parser.add_argument_group("Gather options") group.add_argument( - "--nodeps", action="store_true", help="disable resolving dependencies", + "--nodeps", + action="store_true", + help="disable resolving dependencies", ) group.add_argument( "--selfhosting", @@ -61,7 +68,9 @@ def get_parser(): choices=["none", "all", "build"], ) group.add_argument( - "--multilib", metavar="[METHOD]", action="append", + "--multilib", + metavar="[METHOD]", + action="append", ) group.add_argument( "--tempdir", diff --git a/pungi/util.py b/pungi/util.py index 1965c86a..cf1ec8ee 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -941,7 +941,7 @@ def get_repo_dicts(repos, logger=None): def version_generator(compose, gen): """If ``gen`` is a known generator, create a value. Otherwise return - the argument value unchanged. + the argument value unchanged. """ if gen == "!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN": return "%s.%s" % (compose.image_version, compose.image_release) @@ -963,8 +963,8 @@ def version_generator(compose, gen): def retry(timeout=120, interval=30, wait_on=Exception): - """ A decorator that allows to retry a section of code until success or - timeout. + """A decorator that allows to retry a section of code until success or + timeout. """ def wrapper(function): diff --git a/pungi/wrappers/comps.py b/pungi/wrappers/comps.py index aa7685b4..c9194407 100644 --- a/pungi/wrappers/comps.py +++ b/pungi/wrappers/comps.py @@ -355,7 +355,10 @@ class CompsWrapper(object): if environment.option_ids: append_grouplist( - doc, env_node, set(environment.option_ids), "optionlist", + doc, + env_node, + set(environment.option_ids), + "optionlist", ) if self.comps.langpacks: diff --git a/pungi/wrappers/fus.py b/pungi/wrappers/fus.py index f3685ee9..26c2972d 100644 --- a/pungi/wrappers/fus.py +++ b/pungi/wrappers/fus.py @@ -26,7 +26,12 @@ Pungi). def get_cmd( - conf_file, arch, repos, lookasides, platform=None, filter_packages=None, + conf_file, + arch, + repos, + lookasides, + platform=None, + filter_packages=None, ): cmd = ["fus", "--verbose", "--arch", arch] diff --git a/pungi/wrappers/jigdo.py b/pungi/wrappers/jigdo.py index 5a6c7fee..417762cf 100644 --- a/pungi/wrappers/jigdo.py +++ b/pungi/wrappers/jigdo.py @@ -25,7 +25,7 @@ class JigdoWrapper(kobo.log.LoggingBase): self, image, files, output_dir, cache=None, no_servers=False, report=None ): """ - files: [{"path", "label", "uri"}] + files: [{"path", "label", "uri"}] """ cmd = ["jigdo-file", "make-template"] diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py index e82fa477..212a4298 100644 --- a/pungi/wrappers/kojiwrapper.py +++ b/pungi/wrappers/kojiwrapper.py @@ -202,7 +202,14 @@ class KojiWrapper(object): return cmd def get_pungi_ostree_cmd( - self, target, arch, args, channel=None, packages=None, mounts=None, weight=None, + self, + target, + arch, + args, + channel=None, + packages=None, + mounts=None, + weight=None, ): cmd = self._get_cmd("pungi-ostree", "--nowait", "--task-id") @@ -322,9 +329,11 @@ class KojiWrapper(object): "ksurl", "distro", ) - assert set(min_options).issubset(set(config_options["image-build"].keys())), ( - "image-build requires at least %s got '%s'" - % (", ".join(min_options), config_options) + assert set(min_options).issubset( + set(config_options["image-build"].keys()) + ), "image-build requires at least %s got '%s'" % ( + ", ".join(min_options), + config_options, ) cfg_parser = configparser.ConfigParser() for section, opts in config_options.items(): diff --git a/pungi_utils/orchestrator.py b/pungi_utils/orchestrator.py index 5bf12a05..e63838aa 100644 --- a/pungi_utils/orchestrator.py +++ b/pungi_utils/orchestrator.py @@ -302,8 +302,7 @@ def block_on(parts, name): def check_finished_processes(processes): - """Walk through all active processes and check if something finished. - """ + """Walk through all active processes and check if something finished.""" for proc in processes.keys(): proc.poll() if proc.returncode is not None: diff --git a/tests/helpers.py b/tests/helpers.py index 3fdd1eae..852eb054 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -215,7 +215,10 @@ class DummyCompose(object): self.log_warning = mock.Mock() self.get_image_name = mock.Mock(return_value="image-name") self.image = mock.Mock( - path="Client/i386/iso/image.iso", can_fail=False, size=123, _max_size=None, + path="Client/i386/iso/image.iso", + can_fail=False, + size=123, + _max_size=None, ) self.im = mock.Mock(images={"Client": {"amd64": [self.image]}}) self.old_composes = [] @@ -302,7 +305,10 @@ def mk_boom(cls=Exception, msg="BOOM"): return b -PKGSET_REPOS = dict(pkgset_source="repos", pkgset_repos={},) +PKGSET_REPOS = dict( + pkgset_source="repos", + pkgset_repos={}, +) BASE_CONFIG = dict( release_short="test", diff --git a/tests/test_buildinstall.py b/tests/test_buildinstall.py index e9d27de9..a8c3ddf7 100644 --- a/tests/test_buildinstall.py +++ b/tests/test_buildinstall.py @@ -1920,7 +1920,8 @@ class BuildinstallThreadTestCase(PungiTestCase): "pungi.phases.buildinstall.BuildinstallThread._load_old_buildinstall_metadata" ) def test_reuse_old_buildinstall_result_no_old_compose( - self, load_old_buildinstall_metadata, + self, + load_old_buildinstall_metadata, ): compose, pkgset_phase, cmd = self._prepare_buildinstall_reuse_test() load_old_buildinstall_metadata.return_value = None @@ -1935,7 +1936,8 @@ class BuildinstallThreadTestCase(PungiTestCase): "pungi.phases.buildinstall.BuildinstallThread._load_old_buildinstall_metadata" ) def test_reuse_old_buildinstall_result_different_cmd( - self, load_old_buildinstall_metadata, + self, + load_old_buildinstall_metadata, ): compose, pkgset_phase, cmd = self._prepare_buildinstall_reuse_test() @@ -1958,7 +1960,8 @@ class BuildinstallThreadTestCase(PungiTestCase): "pungi.phases.buildinstall.BuildinstallThread._load_old_buildinstall_metadata" ) def test_reuse_old_buildinstall_result_different_installed_pkgs( - self, load_old_buildinstall_metadata, + self, + load_old_buildinstall_metadata, ): compose, pkgset_phase, cmd = self._prepare_buildinstall_reuse_test() load_old_buildinstall_metadata.return_value = { @@ -1978,7 +1981,9 @@ class BuildinstallThreadTestCase(PungiTestCase): ) @mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper") def test_reuse_old_buildinstall_result_different_buildroot_rpms( - self, KojiWrapperMock, load_old_buildinstall_metadata, + self, + KojiWrapperMock, + load_old_buildinstall_metadata, ): compose, pkgset_phase, cmd = self._prepare_buildinstall_reuse_test() load_old_buildinstall_metadata.return_value = { diff --git a/tests/test_config.py b/tests/test_config.py index a3d8f92c..3bc03328 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -22,7 +22,9 @@ class ConfigTestCase(unittest.TestCase): class PkgsetConfigTestCase(ConfigTestCase): def test_validate_minimal_pkgset_koji(self): - cfg = load_config(pkgset_source="koji",) + cfg = load_config( + pkgset_source="koji", + ) self.assertValidation(cfg) @@ -36,7 +38,9 @@ class PkgsetConfigTestCase(ConfigTestCase): def test_pkgset_mismatch_repos(self): cfg = load_config( - pkgset_source="repos", pkgset_koji_tag="f25", pkgset_koji_inherit=False, + pkgset_source="repos", + pkgset_koji_tag="f25", + pkgset_koji_inherit=False, ) self.assertValidation( @@ -51,7 +55,10 @@ class PkgsetConfigTestCase(ConfigTestCase): ) def test_pkgset_mismatch_koji(self): - cfg = load_config(pkgset_source="koji", pkgset_repos={"whatever": "/foo"},) + cfg = load_config( + pkgset_source="koji", + pkgset_repos={"whatever": "/foo"}, + ) self.assertValidation( cfg, [checks.CONFLICTS.format("pkgset_source", "koji", "pkgset_repos")] @@ -78,7 +85,10 @@ class ReleaseConfigTestCase(ConfigTestCase): ) def test_only_config_base_product_name(self): - cfg = load_config(PKGSET_REPOS, base_product_name="Prod",) + cfg = load_config( + PKGSET_REPOS, + base_product_name="Prod", + ) self.assertValidation( cfg, @@ -99,7 +109,10 @@ class ReleaseConfigTestCase(ConfigTestCase): ) def test_only_config_base_product_short(self): - cfg = load_config(PKGSET_REPOS, base_product_short="bp",) + cfg = load_config( + PKGSET_REPOS, + base_product_short="bp", + ) self.assertValidation( cfg, @@ -118,7 +131,10 @@ class ReleaseConfigTestCase(ConfigTestCase): ) def test_only_config_base_product_version(self): - cfg = load_config(PKGSET_REPOS, base_product_version="1.0",) + cfg = load_config( + PKGSET_REPOS, + base_product_version="1.0", + ) self.assertValidation( cfg, @@ -141,19 +157,28 @@ class ReleaseConfigTestCase(ConfigTestCase): class ImageNameConfigTestCase(ConfigTestCase): def test_image_name_simple_string(self): - cfg = load_config(PKGSET_REPOS, image_name_format="foobar",) + cfg = load_config( + PKGSET_REPOS, + image_name_format="foobar", + ) self.assertValidation(cfg, []) def test_image_name_variant_mapping(self): - cfg = load_config(PKGSET_REPOS, image_name_format={"^Server$": "foobar"},) + cfg = load_config( + PKGSET_REPOS, + image_name_format={"^Server$": "foobar"}, + ) self.assertValidation(cfg, []) class RunrootConfigTestCase(ConfigTestCase): def test_set_runroot_true(self): - cfg = load_config(PKGSET_REPOS, runroot=True,) + cfg = load_config( + PKGSET_REPOS, + runroot=True, + ) self.assertValidation( cfg, @@ -163,7 +188,10 @@ class RunrootConfigTestCase(ConfigTestCase): ) def test_set_runroot_false(self): - cfg = load_config(PKGSET_REPOS, runroot=False,) + cfg = load_config( + PKGSET_REPOS, + runroot=False, + ) self.assertValidation( cfg, @@ -175,7 +203,10 @@ class RunrootConfigTestCase(ConfigTestCase): class BuildinstallConfigTestCase(ConfigTestCase): def test_bootable_deprecated(self): - cfg = load_config(PKGSET_REPOS, bootable=True,) + cfg = load_config( + PKGSET_REPOS, + bootable=True, + ) self.assertValidation( cfg, @@ -185,7 +216,10 @@ class BuildinstallConfigTestCase(ConfigTestCase): ) def test_buildinstall_method_without_bootable(self): - cfg = load_config(PKGSET_REPOS, buildinstall_method="lorax",) + cfg = load_config( + PKGSET_REPOS, + buildinstall_method="lorax", + ) self.assertValidation(cfg, []) @@ -231,7 +265,9 @@ class BuildinstallConfigTestCase(ConfigTestCase): class CreaterepoConfigTestCase(ConfigTestCase): def test_validate_minimal_pkgset_koji(self): cfg = load_config( - pkgset_source="koji", pkgset_koji_tag="f25", product_id_allow_missing=True, + pkgset_source="koji", + pkgset_koji_tag="f25", + product_id_allow_missing=True, ) self.assertValidation( @@ -242,14 +278,20 @@ class CreaterepoConfigTestCase(ConfigTestCase): class GatherConfigTestCase(ConfigTestCase): def test_dnf_backend_is_default_on_py3(self): - cfg = load_config(pkgset_source="koji", pkgset_koji_tag="f27",) + cfg = load_config( + pkgset_source="koji", + pkgset_koji_tag="f27", + ) with mock.patch("six.PY2", new=False): self.assertValidation(cfg, []) self.assertEqual(cfg["gather_backend"], "dnf") def test_yum_backend_is_default_on_py2(self): - cfg = load_config(pkgset_source="koji", pkgset_koji_tag="f27",) + cfg = load_config( + pkgset_source="koji", + pkgset_koji_tag="f27", + ) with mock.patch("six.PY2", new=True): self.assertValidation(cfg, []) @@ -257,7 +299,9 @@ class GatherConfigTestCase(ConfigTestCase): def test_yum_backend_is_rejected_on_py3(self): cfg = load_config( - pkgset_source="koji", pkgset_koji_tag="f27", gather_backend="yum", + pkgset_source="koji", + pkgset_koji_tag="f27", + gather_backend="yum", ) with mock.patch("six.PY2", new=False): @@ -402,7 +446,10 @@ class LiveMediaConfigTestCase(ConfigTestCase): self.assertEqual(cfg["live_media_ksurl"], "git://example.com/repo.git#CAFE") def test_global_config_null_release(self): - cfg = load_config(PKGSET_REPOS, live_media_release=None,) + cfg = load_config( + PKGSET_REPOS, + live_media_release=None, + ) self.assertValidation(cfg) @@ -429,7 +476,8 @@ class TestRegexValidation(ConfigTestCase): class RepoclosureTestCase(ConfigTestCase): def test_invalid_backend(self): cfg = load_config( - PKGSET_REPOS, repoclosure_backend="fnd", # Intentionally with a typo + PKGSET_REPOS, + repoclosure_backend="fnd", # Intentionally with a typo ) options = ["yum", "dnf"] if six.PY2 else ["dnf"] @@ -445,7 +493,10 @@ class RepoclosureTestCase(ConfigTestCase): class VariantAsLookasideTestCase(ConfigTestCase): def test_empty(self): variant_as_lookaside = [] - cfg = load_config(PKGSET_REPOS, variant_as_lookaside=variant_as_lookaside,) + cfg = load_config( + PKGSET_REPOS, + variant_as_lookaside=variant_as_lookaside, + ) self.assertValidation(cfg) def test_basic(self): @@ -454,14 +505,20 @@ class VariantAsLookasideTestCase(ConfigTestCase): ("Server", "Client"), ("Everything", "Spin"), ] - cfg = load_config(PKGSET_REPOS, variant_as_lookaside=variant_as_lookaside,) + cfg = load_config( + PKGSET_REPOS, + variant_as_lookaside=variant_as_lookaside, + ) self.assertValidation(cfg) class SkipPhasesTestCase(ConfigTestCase): def test_empty(self): skip_phases = [] - cfg = load_config(PKGSET_REPOS, skip_phases=skip_phases,) + cfg = load_config( + PKGSET_REPOS, + skip_phases=skip_phases, + ) self.assertValidation(cfg) def test_basic(self): @@ -469,7 +526,10 @@ class SkipPhasesTestCase(ConfigTestCase): "buildinstall", "gather", ] - cfg = load_config(PKGSET_REPOS, skip_phases=skip_phases,) + cfg = load_config( + PKGSET_REPOS, + skip_phases=skip_phases, + ) self.assertValidation(cfg) def test_bad_phase_name(self): @@ -477,5 +537,8 @@ class SkipPhasesTestCase(ConfigTestCase): "gather", "non-existing-phase_name", ] - cfg = load_config(PKGSET_REPOS, skip_phases=skip_phases,) + cfg = load_config( + PKGSET_REPOS, + skip_phases=skip_phases, + ) self.assertNotEqual(checks.validate(cfg), ([], [])) diff --git a/tests/test_createrepophase.py b/tests/test_createrepophase.py index c0617e7f..9bc794ca 100644 --- a/tests/test_createrepophase.py +++ b/tests/test_createrepophase.py @@ -158,7 +158,9 @@ def make_mocked_modifyrepo_cmd(tc, module_artifacts): for ms in module_streams: tc.assertIn(ms.get_stream_name(), module_artifacts) six.assertCountEqual( - tc, ms.get_rpm_artifacts(), module_artifacts[ms.get_stream_name()], + tc, + ms.get_rpm_artifacts(), + module_artifacts[ms.get_stream_name()], ) return mocked_modifyrepo_cmd diff --git a/tests/test_extra_isos_phase.py b/tests/test_extra_isos_phase.py index 83365140..e38ec9da 100644 --- a/tests/test_extra_isos_phase.py +++ b/tests/test_extra_isos_phase.py @@ -596,7 +596,9 @@ class GetExtraFilesTest(helpers.PungiTestCase): get_file.call_args_list, [ mock.call( - cfg1, os.path.join(self.dir, "legalese"), compose=self.compose, + cfg1, + os.path.join(self.dir, "legalese"), + compose=self.compose, ), mock.call(cfg2, self.dir, compose=self.compose), ], @@ -832,7 +834,8 @@ class GetIsoContentsTest(helpers.PungiTestCase): ["Client"], os.path.join(self.topdir, "compose/Server/source/tree/.treeinfo"), os.path.join( - self.topdir, "work/src/Server/extra-iso-extra-files/.treeinfo", + self.topdir, + "work/src/Server/extra-iso-extra-files/.treeinfo", ), ), ], diff --git a/tests/test_fus_wrapper.py b/tests/test_fus_wrapper.py index 2fccd44b..5516fc5a 100644 --- a/tests/test_fus_wrapper.py +++ b/tests/test_fus_wrapper.py @@ -147,7 +147,8 @@ class TestParseOutput(unittest.TestCase): touch(self.file, "*pkg-1.0-1.x86_64@repo-0\n") packages, modules = fus.parse_output(self.file) self.assertEqual( - packages, set([("pkg-1.0-1", "x86_64", frozenset(["modular"]))]), + packages, + set([("pkg-1.0-1", "x86_64", frozenset(["modular"]))]), ) self.assertEqual(modules, set()) diff --git a/tests/test_gather.py b/tests/test_gather.py index 38bc2300..3062c470 100644 --- a/tests/test_gather.py +++ b/tests/test_gather.py @@ -2620,5 +2620,7 @@ class DNFDepsolvingTestCase(DepsolvingBase, unittest.TestCase): six.assertCountEqual(self, pkg_map["rpm"], []) six.assertCountEqual(self, pkg_map["srpm"], []) six.assertCountEqual( - self, pkg_map["debuginfo"], ["dummy-bash-debuginfo-4.2.37-6.x86_64.rpm"], + self, + pkg_map["debuginfo"], + ["dummy-bash-debuginfo-4.2.37-6.x86_64.rpm"], ) diff --git a/tests/test_gather_method_hybrid.py b/tests/test_gather_method_hybrid.py index 80db2913..b4e24a0d 100644 --- a/tests/test_gather_method_hybrid.py +++ b/tests/test_gather_method_hybrid.py @@ -350,7 +350,8 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): ], ) self.assertEqual( - wc.call_args_list, [mock.call(self.config1, ["mod:master"], [])], + wc.call_args_list, + [mock.call(self.config1, ["mod:master"], [])], ) self.assertEqual( gc.call_args_list, @@ -454,7 +455,8 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): ], ) self.assertEqual( - wc.call_args_list, [mock.call(self.config1, [], ["pkg"])], + wc.call_args_list, + [mock.call(self.config1, [], ["pkg"])], ) self.assertEqual( gc.call_args_list, diff --git a/tests/test_image_container_phase.py b/tests/test_image_container_phase.py index 65745b2d..eaea2d24 100644 --- a/tests/test_image_container_phase.py +++ b/tests/test_image_container_phase.py @@ -168,7 +168,10 @@ class ImageContainerThreadTest(helpers.PungiTestCase): [ mock.call.login(), mock.call.koji_proxy.buildContainer( - cfg["url"], cfg["target"], opts, priority=None, + cfg["url"], + cfg["target"], + opts, + priority=None, ), mock.call.watch_task( 12345, diff --git a/tests/test_koji_wrapper.py b/tests/test_koji_wrapper.py index d14fe02a..5f7029e1 100644 --- a/tests/test_koji_wrapper.py +++ b/tests/test_koji_wrapper.py @@ -39,7 +39,9 @@ class KojiWrapperBaseTestCase(unittest.TestCase): koji.get_profile_module = mock.Mock( return_value=mock.Mock( config=DumbMock( - server="koji.example.com", authtype="kerberos", cert="", + server="koji.example.com", + authtype="kerberos", + cert="", ), pathinfo=mock.Mock( work=mock.Mock(return_value="/koji"), diff --git a/tests/test_osbs_phase.py b/tests/test_osbs_phase.py index 03bd6740..0d20e68d 100644 --- a/tests/test_osbs_phase.py +++ b/tests/test_osbs_phase.py @@ -53,7 +53,8 @@ class OSBSPhaseTest(helpers.PungiTestCase): self.assertEqual(data, phase.pool.registries) self.assertEqual( - compose.notifier.call_args_list, [], + compose.notifier.call_args_list, + [], ) diff --git a/tests/test_runroot.py b/tests/test_runroot.py index 95f4ca17..e06cdd31 100644 --- a/tests/test_runroot.py +++ b/tests/test_runroot.py @@ -204,7 +204,8 @@ class TestRunrootKoji(helpers.PungiTestCase): def setUp(self): super(TestRunrootKoji, self).setUp() self.compose = helpers.DummyCompose( - self.topdir, {"runroot": True, "runroot_tag": "f28-build"}, + self.topdir, + {"runroot": True, "runroot_tag": "f28-build"}, ) self.runroot = Runroot(self.compose) diff --git a/tests/test_test_phase.py b/tests/test_test_phase.py index 486d8198..4e82d051 100644 --- a/tests/test_test_phase.py +++ b/tests/test_test_phase.py @@ -267,7 +267,8 @@ class TestCheckImageSanity(PungiTestCase): @mock.patch("pungi.phases.test.check_sanity", new=mock.Mock()) def test_too_big_unified_strict(self): compose = DummyCompose( - self.topdir, {"createiso_max_size_is_strict": [(".*", {"*": True})]}, + self.topdir, + {"createiso_max_size_is_strict": [(".*", {"*": True})]}, ) compose.image.format = "iso" compose.image.bootable = False diff --git a/tests/test_util.py b/tests/test_util.py index a5fc894d..29d85cfa 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -183,7 +183,8 @@ class TestGitRefResolver(unittest.TestCase): def test_resolver_offline_branch(self, mock_resolve_url, mock_resolve_ref): resolver = util.GitUrlResolver(offline=True) self.assertEqual( - resolver("http://example.com/repo.git", "master"), "master", + resolver("http://example.com/repo.git", "master"), + "master", ) self.assertEqual(mock_resolve_url.call_args_list, []) self.assertEqual(mock_resolve_ref.call_args_list, []) @@ -935,7 +936,8 @@ class TestVersionGenerator(unittest.TestCase): def test_version_from_version(self): self.assertEqual( - util.version_generator(self.compose, "!VERSION_FROM_VERSION"), "8", + util.version_generator(self.compose, "!VERSION_FROM_VERSION"), + "8", ) From 2769232b72601ce7776502b23df0aa387b4fd588 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Mon, 1 Mar 2021 10:49:49 +0800 Subject: [PATCH 021/137] runroot: Adjust permissions always Previously commands to adjust permissions do not run when main command failed and then files can't be cleaned up due to Permission Denied problem. JIRA: RHELCMP-4253 Signed-off-by: Haibo Lin --- pungi/runroot.py | 7 +++++-- pungi/wrappers/kojiwrapper.py | 7 +++++-- tests/test_koji_wrapper.py | 2 +- tests/test_runroot.py | 6 ++++-- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/pungi/runroot.py b/pungi/runroot.py index f119b59a..98d7bd38 100644 --- a/pungi/runroot.py +++ b/pungi/runroot.py @@ -174,10 +174,13 @@ class Runroot(kobo.log.LoggingBase): # by the runroot task, so the Pungi user can access them. if chown_paths: paths = " ".join(shlex_quote(pth) for pth in chown_paths) + command += " ; EXIT_CODE=$?" # Make the files world readable - command += " && chmod -R a+r %s" % paths + command += " ; chmod -R a+r %s" % paths # and owned by the same user that is running the process - command += " && chown -R %d %s" % (os.getuid(), paths) + command += " ; chown -R %d %s" % (os.getuid(), paths) + # Exit with code of main command + command += " ; exit $EXIT_CODE" hostname = runroot_ssh_hostnames[arch] user = self.compose.conf.get("runroot_ssh_username", "root") diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py index 212a4298..d5f65ed6 100644 --- a/pungi/wrappers/kojiwrapper.py +++ b/pungi/wrappers/kojiwrapper.py @@ -142,10 +142,13 @@ class KojiWrapper(object): if chown_paths: paths = " ".join(shlex_quote(pth) for pth in chown_paths) + command += " ; EXIT_CODE=$?" # Make the files world readable - command += " && chmod -R a+r %s" % paths + command += " ; chmod -R a+r %s" % paths # and owned by the same user that is running the process - command += " && chown -R %d %s" % (os.getuid(), paths) + command += " ; chown -R %d %s" % (os.getuid(), paths) + # Exit with code of main command + command += " ; exit $EXIT_CODE" cmd.append(command) return cmd diff --git a/tests/test_koji_wrapper.py b/tests/test_koji_wrapper.py index 5f7029e1..fc59e071 100644 --- a/tests/test_koji_wrapper.py +++ b/tests/test_koji_wrapper.py @@ -594,7 +594,7 @@ class RunrootKojiWrapperTest(KojiWrapperBaseTestCase): self.assertEqual(cmd[-2], "s390x") self.assertEqual( cmd[-1], - "rm -f /var/lib/rpm/__db*; rm -rf /var/cache/yum/*; set -x; /bin/echo '&' && chmod -R a+r '/output dir' /foo && chown -R 1010 '/output dir' /foo", # noqa: E501 + "rm -f /var/lib/rpm/__db*; rm -rf /var/cache/yum/*; set -x; /bin/echo '&' ; EXIT_CODE=$? ; chmod -R a+r '/output dir' /foo ; chown -R 1010 '/output dir' /foo ; exit $EXIT_CODE", # noqa: E501 ) six.assertCountEqual( self, diff --git a/tests/test_runroot.py b/tests/test_runroot.py index e06cdd31..4b3f0488 100644 --- a/tests/test_runroot.py +++ b/tests/test_runroot.py @@ -189,8 +189,10 @@ class TestRunrootOpenSSH(helpers.PungiTestCase): run.assert_has_calls( [ self._ssh_call( - "run df -h && chmod -R a+r /mnt/foo/compose /mnt/foo/x && " - "chown -R %d /mnt/foo/compose /mnt/foo/x" % os.getuid() + "run df -h ; EXIT_CODE=$? ; " + "chmod -R a+r /mnt/foo/compose /mnt/foo/x ; " + "chown -R %d /mnt/foo/compose /mnt/foo/x ; exit $EXIT_CODE" + % os.getuid() ), self._ssh_call( "run rpm -qa --qf='%{name}-%{version}-%{release}.%{arch}\n'", From 535034ef91d6d7055af8800da46e259b3cfc9d14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Mon, 8 Mar 2021 11:57:48 +0100 Subject: [PATCH 022/137] image_container: Fix incorrect arch processing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OSBS will reject no scratch builds with arch_override. When the option is not specified in Pungi, it would do `"".split(" ")` to get list of arches, which returns a list with empty string instead of an empty list. With this fixed, it might be possible to have multiple images match the spec (unless arch is used in the filter). To fix that, we can replace arch with $basearch variable. JIRA: RHELCMP-3824 Signed-off-by: Lubomír Sedlář --- pungi/phases/image_container.py | 4 ++-- tests/test_image_container_phase.py | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pungi/phases/image_container.py b/pungi/phases/image_container.py index cb72161f..0e656335 100644 --- a/pungi/phases/image_container.py +++ b/pungi/phases/image_container.py @@ -53,7 +53,7 @@ class ImageContainerThread(WorkerThread): self._get_repo( compose, variant, - config.get("arch_override", "").split(" "), + config.get("arch_override", "").split(), config.pop("image_spec"), ) ] @@ -95,7 +95,7 @@ class ImageContainerThread(WorkerThread): if not re.match(value, getattr(image, key)): break else: - image_paths.add(image.path) + image_paths.add(image.path.replace(arch, "$basearch")) if len(image_paths) != 1: raise RuntimeError( diff --git a/tests/test_image_container_phase.py b/tests/test_image_container_phase.py index eaea2d24..b76d761c 100644 --- a/tests/test_image_container_phase.py +++ b/tests/test_image_container_phase.py @@ -130,7 +130,6 @@ class ImageContainerThreadTest(helpers.PungiTestCase): "target": "f24-docker-candidate", "git_branch": "f24-docker", "image_spec": {"type": "qcow2"}, - "arch_override": "x86_64", } self.compose.im.images["Server"] = { "x86_64": [ @@ -150,7 +149,7 @@ class ImageContainerThreadTest(helpers.PungiTestCase): repo_content = list(f) self.assertIn("[image-to-include]\n", repo_content) self.assertIn( - "baseurl=http://root/compose/Server/x86_64/images/image.qcow2\n", + "baseurl=http://root/compose/Server/$basearch/images/image.qcow2\n", repo_content, ) self.assertIn("enabled=0\n", repo_content) @@ -158,7 +157,6 @@ class ImageContainerThreadTest(helpers.PungiTestCase): def assertKojiCalls(self, cfg, scratch=False): opts = { "git_branch": cfg["git_branch"], - "arch_override": cfg["arch_override"], "yum_repourls": ["http://root/" + self.repofile_path], } if scratch: From edb4517e809e7b0f8c2f7e930a41c9633d515b0d Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Mon, 15 Mar 2021 16:33:06 +0800 Subject: [PATCH 023/137] Add Dockerfile for building testing image There are two images because it's hard to install both Python 2 and Python 3 packages (e.g. libcomps) in latest fedora release. JIRA: RHELCMP-4580 Signed-off-by: Haibo Lin --- tests/Dockerfile-test | 22 ++++++++++++++++++++++ tests/Dockerfile-test-py2 | 25 +++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 tests/Dockerfile-test create mode 100644 tests/Dockerfile-test-py2 diff --git a/tests/Dockerfile-test b/tests/Dockerfile-test new file mode 100644 index 00000000..43b4191d --- /dev/null +++ b/tests/Dockerfile-test @@ -0,0 +1,22 @@ +FROM fedora:33 +LABEL \ + name="Pungi test" \ + description="Run tests using tox with Python 3" \ + vendor="Pungi developers" \ + license="MIT" + +RUN dnf -y update && dnf -y install \ + findutils \ + git \ + koji \ + make \ + python3-createrepo_c \ + python3-tox \ + python3-urlgrabber \ + && dnf clean all + +WORKDIR /src + +COPY . . + +CMD ["tox", "-e", "flake8,black,py3"] diff --git a/tests/Dockerfile-test-py2 b/tests/Dockerfile-test-py2 new file mode 100644 index 00000000..1e41be20 --- /dev/null +++ b/tests/Dockerfile-test-py2 @@ -0,0 +1,25 @@ +FROM centos:7 +LABEL \ + name="Pungi test" \ + description="Run tests using tox with Python 2" \ + vendor="Pungi developers" \ + license="MIT" + +RUN yum -y update && yum -y install \ + git \ + make \ + python3 \ + python-gssapi \ + python-libcomps \ + python-createrepo_c \ + pykickstart \ + && yum clean all + +# python-tox in yum repo is too old, let's install latest version +RUN pip3 install tox + +WORKDIR /src + +COPY . . + +CMD ["tox", "-e", "py27"] From 035b37c56683fd30410d919165a65280aaa12781 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Mon, 22 Mar 2021 15:52:47 +0800 Subject: [PATCH 024/137] Cancel koji tasks when pungi terminated JIRA: RHELCMP-4148 Signed-off-by: Haibo Lin --- pungi/paths.py | 13 +++++++++++ pungi/phases/buildinstall.py | 2 +- pungi/phases/createiso.py | 2 +- pungi/phases/gather/__init__.py | 2 +- pungi/phases/image_build.py | 2 +- pungi/phases/image_container.py | 4 +++- pungi/phases/live_images.py | 2 +- pungi/phases/livemedia_phase.py | 2 +- pungi/phases/osbs.py | 6 ++++-- pungi/phases/osbuild.py | 6 ++++-- pungi/phases/pkgset/pkgsets.py | 4 ---- pungi/phases/pkgset/sources/source_koji.py | 3 +-- pungi/runroot.py | 6 +++--- pungi/scripts/pungi_koji.py | 17 +++++++++++++++ pungi/wrappers/kojiwrapper.py | 25 ++++++++++++++++++---- pungi/wrappers/scm.py | 6 +----- tests/test_image_container_phase.py | 1 + tests/test_koji_wrapper.py | 10 ++++++--- tests/test_osbs_phase.py | 1 + tests/test_osbuild_phase.py | 2 ++ tests/test_pkgset_source_koji.py | 2 +- tests/test_scm.py | 13 +++++------ 22 files changed, 92 insertions(+), 39 deletions(-) diff --git a/pungi/paths.py b/pungi/paths.py index 43b084c7..2916a577 100644 --- a/pungi/paths.py +++ b/pungi/paths.py @@ -103,6 +103,16 @@ class LogPaths(object): makedirs(path) return path + def koji_tasks_dir(self, create_dir=True): + """ + Examples: + logs/global/koji-tasks + """ + path = os.path.join(self.topdir(create_dir=create_dir), "koji-tasks") + if create_dir: + makedirs(path) + return path + def log_file(self, arch, log_name, create_dir=True): arch = arch or "global" if log_name.endswith(".log"): @@ -502,6 +512,9 @@ class WorkPaths(object): """ Returns the path to file in which the cached version of PackageSetBase.file_cache should be stored. + + Example: + work/global/pkgset_f33-compose_file_cache.pickle """ filename = "pkgset_%s_file_cache.pickle" % pkgset_name return os.path.join(self.topdir(arch="global"), filename) diff --git a/pungi/phases/buildinstall.py b/pungi/phases/buildinstall.py index 0eba8fad..04e46870 100644 --- a/pungi/phases/buildinstall.py +++ b/pungi/phases/buildinstall.py @@ -722,7 +722,7 @@ class BuildinstallThread(WorkerThread): # Ask Koji for all the RPMs in the `runroot_tag` and check that # those installed in the old buildinstall buildroot are still in the # very same versions/releases. - koji_wrapper = kojiwrapper.KojiWrapper(compose.conf["koji_profile"]) + koji_wrapper = kojiwrapper.KojiWrapper(compose) rpms = koji_wrapper.koji_proxy.listTaggedRPMS( compose.conf.get("runroot_tag"), inherit=True, latest=True )[0] diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py index cd20080f..083a9a58 100644 --- a/pungi/phases/createiso.py +++ b/pungi/phases/createiso.py @@ -346,7 +346,7 @@ def run_createiso_command( build_arch = arch if runroot.runroot_method == "koji" and not bootable: runroot_tag = compose.conf["runroot_tag"] - koji_wrapper = kojiwrapper.KojiWrapper(compose.conf["koji_profile"]) + koji_wrapper = kojiwrapper.KojiWrapper(compose) koji_proxy = koji_wrapper.koji_proxy tag_info = koji_proxy.getTag(runroot_tag) if not tag_info: diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 9338d5da..7f77f15a 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -607,7 +607,7 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None): ) + "/", "koji": lambda: pungi.wrappers.kojiwrapper.KojiWrapper( - compose.conf["koji_profile"] + compose ).koji_module.config.topdir.rstrip("/") + "/", } diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py index 9818b2d6..127ecf1d 100644 --- a/pungi/phases/image_build.py +++ b/pungi/phases/image_build.py @@ -223,7 +223,7 @@ class CreateImageBuildThread(WorkerThread): ) self.pool.log_info("[BEGIN] %s" % msg) - koji_wrapper = KojiWrapper(compose.conf["koji_profile"]) + koji_wrapper = KojiWrapper(compose) # writes conf file for koji image-build self.pool.log_info( diff --git a/pungi/phases/image_container.py b/pungi/phases/image_container.py index 0e656335..f7139263 100644 --- a/pungi/phases/image_container.py +++ b/pungi/phases/image_container.py @@ -59,12 +59,14 @@ class ImageContainerThread(WorkerThread): ] # Start task - koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"]) + koji = kojiwrapper.KojiWrapper(compose) koji.login() task_id = koji.koji_proxy.buildContainer( source, target, config, priority=priority ) + koji.save_task_id(task_id) + # Wait for it to finish and capture the output into log file (even # though there is not much there). log_dir = os.path.join(compose.paths.log.topdir(), "image_container") diff --git a/pungi/phases/live_images.py b/pungi/phases/live_images.py index d2767aa2..0a4c4108 100644 --- a/pungi/phases/live_images.py +++ b/pungi/phases/live_images.py @@ -186,7 +186,7 @@ class CreateLiveImageThread(WorkerThread): ) self.pool.log_info("[BEGIN] %s" % msg) - koji_wrapper = KojiWrapper(compose.conf["koji_profile"]) + koji_wrapper = KojiWrapper(compose) _, version = compose.compose_id.rsplit("-", 1) name = cmd["name"] or imgname version = cmd["version"] or version diff --git a/pungi/phases/livemedia_phase.py b/pungi/phases/livemedia_phase.py index 50fdb0b8..9796418a 100644 --- a/pungi/phases/livemedia_phase.py +++ b/pungi/phases/livemedia_phase.py @@ -140,7 +140,7 @@ class LiveMediaThread(WorkerThread): ) self.pool.log_info("[BEGIN] %s" % msg) - koji_wrapper = KojiWrapper(compose.conf["koji_profile"]) + koji_wrapper = KojiWrapper(compose) cmd = self._get_cmd(koji_wrapper, config) log_file = self._get_log_file(compose, variant, subvariant, config) diff --git a/pungi/phases/osbs.py b/pungi/phases/osbs.py index a1e0db67..411f05a7 100644 --- a/pungi/phases/osbs.py +++ b/pungi/phases/osbs.py @@ -77,7 +77,7 @@ class OSBSThread(WorkerThread): def worker(self, compose, variant, config): msg = "OSBS task for variant %s" % variant.uid self.pool.log_info("[BEGIN] %s" % msg) - koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"]) + koji = kojiwrapper.KojiWrapper(compose) koji.login() # Start task @@ -98,6 +98,8 @@ class OSBSThread(WorkerThread): source, target, config, priority=priority ) + koji.save_task_id(task_id) + # Wait for it to finish and capture the output into log file (even # though there is not much there). log_dir = os.path.join(compose.paths.log.topdir(), "osbs") @@ -173,7 +175,7 @@ def add_metadata(variant, task_id, compose, is_scratch): # Create new Koji session. The task could take so long to finish that # our session will expire. This second session does not need to be # authenticated since it will only do reading operations. - koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"]) + koji = kojiwrapper.KojiWrapper(compose) # Create metadata metadata = { diff --git a/pungi/phases/osbuild.py b/pungi/phases/osbuild.py index 0d188e8b..15fe1c76 100644 --- a/pungi/phases/osbuild.py +++ b/pungi/phases/osbuild.py @@ -110,7 +110,7 @@ class RunOSBuildThread(WorkerThread): def worker(self, compose, variant, config, arches, version, release, target, repo): msg = "OSBuild task for variant %s" % variant.uid self.pool.log_info("[BEGIN] %s" % msg) - koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"]) + koji = kojiwrapper.KojiWrapper(compose) koji.login() # Start task @@ -127,6 +127,8 @@ class RunOSBuildThread(WorkerThread): opts=opts, ) + koji.save_task_id(task_id) + # Wait for it to finish and capture the output into log file. log_dir = os.path.join(compose.paths.log.topdir(), "osbuild") util.makedirs(log_dir) @@ -141,7 +143,7 @@ class RunOSBuildThread(WorkerThread): # Refresh koji session which may have timed out while the task was # running. Watching is done via a subprocess, so the session is # inactive. - koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"]) + koji = kojiwrapper.KojiWrapper(compose) # Get build id via the task's result json data result = koji.koji_proxy.getTaskResult(task_id) diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index 4b9c3fe8..54dcafde 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -31,7 +31,6 @@ import kobo.rpmlib from kobo.threads import WorkerThread, ThreadPool -import pungi.wrappers.kojiwrapper from pungi.util import pkg_is_srpm, copy_all from pungi.arch import get_valid_arches, is_excluded from pungi.errors import UnsignedPackagesError @@ -391,7 +390,6 @@ class KojiPackageSet(PackageSetBase): def __getstate__(self): result = self.__dict__.copy() - result["koji_profile"] = self.koji_wrapper.profile del result["koji_wrapper"] del result["_logger"] if "cache_region" in result: @@ -399,8 +397,6 @@ class KojiPackageSet(PackageSetBase): return result def __setstate__(self, data): - koji_profile = data.pop("koji_profile") - self.koji_wrapper = pungi.wrappers.kojiwrapper.KojiWrapper(koji_profile) self._logger = None self.__dict__.update(data) diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index f683b4ae..3a0db08a 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -186,8 +186,7 @@ def get_koji_modules(compose, koji_wrapper, event, module_info_str): class PkgsetSourceKoji(pungi.phases.pkgset.source.PkgsetSourceBase): def __call__(self): compose = self.compose - koji_profile = compose.conf["koji_profile"] - self.koji_wrapper = pungi.wrappers.kojiwrapper.KojiWrapper(koji_profile) + self.koji_wrapper = pungi.wrappers.kojiwrapper.KojiWrapper(compose) # path prefix must contain trailing '/' path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip("/") + "/" package_sets = get_pkgset_from_koji( diff --git a/pungi/runroot.py b/pungi/runroot.py index 98d7bd38..95912e90 100644 --- a/pungi/runroot.py +++ b/pungi/runroot.py @@ -110,7 +110,7 @@ class Runroot(kobo.log.LoggingBase): runroot_tag = self.compose.conf["runroot_tag"] log_dir = kwargs.pop("log_dir", None) - koji_wrapper = kojiwrapper.KojiWrapper(self.compose.conf["koji_profile"]) + koji_wrapper = kojiwrapper.KojiWrapper(self.compose) koji_cmd = koji_wrapper.get_runroot_cmd( runroot_tag, arch, @@ -303,7 +303,7 @@ class Runroot(kobo.log.LoggingBase): runroot_channel = self.compose.conf.get("runroot_channel") runroot_tag = self.compose.conf["runroot_tag"] - koji_wrapper = kojiwrapper.KojiWrapper(self.compose.conf["koji_profile"]) + koji_wrapper = kojiwrapper.KojiWrapper(self.compose) koji_cmd = koji_wrapper.get_pungi_buildinstall_cmd( runroot_tag, arch, @@ -337,7 +337,7 @@ class Runroot(kobo.log.LoggingBase): runroot_channel = self.compose.conf.get("runroot_channel") runroot_tag = self.compose.conf["runroot_tag"] - koji_wrapper = kojiwrapper.KojiWrapper(self.compose.conf["koji_profile"]) + koji_wrapper = kojiwrapper.KojiWrapper(self.compose) koji_cmd = koji_wrapper.get_pungi_ostree_cmd( runroot_tag, arch, args, channel=runroot_channel, **kwargs ) diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index dd568128..b63ddd6e 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -22,6 +22,7 @@ from six.moves import shlex_quote from pungi.phases import PHASES_NAMES from pungi import get_full_version, util from pungi.errors import UnsignedPackagesError +from pungi.wrappers import kojiwrapper # force C locales @@ -615,9 +616,25 @@ def try_kill_children(signal): COMPOSE.log_warning("Failed to kill all subprocesses") +def try_kill_koji_tasks(): + try: + if COMPOSE: + koji_tasks_dir = COMPOSE.paths.log.koji_tasks_dir(create_dir=False) + if os.path.exists(koji_tasks_dir): + COMPOSE.log_warning("Trying to kill koji tasks") + koji = kojiwrapper.KojiWrapper(COMPOSE) + koji.login() + for task_id in os.listdir(koji_tasks_dir): + koji.koji_proxy.cancelTask(int(task_id)) + except Exception: + if COMPOSE: + COMPOSE.log_warning("Failed to kill koji tasks") + + def sigterm_handler(signum, frame): if COMPOSE: try_kill_children(signum) + try_kill_koji_tasks() COMPOSE.log_error("Compose run failed: signal %s" % signum) COMPOSE.log_error("Traceback:\n%s" % "\n".join(traceback.format_stack(frame))) COMPOSE.log_critical("Compose failed: %s" % COMPOSE.topdir) diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py index d5f65ed6..b501684d 100644 --- a/pungi/wrappers/kojiwrapper.py +++ b/pungi/wrappers/kojiwrapper.py @@ -36,10 +36,14 @@ KOJI_BUILD_DELETED = koji.BUILD_STATES["DELETED"] class KojiWrapper(object): lock = threading.Lock() - def __init__(self, profile): - self.profile = profile + def __init__(self, compose): + self.compose = compose + try: + self.profile = self.compose.conf["koji_profile"] + except KeyError: + raise RuntimeError("Koji profile must be configured") with self.lock: - self.koji_module = koji.get_profile_module(profile) + self.koji_module = koji.get_profile_module(self.profile) session_opts = {} for key in ( "timeout", @@ -300,6 +304,8 @@ class KojiWrapper(object): task_id = int(match.groups()[0]) + self.save_task_id(task_id) + retcode, output = self._wait_for_task(task_id, logfile=log_file) return { @@ -542,6 +548,8 @@ class KojiWrapper(object): ) task_id = int(match.groups()[0]) + self.save_task_id(task_id) + if retcode != 0 and ( self._has_connection_error(output) or self._has_offline_error(output) ): @@ -827,13 +835,22 @@ class KojiWrapper(object): """ return self.multicall_map(*args, **kwargs) + def save_task_id(self, task_id): + """Save task id by creating a file using task_id as file name + + :param int task_id: ID of koji task + """ + log_dir = self.compose.paths.log.koji_tasks_dir() + with open(os.path.join(log_dir, str(task_id)), "w"): + pass + def get_buildroot_rpms(compose, task_id): """Get build root RPMs - either from runroot or local""" result = [] if task_id: # runroot - koji = KojiWrapper(compose.conf["koji_profile"]) + koji = KojiWrapper(compose) buildroot_infos = koji.koji_proxy.listBuildroots(taskID=task_id) if not buildroot_infos: children_tasks = koji.koji_proxy.getTaskChildren(task_id) diff --git a/pungi/wrappers/scm.py b/pungi/wrappers/scm.py index 5602aafe..5c4b37fb 100644 --- a/pungi/wrappers/scm.py +++ b/pungi/wrappers/scm.py @@ -265,11 +265,7 @@ class RpmScmWrapper(ScmBase): class KojiScmWrapper(ScmBase): def __init__(self, *args, **kwargs): super(KojiScmWrapper, self).__init__(*args, **kwargs) - try: - profile = kwargs["compose"].conf["koji_profile"] - except KeyError: - raise RuntimeError("Koji profile must be configured") - wrapper = KojiWrapper(profile) + wrapper = KojiWrapper(kwargs["compose"]) self.koji = wrapper.koji_module self.proxy = wrapper.koji_proxy diff --git a/tests/test_image_container_phase.py b/tests/test_image_container_phase.py index b76d761c..2bb99c7b 100644 --- a/tests/test_image_container_phase.py +++ b/tests/test_image_container_phase.py @@ -171,6 +171,7 @@ class ImageContainerThreadTest(helpers.PungiTestCase): opts, priority=None, ), + mock.call.save_task_id(12345), mock.call.watch_task( 12345, os.path.join( diff --git a/tests/test_koji_wrapper.py b/tests/test_koji_wrapper.py index fc59e071..1b529473 100644 --- a/tests/test_koji_wrapper.py +++ b/tests/test_koji_wrapper.py @@ -10,6 +10,7 @@ except ImportError: import tempfile import os +import shutil import six @@ -33,7 +34,9 @@ def mock_imagebuild_path(id): class KojiWrapperBaseTestCase(unittest.TestCase): def setUp(self): _, self.tmpfile = tempfile.mkstemp() - self.koji_profile = mock.Mock() + compose = mock.Mock(conf={"koji_profile": "custom-koji"}) + self.tmpdir = tempfile.mkdtemp() + compose.paths.log.koji_tasks_dir.return_value = self.tmpdir with mock.patch("pungi.wrappers.kojiwrapper.koji") as koji: koji.gssapi_login = mock.Mock() koji.get_profile_module = mock.Mock( @@ -51,10 +54,11 @@ class KojiWrapperBaseTestCase(unittest.TestCase): ) ) self.koji_profile = koji.get_profile_module.return_value - self.koji = KojiWrapper("custom-koji") + self.koji = KojiWrapper(compose) def tearDown(self): os.remove(self.tmpfile) + shutil.rmtree(self.tmpdir) class KojiWrapperTest(KojiWrapperBaseTestCase): @@ -1163,7 +1167,7 @@ class TestGetBuildrootRPMs(unittest.TestCase): rpms = get_buildroot_rpms(compose, 1234) - self.assertEqual(KojiWrapper.call_args_list, [mock.call("koji")]) + self.assertEqual(KojiWrapper.call_args_list, [mock.call(compose)]) self.assertEqual( KojiWrapper.return_value.mock_calls, [ diff --git a/tests/test_osbs_phase.py b/tests/test_osbs_phase.py index 0d20e68d..04e98c19 100644 --- a/tests/test_osbs_phase.py +++ b/tests/test_osbs_phase.py @@ -220,6 +220,7 @@ class OSBSThreadTest(helpers.PungiTestCase): options, priority=None, ), + mock.call.save_task_id(12345), mock.call.watch_task( 12345, self.topdir + "/logs/global/osbs/Server-1-watch-task.log" ), diff --git a/tests/test_osbuild_phase.py b/tests/test_osbuild_phase.py index f3b70e61..3dbf6fe5 100644 --- a/tests/test_osbuild_phase.py +++ b/tests/test_osbuild_phase.py @@ -196,6 +196,7 @@ class RunOSBuildThreadTest(helpers.PungiTestCase): "repo": [self.topdir + "/compose/Everything/$arch/os"], }, ), + mock.call.save_task_id(1234), mock.call.watch_task(1234, mock.ANY), mock.call.koji_proxy.getTaskResult(1234), mock.call.koji_proxy.getBuild(build_id), @@ -312,6 +313,7 @@ class RunOSBuildThreadTest(helpers.PungiTestCase): ["aarch64", "x86_64"], opts={"repo": [self.topdir + "/compose/Everything/$arch/os"]}, ), + mock.call.save_task_id(1234), mock.call.watch_task(1234, mock.ANY), mock.call.koji_proxy.getTaskResult(1234), mock.call.koji_proxy.getBuild(build_id), diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index c4f85d39..f45a5ac4 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -446,7 +446,7 @@ class TestSourceKoji(helpers.PungiTestCase): self.assertEqual(pkgsets, gpfk.return_value) self.assertEqual(path_prefix, "/prefix/") - self.assertEqual(KojiWrapper.mock_calls, [mock.call("koji")]) + self.assertEqual(KojiWrapper.mock_calls, [mock.call(compose)]) class TestCorrectNVR(helpers.PungiTestCase): diff --git a/tests/test_scm.py b/tests/test_scm.py index ae18985f..f6307967 100644 --- a/tests/test_scm.py +++ b/tests/test_scm.py @@ -588,9 +588,8 @@ class CvsSCMTestCase(SCMBaseTest): @mock.patch("pungi.wrappers.scm.urlretrieve") -@mock.patch("pungi.wrappers.scm.KojiWrapper") class KojiSCMTestCase(SCMBaseTest): - def test_without_koji_profile(self, KW, dl): + def test_without_koji_profile(self, dl): compose = mock.Mock(conf={}) with self.assertRaises(RuntimeError) as ctx: @@ -600,9 +599,9 @@ class KojiSCMTestCase(SCMBaseTest): compose=compose, ) self.assertIn("Koji profile must be configured", str(ctx.exception)) - self.assertEqual(KW.mock_calls, []) self.assertEqual(dl.mock_calls, []) + @mock.patch("pungi.wrappers.scm.KojiWrapper") def test_doesnt_get_dirs(self, KW, dl): compose = mock.Mock(conf={"koji_profile": "koji"}) @@ -613,7 +612,7 @@ class KojiSCMTestCase(SCMBaseTest): compose=compose, ) self.assertIn("Only files can be exported", str(ctx.exception)) - self.assertEqual(KW.mock_calls, [mock.call("koji")]) + self.assertEqual(KW.mock_calls, [mock.call(compose)]) self.assertEqual(dl.mock_calls, []) def _setup_koji_wrapper(self, KW, build_id, files): @@ -627,6 +626,7 @@ class KojiSCMTestCase(SCMBaseTest): ] KW.return_value.koji_proxy.listTagged.return_value = [buildinfo] + @mock.patch("pungi.wrappers.scm.KojiWrapper") def test_get_from_build(self, KW, dl): compose = mock.Mock(conf={"koji_profile": "koji"}) @@ -646,7 +646,7 @@ class KojiSCMTestCase(SCMBaseTest): self.assertEqual( KW.mock_calls, [ - mock.call("koji"), + mock.call(compose), mock.call().koji_proxy.getBuild("my-build-1.0-2"), mock.call().koji_proxy.listArchives(123), mock.call().koji_module.pathinfo.typedir({"build_id": 123}, "image"), @@ -657,6 +657,7 @@ class KojiSCMTestCase(SCMBaseTest): [mock.call("http://koji.local/koji/images/abc.tar", mock.ANY)], ) + @mock.patch("pungi.wrappers.scm.KojiWrapper") def test_get_from_latest_build(self, KW, dl): compose = mock.Mock(conf={"koji_profile": "koji"}) @@ -676,7 +677,7 @@ class KojiSCMTestCase(SCMBaseTest): self.assertEqual( KW.mock_calls, [ - mock.call("koji"), + mock.call(compose), mock.call().koji_proxy.listTagged( "images", package="my-build", inherit=True, latest=True ), From c8091899b2f0eeabeb270f75c8422393ba1180e5 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Thu, 1 Apr 2021 11:18:18 +0800 Subject: [PATCH 025/137] gather: Copy old logs when reusing gather result This would be helpful for debugging. JIRA: RHELCMP-4594 Signed-off-by: Haibo Lin --- pungi/phases/gather/__init__.py | 30 ++++++++++++++++++++++++++++-- tests/test_gather_phase.py | 28 ++++++++++++++-------------- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 7f77f15a..b95f6e46 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -15,6 +15,7 @@ import json +import glob import os import shutil import threading @@ -192,7 +193,7 @@ def load_old_compose_config(compose): return old_config -def reuse_old_gather_packages(compose, arch, variant, package_sets): +def reuse_old_gather_packages(compose, arch, variant, package_sets, methods): """ Tries to reuse `gather_packages` result from older compose. @@ -200,6 +201,7 @@ def reuse_old_gather_packages(compose, arch, variant, package_sets): :param str arch: Architecture to reuse old gather data for. :param str variant: Variant to reuse old gather data for. :param list package_sets: List of package sets to gather packages from. + :param str methods: Gather method. :return: Old `gather_packages` result or None if old result cannot be used. """ log_msg = "Cannot reuse old GATHER phase results - %s" @@ -372,6 +374,28 @@ def reuse_old_gather_packages(compose, arch, variant, package_sets): compose.log_info(log_msg % "some RPMs have been removed.") return + # Copy old gather log for debugging + try: + if methods == "hybrid": + log_dir = compose.paths.log.topdir(arch, create_dir=False) + old_log_dir = compose.paths.old_compose_path(log_dir) + for log_file in glob.glob( + os.path.join(old_log_dir, "hybrid-depsolver-%s-iter-*" % variant) + ): + compose.log_info( + "Copying old gather log %s to %s" % (log_file, log_dir) + ) + shutil.copy2(log_file, log_dir) + else: + log_dir = os.path.dirname( + compose.paths.work.pungi_log(arch, variant, create_dir=False) + ) + old_log_dir = compose.paths.old_compose_path(log_dir) + compose.log_info("Copying old gather log %s to %s" % (old_log_dir, log_dir)) + shutil.copytree(old_log_dir, log_dir) + except Exception as e: + compose.log_warning("Copying old gather log failed: %s" % str(e)) + return result @@ -398,7 +422,9 @@ def gather_packages(compose, arch, variant, package_sets, fulltree_excludes=None prepopulate = get_prepopulate_packages(compose, arch, variant) fulltree_excludes = fulltree_excludes or set() - reused_result = reuse_old_gather_packages(compose, arch, variant, package_sets) + reused_result = reuse_old_gather_packages( + compose, arch, variant, package_sets, methods + ) if reused_result: result = reused_result elif methods == "hybrid": diff --git a/tests/test_gather_phase.py b/tests/test_gather_phase.py index e120fea2..5ade1c26 100644 --- a/tests/test_gather_phase.py +++ b/tests/test_gather_phase.py @@ -1086,7 +1086,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], [] + compose, "x86_64", compose.variants["Server"], [], "deps" ) self.assertEqual(result, None) @@ -1105,7 +1105,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_compose_config.return_value = None result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], [] + compose, "x86_64", compose.variants["Server"], [], "deps" ) self.assertEqual(result, None) @@ -1126,7 +1126,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_compose_config.return_value = compose_conf_copy result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], [] + compose, "x86_64", compose.variants["Server"], [], "nodeps" ) self.assertEqual(result, None) @@ -1148,7 +1148,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_compose_config.return_value = compose_conf_copy result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], [] + compose, "x86_64", compose.variants["Server"], [], "deps" ) self.assertEqual(result, {"rpm": [], "srpm": [], "debuginfo": []}) @@ -1182,7 +1182,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], package_sets + compose, "x86_64", compose.variants["Server"], package_sets, "deps" ) self.assertEqual( result, @@ -1206,7 +1206,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): gather._update_config(compose, "Server", "x86_64", compose.topdir) result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], package_sets + compose, "x86_64", compose.variants["Server"], package_sets, "deps" ) self.assertEqual( result, @@ -1232,7 +1232,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): gather._update_config(compose, "Server", "x86_64", compose.topdir) result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], package_sets + compose, "x86_64", compose.variants["Server"], package_sets, "deps" ) self.assertEqual(result, None) @@ -1252,7 +1252,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): gather._update_config(compose, "Server", "x86_64", compose.topdir) result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], package_sets + compose, "x86_64", compose.variants["Server"], package_sets, "deps" ) self.assertEqual(result, None) @@ -1271,7 +1271,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], package_sets + compose, "x86_64", compose.variants["Server"], package_sets, "deps" ) self.assertEqual(result, None) @@ -1293,7 +1293,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], package_sets + compose, "x86_64", compose.variants["Server"], package_sets, "deps" ) self.assertEqual(result, None) @@ -1318,7 +1318,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], package_sets + compose, "x86_64", compose.variants["Server"], package_sets, "deps" ) self.assertEqual(result, None) @@ -1335,7 +1335,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], package_sets + compose, "x86_64", compose.variants["Server"], package_sets, "deps" ) self.assertEqual(result, None) @@ -1351,7 +1351,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], package_sets + compose, "x86_64", compose.variants["Server"], package_sets, "deps" ) self.assertEqual(result, None) @@ -1367,7 +1367,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( - compose, "x86_64", compose.variants["Server"], package_sets + compose, "x86_64", compose.variants["Server"], package_sets, "deps" ) self.assertEqual(result, None) From ab1b5b48eca8c58966ac244d97829bae1386b94d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 9 Apr 2021 13:53:31 +0200 Subject: [PATCH 026/137] hybrid: Optimize getting lookaside packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original code ended up downloading all repodata from the lookaside repo. This could cause a lot of memory to be used. The new code only downloads the repomd.xml and then primary record, which is sufficient to obtain all needed information. A lot less memory is used and the code is also significantly faster. Here are some alternative ways of getting a list of packages from the lookaside repo and reasons why they did not work: * dnf repoquery - this doesn't include modular packages unless the stream is default * dnf reposync - requires `--urls` option to only print the names, which is not available on RHEL 7 JIRA: RHELCMP-4761 Signed-off-by: Lubomír Sedlář --- pungi/phases/gather/methods/method_hybrid.py | 42 +++++++++++++------- tests/test_gather_method_hybrid.py | 30 +++----------- 2 files changed, 33 insertions(+), 39 deletions(-) diff --git a/pungi/phases/gather/methods/method_hybrid.py b/pungi/phases/gather/methods/method_hybrid.py index 3c673c0f..3aa60cbd 100644 --- a/pungi/phases/gather/methods/method_hybrid.py +++ b/pungi/phases/gather/methods/method_hybrid.py @@ -499,6 +499,27 @@ def _make_result(paths): return [{"path": path, "flags": []} for path in sorted(paths)] +def get_repo_packages(path): + """Extract file names of all packages in the given repository.""" + + packages = set() + + def callback(pkg): + packages.add(os.path.basename(pkg.location_href)) + + repomd = os.path.join(path, "repodata/repomd.xml") + with as_local_file(repomd) as url_: + repomd = cr.Repomd(url_) + for rec in repomd.records: + if rec.type != "primary": + continue + record_url = os.path.join(path, rec.location_href) + with as_local_file(record_url) as url_: + cr.xml_parse_primary(url_, pkgcb=callback, do_files=False) + + return packages + + def expand_packages(nevra_to_pkg, lookasides, nvrs, filter_packages): """For each package add source RPM.""" # This will serve as the final result. We collect sets of paths to the @@ -509,25 +530,16 @@ def expand_packages(nevra_to_pkg, lookasides, nvrs, filter_packages): filters = set(filter_packages) - # Collect list of all packages in lookaside. These will not be added to the - # result. Fus handles this in part: if a package is explicitly mentioned as - # input (which can happen with comps group expansion), it will be in the - # output even if it's in lookaside. lookaside_packages = set() for repo in lookasides: - md = cr.Metadata() - md.locate_and_load_xml(repo) - for key in md.keys(): - pkg = md.get(key) - url = os.path.join(pkg.location_base or repo, pkg.location_href) - # Strip file:// prefix - lookaside_packages.add(url[7:]) + lookaside_packages.update(get_repo_packages(repo)) for nvr, pkg_arch, flags in nvrs: pkg = nevra_to_pkg["%s.%s" % (nvr, pkg_arch)] - if pkg.file_path in lookaside_packages: - # Package is in lookaside, don't add it and ignore sources and - # debuginfo too. + if os.path.basename(pkg.file_path) in lookaside_packages: + # Fus can return lookaside package in output if the package is + # explicitly listed as input. This can happen during comps + # expansion. continue if pkg_is_debug(pkg): debuginfo.add(pkg.file_path) @@ -540,7 +552,7 @@ def expand_packages(nevra_to_pkg, lookasides, nvrs, filter_packages): if (srpm.name, "src") in filters: # Filtered package, skipping continue - if srpm.file_path not in lookaside_packages: + if os.path.basename(srpm.file_path) not in lookaside_packages: srpms.add(srpm.file_path) except KeyError: # Didn't find source RPM.. this should be logged diff --git a/tests/test_gather_method_hybrid.py b/tests/test_gather_method_hybrid.py index b4e24a0d..bc0a283b 100644 --- a/tests/test_gather_method_hybrid.py +++ b/tests/test_gather_method_hybrid.py @@ -934,20 +934,11 @@ class TestExpandPackages(helpers.PungiTestCase): }, ) - @mock.patch("pungi.phases.gather.methods.method_hybrid.cr") - def test_skip_lookaside_source(self, cr): + @mock.patch("pungi.phases.gather.methods.method_hybrid.get_repo_packages") + def test_skip_lookaside_source(self, get_repo_packages): nevra_to_pkg = self._mk_packages(src=True) lookasides = [mock.Mock()] - repo = { - "abc": NamedMock( - name="pkg", - arch="src", - location_base="file:///tmp/", - location_href="pkg.src.rpm", - ), - } - cr.Metadata.return_value.keys.return_value = repo.keys() - cr.Metadata.return_value.get.side_effect = lambda key: repo[key] + get_repo_packages.return_value = ["pkg.src.rpm"] res = hybrid.expand_packages( nevra_to_pkg, lookasides, [("pkg-3:1-2", "x86_64", [])], [] @@ -962,20 +953,11 @@ class TestExpandPackages(helpers.PungiTestCase): }, ) - @mock.patch("pungi.phases.gather.methods.method_hybrid.cr") - def test_skip_lookaside_packages(self, cr): + @mock.patch("pungi.phases.gather.methods.method_hybrid.get_repo_packages") + def test_skip_lookaside_packages(self, get_repo_packages): nevra_to_pkg = self._mk_packages(debug_arch="x86_64") lookasides = [mock.Mock()] - repo = { - "abc": NamedMock( - name="pkg", - arch="x86_64", - location_base="file:///tmp/", - location_href="pkg.rpm", - ) - } - cr.Metadata.return_value.keys.return_value = repo.keys() - cr.Metadata.return_value.get.side_effect = lambda key: repo[key] + get_repo_packages.return_value = ["pkg.rpm"] res = hybrid.expand_packages( nevra_to_pkg, lookasides, [("pkg-3:1-2", "x86_64", [])], [] From e866d22c045dc2221bf6f94b103caca378249d5c Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Thu, 8 Apr 2021 10:22:15 +0800 Subject: [PATCH 027/137] gather: Adjust reusing with lookaside - Do not reuse when there is any external lookaside repo - Do not reuse when lookaside variant is not reused JIRA: RHELCMP-4596 Signed-off-by: Haibo Lin --- pungi/phases/gather/__init__.py | 51 +++++++++++++++++---------------- tests/test_gather_phase.py | 20 +++++++++++++ 2 files changed, 46 insertions(+), 25 deletions(-) diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index b95f6e46..1578c078 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -19,7 +19,6 @@ import glob import os import shutil import threading -import six from six.moves import cPickle as pickle from kobo.rpmlib import parse_nvra @@ -219,33 +218,33 @@ def reuse_old_gather_packages(compose, arch, variant, package_sets, methods): compose.log_info(log_msg % "no old compose config dump.") return + # Do not reuse when required variant is not reused. + if not hasattr(compose, "_gather_reused_variant_arch"): + setattr(compose, "_gather_reused_variant_arch", []) + variant_as_lookaside = compose.conf.get("variant_as_lookaside", []) + for (requiring, required) in variant_as_lookaside: + if ( + requiring == variant.uid + and (required, arch) not in compose._gather_reused_variant_arch + ): + compose.log_info( + log_msg % "variant %s as lookaside is not reused." % required + ) + return + + # Do not reuse if there's external lookaside repo. + with open(compose.paths.log.log_file("global", "config-dump"), "r") as f: + config_dump = json.load(f) + if config_dump.get("gather_lookaside_repos") or old_config.get( + "gather_lookaside_repos" + ): + compose.log_info(log_msg % "there's external lookaside repo.") + return + # The dumps/loads is needed to convert all unicode strings to non-unicode ones. config = json.loads(json.dumps(compose.conf)) for opt, value in old_config.items(): - # Gather lookaside repos are updated during the gather phase. Check that - # the gather_lookaside_repos except the ones added are the same. - if opt == "gather_lookaside_repos" and opt in config: - value_to_compare = [] - # Filter out repourls which starts with `compose.topdir` and also remove - # their parent list in case it would be empty. - for variant, per_arch_repos in config[opt]: - per_arch_repos_to_compare = {} - for arch, repourl in per_arch_repos.items(): - # The gather_lookaside_repos config allows setting multiple repourls - # using list, but `_update_config` always uses strings. Therefore we - # only try to filter out string_types. - if not isinstance(repourl, six.string_types): - continue - if not repourl.startswith(compose.topdir): - per_arch_repos_to_compare[arch] = repourl - if per_arch_repos_to_compare: - value_to_compare.append([variant, per_arch_repos_to_compare]) - if value != value_to_compare: - compose.log_info( - log_msg - % ("compose configuration option gather_lookaside_repos changed.") - ) - return + if opt == "gather_lookaside_repos": continue # Skip checking for frequently changing configuration options which do *not* @@ -374,6 +373,8 @@ def reuse_old_gather_packages(compose, arch, variant, package_sets, methods): compose.log_info(log_msg % "some RPMs have been removed.") return + compose._gather_reused_variant_arch.append((variant.uid, arch)) + # Copy old gather log for debugging try: if methods == "hybrid": diff --git a/tests/test_gather_phase.py b/tests/test_gather_phase.py index 5ade1c26..4c41e682 100644 --- a/tests/test_gather_phase.py +++ b/tests/test_gather_phase.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import copy +import json import mock import os @@ -1080,11 +1081,17 @@ class TestGatherPackages(helpers.PungiTestCase): class TestReuseOldGatherPackages(helpers.PungiTestCase): + def _save_config_dump(self, compose): + config_dump_full = compose.paths.log.log_file("global", "config-dump") + with open(config_dump_full, "w") as f: + json.dump(compose.conf, f, sort_keys=True, indent=4) + @mock.patch("pungi.phases.gather.load_old_gather_result") def test_reuse_no_old_gather_result(self, load_old_gather_result): load_old_gather_result.return_value = None compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) + self._save_config_dump(compose) result = gather.reuse_old_gather_packages( compose, "x86_64", compose.variants["Server"], [], "deps" ) @@ -1102,6 +1109,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): } compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) + self._save_config_dump(compose) load_old_compose_config.return_value = None result = gather.reuse_old_gather_packages( @@ -1121,6 +1129,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): } compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) + self._save_config_dump(compose) compose_conf_copy = dict(compose.conf) compose_conf_copy["gather_method"] = "nodeps" load_old_compose_config.return_value = compose_conf_copy @@ -1143,6 +1152,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): } compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) + self._save_config_dump(compose) compose_conf_copy = dict(compose.conf) compose_conf_copy[whitelist_opt] = "different" load_old_compose_config.return_value = compose_conf_copy @@ -1179,6 +1189,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_gather_result, requires=[], provides=[] ) compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) + self._save_config_dump(compose) load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( @@ -1202,6 +1213,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_gather_result, requires=[], provides=[] ) compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) + self._save_config_dump(compose) load_old_compose_config.return_value = copy.deepcopy(compose.conf) gather._update_config(compose, "Server", "x86_64", compose.topdir) @@ -1226,6 +1238,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_gather_result, requires=[], provides=[] ) compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) + self._save_config_dump(compose) lookasides = compose.conf["gather_lookaside_repos"] lookasides.append(("^Server$", {"x86_64": "http://localhost/real.repo"})) load_old_compose_config.return_value = copy.deepcopy(compose.conf) @@ -1245,6 +1258,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_gather_result, requires=[], provides=[] ) compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) + self._save_config_dump(compose) lookasides = compose.conf["gather_lookaside_repos"] repos = ["http://localhost/real1.repo", "http://localhost/real2.repo"] lookasides.append(("^Server$", {"x86_64": repos})) @@ -1268,6 +1282,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): "/build/foo-1-1.x86_64.rpm": MockPkg("foo-1-1.x86_64.rpm", sourcerpm="foo") } compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) + self._save_config_dump(compose) load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( @@ -1290,6 +1305,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): pkg_set.old_file_cache["/build/bash-1-2.x86_64.rpm"] = bash_pkg pkg_set.file_cache["/build/bash-1-2.x86_64.rpm"] = bash_pkg compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) + self._save_config_dump(compose) load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( @@ -1315,6 +1331,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): pkg_set.old_file_cache["/build/file-1-1.x86_64.rpm"] = file_pkg pkg_set.file_cache["/build/foo-1-1.x86_64.rpm"] = foo_pkg compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) + self._save_config_dump(compose) load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( @@ -1332,6 +1349,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): ) package_sets[0]["global"].old_file_cache = None compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) + self._save_config_dump(compose) load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( @@ -1348,6 +1366,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_gather_result, requires=["foo"], provides=[] ) compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) + self._save_config_dump(compose) load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( @@ -1364,6 +1383,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): load_old_gather_result, requires=[], provides=["foo"] ) compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) + self._save_config_dump(compose) load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( From 00a9861367f8863436230a805f9514373fda984e Mon Sep 17 00:00:00 2001 From: Lev Veyde Date: Wed, 21 Apr 2021 21:04:01 +0300 Subject: [PATCH 028/137] Updated the deprecated ks argument name (to the current inst.ks) Signed-off-by: Lev Veyde --- pungi/phases/buildinstall.py | 2 +- tests/test_buildinstall.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pungi/phases/buildinstall.py b/pungi/phases/buildinstall.py index 04e46870..a43e3707 100644 --- a/pungi/phases/buildinstall.py +++ b/pungi/phases/buildinstall.py @@ -368,7 +368,7 @@ def tweak_configs(path, volid, ks_file, configs=BOOT_CONFIGS, logger=None): # double-escape volid in yaboot.conf new_volid = volid_escaped_2 if "yaboot" in config else volid_escaped - ks = (" ks=hd:LABEL=%s:/ks.cfg" % new_volid) if ks_file else "" + ks = (" inst.ks=hd:LABEL=%s:/ks.cfg" % new_volid) if ks_file else "" # pre-f18 data = re.sub(r":CDLABEL=[^ \n]*", r":CDLABEL=%s%s" % (new_volid, ks), data) diff --git a/tests/test_buildinstall.py b/tests/test_buildinstall.py index a8c3ddf7..c2051f83 100644 --- a/tests/test_buildinstall.py +++ b/tests/test_buildinstall.py @@ -2161,7 +2161,7 @@ class TestTweakConfigs(PungiTestCase): ) for cfg in configs: self.assertFileContent( - cfg, ":LABEL=new\\x20volid ks=hd:LABEL=new\\x20volid:/ks.cfg\n" + cfg, ":LABEL=new\\x20volid inst.ks=hd:LABEL=new\\x20volid:/ks.cfg\n" ) def test_tweak_configs_yaboot(self): @@ -2173,5 +2173,5 @@ class TestTweakConfigs(PungiTestCase): tweak_configs(self.topdir, "new volid", os.path.join(self.topdir, "ks.cfg")) for cfg in configs: self.assertFileContent( - cfg, ":LABEL=new\\\\x20volid ks=hd:LABEL=new\\\\x20volid:/ks.cfg\n" + cfg, ":LABEL=new\\\\x20volid inst.ks=hd:LABEL=new\\\\x20volid:/ks.cfg\n" ) From da791ed15cbf93d23d7dfa345c01467e7fd52063 Mon Sep 17 00:00:00 2001 From: Romain Forlot Date: Thu, 22 Apr 2021 14:03:12 +0200 Subject: [PATCH 029/137] Fix can't link XDEV using repos as pkgset_sources Trying to compose from external classic repositories return an error trying the hardling from a yum cache directory located in /tmp to the target directory in another filesystem. This commit fixes this using the 'link' method form linker module which handle the link_type configuration parameter instead of the hardcoded method 'hardlink'. Change-Id: Ib79cfbd72f9def6462fddb2ae368730c55f257cd Signed-off-by: Romain Forlot --- pungi/gather_dnf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pungi/gather_dnf.py b/pungi/gather_dnf.py index 5fecc7f0..97ebf48e 100644 --- a/pungi/gather_dnf.py +++ b/pungi/gather_dnf.py @@ -1029,7 +1029,7 @@ class Gather(GatherBase): # Link downloaded package in (or link package from file repo) try: - linker.hardlink(pkg.localPkg(), target) + linker.link(pkg.localPkg(), target) except Exception: self.logger.error("Unable to link %s from the yum cache." % pkg.name) raise From 76d13d0062c0bd1c2126acb4e701cf171dcd8aee Mon Sep 17 00:00:00 2001 From: Ondrej Nosek Date: Thu, 29 Apr 2021 06:43:41 +0200 Subject: [PATCH 030/137] 4.2.9 release Signed-off-by: Ondrej Nosek --- doc/conf.py | 2 +- pungi.spec | 19 +++++++++++++++++-- setup.py | 2 +- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 845600fc..0b5b3a23 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -53,7 +53,7 @@ copyright = u'2016, Red Hat, Inc.' # The short X.Y version. version = '4.2' # The full version, including alpha/beta/rc tags. -release = '4.2.8' +release = '4.2.9' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pungi.spec b/pungi.spec index 22b53512..ce92c16c 100644 --- a/pungi.spec +++ b/pungi.spec @@ -1,5 +1,5 @@ Name: pungi -Version: 4.2.8 +Version: 4.2.9 Release: 1%{?dist} Summary: Distribution compose tool @@ -111,7 +111,22 @@ pytest cd tests && ./test_compose.sh %changelog -* Fri Feb 12 2021 Ondrej Nosek +* Thu Apr 29 2021 Ondrej Nosek - 4.2.9-1 +- Fix can't link XDEV using repos as pkgset_sources (romain.forlot) +- Updated the deprecated ks argument name (to the current inst.ks) (lveyde) +- gather: Adjust reusing with lookaside (hlin) +- hybrid: Optimize getting lookaside packages (lsedlar) +- gather: Copy old logs when reusing gather result (hlin) +- Cancel koji tasks when pungi terminated (hlin) +- Add Dockerfile for building testing image (hlin) +- image_container: Fix incorrect arch processing (lsedlar) +- runroot: Adjust permissions always (hlin) +- Format code (hlin) +- pkgset: Fix meaning of retries (lsedlar) +- pkgset: Store module tag only if module is used (lsedlar) +- Store extended traceback for gather errors (lsedlar) + +* Fri Feb 12 2021 Ondrej Nosek - 4.2.8-1 - pkgset: Add ability to wait for signed packages (lsedlar) - Add image-container phase (lsedlar) - osbs: Move metadata processing to standalone function (lsedlar) diff --git a/setup.py b/setup.py index 072811bf..91dfd15d 100755 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ packages = sorted(packages) setup( name="pungi", - version="4.2.8", + version="4.2.9", description="Distribution compose tool", url="https://pagure.io/pungi", author="Dennis Gilmore", From c27bfe0c59cd626dc40d9f9b25fe4b817ff7710b Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Tue, 27 Apr 2021 18:00:48 +0800 Subject: [PATCH 031/137] Clean up temporary yumroot dir JIRA: RHELCMP-4948 Signed-off-by: Haibo Lin --- pungi/phases/gather/methods/method_deps.py | 16 ++++++++++++++-- pungi/phases/pkgset/sources/source_repos.py | 12 ++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/pungi/phases/gather/methods/method_deps.py b/pungi/phases/gather/methods/method_deps.py index 1f3f9659..4b785629 100644 --- a/pungi/phases/gather/methods/method_deps.py +++ b/pungi/phases/gather/methods/method_deps.py @@ -15,6 +15,7 @@ import os +import shutil from kobo.shortcuts import run from kobo.pkgset import SimpleRpmWrapper, RpmWrapper @@ -241,8 +242,19 @@ def resolve_deps(compose, arch, variant, source_name=None): ) # Use temp working directory directory as workaround for # https://bugzilla.redhat.com/show_bug.cgi?id=795137 - with temp_dir(prefix="pungi_") as tmp_dir: - run(cmd, logfile=pungi_log, show_cmd=True, workdir=tmp_dir, env=os.environ) + with temp_dir(prefix="pungi_") as work_dir: + run(cmd, logfile=pungi_log, show_cmd=True, workdir=work_dir, env=os.environ) + + # Clean up tmp dir + # Workaround for rpm not honoring sgid bit which only appears when yum is used. + yumroot_dir = os.path.join(tmp_dir, "work", arch, "yumroot") + if os.path.isdir(yumroot_dir): + try: + shutil.rmtree(yumroot_dir) + except Exception as e: + compose.log_warning( + "Failed to clean up tmp dir: %s %s" % (yumroot_dir, str(e)) + ) with open(pungi_log, "r") as f: packages, broken_deps, missing_comps_pkgs = pungi_wrapper.parse_log(f) diff --git a/pungi/phases/pkgset/sources/source_repos.py b/pungi/phases/pkgset/sources/source_repos.py index 63c7c4ec..716f6336 100644 --- a/pungi/phases/pkgset/sources/source_repos.py +++ b/pungi/phases/pkgset/sources/source_repos.py @@ -15,6 +15,7 @@ import os +import shutil from kobo.shortcuts import run @@ -110,6 +111,17 @@ def get_pkgset_from_repos(compose): flist.append(dst) pool.queue_put((src, dst)) + # Clean up tmp dir + # Workaround for rpm not honoring sgid bit which only appears when yum is used. + yumroot_dir = os.path.join(pungi_dir, "work", arch, "yumroot") + if os.path.isdir(yumroot_dir): + try: + shutil.rmtree(yumroot_dir) + except Exception as e: + compose.log_warning( + "Failed to clean up tmp dir: %s %s" % (yumroot_dir, str(e)) + ) + msg = "Linking downloaded pkgset packages" compose.log_info("[BEGIN] %s" % msg) pool.start() From 7fe32ae758f15f751f8062cdaff9ef5b7fac37d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 6 May 2021 12:37:01 +0200 Subject: [PATCH 032/137] util: Strip file:// from local urls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make sure that the function returns a path even for local files specified by file:// urls. JIRA: RHELCMP-5340 Signed-off-by: Lubomír Sedlář --- pungi/util.py | 2 ++ tests/test_util.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/pungi/util.py b/pungi/util.py index cf1ec8ee..c5717bde 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -1046,6 +1046,8 @@ def as_local_file(url): yield local_filename finally: os.remove(local_filename) + elif url.startswith("file://"): + yield url[7:] else: # Not a remote url, return unchanged. yield url diff --git a/tests/test_util.py b/tests/test_util.py index 29d85cfa..181c91d8 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1060,3 +1060,8 @@ class TestAsLocalFile(PungiTestCase): self.assertEqual(fn, self.filename) self.assertTrue(os.path.exists(self.filename)) self.assertFalse(os.path.exists(self.filename)) + + def test_file_url(self, urlretrieve): + with util.as_local_file("file:///tmp/foo") as fn: + self.assertEqual(fn, "/tmp/foo") + self.assertEqual(urlretrieve.call_args_list, []) From bf28e8d50c4caa82a211283576c9aa165b1ea859 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 12 May 2021 16:09:47 +0200 Subject: [PATCH 033/137] pkgset: Compare future events correctly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It is possible to try to re-run a compose with old event. When trying to reuse pkgset data, we must use set the bounds not based on current/reused event, but actually check which was first. JIRA: CWFHEALTH-495 Signed-off-by: Lubomír Sedlář --- pungi/phases/pkgset/pkgsets.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index 54dcafde..07f6e8c7 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -745,7 +745,8 @@ class KojiPackageSet(PackageSetBase): changed = self.koji_proxy.queryHistory( tables=["tag_listing", "tag_inheritance"], tag=tag, - afterEvent=old_koji_event, + afterEvent=min(koji_event, old_koji_event), + beforeEvent=max(koji_event, old_koji_event) + 1, ) if changed["tag_listing"]: self.log_debug("Builds under tag %s changed. Can't reuse." % tag) @@ -760,8 +761,8 @@ class KojiPackageSet(PackageSetBase): changed = self.koji_proxy.queryHistory( tables=["tag_listing", "tag_inheritance"], tag=t["name"], - afterEvent=old_koji_event, - beforeEvent=koji_event + 1, + afterEvent=min(koji_event, old_koji_event), + beforeEvent=max(koji_event, old_koji_event) + 1, ) if changed["tag_listing"]: self.log_debug( From 9a5e901cfe20504dbabe78d0cb6e913389ef5831 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Fri, 25 Jun 2021 14:55:23 +0800 Subject: [PATCH 034/137] Log warning when module defined in variants.xml not found JIRA: RHELCMP-5573 Signed-off-by: Haibo Lin --- pungi/phases/pkgset/sources/source_koji.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 3a0db08a..2ee955ce 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -659,7 +659,7 @@ def _get_modules_from_koji_tags( if expected_modules: # There are some module names that were listed in configuration and not # found in any tag... - raise RuntimeError( + compose.log_warning( "Configuration specified patterns (%s) that don't match " "any modules in the configured tags." % ", ".join(expected_modules) ) From edb091b7b15489fc78b76b4345982c3ef13238de Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Fri, 25 Jun 2021 17:24:42 +0800 Subject: [PATCH 035/137] Add task URL to watch task log JIRA: RHELCMP-5666 Signed-off-by: Haibo Lin --- pungi/wrappers/kojiwrapper.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py index b501684d..74e3195d 100644 --- a/pungi/wrappers/kojiwrapper.py +++ b/pungi/wrappers/kojiwrapper.py @@ -564,6 +564,19 @@ class KojiWrapper(object): } def watch_task(self, task_id, log_file=None, max_retries=None): + """Watch and wait for a task to finish. + + :param int task_id: ID of koji task. + :param str log_file: Path to log file. + :param int max_retries: Max times to retry when error occurs, + no limits by default. + """ + if log_file: + task_url = os.path.join( + self.koji_module.config.weburl, "taskinfo?taskID=%d" % task_id + ) + with open(log_file, "a") as f: + f.write("Task URL: %s\n" % task_url) retcode, _ = self._wait_for_task( task_id, logfile=log_file, max_retries=max_retries ) From a435fd58da9a19ef0d9575c55f6726e6083bf7e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Mon, 19 Jul 2021 14:02:16 +0200 Subject: [PATCH 036/137] gather: Add all srpms to variant lookaside repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original code could cause a source RPM to be present in two variants that have a dependency relation. There is always only one source repo for a variant in the final compose. When gathering packages for a variant that depends on another variant, we need to build a temporary lookaside repo that has similar content to the parent variant. This lookaside only contained source RPMs for packages present the the architecture. This could result in duplicated SRPMs in the compose. Example situation: * Variant B depends on variant A. * A contains foo.x86_64.rpm (only on x86_64) * B pulls in subpackage foo-bar.s390x.rpm (on s390x) Source repo for A will correctly contain foo.src.rpm. With original code the srpm would also end up in B.src. By adding all sources to the temporary lookaside Pungi will know that source repo for B doesn't need to duplicate the package. The refactoring to use a set to store the packages is meant to avoid listing the same SRPM multiple times in the repo in the most common situation when SRPM is listed in multiple architectures. JIRA: RHELCMP-6002 Signed-off-by: Lubomír Sedlář --- pungi/phases/gather/__init__.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 1578c078..6520c806 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -639,14 +639,22 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None): + "/", } path_prefix = prefixes[compose.conf["pkgset_source"]]() + package_list = set() + for pkg_arch in pkg_map.keys(): + for pkg_type, packages in pkg_map[pkg_arch][variant.uid].items(): + # We want all packages for current arch, and SRPMs for any + # arch. Ultimately there will only be one source repository, so + # we need a union of all SRPMs. + if pkg_type == "srpm" or pkg_arch == arch: + for pkg in packages: + pkg = pkg["path"] + if path_prefix and pkg.startswith(path_prefix): + pkg = pkg[len(path_prefix) :] + package_list.add(pkg) pkglist = compose.paths.work.lookaside_package_list(arch=arch, variant=variant) with open(pkglist, "w") as f: - for packages in pkg_map[arch][variant.uid].values(): - for pkg in packages: - pkg = pkg["path"] - if path_prefix and pkg.startswith(path_prefix): - pkg = pkg[len(path_prefix) :] - f.write("%s\n" % pkg) + for pkg in sorted(package_list): + f.write("%s\n" % pkg) cr = CreaterepoWrapper(compose.conf["createrepo_c"]) update_metadata = None From 56a55db966370d705a71d0d897cc682fa237b53f Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Wed, 21 Jul 2021 18:16:03 +0800 Subject: [PATCH 037/137] Use cachedir when createrepo Then createrepo can reuse checksum values from cache to make it faster. JIRA: RHELCMP-5984 Signed-off-by: Haibo Lin --- pungi.spec | 2 +- pungi/checks.py | 1 + pungi/phases/createrepo.py | 18 +++++++ tests/test_createrepophase.py | 90 +++++++++++++++++++++++++++++++---- 4 files changed, 100 insertions(+), 11 deletions(-) diff --git a/pungi.spec b/pungi.spec index ce92c16c..4e5705de 100644 --- a/pungi.spec +++ b/pungi.spec @@ -91,7 +91,7 @@ rm -rf %{buildroot} %{_bindir}/comps_filter %{_bindir}/%{name}-make-ostree %{_datadir}/%{name} -/var/cache/%{name} +%dir %attr(1777, root, root) /var/cache/%{name} %files utils %{python_sitelib}/%{name}_utils diff --git a/pungi/checks.py b/pungi/checks.py index d4a8ed26..62724946 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -660,6 +660,7 @@ def make_schema(): "default": "sha256", "enum": ["sha1", "sha256", "sha512"], }, + "createrepo_enable_cache": {"type": "boolean", "default": True}, "createrepo_use_xz": {"type": "boolean", "default": False}, "createrepo_num_threads": {"type": "number", "default": get_num_cpus()}, "createrepo_num_workers": {"type": "number", "default": 3}, diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index 0ad8b3f9..c9ac4746 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -188,6 +188,23 @@ def create_variant_repo( comps_path = None if compose.has_comps and pkg_type == "rpm": comps_path = compose.paths.work.comps(arch=arch, variant=variant) + + if compose.conf["createrepo_enable_cache"]: + cachedir = os.path.join( + "/var/cache/pungi/createrepo_c/", + "%s-%s" % (compose.conf["release_short"], os.getuid()), + ) + if not os.path.exists(cachedir): + try: + os.makedirs(cachedir) + except Exception as e: + compose.log_warning( + "Cache disabled because cannot create cache dir %s %s" + % (cachedir, str(e)) + ) + cachedir = None + else: + cachedir = None cmd = repo.get_createrepo_cmd( repo_dir, update=True, @@ -203,6 +220,7 @@ def create_variant_repo( oldpackagedirs=old_package_dirs, use_xz=compose.conf["createrepo_use_xz"], extra_args=compose.conf["createrepo_extra_args"], + cachedir=cachedir, ) log_file = compose.paths.log.log_file( arch, "createrepo-%s.%s" % (variant, pkg_type) diff --git a/tests/test_createrepophase.py b/tests/test_createrepophase.py index 9bc794ca..963984bc 100644 --- a/tests/test_createrepophase.py +++ b/tests/test_createrepophase.py @@ -176,7 +176,10 @@ class TestCreateVariantRepo(PungiTestCase): @mock.patch("pungi.phases.createrepo.run") @mock.patch("pungi.phases.createrepo.CreaterepoWrapper") def test_variant_repo_rpms(self, CreaterepoWrapperCls, run): - compose = DummyCompose(self.topdir, {"createrepo_checksum": "sha256"}) + compose = DummyCompose( + self.topdir, + {"createrepo_checksum": "sha256"}, + ) compose.has_comps = False repo = CreaterepoWrapperCls.return_value @@ -210,6 +213,10 @@ class TestCreateVariantRepo(PungiTestCase): oldpackagedirs=None, use_xz=False, extra_args=[], + cachedir=os.path.join( + "/var/cache/pungi/createrepo_c/", + "%s-%s" % (compose.conf["release_short"], os.getuid()), + ), ) ], ) @@ -219,7 +226,10 @@ class TestCreateVariantRepo(PungiTestCase): @mock.patch("pungi.phases.createrepo.run") @mock.patch("pungi.phases.createrepo.CreaterepoWrapper") def test_variant_repo_rpms_without_database(self, CreaterepoWrapperCls, run): - compose = DummyCompose(self.topdir, {"createrepo_checksum": "sha256"}) + compose = DummyCompose( + self.topdir, + {"createrepo_checksum": "sha256", "createrepo_enable_cache": False}, + ) compose.should_create_yum_database = False compose.has_comps = False @@ -254,6 +264,7 @@ class TestCreateVariantRepo(PungiTestCase): oldpackagedirs=None, use_xz=False, extra_args=[], + cachedir=None, ) ], ) @@ -263,7 +274,10 @@ class TestCreateVariantRepo(PungiTestCase): @mock.patch("pungi.phases.createrepo.run") @mock.patch("pungi.phases.createrepo.CreaterepoWrapper") def test_variant_repo_source(self, CreaterepoWrapperCls, run): - compose = DummyCompose(self.topdir, {"createrepo_checksum": "sha256"}) + compose = DummyCompose( + self.topdir, + {"createrepo_checksum": "sha256", "createrepo_enable_cache": False}, + ) compose.has_comps = False repo = CreaterepoWrapperCls.return_value @@ -295,6 +309,7 @@ class TestCreateVariantRepo(PungiTestCase): oldpackagedirs=None, use_xz=False, extra_args=[], + cachedir=None, ) ], ) @@ -304,7 +319,10 @@ class TestCreateVariantRepo(PungiTestCase): @mock.patch("pungi.phases.createrepo.run") @mock.patch("pungi.phases.createrepo.CreaterepoWrapper") def test_variant_repo_debug(self, CreaterepoWrapperCls, run): - compose = DummyCompose(self.topdir, {"createrepo_checksum": "sha256"}) + compose = DummyCompose( + self.topdir, + {"createrepo_checksum": "sha256", "createrepo_enable_cache": False}, + ) compose.has_comps = False repo = CreaterepoWrapperCls.return_value @@ -339,6 +357,7 @@ class TestCreateVariantRepo(PungiTestCase): oldpackagedirs=None, use_xz=False, extra_args=[], + cachedir=None, ) ], ) @@ -351,7 +370,12 @@ class TestCreateVariantRepo(PungiTestCase): @mock.patch("pungi.phases.createrepo.CreaterepoWrapper") def test_variant_repo_no_createrepo_c(self, CreaterepoWrapperCls, run): compose = DummyCompose( - self.topdir, {"createrepo_c": False, "createrepo_checksum": "sha256"} + self.topdir, + { + "createrepo_c": False, + "createrepo_enable_cache": False, + "createrepo_checksum": "sha256", + }, ) compose.has_comps = False @@ -386,6 +410,7 @@ class TestCreateVariantRepo(PungiTestCase): oldpackagedirs=None, use_xz=False, extra_args=[], + cachedir=None, ) ], ) @@ -397,7 +422,11 @@ class TestCreateVariantRepo(PungiTestCase): def test_variant_repo_is_idepotent(self, CreaterepoWrapperCls, run): compose = DummyCompose( self.topdir, - {"createrepo_checksum": "sha256", "createrepo_num_workers": 10}, + { + "createrepo_checksum": "sha256", + "createrepo_enable_cache": False, + "createrepo_num_workers": 10, + }, ) compose.has_comps = False @@ -436,6 +465,7 @@ class TestCreateVariantRepo(PungiTestCase): oldpackagedirs=None, use_xz=False, extra_args=[], + cachedir=None, ) ], ) @@ -446,7 +476,12 @@ class TestCreateVariantRepo(PungiTestCase): @mock.patch("pungi.phases.createrepo.CreaterepoWrapper") def test_variant_repo_rpms_with_xz(self, CreaterepoWrapperCls, run): compose = DummyCompose( - self.topdir, {"createrepo_checksum": "sha256", "createrepo_use_xz": True} + self.topdir, + { + "createrepo_checksum": "sha256", + "createrepo_enable_cache": False, + "createrepo_use_xz": True, + }, ) compose.has_comps = False @@ -481,6 +516,7 @@ class TestCreateVariantRepo(PungiTestCase): oldpackagedirs=None, use_xz=True, extra_args=[], + cachedir=None, ) ], ) @@ -491,7 +527,12 @@ class TestCreateVariantRepo(PungiTestCase): @mock.patch("pungi.phases.createrepo.CreaterepoWrapper") def test_variant_repo_rpms_with_deltas(self, CreaterepoWrapperCls, run): compose = DummyCompose( - self.topdir, {"createrepo_checksum": "sha256", "createrepo_deltas": True} + self.topdir, + { + "createrepo_checksum": "sha256", + "createrepo_deltas": True, + "createrepo_enable_cache": False, + }, ) compose.has_comps = False compose.old_composes = [self.topdir + "/old"] @@ -536,6 +577,7 @@ class TestCreateVariantRepo(PungiTestCase): + "/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages", use_xz=False, extra_args=[], + cachedir=None, ) ], ) @@ -552,6 +594,7 @@ class TestCreateVariantRepo(PungiTestCase): { "createrepo_checksum": "sha256", "createrepo_deltas": [("^Server$", {"*": True})], + "createrepo_enable_cache": False, }, ) compose.has_comps = False @@ -596,6 +639,7 @@ class TestCreateVariantRepo(PungiTestCase): + "/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages", use_xz=False, extra_args=[], + cachedir=None, ) ], ) @@ -612,6 +656,7 @@ class TestCreateVariantRepo(PungiTestCase): { "createrepo_checksum": "sha256", "createrepo_deltas": [("^Everything$", {"x86_64": True})], + "createrepo_enable_cache": False, }, ) compose.has_comps = False @@ -652,6 +697,7 @@ class TestCreateVariantRepo(PungiTestCase): oldpackagedirs=None, use_xz=False, extra_args=[], + cachedir=None, ) ], ) @@ -668,6 +714,7 @@ class TestCreateVariantRepo(PungiTestCase): { "createrepo_checksum": "sha256", "createrepo_deltas": [("^Server$", {"s390x": True})], + "createrepo_enable_cache": False, }, ) compose.has_comps = False @@ -708,6 +755,7 @@ class TestCreateVariantRepo(PungiTestCase): oldpackagedirs=None, use_xz=False, extra_args=[], + cachedir=None, ) ], ) @@ -722,6 +770,7 @@ class TestCreateVariantRepo(PungiTestCase): { "createrepo_checksum": "sha256", "createrepo_deltas": True, + "createrepo_enable_cache": False, "hashed_directories": True, }, ) @@ -776,6 +825,7 @@ class TestCreateVariantRepo(PungiTestCase): ], use_xz=False, extra_args=[], + cachedir=None, ) ], ) @@ -792,6 +842,7 @@ class TestCreateVariantRepo(PungiTestCase): { "createrepo_checksum": "sha256", "createrepo_deltas": True, + "createrepo_enable_cache": False, "hashed_directories": True, }, ) @@ -834,6 +885,7 @@ class TestCreateVariantRepo(PungiTestCase): oldpackagedirs=None, use_xz=False, extra_args=[], + cachedir=None, ) ], ) @@ -845,7 +897,12 @@ class TestCreateVariantRepo(PungiTestCase): def test_variant_repo_source_with_deltas(self, CreaterepoWrapperCls, run): # This should not actually create deltas, only binary repos do. compose = DummyCompose( - self.topdir, {"createrepo_checksum": "sha256", "createrepo_deltas": True} + self.topdir, + { + "createrepo_checksum": "sha256", + "createrepo_enable_cache": False, + "createrepo_deltas": True, + }, ) compose.has_comps = False compose.old_composes = [self.topdir + "/old"] @@ -883,6 +940,7 @@ class TestCreateVariantRepo(PungiTestCase): oldpackagedirs=None, use_xz=False, extra_args=[], + cachedir=None, ) ], ) @@ -894,7 +952,12 @@ class TestCreateVariantRepo(PungiTestCase): def test_variant_repo_debug_with_deltas(self, CreaterepoWrapperCls, run): # This should not actually create deltas, only binary repos do. compose = DummyCompose( - self.topdir, {"createrepo_checksum": "sha256", "createrepo_deltas": True} + self.topdir, + { + "createrepo_checksum": "sha256", + "createrepo_deltas": True, + "createrepo_enable_cache": False, + }, ) compose.has_comps = False compose.old_composes = [self.topdir + "/old"] @@ -934,6 +997,7 @@ class TestCreateVariantRepo(PungiTestCase): oldpackagedirs=None, use_xz=False, extra_args=[], + cachedir=None, ) ], ) @@ -949,6 +1013,7 @@ class TestCreateVariantRepo(PungiTestCase): self.topdir, { "createrepo_checksum": "sha256", + "createrepo_enable_cache": False, "product_id": "yes", # Truthy value is enough for this test }, ) @@ -993,6 +1058,7 @@ class TestCreateVariantRepo(PungiTestCase): oldpackagedirs=None, use_xz=False, extra_args=[], + cachedir=None, ) ], ) @@ -1009,6 +1075,7 @@ class TestCreateVariantRepo(PungiTestCase): self.topdir, { "createrepo_checksum": "sha256", + "createrepo_enable_cache": False, "product_id": "yes", # Truthy value is enough for this test }, ) @@ -1046,6 +1113,7 @@ class TestCreateVariantRepo(PungiTestCase): oldpackagedirs=None, use_xz=False, extra_args=[], + cachedir=None, ) ], ) @@ -1061,6 +1129,7 @@ class TestCreateVariantRepo(PungiTestCase): self.topdir, { "createrepo_checksum": "sha256", + "createrepo_enable_cache": False, "product_id": "yes", # Truthy value is enough for this test }, ) @@ -1096,6 +1165,7 @@ class TestCreateVariantRepo(PungiTestCase): oldpackagedirs=None, use_xz=False, extra_args=[], + cachedir=None, ) ], ) From 446334fb958040ed04f2b35d2c65c80e4d6eded5 Mon Sep 17 00:00:00 2001 From: fdiprete Date: Wed, 28 Jul 2021 17:48:10 +0000 Subject: [PATCH 038/137] show and log command when using the run_blocking_cmd() method [RHELCMP-2243] Signed-off-by: fdiprete --- pungi/wrappers/kojiwrapper.py | 1 + tests/test_koji_wrapper.py | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py index 74e3195d..1e67097f 100644 --- a/pungi/wrappers/kojiwrapper.py +++ b/pungi/wrappers/kojiwrapper.py @@ -534,6 +534,7 @@ class KojiWrapper(object): retcode, output = run( command, can_fail=True, + show_cmd=True, logfile=log_file, env=env, buffer_size=-1, diff --git a/tests/test_koji_wrapper.py b/tests/test_koji_wrapper.py index 1b529473..4c533fce 100644 --- a/tests/test_koji_wrapper.py +++ b/tests/test_koji_wrapper.py @@ -720,6 +720,7 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): "cmd", can_fail=True, logfile=None, + show_cmd=True, env={"FOO": "BAR", "PYTHONUNBUFFERED": "1"}, buffer_size=-1, universal_newlines=True, @@ -745,6 +746,7 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): mock.call( "cmd", can_fail=True, + show_cmd=True, logfile=None, env={ "KRB5CCNAME": "DIR:/tmp/foo", @@ -772,6 +774,7 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): mock.call( "cmd", can_fail=True, + show_cmd=True, logfile="logfile", env={"FOO": "BAR", "PYTHONUNBUFFERED": "1"}, buffer_size=-1, @@ -795,6 +798,7 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): mock.call( "cmd", can_fail=True, + show_cmd=True, logfile=None, env={"FOO": "BAR", "PYTHONUNBUFFERED": "1"}, buffer_size=-1, @@ -818,6 +822,7 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): mock.call( "cmd", can_fail=True, + show_cmd=True, logfile=None, env={"FOO": "BAR", "PYTHONUNBUFFERED": "1"}, buffer_size=-1, @@ -842,6 +847,7 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): mock.call( "cmd", can_fail=True, + show_cmd=True, logfile=None, env={"FOO": "BAR", "PYTHONUNBUFFERED": "1"}, buffer_size=-1, @@ -871,6 +877,7 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): mock.call( "cmd", can_fail=True, + show_cmd=True, logfile=None, env={"FOO": "BAR", "PYTHONUNBUFFERED": "1"}, buffer_size=-1, @@ -901,6 +908,7 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): mock.call( "cmd", can_fail=True, + show_cmd=True, logfile=None, env={"FOO": "BAR", "PYTHONUNBUFFERED": "1"}, buffer_size=-1, @@ -944,6 +952,7 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): mock.call( "cmd", can_fail=True, + show_cmd=True, logfile=None, env={"FOO": "BAR", "PYTHONUNBUFFERED": "1"}, buffer_size=-1, @@ -980,6 +989,7 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): mock.call( "cmd", can_fail=True, + show_cmd=True, logfile=None, env={"FOO": "BAR", "PYTHONUNBUFFERED": "1"}, buffer_size=-1, From cf761633f44b2e2e48e9502a55cf5cfcf26de50c Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Wed, 4 Aug 2021 17:23:19 +0800 Subject: [PATCH 039/137] 4.2.10 release JIRA: RHELCMP-6108 Signed-off-by: Haibo Lin --- doc/conf.py | 2 +- pungi.spec | 12 +++++++++++- setup.py | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 0b5b3a23..3e3c7fae 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -53,7 +53,7 @@ copyright = u'2016, Red Hat, Inc.' # The short X.Y version. version = '4.2' # The full version, including alpha/beta/rc tags. -release = '4.2.9' +release = '4.2.10' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pungi.spec b/pungi.spec index 4e5705de..4cc507df 100644 --- a/pungi.spec +++ b/pungi.spec @@ -1,5 +1,5 @@ Name: pungi -Version: 4.2.9 +Version: 4.2.10 Release: 1%{?dist} Summary: Distribution compose tool @@ -111,6 +111,16 @@ pytest cd tests && ./test_compose.sh %changelog +* Wed Aug 04 2021 Haibo Lin - 4.2.10-1 +- Show and log command when using the run_blocking_cmd() method (fdipretre) +- Use cachedir when createrepo (hlin) +- gather: Add all srpms to variant lookaside repo (lsedlar) +- Add task URL to watch task log (hlin) +- Log warning when module defined in variants.xml not found (hlin) +- pkgset: Compare future events correctly (lsedlar) +- util: Strip file:// from local urls (lsedlar) +- Clean up temporary yumroot dir (hlin) + * Thu Apr 29 2021 Ondrej Nosek - 4.2.9-1 - Fix can't link XDEV using repos as pkgset_sources (romain.forlot) - Updated the deprecated ks argument name (to the current inst.ks) (lveyde) diff --git a/setup.py b/setup.py index 91dfd15d..5e0617e1 100755 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ packages = sorted(packages) setup( name="pungi", - version="4.2.9", + version="4.2.10", description="Distribution compose tool", url="https://pagure.io/pungi", author="Dennis Gilmore", From 01a52447bc39615c05652dbb8f3f2c54a2473fd5 Mon Sep 17 00:00:00 2001 From: Ken Dreyer Date: Thu, 5 Aug 2021 12:41:34 -0400 Subject: [PATCH 040/137] doc: explain buildContainer API Explain how to discover the API documentation about the buildContainer method, so users can discover more about how "scratch" and "priority" work. Signed-off-by: Ken Dreyer --- doc/configuration.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 2fd50c63..50ad1890 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1815,7 +1815,8 @@ they are not scratch builds). to create the image will not abort the whole compose. The configuration will pass other attributes directly to the Koji task. - This includes ``scratch`` and ``priority``. + This includes ``scratch`` and ``priority``. See ``koji list-api + buildContainer`` for more details about these options. A value for ``yum_repourls`` will be created automatically and point at a repository in the current compose. You can add extra repositories with From 8a2d0162d9a4876733107867847fb4a2a50be841 Mon Sep 17 00:00:00 2001 From: Dominik Rumian Date: Thu, 5 Aug 2021 14:42:24 +0200 Subject: [PATCH 041/137] Better error message than 'KeyError' in pungi JIRA: RHELCMP-6107 Signed-off-by: Dominik Rumian --- pungi/phases/gather/__init__.py | 58 +++++++++++++++++++++------------ tests/test_gather_phase.py | 23 +++++++++++-- 2 files changed, 58 insertions(+), 23 deletions(-) diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 6520c806..01a2c5ef 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -14,33 +14,34 @@ # along with this program; if not, see . -import json import glob +import json import os import shutil import threading -from six.moves import cPickle as pickle from kobo.rpmlib import parse_nvra from kobo.shortcuts import run from productmd.rpms import Rpms +from six.moves import cPickle as pickle try: from queue import Queue except ImportError: from Queue import Queue -from pungi.wrappers.scm import get_file_from_scm -from .link import link_files -from ...wrappers.createrepo import CreaterepoWrapper import pungi.wrappers.kojiwrapper - -from pungi.compose import get_ordered_variant_uids from pungi.arch import get_compatible_arches, split_name_arch -from pungi.phases.base import PhaseBase -from pungi.util import get_arch_data, get_arch_variant_data, get_variant_data, makedirs +from pungi.compose import get_ordered_variant_uids from pungi.module_util import Modulemd, collect_module_defaults +from pungi.phases.base import PhaseBase from pungi.phases.createrepo import add_modular_metadata +from pungi.util import (get_arch_data, get_arch_variant_data, get_variant_data, + makedirs) +from pungi.wrappers.scm import get_file_from_scm + +from ...wrappers.createrepo import CreaterepoWrapper +from .link import link_files def get_gather_source(name): @@ -81,10 +82,11 @@ class GatherPhase(PhaseBase): if variant.modules: errors.append("Modular compose requires libmodulemd package.") - # check whether variants from configuration value - # 'variant_as_lookaside' are correct variant_as_lookaside = self.compose.conf.get("variant_as_lookaside", []) all_variants = self.compose.all_variants + + # check whether variants from configuration value + # 'variant_as_lookaside' are correct for (requiring, required) in variant_as_lookaside: if requiring in all_variants and required not in all_variants: errors.append( @@ -92,6 +94,15 @@ class GatherPhase(PhaseBase): "required by %r" % (required, requiring) ) + # check whether variants from configuration value + # 'variant_as_lookaside' have same architectures + for (requiring, required) in variant_as_lookaside: + if requiring in all_variants and required in all_variants and \ + sorted(all_variants[requiring].arches) \ + != sorted(all_variants[required].arches): + errors.append("variant_as_lookaside: variant \'%s\' doesn't have same " + "architectures as \'%s\'" % (requiring, required)) + if errors: raise ValueError("\n".join(errors)) @@ -641,16 +652,21 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None): path_prefix = prefixes[compose.conf["pkgset_source"]]() package_list = set() for pkg_arch in pkg_map.keys(): - for pkg_type, packages in pkg_map[pkg_arch][variant.uid].items(): - # We want all packages for current arch, and SRPMs for any - # arch. Ultimately there will only be one source repository, so - # we need a union of all SRPMs. - if pkg_type == "srpm" or pkg_arch == arch: - for pkg in packages: - pkg = pkg["path"] - if path_prefix and pkg.startswith(path_prefix): - pkg = pkg[len(path_prefix) :] - package_list.add(pkg) + try: + for pkg_type, packages in pkg_map[pkg_arch][variant.uid].items(): + # We want all packages for current arch, and SRPMs for any + # arch. Ultimately there will only be one source repository, so + # we need a union of all SRPMs. + if pkg_type == "srpm" or pkg_arch == arch: + for pkg in packages: + pkg = pkg["path"] + if path_prefix and pkg.startswith(path_prefix): + pkg = pkg[len(path_prefix) :] + package_list.add(pkg) + except KeyError: + raise RuntimeError("Variant \'%s\' does not have architecture " + "\'%s\'!" % (variant, pkg_arch)) + pkglist = compose.paths.work.lookaside_package_list(arch=arch, variant=variant) with open(pkglist, "w") as f: for pkg in sorted(package_list): diff --git a/tests/test_gather_phase.py b/tests/test_gather_phase.py index 4c41e682..71c5a1ea 100644 --- a/tests/test_gather_phase.py +++ b/tests/test_gather_phase.py @@ -2,9 +2,10 @@ import copy import json -import mock import os +import mock + try: import unittest2 as unittest except ImportError: @@ -13,8 +14,8 @@ except ImportError: import six from pungi.phases import gather -from pungi.phases.pkgset.common import MaterializedPackageSet from pungi.phases.gather import _mk_pkg_map +from pungi.phases.pkgset.common import MaterializedPackageSet from tests import helpers from tests.helpers import MockPackageSet, MockPkg @@ -1581,6 +1582,24 @@ class TestGatherPhase(helpers.PungiTestCase): phase = gather.GatherPhase(compose, pkgset_phase) phase.validate() + def test_validates_variants_architecture_mismatch(self): + pkgset_phase = mock.Mock() + compose = helpers.DummyCompose( + self.topdir, {"variant_as_lookaside": [("Everything", "Client")]} + ) + phase = gather.GatherPhase(compose, pkgset_phase) + with self.assertRaises(ValueError) as ctx: + phase.validate() + self.assertIn("'Everything' doesn't have", str(ctx.exception)) + + def test_validates_variants_architecture_match(self): + pkgset_phase = mock.Mock() + compose = helpers.DummyCompose( + self.topdir, {"variant_as_lookaside": [("Everything", "Everything")]} + ) + phase = gather.GatherPhase(compose, pkgset_phase) + phase.validate() + class TestGetPackagesToGather(helpers.PungiTestCase): def setUp(self): From 2a679dcb81b8c04726f2038afc22e684ece7fff9 Mon Sep 17 00:00:00 2001 From: Ken Dreyer Date: Wed, 11 Aug 2021 14:07:12 -0400 Subject: [PATCH 042/137] doc: improve signed packages retry docs Reword the signed_packages_retries and signed_packages_wait configuration option documentation to use the active voice. This makes it easier to understand who is doing what in a signing workflow. Signed-off-by: Ken Dreyer --- doc/configuration.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 50ad1890..e42e32cf 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -582,15 +582,16 @@ Options in your new compose. **signed_packages_retries** = 0 - (*int*) -- In automated workflows a compose may start before signed - packages are written to disk. In such case it may make sense to wait for - the package to appear on storage. This option controls how many times to - retry looking for the signed copy. + (*int*) -- In automated workflows, you might start a compose before Koji + has completely written all signed packages to disk. In this case you may + want Pungi to wait for the package to appear in Koji's storage. This + option controls how many times Pungi will retry looking for the signed + copy. **signed_packages_wait** = 30 - (*int*) -- Interval in seconds for how long to wait between attemts to find - signed packages. This option only makes sense when - ``signed_packages_retries`` is set higher than to 0. + (*int*) -- Interval in seconds for how long to wait between attempts to + find signed packages. This option only makes sense when + ``signed_packages_retries`` is set higher than 0. Example From 6afcfef919e61f74a37b049d0bcc9e1766aa9c48 Mon Sep 17 00:00:00 2001 From: Ken Dreyer Date: Wed, 11 Aug 2021 13:22:12 -0400 Subject: [PATCH 043/137] doc: fix typo in additional_packages description not -> nor Signed-off-by: Ken Dreyer --- doc/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index e42e32cf..0a425ffe 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -800,7 +800,7 @@ Options architecture; format: ``[(variant_uid_regex, {arch|*: [package_globs]})]`` The packages specified here are matched against RPM names, not any other - provides in the package not the name of source package. Shell globbing is + provides in the package nor the name of source package. Shell globbing is used, so wildcards are possible. The package can be specified as name only or ``name.arch``. From 5a8df7b69c2ea05b0b0b1b6139af55fadc07a37a Mon Sep 17 00:00:00 2001 From: Ken Dreyer Date: Wed, 11 Aug 2021 13:44:04 -0400 Subject: [PATCH 044/137] doc: more additional_packages documentation Contrast the additional_packages setting with the comps_file setting. Explain what happens when a user lists a package in additional_packages but Pungi cannot find it. Give an example of composing all builds in a Koji tag. Signed-off-by: Ken Dreyer --- doc/configuration.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/doc/configuration.rst b/doc/configuration.rst index 0a425ffe..52b8c9ea 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -799,6 +799,12 @@ Options (*list*) -- additional packages to be included in a variant and architecture; format: ``[(variant_uid_regex, {arch|*: [package_globs]})]`` + In contrast to the ``comps_file`` setting, the ``additional_packages`` + setting merely adds the list of packages to the compose. When a package + is in a comps group, it is visible to users via ``dnf groupinstall`` and + Anaconda's Groups selection, but ``additional_packages`` does not affect + DNF groups. + The packages specified here are matched against RPM names, not any other provides in the package nor the name of source package. Shell globbing is used, so wildcards are possible. The package can be specified as name only @@ -809,6 +815,21 @@ Options it. If you add a debuginfo package that does not have anything else from the same build included in the compose, the sources will not be pulled in. + If you list a package in ``additional_packages`` but Pungi cannot find + it (for example, it's not available in the Koji tag), Pungi will log a + warning in the "work" or "logs" directories and continue without aborting. + + *Example*: This configuration will add all packages in a Koji tag to an + "Everything" variant:: + + additional_packages = [ + ('^Everything$', { + '*': [ + '*', + ], + }) + ] + **filter_packages** (*list*) -- packages to be excluded from a variant and architecture; format: ``[(variant_uid_regex, {arch|*: [package_globs]})]`` From 3349585d788386b421f35d27ac9e8597e44ee719 Mon Sep 17 00:00:00 2001 From: JamesKunstle Date: Tue, 20 Jul 2021 15:57:29 -0400 Subject: [PATCH 045/137] Adding multithreading support for pungi/phases/image_checksum.py Multithreading was added to parallelize the computation of image checksums. Resulting memory structures are protected via synchronization primitives. Max number of threads is uncapped- experiments were done to determine whether a maximum number of threads would yield greater efficiency and there were no gains from this. Likewise, experiments were done to determine whether pools of threads computed in separate processes could likewise decrease compute-time. Evidence did not suggest that this was the case. This indicate that the checksum operation is bounded by I/O read/write times. Merges: https://pagure.io/pungi/pull-request/1520 Jira: RHELCMP-5967 Signed-off-by: James Kunstle jkunstle@redhat.com --- .gitignore | 1 + pungi/phases/image_checksum.py | 64 +++++++++++++++++++++++++--------- 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 07fb4417..101ccee0 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ htmlcov/ .coverage .idea/ .tox +.venv diff --git a/pungi/phases/image_checksum.py b/pungi/phases/image_checksum.py index 5b980597..0277a97c 100644 --- a/pungi/phases/image_checksum.py +++ b/pungi/phases/image_checksum.py @@ -3,6 +3,7 @@ import os from kobo import shortcuts from collections import defaultdict +import threading from .base import PhaseBase from ..util import get_format_substs, get_file_size @@ -68,6 +69,7 @@ class ImageChecksumPhase(PhaseBase): def run(self): topdir = self.compose.paths.compose.topdir() + make_checksums( topdir, self.compose.im, @@ -87,6 +89,8 @@ def _compute_checksums( checksum_types, base_checksum_name_gen, one_file, + results_lock, + cache_lock, ): for image in images: filename = os.path.basename(image.path) @@ -96,14 +100,21 @@ def _compute_checksums( filesize = image.size or get_file_size(full_path) + cache_lock.acquire() if full_path not in cache: + cache_lock.release() # Source ISO is listed under each binary architecture. There's no # point in checksumming it twice, so we can just remember the # digest from first run.. - cache[full_path] = shortcuts.compute_file_checksums( - full_path, checksum_types - ) - digests = cache[full_path] + checksum_value = shortcuts.compute_file_checksums(full_path, checksum_types) + with cache_lock: + cache[full_path] = checksum_value + else: + cache_lock.release() + + with cache_lock: + digests = cache[full_path] + for checksum, digest in digests.items(): # Update metadata with the checksum image.add_checksum(None, checksum, digest) @@ -112,7 +123,10 @@ def _compute_checksums( checksum_filename = os.path.join( path, "%s.%sSUM" % (filename, checksum.upper()) ) - results[checksum_filename].add((filename, filesize, checksum, digest)) + with results_lock: + results[checksum_filename].add( + (filename, filesize, checksum, digest) + ) if one_file: dirname = os.path.basename(path) @@ -125,24 +139,42 @@ def _compute_checksums( checksum_filename = "%s%sSUM" % (base_checksum_name, checksum.upper()) checksum_path = os.path.join(path, checksum_filename) - results[checksum_path].add((filename, filesize, checksum, digest)) + with results_lock: + results[checksum_path].add((filename, filesize, checksum, digest)) def make_checksums(topdir, im, checksum_types, one_file, base_checksum_name_gen): results = defaultdict(set) cache = {} + threads = [] + results_lock = threading.Lock() # lock to synchronize access to the results dict. + cache_lock = threading.Lock() # lock to synchronize access to the cache dict. + + # create all worker threads for (variant, arch, path), images in get_images(topdir, im).items(): - _compute_checksums( - results, - cache, - variant, - arch, - path, - images, - checksum_types, - base_checksum_name_gen, - one_file, + threads.append( + threading.Thread( + target=_compute_checksums, + args=[ + results, + cache, + variant, + arch, + path, + images, + checksum_types, + base_checksum_name_gen, + one_file, + results_lock, + cache_lock, + ], + ) ) + threads[-1].start() + + # wait for all worker threads to finish + for thread in threads: + thread.join() for file in results: dump_checksums(file, results[file]) From 5831d4ae1e98db2366b73238ee79f5cdbc8d6687 Mon Sep 17 00:00:00 2001 From: Dominik Rumian Date: Thu, 12 Aug 2021 11:03:19 +0200 Subject: [PATCH 046/137] Better error message than 'KeyError' in pungi Jira: RHELCMP-6107 Signed-off-by: Dominik Rumian --- pungi/phases/gather/__init__.py | 25 ++++++++++++++++--------- tests/test_gather_phase.py | 8 ++++---- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 01a2c5ef..64310259 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -36,8 +36,7 @@ from pungi.compose import get_ordered_variant_uids from pungi.module_util import Modulemd, collect_module_defaults from pungi.phases.base import PhaseBase from pungi.phases.createrepo import add_modular_metadata -from pungi.util import (get_arch_data, get_arch_variant_data, get_variant_data, - makedirs) +from pungi.util import get_arch_data, get_arch_variant_data, get_variant_data, makedirs from pungi.wrappers.scm import get_file_from_scm from ...wrappers.createrepo import CreaterepoWrapper @@ -97,11 +96,18 @@ class GatherPhase(PhaseBase): # check whether variants from configuration value # 'variant_as_lookaside' have same architectures for (requiring, required) in variant_as_lookaside: - if requiring in all_variants and required in all_variants and \ - sorted(all_variants[requiring].arches) \ - != sorted(all_variants[required].arches): - errors.append("variant_as_lookaside: variant \'%s\' doesn't have same " - "architectures as \'%s\'" % (requiring, required)) + if ( + requiring in all_variants + and required in all_variants + and not set(all_variants[requiring].arches).issubset( + set(all_variants[required].arches) + ) + ): + errors.append( + "variant_as_lookaside: architectures of variant '%s' " + "aren't subset of architectures of variant '%s'" + % (requiring, required) + ) if errors: raise ValueError("\n".join(errors)) @@ -664,8 +670,9 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None): pkg = pkg[len(path_prefix) :] package_list.add(pkg) except KeyError: - raise RuntimeError("Variant \'%s\' does not have architecture " - "\'%s\'!" % (variant, pkg_arch)) + raise RuntimeError( + "Variant '%s' does not have architecture " "'%s'!" % (variant, pkg_arch) + ) pkglist = compose.paths.work.lookaside_package_list(arch=arch, variant=variant) with open(pkglist, "w") as f: diff --git a/tests/test_gather_phase.py b/tests/test_gather_phase.py index 71c5a1ea..2725247f 100644 --- a/tests/test_gather_phase.py +++ b/tests/test_gather_phase.py @@ -1582,7 +1582,7 @@ class TestGatherPhase(helpers.PungiTestCase): phase = gather.GatherPhase(compose, pkgset_phase) phase.validate() - def test_validates_variants_architecture_mismatch(self): + def test_validates_variants_requiring_is_not_subset_of_required(self): pkgset_phase = mock.Mock() compose = helpers.DummyCompose( self.topdir, {"variant_as_lookaside": [("Everything", "Client")]} @@ -1590,12 +1590,12 @@ class TestGatherPhase(helpers.PungiTestCase): phase = gather.GatherPhase(compose, pkgset_phase) with self.assertRaises(ValueError) as ctx: phase.validate() - self.assertIn("'Everything' doesn't have", str(ctx.exception)) + self.assertIn("architectures of variant 'Client'", str(ctx.exception)) - def test_validates_variants_architecture_match(self): + def test_validates_variants_requiring_is_subset_of_required(self): pkgset_phase = mock.Mock() compose = helpers.DummyCompose( - self.topdir, {"variant_as_lookaside": [("Everything", "Everything")]} + self.topdir, {"variant_as_lookaside": [("Client", "Everything")]} ) phase = gather.GatherPhase(compose, pkgset_phase) phase.validate() From a7c111643deeb52ead3428546532d60208d01f81 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 11 Aug 2021 13:15:29 +0200 Subject: [PATCH 047/137] Supersede ModuleStream loading with ModuleIndex - Use ModuleIndex's update_from_file/update_from_string instead of ModuleStream's read_file/read_string which is deprecated. - Extend tests to work with real module streams instead of mocks. Signed-off-by: Filip Valder --- pungi/phases/createrepo.py | 4 +- pungi/phases/pkgset/sources/source_koji.py | 27 ++-- pungi/util.py | 41 ++++++ tests/fixtures/mmds/m1.x86_64.txt | 20 +++ tests/fixtures/mmds/modulemd.armv7hl.txt | 20 +++ tests/fixtures/mmds/modulemd.x86_64.txt | 20 +++ tests/fixtures/mmds/scratch-module.x86_64.txt | 20 +++ tests/test_pkgset_source_koji.py | 136 ++++++------------ 8 files changed, 183 insertions(+), 105 deletions(-) create mode 100644 tests/fixtures/mmds/m1.x86_64.txt create mode 100644 tests/fixtures/mmds/modulemd.armv7hl.txt create mode 100644 tests/fixtures/mmds/modulemd.x86_64.txt create mode 100644 tests/fixtures/mmds/scratch-module.x86_64.txt diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index c9ac4746..83028bb0 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -31,7 +31,7 @@ from kobo.shortcuts import run, relative_path from ..wrappers.scm import get_dir_from_scm from ..wrappers.createrepo import CreaterepoWrapper from .base import PhaseBase -from ..util import get_arch_variant_data, temp_dir +from ..util import get_arch_variant_data, temp_dir, read_single_module_stream_from_file from ..module_util import Modulemd, collect_module_defaults import productmd.rpms @@ -268,7 +268,7 @@ def create_variant_repo( compose.log_debug("Adding extra modulemd for %s.%s", variant.uid, arch) dirname = compose.paths.work.tmp_dir(variant=variant, create_dir=False) for filepath in glob.glob(os.path.join(dirname, arch) + "/*.yaml"): - module_stream = Modulemd.ModuleStream.read_file(filepath, strict=True) + module_stream = read_single_module_stream_from_file(filepath) if not mod_index.add_module_stream(module_stream): raise RuntimeError( "Failed parsing modulemd data from %s" % filepath diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 2ee955ce..1219677c 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -29,7 +29,13 @@ from pungi.wrappers.comps import CompsWrapper from pungi.wrappers.mbs import MBSWrapper import pungi.phases.pkgset.pkgsets from pungi.arch import getBaseArch -from pungi.util import retry, get_arch_variant_data, get_variant_data +from pungi.util import ( + retry, + get_arch_variant_data, + get_variant_data, + read_single_module_stream_from_file, + read_single_module_stream_from_string, +) from pungi.module_util import Modulemd from pungi.phases.pkgset.common import MaterializedPackageSet, get_all_arches @@ -258,17 +264,12 @@ def _add_module_to_variant( compose.log_debug("Module %s is filtered from %s.%s", nsvc, variant, arch) continue - try: - mmd = Modulemd.ModuleStream.read_file( - mmds["modulemd.%s.txt" % arch], strict=True - ) - variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd + mod_stream = read_single_module_stream_from_file( + mmds["modulemd.%s.txt" % arch], compose, arch, build + ) + if mod_stream: added = True - except KeyError: - # There is no modulemd for this arch. This could mean an arch was - # added to the compose after the module was built. We don't want to - # process this, let's skip this module. - pass + variant.arch_mmds.setdefault(arch, {})[nsvc] = mod_stream if not added: # The module is filtered on all arches of this variant. @@ -348,9 +349,7 @@ def _add_scratch_modules_to_variant( tag_to_mmd.setdefault(tag, {}) for arch in variant.arches: try: - mmd = Modulemd.ModuleStream.read_string( - final_modulemd[arch], strict=True - ) + mmd = read_single_module_stream_from_string(final_modulemd[arch]) variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd except KeyError: continue diff --git a/pungi/util.py b/pungi/util.py index c5717bde..da5e82f2 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -34,6 +34,7 @@ import kobo.conf from kobo.shortcuts import run, force_list from kobo.threads import WorkerThread, ThreadPool from productmd.common import get_major_version +from pungi.module_util import Modulemd # Patterns that match all names of debuginfo packages DEBUG_PATTERNS = ["*-debuginfo", "*-debuginfo-*", "*-debugsource"] @@ -1034,6 +1035,46 @@ def load_config(file_path, defaults={}): return conf +def _read_single_module_stream( + file_or_string, compose=None, arch=None, build=None, is_file=True +): + try: + mod_index = Modulemd.ModuleIndex.new() + if is_file: + mod_index.update_from_file(file_or_string, True) + else: + mod_index.update_from_string(file_or_string, True) + mod_names = mod_index.get_module_names() + emit_warning = False + if len(mod_names) > 1: + emit_warning = True + mod_streams = mod_index.get_module(mod_names[0]).get_all_streams() + if len(mod_streams) > 1: + emit_warning = True + if emit_warning and compose: + compose.log_warning( + "Multiple modules/streams for arch: %s. Build: %s. " + "Processing first module/stream only.", + arch, + build, + ) + return mod_streams[0] + except (KeyError, IndexError): + # There is no modulemd for this arch. This could mean an arch was + # added to the compose after the module was built. We don't want to + # process this, let's skip this module. + if compose: + compose.log_info("Skipping arch: %s. Build: %s", arch, build) + + +def read_single_module_stream_from_file(*args, **kwargs): + return _read_single_module_stream(*args, is_file=True, **kwargs) + + +def read_single_module_stream_from_string(*args, **kwargs): + return _read_single_module_stream(*args, is_file=False, **kwargs) + + @contextlib.contextmanager def as_local_file(url): """If URL points to a file over HTTP, the file will be downloaded locally diff --git a/tests/fixtures/mmds/m1.x86_64.txt b/tests/fixtures/mmds/m1.x86_64.txt new file mode 100644 index 00000000..e1989733 --- /dev/null +++ b/tests/fixtures/mmds/m1.x86_64.txt @@ -0,0 +1,20 @@ +--- +document: modulemd +version: 2 +data: + name: m1 + stream: latest + version: 20190101 + context: cafe + arch: x86_64 + summary: Dummy module + description: Dummy module + license: + module: + - Beerware + content: + - Beerware + artifacts: + rpms: + - foobar-0:1.0-1.noarch +... diff --git a/tests/fixtures/mmds/modulemd.armv7hl.txt b/tests/fixtures/mmds/modulemd.armv7hl.txt new file mode 100644 index 00000000..e03147d2 --- /dev/null +++ b/tests/fixtures/mmds/modulemd.armv7hl.txt @@ -0,0 +1,20 @@ +--- +document: modulemd +version: 2 +data: + name: module + stream: master + version: 20190318 + context: abcdef + arch: armhfp + summary: Dummy module + description: Dummy module + license: + module: + - Beerware + content: + - Beerware + artifacts: + rpms: + - foobar-0:1.0-1.noarch +... diff --git a/tests/fixtures/mmds/modulemd.x86_64.txt b/tests/fixtures/mmds/modulemd.x86_64.txt new file mode 100644 index 00000000..b7e3761c --- /dev/null +++ b/tests/fixtures/mmds/modulemd.x86_64.txt @@ -0,0 +1,20 @@ +--- +document: modulemd +version: 2 +data: + name: module + stream: master + version: 20190318 + context: abcdef + arch: x86_64 + summary: Dummy module + description: Dummy module + license: + module: + - Beerware + content: + - Beerware + artifacts: + rpms: + - foobar-0:1.0-1.noarch +... diff --git a/tests/fixtures/mmds/scratch-module.x86_64.txt b/tests/fixtures/mmds/scratch-module.x86_64.txt new file mode 100644 index 00000000..8a13926b --- /dev/null +++ b/tests/fixtures/mmds/scratch-module.x86_64.txt @@ -0,0 +1,20 @@ +--- +document: modulemd +version: 2 +data: + name: scratch-module + stream: master + version: 20200710 + context: abcdef + arch: x86_64 + summary: Dummy module + description: Dummy module + license: + module: + - Beerware + content: + - Beerware + artifacts: + rpms: + - foobar-0:1.0-1.noarch +... diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index f45a5ac4..c4106808 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -14,7 +14,9 @@ except ImportError: from pungi.phases.pkgset.sources import source_koji from tests import helpers from pungi.module_util import Modulemd +from pungi.util import read_single_module_stream_from_file +MMDS_DIR = os.path.join(helpers.FIXTURE_DIR, "mmds") EVENT_INFO = {"id": 15681980, "ts": 1460956382.81936} TAG_INFO = { "maven_support": False, @@ -672,24 +674,12 @@ class TestFilterByWhitelist(unittest.TestCase): self.assertEqual(expected, set()) -class MockModule(object): - def __init__(self, path, strict=True): - self.path = path - - def __repr__(self): - return "MockModule(%r)" % self.path - - def __eq__(self, other): - return self.path == other.path - - -@mock.patch("pungi.module_util.Modulemd.ModuleStream.read_file", new=MockModule) @unittest.skipIf(Modulemd is None, "Skipping tests, no module support") class TestAddModuleToVariant(helpers.PungiTestCase): def setUp(self): super(TestAddModuleToVariant, self).setUp() self.koji = mock.Mock() - self.koji.koji_module.pathinfo.typedir.return_value = "/koji" + self.koji.koji_module.pathinfo.typedir.return_value = MMDS_DIR files = ["modulemd.x86_64.txt", "modulemd.armv7hl.txt", "modulemd.txt"] self.koji.koji_proxy.listArchives.return_value = [ {"btype": "module", "filename": fname} for fname in files @@ -713,50 +703,35 @@ class TestAddModuleToVariant(helpers.PungiTestCase): source_koji._add_module_to_variant(self.koji, variant, self.buildinfo) - self.assertEqual( - variant.arch_mmds, - { - "armhfp": { - "module:master:20190318:abcdef": MockModule( - "/koji/modulemd.armv7hl.txt" - ), - }, - "x86_64": { - "module:master:20190318:abcdef": MockModule( - "/koji/modulemd.x86_64.txt" - ), - }, - }, - ) + mod1 = variant.arch_mmds["armhfp"]["module:master:20190318:abcdef"] + self.assertEqual(mod1.get_NSVCA(), "module:master:20190318:abcdef:armhfp") + mod2 = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"] + self.assertEqual(mod2.get_NSVCA(), "module:master:20190318:abcdef:x86_64") + self.assertEqual(len(variant.arch_mmds), 2) self.assertEqual(variant.modules, []) def test_adding_module_to_existing(self): variant = mock.Mock( arches=["armhfp", "x86_64"], arch_mmds={ - "x86_64": {"m1:latest:20190101:cafe": MockModule("/koji/m1.x86_64.txt")} + "x86_64": { + "m1:latest:20190101:cafe": read_single_module_stream_from_file( + os.path.join(MMDS_DIR, "m1.x86_64.txt") + ) + } }, modules=[{"name": "m1:latest-20190101:cafe", "glob": False}], ) source_koji._add_module_to_variant(self.koji, variant, self.buildinfo) - self.assertEqual( - variant.arch_mmds, - { - "armhfp": { - "module:master:20190318:abcdef": MockModule( - "/koji/modulemd.armv7hl.txt" - ), - }, - "x86_64": { - "module:master:20190318:abcdef": MockModule( - "/koji/modulemd.x86_64.txt" - ), - "m1:latest:20190101:cafe": MockModule("/koji/m1.x86_64.txt"), - }, - }, - ) + mod1 = variant.arch_mmds["armhfp"]["module:master:20190318:abcdef"] + self.assertEqual(mod1.get_NSVCA(), "module:master:20190318:abcdef:armhfp") + mod2 = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"] + self.assertEqual(mod2.get_NSVCA(), "module:master:20190318:abcdef:x86_64") + mod3 = variant.arch_mmds["x86_64"]["m1:latest:20190101:cafe"] + self.assertEqual(mod3.get_NSVCA(), "m1:latest:20190101:cafe:x86_64") + self.assertEqual( variant.modules, [{"name": "m1:latest-20190101:cafe", "glob": False}] ) @@ -768,21 +743,11 @@ class TestAddModuleToVariant(helpers.PungiTestCase): self.koji, variant, self.buildinfo, add_to_variant_modules=True ) - self.assertEqual( - variant.arch_mmds, - { - "armhfp": { - "module:master:20190318:abcdef": MockModule( - "/koji/modulemd.armv7hl.txt" - ), - }, - "x86_64": { - "module:master:20190318:abcdef": MockModule( - "/koji/modulemd.x86_64.txt" - ), - }, - }, - ) + mod1 = variant.arch_mmds["armhfp"]["module:master:20190318:abcdef"] + self.assertEqual(mod1.get_NSVCA(), "module:master:20190318:abcdef:armhfp") + mod2 = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"] + self.assertEqual(mod2.get_NSVCA(), "module:master:20190318:abcdef:x86_64") + self.assertEqual( variant.modules, [{"name": "module:master:20190318:abcdef", "glob": False}] ) @@ -791,7 +756,11 @@ class TestAddModuleToVariant(helpers.PungiTestCase): variant = mock.Mock( arches=["armhfp", "x86_64"], arch_mmds={ - "x86_64": {"m1:latest:20190101:cafe": MockModule("/koji/m1.x86_64.txt")} + "x86_64": { + "m1:latest:20190101:cafe": read_single_module_stream_from_file( + os.path.join(MMDS_DIR, "m1.x86_64.txt") + ) + } }, modules=[{"name": "m1:latest-20190101:cafe", "glob": False}], ) @@ -800,22 +769,13 @@ class TestAddModuleToVariant(helpers.PungiTestCase): self.koji, variant, self.buildinfo, add_to_variant_modules=True ) - self.assertEqual( - variant.arch_mmds, - { - "armhfp": { - "module:master:20190318:abcdef": MockModule( - "/koji/modulemd.armv7hl.txt" - ), - }, - "x86_64": { - "module:master:20190318:abcdef": MockModule( - "/koji/modulemd.x86_64.txt" - ), - "m1:latest:20190101:cafe": MockModule("/koji/m1.x86_64.txt"), - }, - }, - ) + mod1 = variant.arch_mmds["armhfp"]["module:master:20190318:abcdef"] + self.assertEqual(mod1.get_NSVCA(), "module:master:20190318:abcdef:armhfp") + mod2 = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"] + self.assertEqual(mod2.get_NSVCA(), "module:master:20190318:abcdef:x86_64") + mod3 = variant.arch_mmds["x86_64"]["m1:latest:20190101:cafe"] + self.assertEqual(mod3.get_NSVCA(), "m1:latest:20190101:cafe:x86_64") + self.assertEqual( variant.modules, [ @@ -891,12 +851,8 @@ class MockMBS(object): return {"id": 1, "koji_tag": "scratch-module-tag", "name": "scratch-module"} def final_modulemd(self, module_build_id): - return {"x86_64": ""} - - -class MockMmd(object): - def __init__(self, mmd, strict=True): - pass + with open(os.path.join(MMDS_DIR, "scratch-module.x86_64.txt")) as f: + return {"x86_64": f.read()} @mock.patch("pungi.phases.pkgset.sources.source_koji.MBSWrapper", new=MockMBS) @@ -909,10 +865,7 @@ class TestAddScratchModuleToVariant(helpers.PungiTestCase): ) self.nsvc = "scratch-module:master:20200710:abcdef" - @mock.patch( - "pungi.phases.pkgset.sources.source_koji.Modulemd.ModuleStream.read_string" - ) - def test_adding_scratch_module(self, mock_mmd): + def test_adding_scratch_module(self): variant = mock.Mock( arches=["armhfp", "x86_64"], arch_mmds={}, @@ -927,11 +880,16 @@ class TestAddScratchModuleToVariant(helpers.PungiTestCase): self.compose, variant, scratch_modules, variant_tags, tag_to_mmd ) self.assertEqual(variant_tags, {variant: ["scratch-module-tag"]}) + self.assertEqual( - variant.arch_mmds, {"x86_64": {self.nsvc: mock_mmd.return_value}} + variant.arch_mmds["x86_64"][self.nsvc].get_NSVCA(), + "scratch-module:master:20200710:abcdef:x86_64", ) + + self.assertTrue(isinstance(tag_to_mmd["scratch-module-tag"]["x86_64"], set)) self.assertEqual( - tag_to_mmd, {"scratch-module-tag": {"x86_64": set([mock_mmd.return_value])}} + list(tag_to_mmd["scratch-module-tag"]["x86_64"])[0].get_NSVCA(), + "scratch-module:master:20200710:abcdef:x86_64", ) self.assertEqual(variant.modules, []) From efff2c950456fca81aa9ba4e81415a58f6f6e8ff Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Thu, 12 Aug 2021 16:35:47 +0200 Subject: [PATCH 048/137] Use pytest directly incl. support for posargs, e.g.: tox -- -s -vvv tests/path/to/a/single/test_something.py Signed-off-by: Filip Valder --- tox.ini | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 96763345..4ba977cf 100644 --- a/tox.ini +++ b/tox.ini @@ -40,11 +40,10 @@ deps = -rtest-requirements.txt whitelist_externals = sh - make commands = sh -c 'find . -name "__pycache__" -exec rm -rf \{\} +' pip install --force-reinstall pytest mock - make test + pytest {posargs} [flake8] exclude = doc/*,*.pyc,*.py~,*.in,*.spec,*.sh,*.rst From 1bb038ca72c15506efca148f8ce91aad43fe7ca5 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Tue, 17 Aug 2021 10:08:35 +0800 Subject: [PATCH 049/137] Install missing deps in ci image tests requiring libmodulemd are skipped due to missing deps and this patch could fix the issue. Signed-off-by: Haibo Lin --- tests/Dockerfile-test | 2 ++ tests/Dockerfile-test-py2 | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/Dockerfile-test b/tests/Dockerfile-test index 43b4191d..61c99a96 100644 --- a/tests/Dockerfile-test +++ b/tests/Dockerfile-test @@ -7,10 +7,12 @@ LABEL \ RUN dnf -y update && dnf -y install \ findutils \ + libmodulemd \ git \ koji \ make \ python3-createrepo_c \ + python3-gobject-base \ python3-tox \ python3-urlgrabber \ && dnf clean all diff --git a/tests/Dockerfile-test-py2 b/tests/Dockerfile-test-py2 index 1e41be20..84ce1f99 100644 --- a/tests/Dockerfile-test-py2 +++ b/tests/Dockerfile-test-py2 @@ -5,13 +5,15 @@ LABEL \ vendor="Pungi developers" \ license="MIT" -RUN yum -y update && yum -y install \ +RUN yum -y update && yum -y install epel-release && yum -y install \ git \ + libmodulemd2 \ make \ python3 \ + python-createrepo_c \ + python-gobject-base \ python-gssapi \ python-libcomps \ - python-createrepo_c \ pykickstart \ && yum clean all From 795bbe31e3a16029ff4f6f8391f625a802b0ca58 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Tue, 17 Aug 2021 14:22:37 +0800 Subject: [PATCH 050/137] Fix formatting Signed-off-by: Haibo Lin --- pungi/gather.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pungi/gather.py b/pungi/gather.py index 3411e0c3..a77ed6bd 100644 --- a/pungi/gather.py +++ b/pungi/gather.py @@ -35,7 +35,7 @@ from pungi.wrappers.createrepo import CreaterepoWrapper class ReentrantYumLock(object): - """ A lock that can be acquired multiple times by the same process. """ + """A lock that can be acquired multiple times by the same process.""" def __init__(self, lock, log): self.lock = lock @@ -60,7 +60,7 @@ class ReentrantYumLock(object): def yumlocked(method): - """ A locking decorator. """ + """A locking decorator.""" def wrapper(self, *args, **kwargs): with self.yumlock: From 66dacb21e0344b160dc740c6a4d0d4212b62ee5c Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Wed, 18 Aug 2021 09:34:12 +0800 Subject: [PATCH 051/137] Add createrepo_enable_cache to configuration doc JIRA: RHELCMP-5984 Signed-off-by: Haibo Lin --- doc/configuration.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/configuration.rst b/doc/configuration.rst index 52b8c9ea..0535ae97 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -457,6 +457,12 @@ Options cloned files should be split into subdirectories for each architecture of the variant. +**createrepo_enable_cache** = True + (*bool*) -- whether to use ``--cachedir`` option of ``createrepo``. It will + cache and reuse checksum vaules to speed up createrepo phase. + The cache dir is located at ``/var/cache/pungi/createrepo_c/$release_short-$uid`` + e.g. /var/cache/pungi/createrepo_c/Fedora-1000 + **product_id** = None (:ref:`scm_dict `) -- If specified, it should point to a directory with certificates ``*--*.pem``. Pungi will From 980c7ba8fbdf900c1ae6a3d175a664370e8edf2a Mon Sep 17 00:00:00 2001 From: Ozan Unsal Date: Mon, 16 Aug 2021 09:36:13 +0200 Subject: [PATCH 052/137] Handle the pungi failures to ensure creation of log files If the given directory is not a valid git directory, it raises RuntimeError. This can be catched and raised as GitUrlResolveError, so compose can continue to log the failure. Jira: RHELCMP-6077 Signed-off-by: Ozan Unsal --- pungi/scripts/pungi_koji.py | 9 ++++++++- pungi/util.py | 8 ++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index b63ddd6e..c262e775 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -272,7 +272,7 @@ def main(): logger, opts.skip_phase + conf.get("skip_phases", []), opts.just_phase ): sys.exit(1) - errors, warnings = pungi.checks.validate(conf) + errors, warnings = pungi.checks.validate(conf, offline=True) if not opts.quiet: # TODO: workaround for config files containing skip_phase = productimg @@ -328,6 +328,13 @@ def main(): logger=logger, notifier=notifier, ) + errors, warnings = pungi.checks.validate(conf, offline=False) + if errors: + for error in errors: + logger.error("Config validation failed with the error: %s" % error) + fail_to_start("Config validation failed", errors=errors) + sys.exit(1) + notifier.compose = compose COMPOSE = compose try: diff --git a/pungi/util.py b/pungi/util.py index da5e82f2..c6a40d4d 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -288,8 +288,12 @@ def resolve_git_ref(repourl, ref): if re.match(r"^[a-f0-9]{40}$", ref): # This looks like a commit ID already. return ref - - _, output = git_ls_remote(repourl, ref) + try: + _, output = git_ls_remote(repourl, ref) + except RuntimeError as e: + raise GitUrlResolveError( + "ref does not exist in remote repo %s with the error %s" % (repourl, e) + ) lines = [] for line in output.split("\n"): From 9cd42a2b5ee883c7570b03fcf03eafb83eb9b5dc Mon Sep 17 00:00:00 2001 From: Dominik Rumian Date: Mon, 30 Aug 2021 09:41:37 +0200 Subject: [PATCH 053/137] Formatted files according to flake8 and black feedback Signed-off-by: Dominik Rumian --- pungi/phases/createrepo.py | 23 ++++++++++++----------- tests/test_createrepophase.py | 8 ++++---- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index 83028bb0..84ea1f9a 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -16,7 +16,6 @@ __all__ = ("create_variant_repo",) - import copy import errno import glob @@ -25,18 +24,20 @@ import shutil import threading import xml.dom.minidom -from kobo.threads import ThreadPool, WorkerThread -from kobo.shortcuts import run, relative_path - -from ..wrappers.scm import get_dir_from_scm -from ..wrappers.createrepo import CreaterepoWrapper -from .base import PhaseBase -from ..util import get_arch_variant_data, temp_dir, read_single_module_stream_from_file -from ..module_util import Modulemd, collect_module_defaults - -import productmd.rpms import productmd.modules +import productmd.rpms +from kobo.shortcuts import relative_path, run +from kobo.threads import ThreadPool, WorkerThread +from ..module_util import Modulemd, collect_module_defaults +from ..util import ( + get_arch_variant_data, + read_single_module_stream_from_file, + temp_dir, +) +from ..wrappers.createrepo import CreaterepoWrapper +from ..wrappers.scm import get_dir_from_scm +from .base import PhaseBase createrepo_lock = threading.Lock() createrepo_dirs = set() diff --git a/tests/test_createrepophase.py b/tests/test_createrepophase.py index 963984bc..6475c75a 100644 --- a/tests/test_createrepophase.py +++ b/tests/test_createrepophase.py @@ -1,24 +1,24 @@ # -*- coding: utf-8 -*- - try: import unittest2 as unittest except ImportError: import unittest -import mock import glob import os + +import mock import six +from pungi.module_util import Modulemd from pungi.phases.createrepo import ( CreaterepoPhase, + ModulesMetadata, create_variant_repo, get_productids_from_scm, - ModulesMetadata, ) from tests.helpers import DummyCompose, PungiTestCase, copy_fixture, touch -from pungi.module_util import Modulemd class TestCreaterepoPhase(PungiTestCase): From 7c3e8d4276fb3fbf0374b04f5598187dd8d2b9c0 Mon Sep 17 00:00:00 2001 From: Dominik Rumian Date: Mon, 30 Aug 2021 09:41:37 +0200 Subject: [PATCH 054/137] Fix tests for createrepo Tests for createrepo failed when pungi is installed in system. JIRA: RHELCMP-6209 Signed-off-by: Dominik Rumian --- pungi/phases/createrepo.py | 3 ++- tests/test_createrepophase.py | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index 84ea1f9a..c299169f 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -39,6 +39,7 @@ from ..wrappers.createrepo import CreaterepoWrapper from ..wrappers.scm import get_dir_from_scm from .base import PhaseBase +CACHE_TOPDIR = "/var/cache/pungi/createrepo_c/" createrepo_lock = threading.Lock() createrepo_dirs = set() @@ -192,7 +193,7 @@ def create_variant_repo( if compose.conf["createrepo_enable_cache"]: cachedir = os.path.join( - "/var/cache/pungi/createrepo_c/", + CACHE_TOPDIR, "%s-%s" % (compose.conf["release_short"], os.getuid()), ) if not os.path.exists(cachedir): diff --git a/tests/test_createrepophase.py b/tests/test_createrepophase.py index 6475c75a..45c2c25c 100644 --- a/tests/test_createrepophase.py +++ b/tests/test_createrepophase.py @@ -185,13 +185,15 @@ class TestCreateVariantRepo(PungiTestCase): repo = CreaterepoWrapperCls.return_value copy_fixture("server-rpms.json", compose.paths.compose.metadata("rpms.json")) - create_variant_repo( - compose, "x86_64", compose.variants["Server"], "rpm", self.pkgset - ) + with mock.patch("pungi.phases.createrepo.CACHE_TOPDIR", self.topdir): + create_variant_repo( + compose, "x86_64", compose.variants["Server"], "rpm", self.pkgset + ) list_file = ( self.topdir + "/work/x86_64/repo_package_list/Server.x86_64.rpm.conf" ) + self.assertEqual( CreaterepoWrapperCls.mock_calls[0], mock.call(createrepo_c=True) ) @@ -214,7 +216,7 @@ class TestCreateVariantRepo(PungiTestCase): use_xz=False, extra_args=[], cachedir=os.path.join( - "/var/cache/pungi/createrepo_c/", + self.topdir, "%s-%s" % (compose.conf["release_short"], os.getuid()), ), ) From 3d9335e90ebc7eb63721f4ca689c88fa15cca6a2 Mon Sep 17 00:00:00 2001 From: Ozan Unsal Date: Mon, 30 Aug 2021 15:30:28 +0200 Subject: [PATCH 055/137] Use xorriso instead of isoinfo when createiso_use_xorrisofs is enabled Update get_mkisofs_cmd in createiso.py file in order to prevent using default value. With this change it is possible to enable xorriso format Jira: RHELCMP-6325 Signed-off-by: Ozan Unsal --- pungi/createiso.py | 3 ++- pungi/wrappers/iso.py | 20 +++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/pungi/createiso.py b/pungi/createiso.py index 1f7990db..cbd3fef3 100644 --- a/pungi/createiso.py +++ b/pungi/createiso.py @@ -76,6 +76,7 @@ def make_image(f, opts): volid=opts.volid, exclude=["./lost+found"], graft_points=opts.graft_points, + use_xorrisofs=opts.use_xorrisofs, **mkisofs_kwargs ) emit(f, cmd) @@ -97,7 +98,7 @@ def run_isohybrid(f, opts): def make_manifest(f, opts): - emit(f, iso.get_manifest_cmd(opts.iso_name)) + emit(f, iso.get_manifest_cmd(opts.iso_name, opts.use_xorrisofs)) def make_jigdo(f, opts): diff --git a/pungi/wrappers/iso.py b/pungi/wrappers/iso.py index afbdf87b..4cdbd3f6 100644 --- a/pungi/wrappers/iso.py +++ b/pungi/wrappers/iso.py @@ -255,11 +255,21 @@ def get_isohybrid_cmd(iso_path, arch): return cmd -def get_manifest_cmd(iso_name): - return "isoinfo -R -f -i %s | grep -v '/TRANS.TBL$' | sort >> %s.manifest" % ( - shlex_quote(iso_name), - shlex_quote(iso_name), - ) +def get_manifest_cmd(iso_name, xorriso=False): + if xorriso: + return """xorriso -dev %s --find | + tail -n+2 | + tr -d "'" | + cut -c2- | + sort >> %s.manifest""" % ( + shlex_quote(iso_name), + shlex_quote(iso_name), + ) + else: + return "isoinfo -R -f -i %s | grep -v '/TRANS.TBL$' | sort >> %s.manifest" % ( + shlex_quote(iso_name), + shlex_quote(iso_name), + ) def get_volume_id(path): From b7666ba4a45bc5311396277934d9a3ced5822fb2 Mon Sep 17 00:00:00 2001 From: Ozan Unsal Date: Mon, 6 Sep 2021 19:05:49 +0200 Subject: [PATCH 056/137] Enable pungi to send compose_url patches to CTS If cts_keytab is also enabled then the HTTP requests are handled with Kerberos Authentication otherwise no authentication is used. If cts_url is defined in the configuration, translate_paths is required. This is needed in order to get the host and the path of the composes. Jira: RHELCMP-6318 Signed-off-by: Ozan Unsal --- pungi/checks.py | 1 + pungi/compose.py | 24 +++++++++++++++++++++++- pungi/scripts/pungi_koji.py | 5 +++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/pungi/checks.py b/pungi/checks.py index 62724946..581bd9fc 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -1339,6 +1339,7 @@ CONFIG_DEPS = { "requires": ((lambda x: x, ["base_product_name", "base_product_short"]),), "conflicts": ((lambda x: not x, ["base_product_name", "base_product_short"]),), }, + "cts_url": {"requires": ((lambda x: x, ["translate_paths"]),)}, "product_id": {"conflicts": [(lambda x: not x, ["product_id_allow_missing"])]}, "pkgset_scratch_modules": {"requires": ((lambda x: x, ["mbs_api_url"]),)}, "pkgset_source": { diff --git a/pungi/compose.py b/pungi/compose.py index f1229439..a37c5f6d 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -41,6 +41,7 @@ from pungi.util import ( get_arch_variant_data, get_format_substs, get_variant_data, + translate_path_raw, ) from pungi.metadata import compose_to_composeinfo @@ -95,9 +96,12 @@ def get_compose_info( # So at first backup the current environment and revert to it # after the requests.post call. cts_keytab = conf.get("cts_keytab", None) + authentication = None if cts_keytab: environ_copy = dict(os.environ) os.environ["KRB5_CLIENT_KTNAME"] = cts_keytab + # Enables Kerberos Authentication if cts_keytab is specified + authentication = HTTPKerberosAuth() try: # Create compose in CTS and get the reserved compose ID. @@ -108,7 +112,7 @@ def get_compose_info( "parent_compose_ids": parent_compose_ids, "respin_of": respin_of, } - rv = requests.post(url, json=data, auth=HTTPKerberosAuth()) + rv = requests.post(url, json=data, auth=authentication) rv.raise_for_status() finally: if cts_keytab: @@ -120,6 +124,7 @@ def get_compose_info( cts_ci.loads(rv.text) ci.compose.respin = cts_ci.compose.respin ci.compose.id = cts_ci.compose.id + else: ci.compose.id = ci.create_compose_id() @@ -138,6 +143,22 @@ def write_compose_info(compose_dir, ci): ci.dump(os.path.join(work_dir, "composeinfo-base.json")) +def update_compose_url(compose_dir, conf): + import requests + + with open(os.path.join(compose_dir, "COMPOSE_ID"), "r") as f: + compose_id = f.read() + cts_url = conf.get("cts_url", None) + url = os.path.join(cts_url, "api/1/composes", compose_id) + tp = conf.get("translate_paths", None) + compose_url = translate_path_raw(tp, compose_dir) + data = { + "action": "set_url", + "compose_url": compose_url, + } + return requests.patch(url, json=data) + + def get_compose_dir( topdir, conf, @@ -306,6 +327,7 @@ class Compose(kobo.log.LoggingBase): get_compose_info = staticmethod(get_compose_info) write_compose_info = staticmethod(write_compose_info) get_compose_dir = staticmethod(get_compose_dir) + update_compose_url = staticmethod(update_compose_url) def __getitem__(self, name): return self.variants[name] diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index c262e775..b8d899bc 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -328,6 +328,11 @@ def main(): logger=logger, notifier=notifier, ) + + rv = Compose.update_compose_url(compose_dir, conf) + if not rv.ok: + logger.error("CTS compose_url update failed with the error: %s" % rv.text) + errors, warnings = pungi.checks.validate(conf, offline=False) if errors: for error in errors: From e8ddacd10e4a27dbe8ee0e9904b9c927de6c15d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Mon, 30 Aug 2021 08:40:05 +0200 Subject: [PATCH 057/137] Fix type detection for osbuild images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The image type value passed to the task doesn't match the type as it will be recorded by Koji. JIRA: RHELCMP-5727 Signed-off-by: Lubomír Sedlář --- pungi/phases/osbuild.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pungi/phases/osbuild.py b/pungi/phases/osbuild.py index 15fe1c76..89719050 100644 --- a/pungi/phases/osbuild.py +++ b/pungi/phases/osbuild.py @@ -155,7 +155,7 @@ class RunOSBuildThread(WorkerThread): # architecture, but we don't verify that. build_info = koji.koji_proxy.getBuild(build_id) for archive in koji.koji_proxy.listArchives(buildID=build_id): - if archive["type_name"] not in config["image_types"]: + if archive["type_name"] not in EXTENSIONS: # Ignore values that are not of required types. continue @@ -182,8 +182,11 @@ class RunOSBuildThread(WorkerThread): linker.link(src_file, image_dest, link_type=compose.conf["link_type"]) - suffix = archive["filename"].rsplit(".", 1)[-1] - if suffix not in EXTENSIONS[archive["type_name"]]: + for suffix in EXTENSIONS[archive["type_name"]]: + if archive["filename"].endswith(suffix): + break + else: + # No suffix matched. raise RuntimeError( "Failed to generate metadata. Format %s doesn't match type %s" % (suffix, archive["type_name"]) From 904a1c3271a22462c1f92651708c1cba4ee97bfe Mon Sep 17 00:00:00 2001 From: Ozan Unsal Date: Thu, 9 Sep 2021 09:14:18 +0200 Subject: [PATCH 058/137] Add authentication for updating the compose URL in CTS. Put authentication steps in a function in order to prevent code duplication. Jira: RHELCMP-6318 Signed-off-by: Ozan Unsal --- pungi/compose.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/pungi/compose.py b/pungi/compose.py index a37c5f6d..62ce0c1a 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -88,7 +88,6 @@ def get_compose_info( # Import requests and requests-kerberos here so it is not needed # if running without Compose Tracking Service. import requests - from requests_kerberos import HTTPKerberosAuth # Requests-kerberos cannot accept custom keytab, we need to use # environment variable for this. But we need to change environment @@ -96,12 +95,10 @@ def get_compose_info( # So at first backup the current environment and revert to it # after the requests.post call. cts_keytab = conf.get("cts_keytab", None) - authentication = None + authentication = get_authentication(conf) if cts_keytab: environ_copy = dict(os.environ) os.environ["KRB5_CLIENT_KTNAME"] = cts_keytab - # Enables Kerberos Authentication if cts_keytab is specified - authentication = HTTPKerberosAuth() try: # Create compose in CTS and get the reserved compose ID. @@ -131,6 +128,16 @@ def get_compose_info( return ci +def get_authentication(conf): + from requests_kerberos import HTTPKerberosAuth + + authentication = None + cts_keytab = conf.get("cts_keytab", None) + if cts_keytab: + authentication = HTTPKerberosAuth() + return authentication + + def write_compose_info(compose_dir, ci): """ Write ComposeInfo `ci` to `compose_dir` subdirectories. @@ -148,6 +155,7 @@ def update_compose_url(compose_dir, conf): with open(os.path.join(compose_dir, "COMPOSE_ID"), "r") as f: compose_id = f.read() + authentication = get_authentication(conf) cts_url = conf.get("cts_url", None) url = os.path.join(cts_url, "api/1/composes", compose_id) tp = conf.get("translate_paths", None) @@ -156,7 +164,7 @@ def update_compose_url(compose_dir, conf): "action": "set_url", "compose_url": compose_url, } - return requests.patch(url, json=data) + return requests.patch(url, json=data, auth=authentication) def get_compose_dir( From d8d1cc520be4dea60ac2ce2494c62c054e43b16a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 6 Aug 2021 11:22:54 +0200 Subject: [PATCH 059/137] paths: Allow customizing log file extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the file contents is JSON, it would be nice to have matching extension. Signed-off-by: Lubomír Sedlář --- pungi/paths.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pungi/paths.py b/pungi/paths.py index 2916a577..6049b667 100644 --- a/pungi/paths.py +++ b/pungi/paths.py @@ -113,12 +113,13 @@ class LogPaths(object): makedirs(path) return path - def log_file(self, arch, log_name, create_dir=True): + def log_file(self, arch, log_name, create_dir=True, ext=None): + ext = ext or "log" arch = arch or "global" if log_name.endswith(".log"): log_name = log_name[:-4] return os.path.join( - self.topdir(arch, create_dir=create_dir), "%s.%s.log" % (log_name, arch) + self.topdir(arch, create_dir=create_dir), "%s.%s.%s" % (log_name, arch, ext) ) From 20dc4beb6b6ca3b50e45955b46bab541a36f145b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 6 Aug 2021 14:02:54 +0200 Subject: [PATCH 060/137] Make getting old compose config reusable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The file will only be loaded once, it gets cached afterwards. Signed-off-by: Lubomír Sedlář --- pungi/compose.py | 18 +++++++ pungi/phases/gather/__init__.py | 17 +------ tests/helpers.py | 1 + tests/test_gather_phase.py | 84 ++++++++++----------------------- 4 files changed, 45 insertions(+), 75 deletions(-) diff --git a/pungi/compose.py b/pungi/compose.py index 62ce0c1a..24cc5d29 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -252,6 +252,8 @@ class Compose(kobo.log.LoggingBase): self.koji_event = koji_event or conf.get("koji_event") self.notifier = notifier + self._old_config = None + # path definitions self.paths = Paths(self) @@ -635,6 +637,22 @@ class Compose(kobo.log.LoggingBase): with open(tb_path, "wb") as f: f.write(kobo.tback.Traceback().get_traceback()) + def load_old_compose_config(self): + """ + Helper method to load Pungi config dump from old compose. + """ + if not self._old_config: + config_dump_full = self.paths.log.log_file("global", "config-dump") + config_dump_full = self.paths.old_compose_path(config_dump_full) + if not config_dump_full: + return None + + self.log_info("Loading old config file: %s", config_dump_full) + with open(config_dump_full, "r") as f: + self._old_config = json.load(f) + + return self._old_config + def get_ordered_variant_uids(compose): if not hasattr(compose, "_ordered_variant_uids"): diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 64310259..00af2ff1 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -194,21 +194,6 @@ def load_old_gather_result(compose, arch, variant): return old_result -def load_old_compose_config(compose): - """ - Helper method to load Pungi config dump from old compose. - """ - config_dump_full = compose.paths.log.log_file("global", "config-dump") - config_dump_full = compose.paths.old_compose_path(config_dump_full) - if not config_dump_full: - return None - - compose.log_info("Loading old config file: %s", config_dump_full) - with open(config_dump_full, "r") as f: - old_config = json.load(f) - return old_config - - def reuse_old_gather_packages(compose, arch, variant, package_sets, methods): """ Tries to reuse `gather_packages` result from older compose. @@ -230,7 +215,7 @@ def reuse_old_gather_packages(compose, arch, variant, package_sets, methods): compose.log_info(log_msg % "no old gather results.") return - old_config = load_old_compose_config(compose) + old_config = compose.load_old_compose_config() if old_config is None: compose.log_info(log_msg % "no old compose config dump.") return diff --git a/tests/helpers.py b/tests/helpers.py index 852eb054..7aa7452d 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -230,6 +230,7 @@ class DummyCompose(object): self.should_create_yum_database = True self.cache_region = None self.containers_metadata = {} + self.load_old_compose_config = mock.Mock(return_value=None) def setup_optional(self): self.all_variants["Server-optional"] = MockVariant( diff --git a/tests/test_gather_phase.py b/tests/test_gather_phase.py index 2725247f..a5d59730 100644 --- a/tests/test_gather_phase.py +++ b/tests/test_gather_phase.py @@ -1099,10 +1099,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): self.assertEqual(result, None) @mock.patch("pungi.phases.gather.load_old_gather_result") - @mock.patch("pungi.phases.gather.load_old_compose_config") - def test_reuse_no_old_compose_config( - self, load_old_compose_config, load_old_gather_result - ): + def test_reuse_no_old_compose_config(self, load_old_gather_result): load_old_gather_result.return_value = { "rpm": [{"path": "/build/bash-1.0.0-1.x86_64.rpm"}], "srpm": [], @@ -1111,7 +1108,6 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) self._save_config_dump(compose) - load_old_compose_config.return_value = None result = gather.reuse_old_gather_packages( compose, "x86_64", compose.variants["Server"], [], "deps" @@ -1119,10 +1115,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): self.assertEqual(result, None) @mock.patch("pungi.phases.gather.load_old_gather_result") - @mock.patch("pungi.phases.gather.load_old_compose_config") - def test_reuse_compose_config_different( - self, load_old_compose_config, load_old_gather_result - ): + def test_reuse_compose_config_different(self, load_old_gather_result): load_old_gather_result.return_value = { "rpm": [{"path": "/build/bash-1.0.0-1.x86_64.rpm"}], "srpm": [], @@ -1133,7 +1126,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): self._save_config_dump(compose) compose_conf_copy = dict(compose.conf) compose_conf_copy["gather_method"] = "nodeps" - load_old_compose_config.return_value = compose_conf_copy + compose.load_old_compose_config.return_value = compose_conf_copy result = gather.reuse_old_gather_packages( compose, "x86_64", compose.variants["Server"], [], "nodeps" @@ -1141,10 +1134,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): self.assertEqual(result, None) @mock.patch("pungi.phases.gather.load_old_gather_result") - @mock.patch("pungi.phases.gather.load_old_compose_config") - def test_reuse_compose_config_different_whitelist( - self, load_old_compose_config, load_old_gather_result - ): + def test_reuse_compose_config_different_whitelist(self, load_old_gather_result): for whitelist_opt in ["product_id", "pkgset_koji_builds"]: load_old_gather_result.return_value = { "rpm": [{"path": "/build/bash-1.0.0-1.x86_64.rpm"}], @@ -1156,7 +1146,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): self._save_config_dump(compose) compose_conf_copy = dict(compose.conf) compose_conf_copy[whitelist_opt] = "different" - load_old_compose_config.return_value = compose_conf_copy + compose.load_old_compose_config.return_value = compose_conf_copy result = gather.reuse_old_gather_packages( compose, "x86_64", compose.variants["Server"], [], "deps" @@ -1184,14 +1174,13 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): return package_sets @mock.patch("pungi.phases.gather.load_old_gather_result") - @mock.patch("pungi.phases.gather.load_old_compose_config") - def test_reuse(self, load_old_compose_config, load_old_gather_result): + def test_reuse(self, load_old_gather_result): package_sets = self._prepare_package_sets( load_old_gather_result, requires=[], provides=[] ) compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) self._save_config_dump(compose) - load_old_compose_config.return_value = compose.conf + compose.load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( compose, "x86_64", compose.variants["Server"], package_sets, "deps" @@ -1206,16 +1195,13 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): ) @mock.patch("pungi.phases.gather.load_old_gather_result") - @mock.patch("pungi.phases.gather.load_old_compose_config") - def test_reuse_update_gather_lookaside_repos( - self, load_old_compose_config, load_old_gather_result - ): + def test_reuse_update_gather_lookaside_repos(self, load_old_gather_result): package_sets = self._prepare_package_sets( load_old_gather_result, requires=[], provides=[] ) compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) self._save_config_dump(compose) - load_old_compose_config.return_value = copy.deepcopy(compose.conf) + compose.load_old_compose_config.return_value = copy.deepcopy(compose.conf) gather._update_config(compose, "Server", "x86_64", compose.topdir) result = gather.reuse_old_gather_packages( @@ -1231,9 +1217,8 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): ) @mock.patch("pungi.phases.gather.load_old_gather_result") - @mock.patch("pungi.phases.gather.load_old_compose_config") def test_reuse_update_gather_lookaside_repos_different_initial_repos( - self, load_old_compose_config, load_old_gather_result + self, load_old_gather_result ): package_sets = self._prepare_package_sets( load_old_gather_result, requires=[], provides=[] @@ -1242,7 +1227,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): self._save_config_dump(compose) lookasides = compose.conf["gather_lookaside_repos"] lookasides.append(("^Server$", {"x86_64": "http://localhost/real.repo"})) - load_old_compose_config.return_value = copy.deepcopy(compose.conf) + compose.load_old_compose_config.return_value = copy.deepcopy(compose.conf) gather._update_config(compose, "Server", "x86_64", compose.topdir) result = gather.reuse_old_gather_packages( @@ -1251,9 +1236,8 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): self.assertEqual(result, None) @mock.patch("pungi.phases.gather.load_old_gather_result") - @mock.patch("pungi.phases.gather.load_old_compose_config") def test_reuse_update_gather_lookaside_repos_different_initial_repos_list( - self, load_old_compose_config, load_old_gather_result + self, load_old_gather_result ): package_sets = self._prepare_package_sets( load_old_gather_result, requires=[], provides=[] @@ -1263,7 +1247,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): lookasides = compose.conf["gather_lookaside_repos"] repos = ["http://localhost/real1.repo", "http://localhost/real2.repo"] lookasides.append(("^Server$", {"x86_64": repos})) - load_old_compose_config.return_value = copy.deepcopy(compose.conf) + compose.load_old_compose_config.return_value = copy.deepcopy(compose.conf) gather._update_config(compose, "Server", "x86_64", compose.topdir) result = gather.reuse_old_gather_packages( @@ -1272,10 +1256,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): self.assertEqual(result, None) @mock.patch("pungi.phases.gather.load_old_gather_result") - @mock.patch("pungi.phases.gather.load_old_compose_config") - def test_reuse_no_old_file_cache( - self, load_old_compose_config, load_old_gather_result - ): + def test_reuse_no_old_file_cache(self, load_old_gather_result): package_sets = self._prepare_package_sets( load_old_gather_result, requires=[], provides=[] ) @@ -1284,7 +1265,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): } compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) self._save_config_dump(compose) - load_old_compose_config.return_value = compose.conf + compose.load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( compose, "x86_64", compose.variants["Server"], package_sets, "deps" @@ -1292,10 +1273,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): self.assertEqual(result, None) @mock.patch("pungi.phases.gather.load_old_gather_result") - @mock.patch("pungi.phases.gather.load_old_compose_config") - def test_reuse_two_rpms_from_same_source( - self, load_old_compose_config, load_old_gather_result - ): + def test_reuse_two_rpms_from_same_source(self, load_old_gather_result): package_sets = self._prepare_package_sets( load_old_gather_result, requires=[], provides=[] ) @@ -1307,7 +1285,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): pkg_set.file_cache["/build/bash-1-2.x86_64.rpm"] = bash_pkg compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) self._save_config_dump(compose) - load_old_compose_config.return_value = compose.conf + compose.load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( compose, "x86_64", compose.variants["Server"], package_sets, "deps" @@ -1315,10 +1293,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): self.assertEqual(result, None) @mock.patch("pungi.phases.gather.load_old_gather_result") - @mock.patch("pungi.phases.gather.load_old_compose_config") - def test_reuse_rpm_added_removed( - self, load_old_compose_config, load_old_gather_result - ): + def test_reuse_rpm_added_removed(self, load_old_gather_result): package_sets = self._prepare_package_sets( load_old_gather_result, requires=[], provides=[] ) @@ -1333,7 +1308,7 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): pkg_set.file_cache["/build/foo-1-1.x86_64.rpm"] = foo_pkg compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) self._save_config_dump(compose) - load_old_compose_config.return_value = compose.conf + compose.load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( compose, "x86_64", compose.variants["Server"], package_sets, "deps" @@ -1341,17 +1316,14 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): self.assertEqual(result, None) @mock.patch("pungi.phases.gather.load_old_gather_result") - @mock.patch("pungi.phases.gather.load_old_compose_config") - def test_reuse_different_packages( - self, load_old_compose_config, load_old_gather_result - ): + def test_reuse_different_packages(self, load_old_gather_result): package_sets = self._prepare_package_sets( load_old_gather_result, requires=[], provides=["foo"] ) package_sets[0]["global"].old_file_cache = None compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) self._save_config_dump(compose) - load_old_compose_config.return_value = compose.conf + compose.load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( compose, "x86_64", compose.variants["Server"], package_sets, "deps" @@ -1359,16 +1331,13 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): self.assertEqual(result, None) @mock.patch("pungi.phases.gather.load_old_gather_result") - @mock.patch("pungi.phases.gather.load_old_compose_config") - def test_reuse_requires_changed( - self, load_old_compose_config, load_old_gather_result - ): + def test_reuse_requires_changed(self, load_old_gather_result): package_sets = self._prepare_package_sets( load_old_gather_result, requires=["foo"], provides=[] ) compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) self._save_config_dump(compose) - load_old_compose_config.return_value = compose.conf + compose.load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( compose, "x86_64", compose.variants["Server"], package_sets, "deps" @@ -1376,16 +1345,13 @@ class TestReuseOldGatherPackages(helpers.PungiTestCase): self.assertEqual(result, None) @mock.patch("pungi.phases.gather.load_old_gather_result") - @mock.patch("pungi.phases.gather.load_old_compose_config") - def test_reuse_provides_changed( - self, load_old_compose_config, load_old_gather_result - ): + def test_reuse_provides_changed(self, load_old_gather_result): package_sets = self._prepare_package_sets( load_old_gather_result, requires=[], provides=["foo"] ) compose = helpers.DummyCompose(self.topdir, {"gather_allow_reuse": True}) self._save_config_dump(compose) - load_old_compose_config.return_value = compose.conf + compose.load_old_compose_config.return_value = compose.conf result = gather.reuse_old_gather_packages( compose, "x86_64", compose.variants["Server"], package_sets, "deps" From 195bfbefa4f4936c235d83993a7da39cdeb16533 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Thu, 9 Sep 2021 08:49:31 +0200 Subject: [PATCH 061/137] Allow specifying $COMPOSE_ID in the `repo` value for osbs phase. There should be an option for `yum_repourls` to point to static URL, for example when CTS is used. The idea is that instead of setting `repo` to `AppStream`, we could use link similar to this one: `https://cts.localhost/api/1/composes/$COMPOSE_ID/repo/?variant=AppStream` This would be translated to real static link during the OSBS phase: `https://cts.localhost/api/1/composes/CentOS-Stream-9-20210803.0/repo/?variant=AppStream` That way this statis link would appear in the yum_repourls. Merges: https://pagure.io/pungi/pull-request/1543 Signed-off-by: Jan Kaluza --- doc/configuration.rst | 6 ++++-- pungi/phases/osbs.py | 2 +- tests/test_osbs_phase.py | 8 +++++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 0535ae97..ba0f5040 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1849,8 +1849,10 @@ they are not scratch builds). A value for ``yum_repourls`` will be created automatically and point at a repository in the current compose. You can add extra repositories with ``repo`` key having a list of urls pointing to ``.repo`` files or just - variant uid, Pungi will create the .repo file for that variant. ``gpgkey`` - can be specified to enable gpgcheck in repo files for variants. + variant uid, Pungi will create the .repo file for that variant. If + specific URL is used in the ``repo``, the ``$COMPOSE_ID`` variable in + the ``repo`` string will be replaced with the real compose ID. + ``gpgkey`` can be specified to enable gpgcheck in repo files for variants. **osbs_registries** (*dict*) -- It is possible to configure extra information about where to diff --git a/pungi/phases/osbs.py b/pungi/phases/osbs.py index 411f05a7..4f4a2980 100644 --- a/pungi/phases/osbs.py +++ b/pungi/phases/osbs.py @@ -128,7 +128,7 @@ class OSBSThread(WorkerThread): file pointing to that location and return the URL to .repo file. """ if "://" in repo: - return repo + return repo.replace("$COMPOSE_ID", compose.compose_id) if repo.startswith("/"): # The repo is an absolute path on the filesystem diff --git a/tests/test_osbs_phase.py b/tests/test_osbs_phase.py index 04e98c19..b37fa178 100644 --- a/tests/test_osbs_phase.py +++ b/tests/test_osbs_phase.py @@ -330,7 +330,12 @@ class OSBSThreadTest(helpers.PungiTestCase): "git_branch": "f24-docker", "name": "my-name", "version": "1.0", - "repo": ["Everything", "http://pkgs.example.com/my.repo", "/extra/repo"], + "repo": [ + "Everything", + "http://pkgs.example.com/my.repo", + "/extra/repo", + "http://cts.localhost/$COMPOSE_ID/repo", + ], } self.compose.conf["translate_paths"].append(("/extra", "http://example.com")) self._setupMock(KojiWrapper) @@ -347,6 +352,7 @@ class OSBSThreadTest(helpers.PungiTestCase): "http://root/work/global/tmp-Everything/compose-rpms-Everything-1.repo", "http://pkgs.example.com/my.repo", "http://root/work/global/tmp/compose-rpms-local-1.repo", + "http://cts.localhost/%s/repo" % self.compose.compose_id, ], } self._assertCorrectCalls(options) From 5c26aa9127f00f2d6f0a35d8563176927b160f44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 10 Sep 2021 09:59:10 +0200 Subject: [PATCH 062/137] Require requests_kerberos only when needed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If CTS integration is not used, let's not import a module that is not needed. JIRA: RHELCMP-6611 Signed-off-by: Lubomír Sedlář --- pungi/compose.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pungi/compose.py b/pungi/compose.py index 24cc5d29..a43d899e 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -129,11 +129,11 @@ def get_compose_info( def get_authentication(conf): - from requests_kerberos import HTTPKerberosAuth - authentication = None cts_keytab = conf.get("cts_keytab", None) if cts_keytab: + from requests_kerberos import HTTPKerberosAuth + authentication = HTTPKerberosAuth() return authentication From a1ebd234a4f9e64f4476d827e5c450a7c02157c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 10 Sep 2021 11:29:31 +0200 Subject: [PATCH 063/137] Only build CTS url when configured MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JIRA: RHELCMP-6611 Signed-off-by: Lubomír Sedlář --- pungi/compose.py | 17 +++++++++-------- pungi/scripts/pungi_koji.py | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/pungi/compose.py b/pungi/compose.py index a43d899e..37f63449 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -157,14 +157,15 @@ def update_compose_url(compose_dir, conf): compose_id = f.read() authentication = get_authentication(conf) cts_url = conf.get("cts_url", None) - url = os.path.join(cts_url, "api/1/composes", compose_id) - tp = conf.get("translate_paths", None) - compose_url = translate_path_raw(tp, compose_dir) - data = { - "action": "set_url", - "compose_url": compose_url, - } - return requests.patch(url, json=data, auth=authentication) + if cts_url: + url = os.path.join(cts_url, "api/1/composes", compose_id) + tp = conf.get("translate_paths", None) + compose_url = translate_path_raw(tp, compose_dir) + data = { + "action": "set_url", + "compose_url": compose_url, + } + return requests.patch(url, json=data, auth=authentication) def get_compose_dir( diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index b8d899bc..6f1b92a6 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -330,7 +330,7 @@ def main(): ) rv = Compose.update_compose_url(compose_dir, conf) - if not rv.ok: + if rv and not rv.ok: logger.error("CTS compose_url update failed with the error: %s" % rv.text) errors, warnings = pungi.checks.validate(conf, offline=False) From 72bcee01be96033fb6519a90da920dac3af8e5ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 10 Sep 2021 09:48:16 +0200 Subject: [PATCH 064/137] 4.3.0 release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JIRA: RHELCMP-6614 Signed-off-by: Lubomír Sedlář --- doc/conf.py | 2 +- pungi.spec | 30 +++++++++++++++++++++++++++++- setup.py | 2 +- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 3e3c7fae..7c95c1af 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -53,7 +53,7 @@ copyright = u'2016, Red Hat, Inc.' # The short X.Y version. version = '4.2' # The full version, including alpha/beta/rc tags. -release = '4.2.10' +release = '4.3.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pungi.spec b/pungi.spec index 4cc507df..180c088a 100644 --- a/pungi.spec +++ b/pungi.spec @@ -1,5 +1,5 @@ Name: pungi -Version: 4.2.10 +Version: 4.3.0 Release: 1%{?dist} Summary: Distribution compose tool @@ -111,6 +111,34 @@ pytest cd tests && ./test_compose.sh %changelog +* Fri Sep 10 2021 Lubomír Sedlář - 4.3.0-1 +- Only build CTS url when configured (lsedlar) +- Require requests_kerberos only when needed (lsedlar) +- Allow specifying $COMPOSE_ID in the `repo` value for osbs phase. (jkaluza) +- Make getting old compose config reusable (lsedlar) +- paths: Allow customizing log file extension (lsedlar) +- Add authentication for updating the compose URL in CTS. (ounsal) +- Fix type detection for osbuild images (lsedlar) +- Enable pungi to send compose_url patches to CTS (ounsal) +- Use xorriso instead of isoinfo when createiso_use_xorrisofs is enabled + (ounsal) +- Fix tests for createrepo (drumian) +- Formatted files according to flake8 and black feedback (drumian) +- Handle the pungi failures to ensure creation of log files (ounsal) +- Add createrepo_enable_cache to configuration doc (hlin) +- Fix formatting (hlin) +- Install missing deps in ci image (hlin) +- Use pytest directly incl. support for posargs, e.g.: tox -- -s -vvv + tests/path/to/a/single/test_something.py (fvalder) +- Supersede ModuleStream loading with ModuleIndex (fvalder) +- Better error message than 'KeyError' in pungi (drumian) +- Adding multithreading support for pungi/phases/image_checksum.py (jkunstle) +- doc: more additional_packages documentation (kdreyer) +- doc: fix typo in additional_packages description (kdreyer) +- doc: improve signed packages retry docs (kdreyer) +- Better error message than 'KeyError' in pungi (drumian) +- doc: explain buildContainer API (kdreyer) + * Wed Aug 04 2021 Haibo Lin - 4.2.10-1 - Show and log command when using the run_blocking_cmd() method (fdipretre) - Use cachedir when createrepo (hlin) diff --git a/setup.py b/setup.py index 5e0617e1..a6578563 100755 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ packages = sorted(packages) setup( name="pungi", - version="4.2.10", + version="4.3.0", description="Distribution compose tool", url="https://pagure.io/pungi", author="Dennis Gilmore", From ba6f7429eee6a745f894a31f63f4527fbbd5453d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Tue, 14 Sep 2021 09:57:32 +0200 Subject: [PATCH 065/137] buildinstall: Add easy way to check if previous result was reused MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lubomír Sedlář --- pungi/phases/buildinstall.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pungi/phases/buildinstall.py b/pungi/phases/buildinstall.py index a43e3707..9f34423c 100644 --- a/pungi/phases/buildinstall.py +++ b/pungi/phases/buildinstall.py @@ -50,6 +50,9 @@ class BuildinstallPhase(PhaseBase): # A set of (variant_uid, arch) pairs that completed successfully. This # is needed to skip copying files for failed tasks. self.pool.finished_tasks = set() + # A set of (variant_uid, arch) pairs that were reused from previous + # compose. + self.pool.reused_tasks = set() self.buildinstall_method = self.compose.conf.get("buildinstall_method") self.lorax_use_koji_plugin = self.compose.conf.get("lorax_use_koji_plugin") self.used_lorax = self.buildinstall_method == "lorax" @@ -312,6 +315,18 @@ class BuildinstallPhase(PhaseBase): in self.pool.finished_tasks ) + def reused(self, variant, arch): + """ + Check if buildinstall phase reused previous results for given variant + and arch. If the phase is skipped, the results will be considered + reused as well. + """ + return ( + super(BuildinstallPhase, self).skip() + or (variant.uid if self.used_lorax else None, arch) + in self.pool.reused_tasks + ) + def get_kickstart_file(compose): scm_dict = compose.conf.get("buildinstall_kickstart") @@ -800,6 +815,7 @@ class BuildinstallThread(WorkerThread): ): self.copy_files(compose, variant, arch) self.pool.finished_tasks.add((variant.uid if variant else None, arch)) + self.pool.reused_tasks.add((variant.uid if variant else None, arch)) self.pool.log_info("[DONE ] %s" % msg) return From 9612241396a343b7635252fedb1c3f8014d7c6e1 Mon Sep 17 00:00:00 2001 From: Ozan Unsal Date: Tue, 21 Sep 2021 10:48:35 +0200 Subject: [PATCH 066/137] Add COMPOSE_ID into the pungi log file Jira: RHELCMP-6739 Signed-off-by: Ozan Unsal --- pungi/scripts/pungi_koji.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index 6f1b92a6..aaa5773a 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -377,6 +377,8 @@ def run_compose( ) compose.log_info("Compose top directory: %s" % compose.topdir) compose.log_info("Current timezone offset: %s" % pungi.util.get_tz_offset()) + compose.log_info("COMPOSE_ID=%s" % compose.compose_id) + compose.read_variants() # dump the config file From 0530cf27126a452b26031bb3deeba919d6908cd3 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Fri, 24 Sep 2021 10:17:40 +0200 Subject: [PATCH 067/137] When `cts_url` is configured, use CTS `/repo` API for buildContainer yum_repourls. Signed-off-by: Jan Kaluza --- pungi/phases/osbs.py | 9 +++++++++ tests/test_osbs_phase.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/pungi/phases/osbs.py b/pungi/phases/osbs.py index 4f4a2980..8a395238 100644 --- a/pungi/phases/osbs.py +++ b/pungi/phases/osbs.py @@ -147,6 +147,15 @@ class OSBSThread(WorkerThread): raise RuntimeError( "There is no variant %s to get repo from to pass to OSBS." % repo ) + cts_url = compose.conf.get("cts_url", None) + if cts_url: + return os.path.join( + cts_url, + "api/1/composes", + compose.compose_id, + "repo/?variant=%s" % variant, + ) + repo_path = compose.paths.compose.repository( "$basearch", variant, create_dir=False ) diff --git a/tests/test_osbs_phase.py b/tests/test_osbs_phase.py index b37fa178..fde5753d 100644 --- a/tests/test_osbs_phase.py +++ b/tests/test_osbs_phase.py @@ -364,6 +364,37 @@ class OSBSThreadTest(helpers.PungiTestCase): ) as f: self.assertIn("baseurl=http://example.com/repo\n", f) + @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") + def test_run_with_extra_repos_with_cts(self, KojiWrapper): + cfg = { + "url": "git://example.com/repo?#BEEFCAFE", + "target": "f24-docker-candidate", + "git_branch": "f24-docker", + "name": "my-name", + "version": "1.0", + "repo": [ + "Everything", + ], + } + self.compose.conf["cts_url"] = "http://cts.localhost" + self._setupMock(KojiWrapper) + self._assertConfigCorrect(cfg) + + self.t.process((self.compose, self.compose.variants["Server"], cfg), 1) + + cts_url = "http://cts.localhost/api/1/composes/%s" % self.compose.compose_id + options = { + "name": "my-name", + "version": "1.0", + "git_branch": "f24-docker", + "yum_repourls": [ + "%s/repo/?variant=Server" % cts_url, + "%s/repo/?variant=Everything" % cts_url, + ], + } + self._assertCorrectCalls(options) + self._assertCorrectMetadata() + @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") def test_run_with_deprecated_registry(self, KojiWrapper): cfg = { From ac061b2ea8429b6b6304a1c9c8dd99acffeb83f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Tue, 14 Sep 2021 11:44:45 +0200 Subject: [PATCH 068/137] Work around ODCS creating COMPOSE_ID later MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When ODCS starts a compose, it will provide base composeinfo file, but it doesn't create COMPOSE_ID. This leads to a crash when updating CTS, since the compose id can't be read from the file. We can instead use the value we already have in memory. Signed-off-by: Lubomír Sedlář --- pungi/compose.py | 4 +--- pungi/scripts/pungi_koji.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pungi/compose.py b/pungi/compose.py index 37f63449..f02df9d3 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -150,11 +150,9 @@ def write_compose_info(compose_dir, ci): ci.dump(os.path.join(work_dir, "composeinfo-base.json")) -def update_compose_url(compose_dir, conf): +def update_compose_url(compose_id, compose_dir, conf): import requests - with open(os.path.join(compose_dir, "COMPOSE_ID"), "r") as f: - compose_id = f.read() authentication = get_authentication(conf) cts_url = conf.get("cts_url", None) if cts_url: diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index aaa5773a..07f8cf83 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -329,7 +329,7 @@ def main(): notifier=notifier, ) - rv = Compose.update_compose_url(compose_dir, conf) + rv = Compose.update_compose_url(compose.compose_id, compose_dir, conf) if rv and not rv.ok: logger.error("CTS compose_url update failed with the error: %s" % rv.text) From 7475d2a3a9f34a51b236d2a5823336acf56a4806 Mon Sep 17 00:00:00 2001 From: Ozan Unsal Date: Fri, 8 Oct 2021 15:53:27 +0200 Subject: [PATCH 069/137] Allow ISO-Level configuration within the config file In order to enable this feature set "iso_level=" in config file Jira: RHELCMP-6880 Signed-off-by: Ozan Unsal --- doc/configuration.rst | 3 +++ pungi/checks.py | 4 ++++ pungi/createiso.py | 2 ++ pungi/phases/createiso.py | 1 + pungi/phases/extra_isos.py | 1 + pungi/wrappers/iso.py | 5 +++++ 6 files changed, 16 insertions(+) diff --git a/doc/configuration.rst b/doc/configuration.rst index ba0f5040..f52257fb 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1267,6 +1267,9 @@ Options meaning size in bytes, or it can be a string with ``k``, ``M``, ``G`` suffix (using multiples of 1024). +**iso_level** + (*int*) [optional] -- Set the ISO9660 conformance level. Valid numbers are 1 to 4. + **split_iso_reserve** = 10MiB (*int|str*) -- how much free space should be left on each disk. The format is the same as for ``iso_size`` option. diff --git a/pungi/checks.py b/pungi/checks.py index 581bd9fc..a2434fce 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -746,6 +746,10 @@ def make_schema(): ), "createiso_break_hardlinks": {"type": "boolean", "default": False}, "createiso_use_xorrisofs": {"type": "boolean", "default": False}, + "iso_level": { + "type": "number", + "enum": [1, 2, 3, 4], + }, "iso_hfs_ppc64le_compatible": {"type": "boolean", "default": True}, "multilib": _variant_arch_mapping( {"$ref": "#/definitions/list_of_strings"} diff --git a/pungi/createiso.py b/pungi/createiso.py index cbd3fef3..4e1a5336 100644 --- a/pungi/createiso.py +++ b/pungi/createiso.py @@ -25,6 +25,7 @@ CreateIsoOpts = namedtuple( "os_tree", "hfs_compat", "use_xorrisofs", + "iso_level", ], ) CreateIsoOpts.__new__.__defaults__ = (None,) * len(CreateIsoOpts._fields) @@ -77,6 +78,7 @@ def make_image(f, opts): exclude=["./lost+found"], graft_points=opts.graft_points, use_xorrisofs=opts.use_xorrisofs, + iso_level=opts.iso_level, **mkisofs_kwargs ) emit(f, cmd) diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py index 083a9a58..529101f5 100644 --- a/pungi/phases/createiso.py +++ b/pungi/phases/createiso.py @@ -172,6 +172,7 @@ class CreateisoPhase(PhaseLoggerMixin, PhaseBase): supported=self.compose.supported, hfs_compat=self.compose.conf["iso_hfs_ppc64le_compatible"], use_xorrisofs=self.compose.conf.get("createiso_use_xorrisofs"), + iso_level=self.compose.conf.get("iso_level"), ) if bootable: diff --git a/pungi/phases/extra_isos.py b/pungi/phases/extra_isos.py index abca3b1f..5db71a6f 100644 --- a/pungi/phases/extra_isos.py +++ b/pungi/phases/extra_isos.py @@ -115,6 +115,7 @@ class ExtraIsosThread(WorkerThread): supported=compose.supported, hfs_compat=compose.conf["iso_hfs_ppc64le_compatible"], use_xorrisofs=compose.conf.get("createiso_use_xorrisofs"), + iso_level=compose.conf.get("iso_level"), ) if compose.conf["create_jigdo"]: jigdo_dir = compose.paths.compose.jigdo_dir(arch, variant) diff --git a/pungi/wrappers/iso.py b/pungi/wrappers/iso.py index 4cdbd3f6..3f438f74 100644 --- a/pungi/wrappers/iso.py +++ b/pungi/wrappers/iso.py @@ -146,6 +146,7 @@ def get_mkisofs_cmd( input_charset="utf-8", graft_points=None, use_xorrisofs=False, + iso_level=None, ): # following options are always enabled untranslated_filenames = True @@ -155,6 +156,10 @@ def get_mkisofs_cmd( rock = True cmd = ["/usr/bin/xorrisofs" if use_xorrisofs else "/usr/bin/genisoimage"] + + if iso_level: + cmd.extend(["-iso-level", str(iso_level)]) + if appid: cmd.extend(["-appid", appid]) From e42e65783d51a803d6b5fdced8c38b17e9861b9e Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Fri, 20 Aug 2021 19:35:36 +0800 Subject: [PATCH 070/137] image_build: Allow reusing old image_build results JIRA: RHELCMP-5970 Signed-off-by: Haibo Lin --- pungi/checks.py | 1 + pungi/phases/image_build.py | 225 ++++++++++++++++++++++++- pungi/scripts/pungi_koji.py | 2 +- tests/test_imagebuildphase.py | 307 +++++++++++++++++----------------- 4 files changed, 369 insertions(+), 166 deletions(-) diff --git a/pungi/checks.py b/pungi/checks.py index a2434fce..c79f2bbc 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -1080,6 +1080,7 @@ def make_schema(): "live_images": _variant_arch_mapping( _one_or_list({"$ref": "#/definitions/live_image_config"}) ), + "image_build_allow_reuse": {"type": "boolean", "default": False}, "image_build": { "type": "object", "patternProperties": { diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py index 127ecf1d..e0dcb02b 100644 --- a/pungi/phases/image_build.py +++ b/pungi/phases/image_build.py @@ -1,18 +1,22 @@ # -*- coding: utf-8 -*- import copy +import hashlib +import json import os +import shutil import time from kobo import shortcuts from pungi.util import makedirs, get_mtime, get_file_size, failable, log_failed_task -from pungi.util import translate_path, get_repo_urls, version_generator +from pungi.util import as_local_file, translate_path, get_repo_urls, version_generator from pungi.phases import base from pungi.linker import Linker from pungi.wrappers.kojiwrapper import KojiWrapper from kobo.threads import ThreadPool, WorkerThread from kobo.shortcuts import force_list from productmd.images import Image +from productmd.rpms import Rpms # This is a mapping from formats to file extensions. The format is what koji @@ -46,9 +50,10 @@ class ImageBuildPhase( name = "image_build" - def __init__(self, compose): + def __init__(self, compose, buildinstall_phase=None): super(ImageBuildPhase, self).__init__(compose) self.pool = ThreadPool(logger=self.logger) + self.buildinstall_phase = buildinstall_phase def _get_install_tree(self, image_conf, variant): """ @@ -117,6 +122,7 @@ class ImageBuildPhase( # prevent problems in next iteration where the original # value is needed. image_conf = copy.deepcopy(image_conf) + original_image_conf = copy.deepcopy(image_conf) # image_conf is passed to get_image_build_cmd as dict @@ -167,6 +173,7 @@ class ImageBuildPhase( image_conf["image-build"]["can_fail"] = sorted(can_fail) cmd = { + "original_image_conf": original_image_conf, "image_conf": image_conf, "conf_file": self.compose.paths.work.image_build_conf( image_conf["image-build"]["variant"], @@ -182,7 +189,7 @@ class ImageBuildPhase( "scratch": image_conf["image-build"].pop("scratch", False), } self.pool.add(CreateImageBuildThread(self.pool)) - self.pool.queue_put((self.compose, cmd)) + self.pool.queue_put((self.compose, cmd, self.buildinstall_phase)) self.pool.start() @@ -192,7 +199,7 @@ class CreateImageBuildThread(WorkerThread): self.pool.log_error("CreateImageBuild failed.") def process(self, item, num): - compose, cmd = item + compose, cmd, buildinstall_phase = item variant = cmd["image_conf"]["image-build"]["variant"] subvariant = cmd["image_conf"]["image-build"].get("subvariant", variant.uid) self.failable_arches = cmd["image_conf"]["image-build"].get("can_fail", "") @@ -208,15 +215,47 @@ class CreateImageBuildThread(WorkerThread): subvariant, logger=self.pool._logger, ): - self.worker(num, compose, variant, subvariant, cmd) + self.worker(num, compose, variant, subvariant, cmd, buildinstall_phase) - def worker(self, num, compose, variant, subvariant, cmd): + def worker(self, num, compose, variant, subvariant, cmd, buildinstall_phase): arches = cmd["image_conf"]["image-build"]["arches"] formats = "-".join(cmd["image_conf"]["image-build"]["format"]) dash_arches = "-".join(arches) log_file = compose.paths.log.log_file( dash_arches, "imagebuild-%s-%s-%s" % (variant.uid, subvariant, formats) ) + metadata_file = log_file[:-4] + ".reuse.json" + + external_repo_checksum = {} + try: + for repo in cmd["original_image_conf"]["image-build"]["repo"]: + if repo in compose.all_variants: + continue + with as_local_file( + os.path.join(repo, "repodata/repomd.xml") + ) as filename: + with open(filename, "rb") as f: + external_repo_checksum[repo] = hashlib.sha256( + f.read() + ).hexdigest() + except Exception as e: + external_repo_checksum = None + self.pool.log_info( + "Can't calculate checksum of repomd.xml of external repo - %s" % str(e) + ) + + if self._try_to_reuse( + compose, + variant, + subvariant, + metadata_file, + log_file, + cmd, + external_repo_checksum, + buildinstall_phase, + ): + return + msg = ( "Creating image (formats: %s, arches: %s, variant: %s, subvariant: %s)" % (formats, dash_arches, variant, subvariant) @@ -275,6 +314,22 @@ class CreateImageBuildThread(WorkerThread): ) break + self._link_images(compose, variant, subvariant, cmd, image_infos) + self._write_reuse_metadata( + compose, metadata_file, cmd, image_infos, external_repo_checksum + ) + + self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, output["task_id"])) + + def _link_images(self, compose, variant, subvariant, cmd, image_infos): + """Link images to compose and update image manifest. + + :param Compose compose: Current compose. + :param Variant variant: Current variant. + :param str subvariant: + :param dict cmd: Dict of params for image-build. + :param dict image_infos: Dict contains image info. + """ # The usecase here is that you can run koji image-build with multiple --format # It's ok to do it serialized since we're talking about max 2 images per single # image_build record @@ -308,4 +363,160 @@ class CreateImageBuildThread(WorkerThread): setattr(img, "deliverable", "image-build") compose.im.add(variant=variant.uid, arch=image_info["arch"], image=img) - self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, output["task_id"])) + def _try_to_reuse( + self, + compose, + variant, + subvariant, + metadata_file, + log_file, + cmd, + external_repo_checksum, + buildinstall_phase, + ): + """Try to reuse images from old compose. + + :param Compose compose: Current compose. + :param Variant variant: Current variant. + :param str subvariant: + :param str metadata_file: Path to reuse metadata file. + :param str log_file: Path to log file. + :param dict cmd: Dict of params for image-build. + :param dict external_repo_checksum: Dict contains checksum of repomd.xml + or None if can't get checksum. + :param BuildinstallPhase buildinstall_phase: buildinstall phase of + current compose. + """ + log_msg = "Cannot reuse old image_build phase results - %s" + if not compose.conf["image_build_allow_reuse"]: + self.pool.log_info( + log_msg % "reuse of old image_build results is disabled." + ) + return False + + if external_repo_checksum is None: + self.pool.log_info( + log_msg % "Can't ensure that external repo is not changed." + ) + return False + + old_metadata_file = compose.paths.old_compose_path(metadata_file) + if not old_metadata_file: + self.pool.log_info(log_msg % "Can't find old reuse metadata file") + return False + + try: + old_metadata = self._load_reuse_metadata(old_metadata_file) + except Exception as e: + self.pool.log_info( + log_msg % "Can't load old reuse metadata file: %s" % str(e) + ) + return False + + if old_metadata["cmd"]["original_image_conf"] != cmd["original_image_conf"]: + self.pool.log_info(log_msg % "image_build config changed") + return False + + # Make sure external repo does not change + if ( + old_metadata["external_repo_checksum"] is None + or old_metadata["external_repo_checksum"] != external_repo_checksum + ): + self.pool.log_info(log_msg % "External repo may be changed") + return False + + # Make sure buildinstall phase is reused + for arch in cmd["image_conf"]["image-build"]["arches"]: + if buildinstall_phase and not buildinstall_phase.reused(variant, arch): + self.pool.log_info(log_msg % "buildinstall phase changed") + return False + + # Make sure packages in variant not change + rpm_manifest_file = compose.paths.compose.metadata("rpms.json") + rpm_manifest = Rpms() + rpm_manifest.load(rpm_manifest_file) + + old_rpm_manifest_file = compose.paths.old_compose_path(rpm_manifest_file) + old_rpm_manifest = Rpms() + old_rpm_manifest.load(old_rpm_manifest_file) + + for repo in cmd["original_image_conf"]["image-build"]["repo"]: + if repo not in compose.all_variants: + # External repos are checked using other logic. + continue + for arch in cmd["image_conf"]["image-build"]["arches"]: + if ( + rpm_manifest.rpms[variant.uid][arch] + != old_rpm_manifest.rpms[variant.uid][arch] + ): + self.pool.log_info( + log_msg % "Packages in %s.%s changed." % (variant.uid, arch) + ) + return False + + self.pool.log_info( + "Reusing images from old compose for variant %s" % variant.uid + ) + try: + self._link_images( + compose, variant, subvariant, cmd, old_metadata["image_infos"] + ) + except Exception as e: + self.pool.log_info(log_msg % "Can't link images %s" % str(e)) + return False + + old_log_file = compose.paths.old_compose_path(log_file) + try: + shutil.copy2(old_log_file, log_file) + except Exception as e: + self.pool.log_info( + log_msg % "Can't copy old log_file: %s %s" % (old_log_file, str(e)) + ) + return False + + self._write_reuse_metadata( + compose, + metadata_file, + cmd, + old_metadata["image_infos"], + external_repo_checksum, + ) + + return True + + def _write_reuse_metadata( + self, compose, metadata_file, cmd, image_infos, external_repo_checksum + ): + """Write metadata file. + + :param Compose compose: Current compose. + :param str metadata_file: Path to reuse metadata file. + :param dict cmd: Dict of params for image-build. + :param dict image_infos: Dict contains image info. + :param dict external_repo_checksum: Dict contains checksum of repomd.xml + or None if can't get checksum. + """ + msg = "Writing reuse metadata file: %s" % metadata_file + self.pool.log_info(msg) + + cmd_copy = copy.deepcopy(cmd) + del cmd_copy["image_conf"]["image-build"]["variant"] + + data = { + "cmd": cmd_copy, + "image_infos": image_infos, + "external_repo_checksum": external_repo_checksum, + } + try: + with open(metadata_file, "w") as f: + json.dump(data, f, indent=4) + except Exception as e: + self.pool.log_info("%s Failed: %s" % (msg, str(e))) + + def _load_reuse_metadata(self, metadata_file): + """Load metadata file. + + :param str metadata_file: Path to reuse metadata file. + """ + with open(metadata_file, "r") as f: + return json.load(f) diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index 07f8cf83..ed3b0815 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -406,7 +406,7 @@ def run_compose( extra_isos_phase = pungi.phases.ExtraIsosPhase(compose) liveimages_phase = pungi.phases.LiveImagesPhase(compose) livemedia_phase = pungi.phases.LiveMediaPhase(compose) - image_build_phase = pungi.phases.ImageBuildPhase(compose) + image_build_phase = pungi.phases.ImageBuildPhase(compose, buildinstall_phase) osbuild_phase = pungi.phases.OSBuildPhase(compose) osbs_phase = pungi.phases.OSBSPhase(compose) image_container_phase = pungi.phases.ImageContainerPhase(compose) diff --git a/tests/test_imagebuildphase.py b/tests/test_imagebuildphase.py index 711e0f7d..c0f5ac7d 100644 --- a/tests/test_imagebuildphase.py +++ b/tests/test_imagebuildphase.py @@ -17,26 +17,23 @@ class TestImageBuildPhase(PungiTestCase): @mock.patch("pungi.phases.image_build.ThreadPool") def test_image_build(self, ThreadPool): + original_image_conf = { + "image-build": { + "format": [("docker", "tar.xz")], + "name": "Fedora-Docker-Base", + "target": "f24", + "version": "Rawhide", + "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 + "kickstart": "fedora-docker-base.ks", + "distro": "Fedora-20", + "disk_size": 3, + "failable": ["x86_64"], + } + } compose = DummyCompose( self.topdir, { - "image_build": { - "^Client|Server$": [ - { - "image-build": { - "format": [("docker", "tar.xz")], - "name": "Fedora-Docker-Base", - "target": "f24", - "version": "Rawhide", - "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 - "kickstart": "fedora-docker-base.ks", - "distro": "Fedora-20", - "disk_size": 3, - "failable": ["x86_64"], - } - } - ] - }, + "image_build": {"^Client|Server$": [original_image_conf]}, "koji_profile": "koji", }, ) @@ -50,6 +47,7 @@ class TestImageBuildPhase(PungiTestCase): # assert at least one thread was started self.assertTrue(phase.pool.add.called) client_args = { + "original_image_conf": original_image_conf, "image_conf": { "image-build": { "install_tree": self.topdir + "/compose/Client/$arch/os", @@ -75,6 +73,7 @@ class TestImageBuildPhase(PungiTestCase): "scratch": False, } server_args = { + "original_image_conf": original_image_conf, "image_conf": { "image-build": { "install_tree": self.topdir + "/compose/Server/$arch/os", @@ -102,11 +101,23 @@ class TestImageBuildPhase(PungiTestCase): six.assertCountEqual( self, phase.pool.queue_put.mock_calls, - [mock.call((compose, client_args)), mock.call((compose, server_args))], + [ + mock.call((compose, client_args, phase.buildinstall_phase)), + mock.call((compose, server_args, phase.buildinstall_phase)), + ], ) @mock.patch("pungi.phases.image_build.ThreadPool") def test_image_build_phase_global_options(self, ThreadPool): + original_image_conf = { + "image-build": { + "format": ["docker"], + "name": "Fedora-Docker-Base", + "kickstart": "fedora-docker-base.ks", + "distro": "Fedora-20", + "disk_size": 3, + } + } compose = DummyCompose( self.topdir, { @@ -114,19 +125,7 @@ class TestImageBuildPhase(PungiTestCase): "image_build_release": "!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN", "image_build_target": "f24", "image_build_version": "Rawhide", - "image_build": { - "^Server$": [ - { - "image-build": { - "format": ["docker"], - "name": "Fedora-Docker-Base", - "kickstart": "fedora-docker-base.ks", - "distro": "Fedora-20", - "disk_size": 3, - } - } - ] - }, + "image_build": {"^Server$": [original_image_conf]}, "koji_profile": "koji", }, ) @@ -140,6 +139,7 @@ class TestImageBuildPhase(PungiTestCase): # assert at least one thread was started self.assertTrue(phase.pool.add.called) server_args = { + "original_image_conf": original_image_conf, "image_conf": { "image-build": { "install_tree": self.topdir + "/compose/Server/$arch/os", @@ -165,30 +165,28 @@ class TestImageBuildPhase(PungiTestCase): "scratch": False, } self.assertEqual( - phase.pool.queue_put.mock_calls, [mock.call((compose, server_args))] + phase.pool.queue_put.mock_calls, + [mock.call((compose, server_args, phase.buildinstall_phase))], ) @mock.patch("pungi.phases.image_build.ThreadPool") def test_image_build_phase_missing_version(self, ThreadPool): + original_image_conf = { + "image-build": { + "format": "docker", + "name": "Fedora-Docker-Base", + "kickstart": "fedora-docker-base.ks", + "distro": "Fedora-20", + "disk_size": 3, + } + } compose = DummyCompose( self.topdir, { "image_build_ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 "image_build_release": "!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN", "image_build_target": "f24", - "image_build": { - "^Server$": [ - { - "image-build": { - "format": "docker", - "name": "Fedora-Docker-Base", - "kickstart": "fedora-docker-base.ks", - "distro": "Fedora-20", - "disk_size": 3, - } - } - ] - }, + "image_build": {"^Server$": [original_image_conf]}, "koji_profile": "koji", }, ) @@ -200,6 +198,7 @@ class TestImageBuildPhase(PungiTestCase): # assert at least one thread was started self.assertTrue(phase.pool.add.called) server_args = { + "original_image_conf": original_image_conf, "image_conf": { "image-build": { "install_tree": self.topdir + "/compose/Server/$arch/os", @@ -225,7 +224,8 @@ class TestImageBuildPhase(PungiTestCase): "scratch": False, } self.assertEqual( - phase.pool.queue_put.mock_calls, [mock.call((compose, server_args))] + phase.pool.queue_put.mock_calls, + [mock.call((compose, server_args, phase.buildinstall_phase))], ) @mock.patch("pungi.phases.image_build.ThreadPool") @@ -266,27 +266,25 @@ class TestImageBuildPhase(PungiTestCase): @mock.patch("pungi.phases.image_build.ThreadPool") def test_image_build_set_install_tree(self, ThreadPool): + original_image_conf = { + "image-build": { + "format": ["docker"], + "name": "Fedora-Docker-Base", + "target": "f24", + "version": "Rawhide", + "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 + "kickstart": "fedora-docker-base.ks", + "distro": "Fedora-20", + "disk_size": 3, + "arches": ["x86_64"], + "install_tree_from": "Server-optional", + } + } + compose = DummyCompose( self.topdir, { - "image_build": { - "^Server$": [ - { - "image-build": { - "format": ["docker"], - "name": "Fedora-Docker-Base", - "target": "f24", - "version": "Rawhide", - "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 - "kickstart": "fedora-docker-base.ks", - "distro": "Fedora-20", - "disk_size": 3, - "arches": ["x86_64"], - "install_tree_from": "Server-optional", - } - } - ] - }, + "image_build": {"^Server$": [original_image_conf]}, "koji_profile": "koji", }, ) @@ -307,6 +305,7 @@ class TestImageBuildPhase(PungiTestCase): self.assertDictEqual( args[0][1], { + "original_image_conf": original_image_conf, "image_conf": { "image-build": { "install_tree": self.topdir @@ -335,27 +334,24 @@ class TestImageBuildPhase(PungiTestCase): @mock.patch("pungi.phases.image_build.ThreadPool") def test_image_build_set_install_tree_from_path(self, ThreadPool): + original_image_conf = { + "image-build": { + "format": ["docker"], + "name": "Fedora-Docker-Base", + "target": "f24", + "version": "Rawhide", + "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 + "kickstart": "fedora-docker-base.ks", + "distro": "Fedora-20", + "disk_size": 3, + "arches": ["x86_64"], + "install_tree_from": "/my/tree", + } + } compose = DummyCompose( self.topdir, { - "image_build": { - "^Server$": [ - { - "image-build": { - "format": ["docker"], - "name": "Fedora-Docker-Base", - "target": "f24", - "version": "Rawhide", - "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 - "kickstart": "fedora-docker-base.ks", - "distro": "Fedora-20", - "disk_size": 3, - "arches": ["x86_64"], - "install_tree_from": "/my/tree", - } - } - ] - }, + "image_build": {"^Server$": [original_image_conf]}, "koji_profile": "koji", "translate_paths": [("/my", "http://example.com")], }, @@ -376,6 +372,7 @@ class TestImageBuildPhase(PungiTestCase): self.assertDictEqual( args[0][1], { + "original_image_conf": original_image_conf, "image_conf": { "image-build": { "install_tree": "http://example.com/tree", @@ -403,27 +400,24 @@ class TestImageBuildPhase(PungiTestCase): @mock.patch("pungi.phases.image_build.ThreadPool") def test_image_build_set_extra_repos(self, ThreadPool): + original_image_conf = { + "image-build": { + "format": ["docker"], + "name": "Fedora-Docker-Base", + "target": "f24", + "version": "Rawhide", + "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 + "kickstart": "fedora-docker-base.ks", + "distro": "Fedora-20", + "disk_size": 3, + "arches": ["x86_64"], + "repo_from": ["Everything", "Server-optional"], + } + } compose = DummyCompose( self.topdir, { - "image_build": { - "^Server$": [ - { - "image-build": { - "format": ["docker"], - "name": "Fedora-Docker-Base", - "target": "f24", - "version": "Rawhide", - "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 - "kickstart": "fedora-docker-base.ks", - "distro": "Fedora-20", - "disk_size": 3, - "arches": ["x86_64"], - "repo_from": ["Everything", "Server-optional"], - } - } - ] - }, + "image_build": {"^Server$": [original_image_conf]}, "koji_profile": "koji", }, ) @@ -444,6 +438,7 @@ class TestImageBuildPhase(PungiTestCase): self.assertDictEqual( args[0][1], { + "original_image_conf": original_image_conf, "image_conf": { "image-build": { "install_tree": self.topdir + "/compose/Server/$arch/os", @@ -477,27 +472,24 @@ class TestImageBuildPhase(PungiTestCase): @mock.patch("pungi.phases.image_build.ThreadPool") def test_image_build_set_external_install_tree(self, ThreadPool): + original_image_conf = { + "image-build": { + "format": ["docker"], + "name": "Fedora-Docker-Base", + "target": "f24", + "version": "Rawhide", + "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 + "kickstart": "fedora-docker-base.ks", + "distro": "Fedora-20", + "disk_size": 3, + "arches": ["x86_64"], + "install_tree_from": "http://example.com/install-tree/", + } + } compose = DummyCompose( self.topdir, { - "image_build": { - "^Server$": [ - { - "image-build": { - "format": ["docker"], - "name": "Fedora-Docker-Base", - "target": "f24", - "version": "Rawhide", - "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 - "kickstart": "fedora-docker-base.ks", - "distro": "Fedora-20", - "disk_size": 3, - "arches": ["x86_64"], - "install_tree_from": "http://example.com/install-tree/", - } - } - ] - }, + "image_build": {"^Server$": [original_image_conf]}, "koji_profile": "koji", }, ) @@ -517,6 +509,7 @@ class TestImageBuildPhase(PungiTestCase): self.assertDictEqual( args[0][1], { + "original_image_conf": original_image_conf, "image_conf": { "image-build": { "install_tree": "http://example.com/install-tree/", @@ -670,26 +663,23 @@ class TestImageBuildPhase(PungiTestCase): @mock.patch("pungi.phases.image_build.ThreadPool") def test_image_build_optional(self, ThreadPool): + original_image_conf = { + "image-build": { + "format": ["docker"], + "name": "Fedora-Docker-Base", + "target": "f24", + "version": "Rawhide", + "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 + "kickstart": "fedora-docker-base.ks", + "distro": "Fedora-20", + "disk_size": 3, + "failable": ["x86_64"], + } + } compose = DummyCompose( self.topdir, { - "image_build": { - "^Server-optional$": [ - { - "image-build": { - "format": ["docker"], - "name": "Fedora-Docker-Base", - "target": "f24", - "version": "Rawhide", - "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 - "kickstart": "fedora-docker-base.ks", - "distro": "Fedora-20", - "disk_size": 3, - "failable": ["x86_64"], - } - } - ] - }, + "image_build": {"^Server-optional$": [original_image_conf]}, "koji_profile": "koji", }, ) @@ -704,6 +694,7 @@ class TestImageBuildPhase(PungiTestCase): # assert at least one thread was started self.assertTrue(phase.pool.add.called) server_args = { + "original_image_conf": original_image_conf, "image_conf": { "image-build": { "install_tree": self.topdir + "/compose/Server/$arch/os", @@ -729,31 +720,29 @@ class TestImageBuildPhase(PungiTestCase): "scratch": False, } self.assertEqual( - phase.pool.queue_put.mock_calls, [mock.call((compose, server_args))] + phase.pool.queue_put.mock_calls, + [mock.call((compose, server_args, phase.buildinstall_phase))], ) @mock.patch("pungi.phases.image_build.ThreadPool") def test_failable_star(self, ThreadPool): + original_image_conf = { + "image-build": { + "format": ["docker"], + "name": "Fedora-Docker-Base", + "target": "f24", + "version": "Rawhide", + "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 + "kickstart": "fedora-docker-base.ks", + "distro": "Fedora-20", + "disk_size": 3, + "failable": ["*"], + } + } compose = DummyCompose( self.topdir, { - "image_build": { - "^Server$": [ - { - "image-build": { - "format": ["docker"], - "name": "Fedora-Docker-Base", - "target": "f24", - "version": "Rawhide", - "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 - "kickstart": "fedora-docker-base.ks", - "distro": "Fedora-20", - "disk_size": 3, - "failable": ["*"], - } - } - ] - }, + "image_build": {"^Server$": [original_image_conf]}, "koji_profile": "koji", }, ) @@ -768,6 +757,7 @@ class TestImageBuildPhase(PungiTestCase): # assert at least one thread was started self.assertTrue(phase.pool.add.called) server_args = { + "original_image_conf": original_image_conf, "image_conf": { "image-build": { "install_tree": self.topdir + "/compose/Server/$arch/os", @@ -793,7 +783,8 @@ class TestImageBuildPhase(PungiTestCase): "scratch": False, } self.assertEqual( - phase.pool.queue_put.mock_calls, [mock.call((compose, server_args))] + phase.pool.queue_put.mock_calls, + [mock.call((compose, server_args, phase.buildinstall_phase))], ) @@ -854,7 +845,7 @@ class TestCreateImageBuildThread(PungiTestCase): t = CreateImageBuildThread(pool) with mock.patch("time.sleep"): - t.process((compose, cmd), 1) + t.process((compose, cmd, None), 1) self.assertEqual( koji_wrapper.get_image_build_cmd.call_args_list, @@ -987,7 +978,7 @@ class TestCreateImageBuildThread(PungiTestCase): t = CreateImageBuildThread(pool) with mock.patch("time.sleep"): - t.process((compose, cmd), 1) + t.process((compose, cmd, None), 1) pool._logger.error.assert_has_calls( [ @@ -1041,7 +1032,7 @@ class TestCreateImageBuildThread(PungiTestCase): t = CreateImageBuildThread(pool) with mock.patch("time.sleep"): - t.process((compose, cmd), 1) + t.process((compose, cmd, None), 1) pool._logger.error.assert_has_calls( [ @@ -1092,4 +1083,4 @@ class TestCreateImageBuildThread(PungiTestCase): t = CreateImageBuildThread(pool) with self.assertRaises(RuntimeError): with mock.patch("time.sleep"): - t.process((compose, cmd), 1) + t.process((compose, cmd, None), 1) From 8133676270eb0c37c3e14a873b9f490d6541e5ef Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Thu, 9 Sep 2021 13:58:32 +0800 Subject: [PATCH 071/137] osbs: Reuse images from old compose JIRA: RHELCMP-5972 Signed-off-by: Haibo Lin --- pungi/checks.py | 1 + pungi/phases/osbs.py | 247 ++++++++++++++++++++++++++++++++++-- pungi/scripts/pungi_koji.py | 2 +- tests/test_osbs_phase.py | 43 +++++-- 4 files changed, 264 insertions(+), 29 deletions(-) diff --git a/pungi/checks.py b/pungi/checks.py index c79f2bbc..e5c2f225 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -1202,6 +1202,7 @@ def make_schema(): "anyOf": [{"type": "string"}, {"type": "number"}], "default": 10 * 1024 * 1024, }, + "osbs_allow_reuse": {"type": "boolean", "default": False}, "osbs": { "type": "object", "patternProperties": { diff --git a/pungi/phases/osbs.py b/pungi/phases/osbs.py index 8a395238..9db4371e 100644 --- a/pungi/phases/osbs.py +++ b/pungi/phases/osbs.py @@ -1,23 +1,29 @@ # -*- coding: utf-8 -*- +import copy import fnmatch import json import os from kobo.threads import ThreadPool, WorkerThread from kobo import shortcuts +from productmd.rpms import Rpms +from six.moves import configparser from .base import ConfigGuardedPhase, PhaseLoggerMixin from .. import util from ..wrappers import kojiwrapper +from ..wrappers.scm import get_file_from_scm class OSBSPhase(PhaseLoggerMixin, ConfigGuardedPhase): name = "osbs" - def __init__(self, compose): + def __init__(self, compose, pkgset_phase, buildinstall_phase): super(OSBSPhase, self).__init__(compose) self.pool = ThreadPool(logger=self.logger) self.pool.registries = {} + self.pool.pkgset_phase = pkgset_phase + self.pool.buildinstall_phase = buildinstall_phase def run(self): for variant in self.compose.get_variants(): @@ -77,8 +83,8 @@ class OSBSThread(WorkerThread): def worker(self, compose, variant, config): msg = "OSBS task for variant %s" % variant.uid self.pool.log_info("[BEGIN] %s" % msg) - koji = kojiwrapper.KojiWrapper(compose) - koji.login() + + original_config = copy.deepcopy(config) # Start task source = config.pop("url") @@ -94,33 +100,99 @@ class OSBSThread(WorkerThread): config["yum_repourls"] = repos - task_id = koji.koji_proxy.buildContainer( - source, target, config, priority=priority - ) - - koji.save_task_id(task_id) - - # Wait for it to finish and capture the output into log file (even - # though there is not much there). log_dir = os.path.join(compose.paths.log.topdir(), "osbs") util.makedirs(log_dir) log_file = os.path.join( log_dir, "%s-%s-watch-task.log" % (variant.uid, self.num) ) + reuse_file = log_file[:-4] + ".reuse.json" + + try: + image_conf = self._get_image_conf(compose, original_config) + except Exception as e: + image_conf = None + self.pool.log_info( + "Can't get image-build.conf for variant: %s source: %s - %s" + % (variant.uid, source, str(e)) + ) + + koji = kojiwrapper.KojiWrapper(compose) + koji.login() + + task_id = self._try_to_reuse( + compose, variant, original_config, image_conf, reuse_file + ) + + if not task_id: + task_id = koji.koji_proxy.buildContainer( + source, target, config, priority=priority + ) + + koji.save_task_id(task_id) + + # Wait for it to finish and capture the output into log file (even + # though there is not much there). if koji.watch_task(task_id, log_file) != 0: raise RuntimeError( "OSBS: task %s failed: see %s for details" % (task_id, log_file) ) scratch = config.get("scratch", False) - nvr = add_metadata(variant, task_id, compose, scratch) + nvr, archive_ids = add_metadata(variant, task_id, compose, scratch) if nvr: registry = get_registry(compose, nvr, registry) if registry: self.pool.registries[nvr] = registry + self._write_reuse_metadata( + compose, + variant, + original_config, + image_conf, + task_id, + archive_ids, + reuse_file, + ) + self.pool.log_info("[DONE ] %s" % msg) + def _get_image_conf(self, compose, config): + """Get image-build.conf from git repo. + + :param Compose compose: Current compose. + :param dict config: One osbs config item of compose.conf["osbs"][$variant] + """ + tmp_dir = compose.mkdtemp(prefix="osbs_") + + url = config["url"].split("#") + if len(url) == 1: + url.append(config["git_branch"]) + + filename = "image-build.conf" + get_file_from_scm( + { + "scm": "git", + "repo": url[0], + "branch": url[1], + "file": [filename], + }, + tmp_dir, + ) + + c = configparser.ConfigParser() + c.read(os.path.join(tmp_dir, filename)) + return c + + def _get_ksurl(self, image_conf): + """Get ksurl from image-build.conf""" + ksurl = image_conf.get("image-build", "ksurl") + + if ksurl: + resolver = util.GitUrlResolver(offline=False) + return resolver(ksurl) + else: + return None + def _get_repo(self, compose, repo, gpgkey=None): """ Return repo file URL of repo, if repo contains "://", it's already a @@ -177,6 +249,151 @@ class OSBSThread(WorkerThread): return util.translate_path(compose, repo_file) + def _try_to_reuse(self, compose, variant, config, image_conf, reuse_file): + """Try to reuse results of old compose. + + :param Compose compose: Current compose. + :param Variant variant: Current variant. + :param dict config: One osbs config item of compose.conf["osbs"][$variant] + :param ConfigParser image_conf: ConfigParser obj of image-build.conf. + :param str reuse_file: Path to reuse metadata file + """ + log_msg = "Cannot reuse old osbs phase results - %s" + + if not compose.conf["osbs_allow_reuse"]: + self.pool.log_info(log_msg % "reuse of old osbs results is disabled.") + return False + + old_reuse_file = compose.paths.old_compose_path(reuse_file) + if not old_reuse_file: + self.pool.log_info(log_msg % "Can't find old reuse metadata file") + return False + + try: + with open(old_reuse_file) as f: + old_reuse_metadata = json.load(f) + except Exception as e: + self.pool.log_info( + log_msg % "Can't load old reuse metadata file: %s" % str(e) + ) + return False + + if old_reuse_metadata["config"] != config: + self.pool.log_info(log_msg % "osbs config changed") + return False + + if not image_conf: + self.pool.log_info(log_msg % "Can't get image-build.conf") + return False + + # Make sure ksurl not change + try: + ksurl = self._get_ksurl(image_conf) + except Exception as e: + self.pool.log_info( + log_msg % "Can't get ksurl from image-build.conf - %s" % str(e) + ) + return False + + if not old_reuse_metadata["ksurl"]: + self.pool.log_info( + log_msg % "Can't get ksurl from old compose reuse metadata." + ) + return False + + if ksurl != old_reuse_metadata["ksurl"]: + self.pool.log_info(log_msg % "ksurl changed") + return False + + # Make sure buildinstall phase is reused + try: + arches = image_conf.get("image-build", "arches").split(",") + except Exception as e: + self.pool.log_info( + log_msg % "Can't get arches from image-build.conf - %s" % str(e) + ) + for arch in arches: + if not self.pool.buildinstall_phase.reused(variant, arch): + self.pool.log_info( + log_msg % "buildinstall phase changed %s.%s" % (variant, arch) + ) + return False + + # Make sure rpms installed in image exists in current compose + rpm_manifest_file = compose.paths.compose.metadata("rpms.json") + rpm_manifest = Rpms() + rpm_manifest.load(rpm_manifest_file) + rpms = set() + for variant in rpm_manifest.rpms: + for arch in rpm_manifest.rpms[variant]: + for src in rpm_manifest.rpms[variant][arch]: + for nevra in rpm_manifest.rpms[variant][arch][src]: + rpms.add(nevra) + + for nevra in old_reuse_metadata["rpmlist"]: + if nevra not in rpms: + self.pool.log_info( + log_msg % "%s does not exist in current compose" % nevra + ) + return False + + self.pool.log_info( + "Reusing old OSBS task %d result" % old_reuse_file["task_id"] + ) + return old_reuse_file["task_id"] + + def _write_reuse_metadata( + self, compose, variant, config, image_conf, task_id, archive_ids, reuse_file + ): + """Write metadata to file for reusing. + + :param Compose compose: Current compose. + :param Variant variant: Current variant. + :param dict config: One osbs config item of compose.conf["osbs"][$variant] + :param ConfigParser image_conf: ConfigParser obj of image-build.conf. + :param int task_id: Koji task id of osbs task. + :param list archive_ids: List of koji archive id + :param str reuse_file: Path to reuse metadata file. + """ + msg = "Writing reuse metadata file %s" % reuse_file + compose.log_info(msg) + + rpmlist = set() + koji = kojiwrapper.KojiWrapper(compose) + for archive_id in archive_ids: + rpms = koji.koji_proxy.listRPMs(imageID=archive_id) + for item in rpms: + if item["epoch"]: + rpmlist.add( + "%s:%s-%s-%s.%s" + % ( + item["name"], + item["epoch"], + item["version"], + item["release"], + item["arch"], + ) + ) + else: + rpmlist.add("%s.%s" % (item["nvr"], item["arch"])) + + try: + ksurl = self._get_ksurl(image_conf) + except Exception: + ksurl = None + + data = { + "config": config, + "ksurl": ksurl, + "rpmlist": sorted(rpmlist), + "task_id": task_id, + } + try: + with open(reuse_file, "w") as f: + json.dump(data, f, indent=4) + except Exception as e: + compose.log_info(msg + " failed - %s" % str(e)) + def add_metadata(variant, task_id, compose, is_scratch): """Given a task ID, find details about the container and add it to global @@ -200,7 +417,7 @@ def add_metadata(variant, task_id, compose, is_scratch): compose.containers_metadata.setdefault(variant.uid, {}).setdefault( "scratch", [] ).append(metadata) - return None + return None, [] else: build_id = int(result["koji_builds"][0]) @@ -218,6 +435,7 @@ def add_metadata(variant, task_id, compose, is_scratch): "creation_time": buildinfo["creation_time"], } ) + archive_ids = [] for archive in archives: data = { "filename": archive["filename"], @@ -234,4 +452,5 @@ def add_metadata(variant, task_id, compose, is_scratch): compose.containers_metadata.setdefault(variant.uid, {}).setdefault( arch, [] ).append(data) - return nvr + archive_ids.append(archive["id"]) + return nvr, archive_ids diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index ed3b0815..9570e3c5 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -408,7 +408,7 @@ def run_compose( livemedia_phase = pungi.phases.LiveMediaPhase(compose) image_build_phase = pungi.phases.ImageBuildPhase(compose, buildinstall_phase) osbuild_phase = pungi.phases.OSBuildPhase(compose) - osbs_phase = pungi.phases.OSBSPhase(compose) + osbs_phase = pungi.phases.OSBSPhase(compose, pkgset_phase, buildinstall_phase) image_container_phase = pungi.phases.ImageContainerPhase(compose) image_checksum_phase = pungi.phases.ImageChecksumPhase(compose) repoclosure_phase = pungi.phases.RepoclosurePhase(compose) diff --git a/tests/test_osbs_phase.py b/tests/test_osbs_phase.py index fde5753d..456fe1f6 100644 --- a/tests/test_osbs_phase.py +++ b/tests/test_osbs_phase.py @@ -19,7 +19,7 @@ class OSBSPhaseTest(helpers.PungiTestCase): pool = ThreadPool.return_value - phase = osbs.OSBSPhase(compose) + phase = osbs.OSBSPhase(compose, None, None) phase.run() self.assertEqual(len(pool.add.call_args_list), 1) @@ -33,7 +33,7 @@ class OSBSPhaseTest(helpers.PungiTestCase): compose = helpers.DummyCompose(self.topdir, {}) compose.just_phases = None compose.skip_phases = [] - phase = osbs.OSBSPhase(compose) + phase = osbs.OSBSPhase(compose, None, None) self.assertTrue(phase.skip()) @mock.patch("pungi.phases.osbs.ThreadPool") @@ -42,7 +42,7 @@ class OSBSPhaseTest(helpers.PungiTestCase): compose.just_phases = None compose.skip_phases = [] compose.notifier = mock.Mock() - phase = osbs.OSBSPhase(compose) + phase = osbs.OSBSPhase(compose, None, None) phase.start() phase.stop() phase.pool.registries = {"foo": "bar"} @@ -139,6 +139,8 @@ METADATA = { } } +RPMS = [] + SCRATCH_TASK_RESULT = { "koji_builds": [], "repositories": [ @@ -182,6 +184,7 @@ class OSBSThreadTest(helpers.PungiTestCase): self.wrapper.koji_proxy.getTaskResult.return_value = TASK_RESULT self.wrapper.koji_proxy.getBuild.return_value = BUILD_INFO self.wrapper.koji_proxy.listArchives.return_value = ARCHIVES + self.wrapper.koji_proxy.listRPMs.return_value = RPMS self.wrapper.koji_proxy.getLatestBuilds.return_value = [ mock.Mock(), mock.Mock(), @@ -233,6 +236,7 @@ class OSBSThreadTest(helpers.PungiTestCase): [ mock.call.koji_proxy.getBuild(54321), mock.call.koji_proxy.listArchives(54321), + mock.call.koji_proxy.listRPMs(imageID=1436049), ] ) self.assertEqual(self.wrapper.mock_calls, expect_calls) @@ -269,8 +273,9 @@ class OSBSThreadTest(helpers.PungiTestCase): self.assertIn(" Possible reason: %r is a required property" % key, errors) self.assertEqual([], warnings) + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_minimal_run(self, KojiWrapper): + def test_minimal_run(self, KojiWrapper, get_file_from_scm): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "f24-docker-candidate", @@ -285,8 +290,9 @@ class OSBSThreadTest(helpers.PungiTestCase): self._assertCorrectMetadata() self._assertRepoFile() + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_run_failable(self, KojiWrapper): + def test_run_failable(self, KojiWrapper, get_file_from_scm): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "f24-docker-candidate", @@ -302,8 +308,9 @@ class OSBSThreadTest(helpers.PungiTestCase): self._assertCorrectMetadata() self._assertRepoFile() + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_run_with_more_args(self, KojiWrapper): + def test_run_with_more_args(self, KojiWrapper, get_file_from_scm): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "f24-docker-candidate", @@ -322,8 +329,9 @@ class OSBSThreadTest(helpers.PungiTestCase): self._assertCorrectMetadata() self._assertRepoFile() + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_run_with_extra_repos(self, KojiWrapper): + def test_run_with_extra_repos(self, KojiWrapper, get_file_from_scm): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "f24-docker-candidate", @@ -395,8 +403,9 @@ class OSBSThreadTest(helpers.PungiTestCase): self._assertCorrectCalls(options) self._assertCorrectMetadata() + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_run_with_deprecated_registry(self, KojiWrapper): + def test_run_with_deprecated_registry(self, KojiWrapper, get_file_from_scm): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "f24-docker-candidate", @@ -426,8 +435,9 @@ class OSBSThreadTest(helpers.PungiTestCase): self._assertRepoFile(["Server", "Everything"]) self.assertEqual(self.t.pool.registries, {"my-name-1.0-1": {"foo": "bar"}}) + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_run_with_registry(self, KojiWrapper): + def test_run_with_registry(self, KojiWrapper, get_file_from_scm): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "f24-docker-candidate", @@ -457,8 +467,9 @@ class OSBSThreadTest(helpers.PungiTestCase): self._assertRepoFile(["Server", "Everything"]) self.assertEqual(self.t.pool.registries, {"my-name-1.0-1": [{"foo": "bar"}]}) + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_run_with_extra_repos_in_list(self, KojiWrapper): + def test_run_with_extra_repos_in_list(self, KojiWrapper, get_file_from_scm): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "f24-docker-candidate", @@ -487,8 +498,9 @@ class OSBSThreadTest(helpers.PungiTestCase): self._assertCorrectMetadata() self._assertRepoFile(["Server", "Everything", "Client"]) + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_run_with_gpgkey_enabled(self, KojiWrapper): + def test_run_with_gpgkey_enabled(self, KojiWrapper, get_file_from_scm): gpgkey = "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release" cfg = { "url": "git://example.com/repo?#BEEFCAFE", @@ -547,8 +559,9 @@ class OSBSThreadTest(helpers.PungiTestCase): } self._assertConfigMissing(cfg, "git_branch") + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_failing_task(self, KojiWrapper): + def test_failing_task(self, KojiWrapper, get_file_from_scm): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "fedora-24-docker-candidate", @@ -563,8 +576,9 @@ class OSBSThreadTest(helpers.PungiTestCase): self.assertRegex(str(ctx.exception), r"task 12345 failed: see .+ for details") + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_failing_task_with_failable(self, KojiWrapper): + def test_failing_task_with_failable(self, KojiWrapper, get_file_from_scm): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "fedora-24-docker-candidate", @@ -577,8 +591,9 @@ class OSBSThreadTest(helpers.PungiTestCase): self.t.process((self.compose, self.compose.variants["Server"], cfg), 1) + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_scratch_metadata(self, KojiWrapper): + def test_scratch_metadata(self, KojiWrapper, get_file_from_scm): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "f24-docker-candidate", From 204d88a3514922025a7e91f474d79837ac03979c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 21 Oct 2021 13:34:37 +0200 Subject: [PATCH 072/137] Add missing mock to osbs tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We don't want the test to try to a dummy URL. Signed-off-by: Lubomír Sedlář --- tests/test_osbs_phase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_osbs_phase.py b/tests/test_osbs_phase.py index 456fe1f6..5ad64d83 100644 --- a/tests/test_osbs_phase.py +++ b/tests/test_osbs_phase.py @@ -372,8 +372,9 @@ class OSBSThreadTest(helpers.PungiTestCase): ) as f: self.assertIn("baseurl=http://example.com/repo\n", f) + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_run_with_extra_repos_with_cts(self, KojiWrapper): + def test_run_with_extra_repos_with_cts(self, KojiWrapper, get_file_from_scm): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "f24-docker-candidate", From ab1904377327cbd2b8d5f881acc56d79695be0f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dan=20=C4=8Cerm=C3=A1k?= Date: Fri, 22 Oct 2021 22:47:59 +0200 Subject: [PATCH 073/137] Correct irc network name & add matrix room MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dan Čermák --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5828077b..ae7d9a48 100644 --- a/README.md +++ b/README.md @@ -34,4 +34,6 @@ also moves the artifacts to correct locations. - Documentation: https://docs.pagure.org/pungi/ - Upstream GIT: https://pagure.io/pungi/ - Issue tracker: https://pagure.io/pungi/issues -- Questions can be asked on *#fedora-releng* IRC channel on FreeNode +- Questions can be asked in the *#fedora-releng* IRC channel on irc.libera.chat + or in the matrix room + [`#releng:fedoraproject.org`](https://matrix.to/#/#releng:fedoraproject.org) From b03490bf180d8e514db6b15f4456ab765537ef65 Mon Sep 17 00:00:00 2001 From: Ozan Unsal Date: Mon, 25 Oct 2021 15:18:26 +0200 Subject: [PATCH 074/137] 4.3.1 release JIRA: RHELCMP-7116 Signed-off-by: Ozan Unsal --- doc/conf.py | 2 +- pungi.spec | 14 +++++++++++++- setup.py | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 7c95c1af..3b34418a 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -53,7 +53,7 @@ copyright = u'2016, Red Hat, Inc.' # The short X.Y version. version = '4.2' # The full version, including alpha/beta/rc tags. -release = '4.3.0' +release = '4.3.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pungi.spec b/pungi.spec index 180c088a..bcd6d86a 100644 --- a/pungi.spec +++ b/pungi.spec @@ -1,5 +1,5 @@ Name: pungi -Version: 4.3.0 +Version: 4.3.1 Release: 1%{?dist} Summary: Distribution compose tool @@ -111,6 +111,18 @@ pytest cd tests && ./test_compose.sh %changelog +* Mon Oct 25 2021 Ozan Unsal - 4.3.1-1 +- Correct irc network name & add matrix room (dan.cermak) +- Add missing mock to osbs tests (lsedlar) +- osbs: Reuse images from old compose (hlin) +- image_build: Allow reusing old image_build results (hlin) +- Allow ISO-Level configuration within the config file (ounsal) +- Work around ODCS creating COMPOSE_ID later (lsedlar) +- When `cts_url` is configured, use CTS `/repo` API for buildContainer + yum_repourls. (jkaluza) +- Add COMPOSE_ID into the pungi log file (ounsal) +- buildinstall: Add easy way to check if previous result was reused (lsedlar) + * Fri Sep 10 2021 Lubomír Sedlář - 4.3.0-1 - Only build CTS url when configured (lsedlar) - Require requests_kerberos only when needed (lsedlar) diff --git a/setup.py b/setup.py index a6578563..37212b92 100755 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ packages = sorted(packages) setup( name="pungi", - version="4.3.0", + version="4.3.1", description="Distribution compose tool", url="https://pagure.io/pungi", author="Dennis Gilmore", From eb61c97cdb08ac6d8b0f36ed23dd9550a059c89c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 21 Oct 2021 13:22:33 +0200 Subject: [PATCH 075/137] Remove default runroot channel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the value is not specified in the configuration file, let Koji pick the default channel. JIRA: RHELBLD-8088 Signed-off-by: Lubomír Sedlář --- pungi/wrappers/kojiwrapper.py | 6 ------ tests/test_koji_wrapper.py | 6 ++---- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py index 1e67097f..140b2bfc 100644 --- a/pungi/wrappers/kojiwrapper.py +++ b/pungi/wrappers/kojiwrapper.py @@ -115,8 +115,6 @@ class KojiWrapper(object): if channel: cmd.append("--channel-override=%s" % channel) - else: - cmd.append("--channel-override=runroot-local") if weight: cmd.append("--weight=%s" % int(weight)) @@ -172,8 +170,6 @@ class KojiWrapper(object): if channel: cmd.append("--channel-override=%s" % channel) - else: - cmd.append("--channel-override=runroot-local") if weight: cmd.append("--weight=%s" % int(weight)) @@ -222,8 +218,6 @@ class KojiWrapper(object): if channel: cmd.append("--channel-override=%s" % channel) - else: - cmd.append("--channel-override=runroot-local") if weight: cmd.append("--weight=%s" % int(weight)) diff --git a/tests/test_koji_wrapper.py b/tests/test_koji_wrapper.py index 4c533fce..2a7c2df9 100644 --- a/tests/test_koji_wrapper.py +++ b/tests/test_koji_wrapper.py @@ -533,7 +533,7 @@ class LiveImageKojiWrapperTest(KojiWrapperBaseTestCase): class RunrootKojiWrapperTest(KojiWrapperBaseTestCase): def test_get_cmd_minimal(self): cmd = self.koji.get_runroot_cmd("tgt", "s390x", "date", use_shell=False) - self.assertEqual(len(cmd), 9) + self.assertEqual(len(cmd), 8) self.assertEqual( cmd[:5], ["koji", "--profile=custom-koji", "runroot", "--nowait", "--task-id"], @@ -543,7 +543,7 @@ class RunrootKojiWrapperTest(KojiWrapperBaseTestCase): self.assertEqual( cmd[-1], "rm -f /var/lib/rpm/__db*; rm -rf /var/cache/yum/*; set -x; date" ) - six.assertCountEqual(self, cmd[5:-3], ["--channel-override=runroot-local"]) + six.assertCountEqual(self, cmd[5:-3], []) def test_get_cmd_full(self): cmd = self.koji.get_runroot_cmd( @@ -1023,7 +1023,6 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): "pungi-buildinstall", "--nowait", "--task-id", - "--channel-override=runroot-local", "--weight=123", "--package=lorax", "--mount=/tmp", @@ -1052,7 +1051,6 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): "pungi-ostree", "--nowait", "--task-id", - "--channel-override=runroot-local", "--weight=123", "--package=lorax", "--mount=/tmp", From ac66c3d7f31dc04a589ff4a6e726929da0a4df5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 6 Aug 2021 14:42:42 +0200 Subject: [PATCH 076/137] createiso: Allow reusing old images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch allows Pungi to reuse ISO image created in previous compose if a list of assumptions proves to hold: * If image is bootable, buildinstall phase must have been reused too. * Compose configuration must have not changed (except for a few whitelisted options). * Volume ID of the ISO much not have changed. * No RPM on the ISO must have changed. The ISO also contains other files. Changes in extra files and product ID certificates should be visible in configuration (the SHA will differ). Similarly any repodata configuration would be reflected in configuration. JIRA: RHELCMP-5969 Signed-off-by: Lubomír Sedlář --- pungi/checks.py | 1 + pungi/phases/createiso.py | 213 ++++++++++++++++++++++++++++++++ pungi/util.py | 6 + tests/test_createiso_phase.py | 225 ++++++++++++++++++++++++++++++++++ 4 files changed, 445 insertions(+) diff --git a/pungi/checks.py b/pungi/checks.py index e5c2f225..d262f313 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -653,6 +653,7 @@ def make_schema(): "gather_profiler": {"type": "boolean", "default": False}, "gather_allow_reuse": {"type": "boolean", "default": False}, "pkgset_allow_reuse": {"type": "boolean", "default": True}, + "createiso_allow_reuse": {"type": "boolean", "default": True}, "pkgset_source": {"type": "string", "enum": ["koji", "repos"]}, "createrepo_c": {"type": "boolean", "default": True}, "createrepo_checksum": { diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py index 529101f5..706ba577 100644 --- a/pungi/phases/createiso.py +++ b/pungi/phases/createiso.py @@ -18,6 +18,7 @@ import os import random import shutil import stat +import json import productmd.treeinfo from productmd.images import Image @@ -36,6 +37,7 @@ from pungi.util import ( failable, get_file_size, get_mtime, + read_json_file, ) from pungi.media_split import MediaSplitter, convert_media_size from pungi.compose_metadata.discinfo import read_discinfo, write_discinfo @@ -73,6 +75,168 @@ class CreateisoPhase(PhaseLoggerMixin, PhaseBase): return False return bool(self.compose.conf.get("buildinstall_method", "")) + def _metadata_path(self, variant, arch, disc_num, disc_count): + return self.compose.paths.log.log_file( + arch, + "createiso-%s-%d-%d" % (variant.uid, disc_num, disc_count), + ext="json", + ) + + def save_reuse_metadata(self, cmd, variant, arch, opts): + """Save metadata for future composes to verify if the compose can be reused.""" + metadata = { + "cmd": cmd, + "opts": opts._asdict(), + } + + metadata_path = self._metadata_path( + variant, arch, cmd["disc_num"], cmd["disc_count"] + ) + with open(metadata_path, "w") as f: + json.dump(metadata, f, indent=2) + return metadata + + def _load_old_metadata(self, cmd, variant, arch): + metadata_path = self._metadata_path( + variant, arch, cmd["disc_num"], cmd["disc_count"] + ) + old_path = self.compose.paths.old_compose_path(metadata_path) + self.logger.info( + "Loading old metadata for %s.%s from: %s", variant, arch, old_path + ) + try: + return read_json_file(old_path) + except Exception: + return None + + def perform_reuse(self, cmd, variant, arch, opts, iso_path): + """ + Copy all related files from old compose to the new one. As a last step + add the new image to metadata. + """ + linker = OldFileLinker(self.logger) + old_file_name = os.path.basename(iso_path) + current_file_name = os.path.basename(cmd["iso_path"]) + try: + # Hardlink ISO and manifest + for suffix in ("", ".manifest"): + linker.link(iso_path + suffix, cmd["iso_path"] + suffix) + # Copy log files + # The log file name includes filename of the image, so we need to + # find old file with the old name, and rename it to the new name. + log_file = self.compose.paths.log.log_file( + arch, "createiso-%s" % current_file_name + ) + old_log_file = self.compose.paths.old_compose_path( + self.compose.paths.log.log_file(arch, "createiso-%s" % old_file_name) + ) + linker.link(old_log_file, log_file) + # Copy jigdo files + if opts.jigdo_dir: + old_jigdo_dir = self.compose.paths.old_compose_path(opts.jigdo_dir) + for suffix in (".template", ".jigdo"): + linker.link( + os.path.join(old_jigdo_dir, old_file_name) + suffix, + os.path.join(opts.jigdo_dir, current_file_name) + suffix, + ) + except Exception: + # A problem happened while linking some file, let's clean up + # everything. + linker.abort() + raise + # Add image to manifest + add_iso_to_metadata( + self.compose, + variant, + arch, + cmd["iso_path"], + bootable=cmd["bootable"], + disc_num=cmd["disc_num"], + disc_count=cmd["disc_count"], + ) + + def try_reuse(self, cmd, variant, arch, opts): + """Try to reuse image from previous compose. + + :returns bool: True if reuse was successful, False otherwise + """ + if not self.compose.conf["createiso_allow_reuse"]: + return + + log_msg = "Cannot reuse ISO for %s.%s" % (variant, arch) + current_metadata = self.save_reuse_metadata(cmd, variant, arch, opts) + + if opts.buildinstall_method and not self.bi.reused(variant, arch): + # If buildinstall phase was not reused for some reason, we can not + # reuse any bootable image. If a package change caused rebuild of + # boot.iso, we would catch it here too, but there could be a + # configuration change in lorax template which would remain + # undetected. + self.logger.info("%s - boot configuration changed", log_msg) + return False + + # Check old compose configuration: extra_files and product_ids can be + # reflected on ISO. + old_config = self.compose.load_old_compose_config() + if not old_config: + self.logger.info("%s - no config for old compose", log_msg) + return False + # Convert current configuration to JSON and back to encode it similarly + # to the old one + config = json.loads(json.dumps(self.compose.conf)) + for opt in self.compose.conf: + # Skip a selection of options: these affect what packages can be + # included, which we explicitly check later on. + config_whitelist = set( + [ + "gather_lookaside_repos", + "pkgset_koji_builds", + "pkgset_koji_scratch_tasks", + "pkgset_koji_module_builds", + ] + ) + if opt in config_whitelist: + continue + + if old_config.get(opt) != config.get(opt): + self.logger.info("%s - option %s differs", log_msg, opt) + return False + + old_metadata = self._load_old_metadata(cmd, variant, arch) + if not old_metadata: + self.logger.info("%s - no old metadata found", log_msg) + return False + + # Test if volume ID matches - volid can be generated dynamically based on + # other values, and could change even if nothing else is different. + if current_metadata["opts"]["volid"] != old_metadata["opts"]["volid"]: + self.logger.info("%s - volume ID differs", log_msg) + return False + + # Compare packages on the ISO. + if compare_packages( + old_metadata["opts"]["graft_points"], + current_metadata["opts"]["graft_points"], + ): + self.logger.info("%s - packages differ", log_msg) + return False + + try: + self.perform_reuse( + cmd, + variant, + arch, + opts, + old_metadata["cmd"]["iso_path"], + ) + return True + except Exception as exc: + self.compose.log_error( + "Error while reusing ISO for %s.%s: %s", variant, arch, exc + ) + self.compose.traceback("createiso-reuse-%s-%s" % (variant, arch)) + return False + def run(self): symlink_isos_to = self.compose.conf.get("symlink_isos_to") disc_type = self.compose.conf["disc_types"].get("dvd", "dvd") @@ -184,6 +348,11 @@ class CreateisoPhase(PhaseLoggerMixin, PhaseBase): jigdo_dir = self.compose.paths.compose.jigdo_dir(arch, variant) opts = opts._replace(jigdo_dir=jigdo_dir, os_tree=os_tree) + # Try to reuse + if self.try_reuse(cmd, variant, arch, opts): + # Reuse was successful, go to next ISO + continue + script_file = os.path.join( self.compose.paths.work.tmp_dir(arch, variant), "createiso-%s.sh" % filename, @@ -203,6 +372,29 @@ class CreateisoPhase(PhaseLoggerMixin, PhaseBase): self.pool.start() +def read_packages(graft_points): + """Read packages that were listed in given graft points file. + + Only files under Packages directory are considered. Particularly this + excludes .discinfo, .treeinfo and media.repo as well as repodata and + any extra files. + + Extra files are easier to check by configuration (same name doesn't + imply same content). Repodata depend entirely on included packages (and + possibly product id certificate), but are affected by current time + which can change checksum despite data being the same. + """ + with open(graft_points) as f: + return set(line.split("=", 1)[0] for line in f if line.startswith("Packages/")) + + +def compare_packages(old_graft_points, new_graft_points): + """Read packages from the two files and compare them.""" + old_files = read_packages(old_graft_points) + new_files = read_packages(new_graft_points) + return old_files != new_files + + class CreateIsoThread(WorkerThread): def fail(self, compose, cmd, variant, arch): self.pool.log_error("CreateISO failed, removing ISO: %s" % cmd["iso_path"]) @@ -599,3 +791,24 @@ def create_hardlinks(staging_dir, log_file): """ cmd = ["/usr/sbin/hardlink", "-c", "-vv", staging_dir] run(cmd, logfile=log_file, show_cmd=True) + + +class OldFileLinker(object): + """ + A wrapper around os.link that remembers which files were linked and can + clean them up. + """ + + def __init__(self, logger): + self.logger = logger + self.linked_files = [] + + def link(self, src, dst): + self.logger.debug("Hardlinking %s to %s", src, dst) + os.link(src, dst) + self.linked_files.append(dst) + + def abort(self): + """Clean up all files created by this instance.""" + for f in self.linked_files: + os.unlink(f) diff --git a/pungi/util.py b/pungi/util.py index c6a40d4d..06b657ad 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -1130,3 +1130,9 @@ class PartialFuncThreadPool(ThreadPool): @property def results(self): return self._results + + +def read_json_file(file_path): + """A helper function to read a JSON file.""" + with open(file_path) as f: + return json.load(f) diff --git a/tests/test_createiso_phase.py b/tests/test_createiso_phase.py index 48bf37b7..180760a5 100644 --- a/tests/test_createiso_phase.py +++ b/tests/test_createiso_phase.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- +import logging import mock import six @@ -1322,3 +1323,227 @@ class TweakTreeinfo(helpers.PungiTestCase): ti.dump(output) self.assertFilesEqual(output, expected) + + +class CreateisoTryReusePhaseTest(helpers.PungiTestCase): + def setUp(self): + super(CreateisoTryReusePhaseTest, self).setUp() + self.logger = logging.getLogger() + self.logger.setLevel(logging.DEBUG) + self.logger.addHandler(logging.NullHandler()) + + def test_disabled(self): + compose = helpers.DummyCompose(self.topdir, {"createiso_allow_reuse": False}) + phase = createiso.CreateisoPhase(compose, mock.Mock()) + + self.assertFalse(phase.try_reuse(mock.Mock(), "Server", "x86_64", mock.Mock())) + + def test_buildinstall_changed(self): + compose = helpers.DummyCompose(self.topdir, {"createiso_allow_reuse": True}) + phase = createiso.CreateisoPhase(compose, mock.Mock()) + phase.logger = self.logger + phase.bi = mock.Mock() + phase.bi.reused.return_value = False + cmd = {"disc_num": 1, "disc_count": 1} + opts = CreateIsoOpts(buildinstall_method="lorax") + + self.assertFalse( + phase.try_reuse(cmd, compose.variants["Server"], "x86_64", opts) + ) + + def test_no_old_config(self): + compose = helpers.DummyCompose(self.topdir, {"createiso_allow_reuse": True}) + phase = createiso.CreateisoPhase(compose, mock.Mock()) + phase.logger = self.logger + cmd = {"disc_num": 1, "disc_count": 1} + opts = CreateIsoOpts() + + self.assertFalse( + phase.try_reuse(cmd, compose.variants["Server"], "x86_64", opts) + ) + + def test_old_config_changed(self): + compose = helpers.DummyCompose(self.topdir, {"createiso_allow_reuse": True}) + old_config = compose.conf.copy() + old_config["release_version"] = "2" + compose.load_old_compose_config.return_value = old_config + phase = createiso.CreateisoPhase(compose, mock.Mock()) + phase.logger = self.logger + cmd = {"disc_num": 1, "disc_count": 1} + opts = CreateIsoOpts() + + self.assertFalse( + phase.try_reuse(cmd, compose.variants["Server"], "x86_64", opts) + ) + + def test_no_old_metadata(self): + compose = helpers.DummyCompose(self.topdir, {"createiso_allow_reuse": True}) + compose.load_old_compose_config.return_value = compose.conf.copy() + phase = createiso.CreateisoPhase(compose, mock.Mock()) + phase.logger = self.logger + cmd = {"disc_num": 1, "disc_count": 1} + opts = CreateIsoOpts() + + self.assertFalse( + phase.try_reuse(cmd, compose.variants["Server"], "x86_64", opts) + ) + + @mock.patch("pungi.phases.createiso.read_json_file") + def test_volume_id_differs(self, read_json_file): + compose = helpers.DummyCompose(self.topdir, {"createiso_allow_reuse": True}) + compose.load_old_compose_config.return_value = compose.conf.copy() + phase = createiso.CreateisoPhase(compose, mock.Mock()) + phase.logger = self.logger + cmd = {"disc_num": 1, "disc_count": 1} + + opts = CreateIsoOpts(volid="new-volid") + + read_json_file.return_value = {"opts": {"volid": "old-volid"}} + + self.assertFalse( + phase.try_reuse(cmd, compose.variants["Server"], "x86_64", opts) + ) + + @mock.patch("pungi.phases.createiso.read_json_file") + def test_packages_differ(self, read_json_file): + compose = helpers.DummyCompose(self.topdir, {"createiso_allow_reuse": True}) + compose.load_old_compose_config.return_value = compose.conf.copy() + phase = createiso.CreateisoPhase(compose, mock.Mock()) + phase.logger = self.logger + cmd = {"disc_num": 1, "disc_count": 1} + + new_graft_points = os.path.join(self.topdir, "new_graft_points") + helpers.touch(new_graft_points, "Packages/f/foo-1-1.x86_64.rpm\n") + opts = CreateIsoOpts(graft_points=new_graft_points, volid="volid") + + old_graft_points = os.path.join(self.topdir, "old_graft_points") + helpers.touch(old_graft_points, "Packages/f/foo-1-2.x86_64.rpm\n") + read_json_file.return_value = { + "opts": {"graft_points": old_graft_points, "volid": "volid"} + } + + self.assertFalse( + phase.try_reuse(cmd, compose.variants["Server"], "x86_64", opts) + ) + + @mock.patch("pungi.phases.createiso.read_json_file") + def test_runs_perform_reuse(self, read_json_file): + compose = helpers.DummyCompose(self.topdir, {"createiso_allow_reuse": True}) + compose.load_old_compose_config.return_value = compose.conf.copy() + phase = createiso.CreateisoPhase(compose, mock.Mock()) + phase.logger = self.logger + phase.perform_reuse = mock.Mock() + cmd = {"disc_num": 1, "disc_count": 1} + + new_graft_points = os.path.join(self.topdir, "new_graft_points") + helpers.touch(new_graft_points) + opts = CreateIsoOpts(graft_points=new_graft_points, volid="volid") + + old_graft_points = os.path.join(self.topdir, "old_graft_points") + helpers.touch(old_graft_points) + dummy_iso_path = "dummy-iso-path" + read_json_file.return_value = { + "opts": { + "graft_points": old_graft_points, + "volid": "volid", + }, + "cmd": {"iso_path": dummy_iso_path}, + } + + self.assertTrue( + phase.try_reuse(cmd, compose.variants["Server"], "x86_64", opts) + ) + self.assertEqual( + phase.perform_reuse.call_args_list, + [ + mock.call( + cmd, + compose.variants["Server"], + "x86_64", + opts, + dummy_iso_path, + ) + ], + ) + + +@mock.patch("pungi.phases.createiso.OldFileLinker") +@mock.patch("pungi.phases.createiso.add_iso_to_metadata") +class CreateisoPerformReusePhaseTest(helpers.PungiTestCase): + def test_success(self, add_iso_to_metadata, OldFileLinker): + compose = helpers.DummyCompose(self.topdir, {"createiso_allow_reuse": True}) + phase = createiso.CreateisoPhase(compose, mock.Mock()) + cmd = { + "iso_path": "target/image.iso", + "bootable": False, + "disc_num": 1, + "disc_count": 2, + } + opts = CreateIsoOpts() + + phase.perform_reuse( + cmd, + compose.variants["Server"], + "x86_64", + opts, + "old/image.iso", + ) + + self.assertEqual( + add_iso_to_metadata.call_args_list, + [ + mock.call( + compose, + compose.variants["Server"], + "x86_64", + cmd["iso_path"], + bootable=False, + disc_count=2, + disc_num=1, + ), + ], + ) + self.assertEqual( + OldFileLinker.return_value.mock_calls, + [ + mock.call.link("old/image.iso", "target/image.iso"), + mock.call.link("old/image.iso.manifest", "target/image.iso.manifest"), + # The old log file doesn't exist in the test scenario. + mock.call.link( + None, + os.path.join( + self.topdir, "logs/x86_64/createiso-image.iso.x86_64.log" + ), + ), + ], + ) + + def test_failure(self, add_iso_to_metadata, OldFileLinker): + OldFileLinker.return_value.link.side_effect = helpers.mk_boom() + compose = helpers.DummyCompose(self.topdir, {"createiso_allow_reuse": True}) + phase = createiso.CreateisoPhase(compose, mock.Mock()) + cmd = { + "iso_path": "target/image.iso", + "bootable": False, + "disc_num": 1, + "disc_count": 2, + } + opts = CreateIsoOpts() + + with self.assertRaises(Exception): + phase.perform_reuse( + cmd, + compose.variants["Server"], + "x86_64", + opts, + "old/image.iso", + ) + + self.assertEqual(add_iso_to_metadata.call_args_list, []) + self.assertEqual( + OldFileLinker.return_value.mock_calls, + [ + mock.call.link("old/image.iso", "target/image.iso"), + mock.call.abort(), + ], + ) From e8305f3978e20ae386d375d71119fe09a1f7fd8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Mon, 13 Sep 2021 15:28:45 +0200 Subject: [PATCH 077/137] extra_isos: Allow reusing old images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When nothing in configuration or the image itself changed, let's just copy the older one. JIRA: RHELCMP-5969 Signed-off-by: Lubomír Sedlář --- pungi/checks.py | 1 + pungi/phases/extra_isos.py | 206 ++++++++++++++++++++++++--- pungi/scripts/config_validate.py | 2 +- pungi/scripts/pungi_koji.py | 2 +- tests/test_extra_isos_phase.py | 237 +++++++++++++++++++++++++++++-- 5 files changed, 414 insertions(+), 34 deletions(-) diff --git a/pungi/checks.py b/pungi/checks.py index d262f313..66f852b0 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -654,6 +654,7 @@ def make_schema(): "gather_allow_reuse": {"type": "boolean", "default": False}, "pkgset_allow_reuse": {"type": "boolean", "default": True}, "createiso_allow_reuse": {"type": "boolean", "default": True}, + "extraiso_allow_reuse": {"type": "boolean", "default": True}, "pkgset_source": {"type": "string", "enum": ["koji", "repos"]}, "createrepo_c": {"type": "boolean", "default": True}, "createrepo_checksum": { diff --git a/pungi/phases/extra_isos.py b/pungi/phases/extra_isos.py index 5db71a6f..31139159 100644 --- a/pungi/phases/extra_isos.py +++ b/pungi/phases/extra_isos.py @@ -14,6 +14,8 @@ # along with this program; if not, see . import os +import hashlib +import json from kobo.shortcuts import force_list from kobo.threads import ThreadPool, WorkerThread @@ -28,8 +30,16 @@ from pungi.phases.createiso import ( copy_boot_images, run_createiso_command, load_and_tweak_treeinfo, + compare_packages, + OldFileLinker, +) +from pungi.util import ( + failable, + get_format_substs, + get_variant_data, + get_volid, + read_json_file, ) -from pungi.util import failable, get_format_substs, get_variant_data, get_volid from pungi.wrappers import iso from pungi.wrappers.scm import get_dir_from_scm, get_file_from_scm @@ -37,9 +47,10 @@ from pungi.wrappers.scm import get_dir_from_scm, get_file_from_scm class ExtraIsosPhase(PhaseLoggerMixin, ConfigGuardedPhase, PhaseBase): name = "extra_isos" - def __init__(self, compose): + def __init__(self, compose, buildinstall_phase): super(ExtraIsosPhase, self).__init__(compose) self.pool = ThreadPool(logger=self.logger) + self.bi = buildinstall_phase def validate(self): for variant in self.compose.get_variants(types=["variant"]): @@ -65,13 +76,17 @@ class ExtraIsosPhase(PhaseLoggerMixin, ConfigGuardedPhase, PhaseBase): commands.append((config, variant, arch)) for (config, variant, arch) in commands: - self.pool.add(ExtraIsosThread(self.pool)) + self.pool.add(ExtraIsosThread(self.pool, self.bi)) self.pool.queue_put((self.compose, config, variant, arch)) self.pool.start() class ExtraIsosThread(WorkerThread): + def __init__(self, pool, buildinstall_phase): + super(ExtraIsosThread, self).__init__(pool) + self.bi = buildinstall_phase + def process(self, item, num): self.num = num compose, config, variant, arch = item @@ -127,24 +142,30 @@ class ExtraIsosThread(WorkerThread): buildinstall_method=compose.conf["buildinstall_method"] ) - script_file = os.path.join( - compose.paths.work.tmp_dir(arch, variant), "extraiso-%s.sh" % filename - ) - with open(script_file, "w") as f: - createiso.write_script(opts, f) + # Check if it can be reused. + hash = hashlib.sha256() + hash.update(json.dumps(config, sort_keys=True).encode("utf-8")) + config_hash = hash.hexdigest() - run_createiso_command( - self.num, - compose, - bootable, - arch, - ["bash", script_file], - [compose.topdir], - log_file=compose.paths.log.log_file( - arch, "extraiso-%s" % os.path.basename(iso_path) - ), - with_jigdo=compose.conf["create_jigdo"], - ) + if not self.try_reuse(compose, variant, arch, config_hash, opts): + script_file = os.path.join( + compose.paths.work.tmp_dir(arch, variant), "extraiso-%s.sh" % filename + ) + with open(script_file, "w") as f: + createiso.write_script(opts, f) + + run_createiso_command( + self.num, + compose, + bootable, + arch, + ["bash", script_file], + [compose.topdir], + log_file=compose.paths.log.log_file( + arch, "extraiso-%s" % os.path.basename(iso_path) + ), + with_jigdo=compose.conf["create_jigdo"], + ) img = add_iso_to_metadata( compose, @@ -156,8 +177,153 @@ class ExtraIsosThread(WorkerThread): ) img._max_size = config.get("max_size") + save_reuse_metadata(compose, variant, arch, config_hash, opts, iso_path) + self.pool.log_info("[DONE ] %s" % msg) + def try_reuse(self, compose, variant, arch, config_hash, opts): + # Check explicit config + if not compose.conf["extraiso_allow_reuse"]: + return + + log_msg = "Cannot reuse ISO for %s.%s" % (variant, arch) + + if opts.buildinstall_method and not self.bi.reused(variant, arch): + # If buildinstall phase was not reused for some reason, we can not + # reuse any bootable image. If a package change caused rebuild of + # boot.iso, we would catch it here too, but there could be a + # configuration change in lorax template which would remain + # undetected. + self.pool.log_info("%s - boot configuration changed", log_msg) + return False + + # Check old compose configuration: extra_files and product_ids can be + # reflected on ISO. + old_config = compose.load_old_compose_config() + if not old_config: + self.pool.log_info("%s - no config for old compose", log_msg) + return False + # Convert current configuration to JSON and back to encode it similarly + # to the old one + config = json.loads(json.dumps(compose.conf)) + for opt in compose.conf: + # Skip a selection of options: these affect what packages can be + # included, which we explicitly check later on. + config_whitelist = set( + [ + "gather_lookaside_repos", + "pkgset_koji_builds", + "pkgset_koji_scratch_tasks", + "pkgset_koji_module_builds", + ] + ) + if opt in config_whitelist: + continue + + if old_config.get(opt) != config.get(opt): + self.pool.log_info("%s - option %s differs", log_msg, opt) + return False + + old_metadata = load_old_metadata(compose, variant, arch, config_hash) + if not old_metadata: + self.pool.log_info("%s - no old metadata found", log_msg) + return False + + # Test if volume ID matches - volid can be generated dynamically based on + # other values, and could change even if nothing else is different. + if opts.volid != old_metadata["opts"]["volid"]: + self.pool.log_info("%s - volume ID differs", log_msg) + return False + + # Compare packages on the ISO. + if compare_packages( + old_metadata["opts"]["graft_points"], + opts.graft_points, + ): + self.pool.log_info("%s - packages differ", log_msg) + return False + + try: + self.perform_reuse( + compose, + variant, + arch, + opts, + old_metadata["opts"]["output_dir"], + old_metadata["opts"]["iso_name"], + ) + return True + except Exception as exc: + self.pool.log_error( + "Error while reusing ISO for %s.%s: %s", variant, arch, exc + ) + compose.traceback("extraiso-reuse-%s-%s-%s" % (variant, arch, config_hash)) + return False + + def perform_reuse(self, compose, variant, arch, opts, old_iso_dir, old_file_name): + """ + Copy all related files from old compose to the new one. As a last step + add the new image to metadata. + """ + linker = OldFileLinker(self.pool._logger) + old_iso_path = os.path.join(old_iso_dir, old_file_name) + iso_path = os.path.join(opts.output_dir, opts.iso_name) + try: + # Hardlink ISO and manifest + for suffix in ("", ".manifest"): + linker.link(old_iso_path + suffix, iso_path + suffix) + # Copy log files + # The log file name includes filename of the image, so we need to + # find old file with the old name, and rename it to the new name. + log_file = compose.paths.log.log_file(arch, "extraiso-%s" % opts.iso_name) + old_log_file = compose.paths.old_compose_path( + compose.paths.log.log_file(arch, "extraiso-%s" % old_file_name) + ) + linker.link(old_log_file, log_file) + # Copy jigdo files + if opts.jigdo_dir: + old_jigdo_dir = compose.paths.old_compose_path(opts.jigdo_dir) + for suffix in (".template", ".jigdo"): + linker.link( + os.path.join(old_jigdo_dir, old_file_name) + suffix, + os.path.join(opts.jigdo_dir, opts.iso_name) + suffix, + ) + except Exception: + # A problem happened while linking some file, let's clean up + # everything. + linker.abort() + raise + + +def save_reuse_metadata(compose, variant, arch, config_hash, opts, iso_path): + """ + Save metadata for possible reuse of this image. The file name is determined + from the hash of a configuration snippet for this image. Any change in that + configuration in next compose will change the hash and thus reuse will be + blocked. + """ + metadata = {"opts": opts._asdict()} + metadata_path = compose.paths.log.log_file( + arch, + "extraiso-reuse-%s-%s-%s" % (variant.uid, arch, config_hash), + ext="json", + ) + with open(metadata_path, "w") as f: + json.dump(metadata, f, indent=2) + + +def load_old_metadata(compose, variant, arch, config_hash): + metadata_path = compose.paths.log.log_file( + arch, + "extraiso-reuse-%s-%s-%s" % (variant.uid, arch, config_hash), + ext="json", + ) + old_path = compose.paths.old_compose_path(metadata_path) + try: + return read_json_file(old_path) + except Exception: + return None + def get_extra_files(compose, variant, arch, extra_files): """Clone the configured files into a directory from where they can be diff --git a/pungi/scripts/config_validate.py b/pungi/scripts/config_validate.py index d4b9b5b5..b4bdb1eb 100644 --- a/pungi/scripts/config_validate.py +++ b/pungi/scripts/config_validate.py @@ -127,7 +127,7 @@ def run(config, topdir, has_old, offline, defined_variables, schema_overrides): pungi.phases.OstreeInstallerPhase(compose, buildinstall_phase), pungi.phases.OSTreePhase(compose), pungi.phases.CreateisoPhase(compose, buildinstall_phase), - pungi.phases.ExtraIsosPhase(compose), + pungi.phases.ExtraIsosPhase(compose, buildinstall_phase), pungi.phases.LiveImagesPhase(compose), pungi.phases.LiveMediaPhase(compose), pungi.phases.ImageBuildPhase(compose), diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index 9570e3c5..e393220d 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -403,7 +403,7 @@ def run_compose( ) ostree_phase = pungi.phases.OSTreePhase(compose, pkgset_phase) createiso_phase = pungi.phases.CreateisoPhase(compose, buildinstall_phase) - extra_isos_phase = pungi.phases.ExtraIsosPhase(compose) + extra_isos_phase = pungi.phases.ExtraIsosPhase(compose, buildinstall_phase) liveimages_phase = pungi.phases.LiveImagesPhase(compose) livemedia_phase = pungi.phases.LiveMediaPhase(compose) image_build_phase = pungi.phases.ImageBuildPhase(compose, buildinstall_phase) diff --git a/tests/test_extra_isos_phase.py b/tests/test_extra_isos_phase.py index e38ec9da..7b7c7346 100644 --- a/tests/test_extra_isos_phase.py +++ b/tests/test_extra_isos_phase.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- - +import logging import mock import six import os from tests import helpers +from pungi.createiso import CreateIsoOpts from pungi.phases import extra_isos @@ -19,7 +20,7 @@ class ExtraIsosPhaseTest(helpers.PungiTestCase): } compose = helpers.DummyCompose(self.topdir, {"extra_isos": {"^Server$": [cfg]}}) - phase = extra_isos.ExtraIsosPhase(compose) + phase = extra_isos.ExtraIsosPhase(compose, mock.Mock()) phase.validate() self.assertEqual(len(compose.log_warning.call_args_list), 1) @@ -30,7 +31,7 @@ class ExtraIsosPhaseTest(helpers.PungiTestCase): } compose = helpers.DummyCompose(self.topdir, {"extra_isos": {"^Server$": [cfg]}}) - phase = extra_isos.ExtraIsosPhase(compose) + phase = extra_isos.ExtraIsosPhase(compose, mock.Mock()) phase.run() self.assertEqual(len(ThreadPool.return_value.add.call_args_list), 3) @@ -51,7 +52,7 @@ class ExtraIsosPhaseTest(helpers.PungiTestCase): } compose = helpers.DummyCompose(self.topdir, {"extra_isos": {"^Server$": [cfg]}}) - phase = extra_isos.ExtraIsosPhase(compose) + phase = extra_isos.ExtraIsosPhase(compose, mock.Mock()) phase.run() self.assertEqual(len(ThreadPool.return_value.add.call_args_list), 2) @@ -71,7 +72,7 @@ class ExtraIsosPhaseTest(helpers.PungiTestCase): } compose = helpers.DummyCompose(self.topdir, {"extra_isos": {"^Server$": [cfg]}}) - phase = extra_isos.ExtraIsosPhase(compose) + phase = extra_isos.ExtraIsosPhase(compose, mock.Mock()) phase.run() self.assertEqual(len(ThreadPool.return_value.add.call_args_list), 2) @@ -106,7 +107,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase): gvi.return_value = "my volume id" gic.return_value = "/tmp/iso-graft-points" - t = extra_isos.ExtraIsosThread(mock.Mock()) + t = extra_isos.ExtraIsosThread(mock.Mock(), mock.Mock()) with mock.patch("time.sleep"): t.process((compose, cfg, server, "x86_64"), 1) @@ -182,7 +183,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase): gvi.return_value = "my volume id" gic.return_value = "/tmp/iso-graft-points" - t = extra_isos.ExtraIsosThread(mock.Mock()) + t = extra_isos.ExtraIsosThread(mock.Mock(), mock.Mock()) with mock.patch("time.sleep"): t.process((compose, cfg, server, "x86_64"), 1) @@ -256,7 +257,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase): gvi.return_value = "my volume id" gic.return_value = "/tmp/iso-graft-points" - t = extra_isos.ExtraIsosThread(mock.Mock()) + t = extra_isos.ExtraIsosThread(mock.Mock(), mock.Mock()) with mock.patch("time.sleep"): t.process((compose, cfg, server, "x86_64"), 1) @@ -330,7 +331,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase): gvi.return_value = "my volume id" gic.return_value = "/tmp/iso-graft-points" - t = extra_isos.ExtraIsosThread(mock.Mock()) + t = extra_isos.ExtraIsosThread(mock.Mock(), mock.Mock()) with mock.patch("time.sleep"): t.process((compose, cfg, server, "x86_64"), 1) @@ -405,7 +406,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase): gvi.return_value = "my volume id" gic.return_value = "/tmp/iso-graft-points" - t = extra_isos.ExtraIsosThread(mock.Mock()) + t = extra_isos.ExtraIsosThread(mock.Mock(), mock.Mock()) with mock.patch("time.sleep"): t.process((compose, cfg, server, "src"), 1) @@ -476,7 +477,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase): gic.return_value = "/tmp/iso-graft-points" rcc.side_effect = helpers.mk_boom() - t = extra_isos.ExtraIsosThread(mock.Mock()) + t = extra_isos.ExtraIsosThread(mock.Mock(), mock.Mock()) with mock.patch("time.sleep"): t.process((compose, cfg, server, "x86_64"), 1) @@ -494,7 +495,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase): gic.return_value = "/tmp/iso-graft-points" rcc.side_effect = helpers.mk_boom(RuntimeError) - t = extra_isos.ExtraIsosThread(mock.Mock()) + t = extra_isos.ExtraIsosThread(mock.Mock(), mock.Mock()) with self.assertRaises(RuntimeError): with mock.patch("time.sleep"): t.process((compose, cfg, server, "x86_64"), 1) @@ -1061,3 +1062,215 @@ class PrepareMetadataTest(helpers.PungiTestCase): ), ], ) + + +class ExtraisoTryReusePhaseTest(helpers.PungiTestCase): + def setUp(self): + super(ExtraisoTryReusePhaseTest, self).setUp() + self.logger = logging.getLogger() + self.logger.setLevel(logging.DEBUG) + self.logger.addHandler(logging.NullHandler()) + + def test_disabled(self): + compose = helpers.DummyCompose(self.topdir, {"extraiso_allow_reuse": False}) + thread = extra_isos.ExtraIsosThread(compose, mock.Mock()) + opts = CreateIsoOpts() + + self.assertFalse( + thread.try_reuse( + compose, compose.variants["Server"], "x86_64", "abcdef", opts + ) + ) + + def test_buildinstall_changed(self): + compose = helpers.DummyCompose(self.topdir, {"extraiso_allow_reuse": True}) + thread = extra_isos.ExtraIsosThread(compose, mock.Mock()) + thread.logger = self.logger + thread.bi = mock.Mock() + thread.bi.reused.return_value = False + opts = CreateIsoOpts(buildinstall_method="lorax") + + self.assertFalse( + thread.try_reuse( + compose, compose.variants["Server"], "x86_64", "abcdef", opts + ) + ) + + def test_no_old_config(self): + compose = helpers.DummyCompose(self.topdir, {"extraiso_allow_reuse": True}) + thread = extra_isos.ExtraIsosThread(compose, mock.Mock()) + thread.logger = self.logger + opts = CreateIsoOpts() + + self.assertFalse( + thread.try_reuse( + compose, compose.variants["Server"], "x86_64", "abcdef", opts + ) + ) + + def test_old_config_changed(self): + compose = helpers.DummyCompose(self.topdir, {"extraiso_allow_reuse": True}) + old_config = compose.conf.copy() + old_config["release_version"] = "2" + compose.load_old_compose_config.return_value = old_config + thread = extra_isos.ExtraIsosThread(compose, mock.Mock()) + thread.logger = self.logger + opts = CreateIsoOpts() + + self.assertFalse( + thread.try_reuse( + compose, compose.variants["Server"], "x86_64", "abcdef", opts + ) + ) + + def test_no_old_metadata(self): + compose = helpers.DummyCompose(self.topdir, {"extraiso_allow_reuse": True}) + compose.load_old_compose_config.return_value = compose.conf.copy() + thread = extra_isos.ExtraIsosThread(compose, mock.Mock()) + thread.logger = self.logger + opts = CreateIsoOpts() + + self.assertFalse( + thread.try_reuse( + compose, compose.variants["Server"], "x86_64", "abcdef", opts + ) + ) + + @mock.patch("pungi.phases.extra_isos.read_json_file") + def test_volume_id_differs(self, read_json_file): + compose = helpers.DummyCompose(self.topdir, {"extraiso_allow_reuse": True}) + compose.load_old_compose_config.return_value = compose.conf.copy() + thread = extra_isos.ExtraIsosThread(compose, mock.Mock()) + thread.logger = self.logger + + opts = CreateIsoOpts(volid="new-volid") + + read_json_file.return_value = {"opts": {"volid": "old-volid"}} + + self.assertFalse( + thread.try_reuse( + compose, compose.variants["Server"], "x86_64", "abcdef", opts + ) + ) + + @mock.patch("pungi.phases.extra_isos.read_json_file") + def test_packages_differ(self, read_json_file): + compose = helpers.DummyCompose(self.topdir, {"extraiso_allow_reuse": True}) + compose.load_old_compose_config.return_value = compose.conf.copy() + thread = extra_isos.ExtraIsosThread(compose, mock.Mock()) + thread.logger = self.logger + + new_graft_points = os.path.join(self.topdir, "new_graft_points") + helpers.touch(new_graft_points, "Packages/f/foo-1-1.x86_64.rpm\n") + opts = CreateIsoOpts(graft_points=new_graft_points, volid="volid") + + old_graft_points = os.path.join(self.topdir, "old_graft_points") + helpers.touch(old_graft_points, "Packages/f/foo-1-2.x86_64.rpm\n") + read_json_file.return_value = { + "opts": {"graft_points": old_graft_points, "volid": "volid"} + } + + self.assertFalse( + thread.try_reuse( + compose, compose.variants["Server"], "x86_64", "abcdef", opts + ) + ) + + @mock.patch("pungi.phases.extra_isos.read_json_file") + def test_runs_perform_reuse(self, read_json_file): + compose = helpers.DummyCompose(self.topdir, {"extraiso_allow_reuse": True}) + compose.load_old_compose_config.return_value = compose.conf.copy() + thread = extra_isos.ExtraIsosThread(compose, mock.Mock()) + thread.logger = self.logger + thread.perform_reuse = mock.Mock() + + new_graft_points = os.path.join(self.topdir, "new_graft_points") + helpers.touch(new_graft_points) + opts = CreateIsoOpts(graft_points=new_graft_points, volid="volid") + + old_graft_points = os.path.join(self.topdir, "old_graft_points") + helpers.touch(old_graft_points) + dummy_iso_path = "dummy-iso-path/dummy.iso" + read_json_file.return_value = { + "opts": { + "graft_points": old_graft_points, + "volid": "volid", + "output_dir": os.path.dirname(dummy_iso_path), + "iso_name": os.path.basename(dummy_iso_path), + }, + } + + self.assertTrue( + thread.try_reuse( + compose, compose.variants["Server"], "x86_64", "abcdef", opts + ) + ) + self.assertEqual( + thread.perform_reuse.call_args_list, + [ + mock.call( + compose, + compose.variants["Server"], + "x86_64", + opts, + "dummy-iso-path", + "dummy.iso", + ) + ], + ) + + +@mock.patch("pungi.phases.extra_isos.OldFileLinker") +class ExtraIsoPerformReusePhaseTest(helpers.PungiTestCase): + def test_success(self, OldFileLinker): + compose = helpers.DummyCompose(self.topdir, {"extraiso_allow_reuse": True}) + thread = extra_isos.ExtraIsosThread(compose, mock.Mock()) + opts = CreateIsoOpts(output_dir="new/path", iso_name="new.iso") + + thread.perform_reuse( + compose, + compose.variants["Server"], + "x86_64", + opts, + "old", + "image.iso", + ) + + self.assertEqual( + OldFileLinker.return_value.mock_calls, + [ + mock.call.link("old/image.iso", "new/path/new.iso"), + mock.call.link("old/image.iso.manifest", "new/path/new.iso.manifest"), + # The old log file doesn't exist in the test scenario. + mock.call.link( + None, + os.path.join( + self.topdir, "logs/x86_64/extraiso-new.iso.x86_64.log" + ), + ), + ], + ) + + def test_failure(self, OldFileLinker): + OldFileLinker.return_value.link.side_effect = helpers.mk_boom() + compose = helpers.DummyCompose(self.topdir, {"extraiso_allow_reuse": True}) + thread = extra_isos.ExtraIsosThread(compose, mock.Mock()) + opts = CreateIsoOpts(output_dir="new/path", iso_name="new.iso") + + with self.assertRaises(Exception): + thread.perform_reuse( + compose, + compose.variants["Server"], + "x86_64", + opts, + "old", + "image.iso", + ) + + self.assertEqual( + OldFileLinker.return_value.mock_calls, + [ + mock.call.link("old/image.iso", "new/path/new.iso"), + mock.call.abort(), + ], + ) From e2b3002726e6b25f2e8aa15d4ce7256fecd3ce4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Tue, 2 Nov 2021 08:49:04 +0100 Subject: [PATCH 078/137] repoclosure: Use --forcearch for dnf repoclosure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DNF repoclosure requires this option when checking a repository that is not compatible with host architecture. It seems that when it is compatible, it works as well. Based on how the list of architectures is generated, we know that the main one will always be first. Fixes: https://pagure.io/pungi/issue/1562 Signed-off-by: Lubomír Sedlář --- pungi/wrappers/repoclosure.py | 6 ++++- tests/test_repoclosure_wrapper.py | 38 +++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/pungi/wrappers/repoclosure.py b/pungi/wrappers/repoclosure.py index f62b3da4..268df094 100644 --- a/pungi/wrappers/repoclosure.py +++ b/pungi/wrappers/repoclosure.py @@ -40,9 +40,13 @@ def get_repoclosure_cmd(backend="yum", arch=None, repos=None, lookaside=None): # There are options that are not exposed here, because we don't need # them. - for i in force_list(arch or []): + arches = force_list(arch or []) + for i in arches: cmd.append("--arch=%s" % i) + if backend == "dnf" and arches: + cmd.append("--forcearch=%s" % arches[0]) + repos = repos or {} for repo_id, repo_path in repos.items(): cmd.append("--repofrompath=%s,%s" % (repo_id, _to_url(repo_path))) diff --git a/tests/test_repoclosure_wrapper.py b/tests/test_repoclosure_wrapper.py index af2dc3e0..361d5846 100755 --- a/tests/test_repoclosure_wrapper.py +++ b/tests/test_repoclosure_wrapper.py @@ -25,8 +25,14 @@ class RepoclosureWrapperTestCase(helpers.BaseTestCase): def test_multiple_arches(self): self.assertEqual( - rc.get_repoclosure_cmd(arch=["x86_64", "ppc64"]), - ["/usr/bin/repoclosure", "--tempcache", "--arch=x86_64", "--arch=ppc64"], + rc.get_repoclosure_cmd(arch=["x86_64", "i686", "noarch"]), + [ + "/usr/bin/repoclosure", + "--tempcache", + "--arch=x86_64", + "--arch=i686", + "--arch=noarch", + ], ) def test_full_command(self): @@ -61,6 +67,34 @@ class RepoclosureWrapperTestCase(helpers.BaseTestCase): cmd[2:], [ "--arch=x86_64", + "--forcearch=x86_64", + "--repofrompath=my-repo,file:///mnt/koji/repo", + "--repofrompath=fedora,http://kojipkgs.fp.o/repo", + "--repo=my-repo", + "--check=my-repo", + "--repo=fedora", + ], + ) + + def test_dnf_command_with_multiple_arches(self): + repos = {"my-repo": "/mnt/koji/repo"} + lookaside = {"fedora": "http://kojipkgs.fp.o/repo"} + + cmd = rc.get_repoclosure_cmd( + backend="dnf", + arch=["x86_64", "i686", "noarch"], + repos=repos, + lookaside=lookaside, + ) + self.assertEqual(cmd[:2], ["dnf", "repoclosure"]) + six.assertCountEqual( + self, + cmd[2:], + [ + "--arch=x86_64", + "--arch=i686", + "--arch=noarch", + "--forcearch=x86_64", "--repofrompath=my-repo,file:///mnt/koji/repo", "--repofrompath=fedora,http://kojipkgs.fp.o/repo", "--repo=my-repo", From 7b9e08ab28668ae3a1cffc4a6b13a9cae699e9c8 Mon Sep 17 00:00:00 2001 From: fdiprete Date: Mon, 1 Nov 2021 16:54:34 +0000 Subject: [PATCH 079/137] test images for metadata deserialization error Merges: https://pagure.io/pungi/pull-request/1559 Jira: https://issues.redhat.com/browse/RHELCMP-6685 Signed-off-by: fdiprete --- pungi/phases/test.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pungi/phases/test.py b/pungi/phases/test.py index 5d3a483e..b4e87ab1 100644 --- a/pungi/phases/test.py +++ b/pungi/phases/test.py @@ -18,6 +18,7 @@ import os from pungi.phases.base import PhaseBase from pungi.util import failable, get_arch_variant_data +import productmd.compose class TestPhase(PhaseBase): @@ -25,6 +26,7 @@ class TestPhase(PhaseBase): def run(self): check_image_sanity(self.compose) + check_image_metadata(self.compose) def check_image_sanity(compose): @@ -45,6 +47,17 @@ def check_image_sanity(compose): check_size_limit(compose, variant, arch, img) +def check_image_metadata(compose): + """ + Check the images metadata for entries that cannot be serialized. + Often caused by isos with duplicate metadata. + Accessing the `images` attribute will raise an exception if there's a problem + """ + + compose = productmd.compose.Compose(compose.paths.compose.topdir()) + return compose.images + + def check_sanity(compose, variant, arch, image): path = os.path.join(compose.paths.compose.topdir(), image.path) deliverable = getattr(image, "deliverable") From 9d02f87c9920275cf9820afd88cd3bea9d50b053 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 4 Nov 2021 09:01:28 +0100 Subject: [PATCH 080/137] Stop trying to validate non-existent metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a compose doesn't build any images, it won't produce any metadata file for them, and thus it makes no sense to validate it. Signed-off-by: Lubomír Sedlář Fixes: https://pagure.io/pungi/issue/1565 --- pungi/phases/test.py | 6 +- .../compose/metadata/images.json | 58 +++++++++++++++++++ tests/test_test_phase.py | 30 +++++++++- 3 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 tests/fixtures/invalid-image-metadata/compose/metadata/images.json diff --git a/pungi/phases/test.py b/pungi/phases/test.py index b4e87ab1..099af558 100644 --- a/pungi/phases/test.py +++ b/pungi/phases/test.py @@ -53,9 +53,9 @@ def check_image_metadata(compose): Often caused by isos with duplicate metadata. Accessing the `images` attribute will raise an exception if there's a problem """ - - compose = productmd.compose.Compose(compose.paths.compose.topdir()) - return compose.images + if compose.im.images: + compose = productmd.compose.Compose(compose.paths.compose.topdir()) + return compose.images def check_sanity(compose, variant, arch, image): diff --git a/tests/fixtures/invalid-image-metadata/compose/metadata/images.json b/tests/fixtures/invalid-image-metadata/compose/metadata/images.json new file mode 100644 index 00000000..025d329e --- /dev/null +++ b/tests/fixtures/invalid-image-metadata/compose/metadata/images.json @@ -0,0 +1,58 @@ +{ + "header": { + "type": "productmd.images", + "version": "1.2" + }, + "payload": { + "compose": { + "date": "20181001", + "id": "Mixed-1.0-20181001.n.0", + "respin": 0, + "type": "nightly" + }, + "images": { + "Server": { + "x86_64": [ + { + "arch": "x86_64", + "bootable": false, + "checksums": { + "md5": "c7977d67f6522bce7fb04c0818a3c744", + "sha1": "c7d65673b2eb477016f9e09f321935bace545515", + "sha256": "6d9cfc9be59cba96763dcca5d1b5759127d2f7920055b663dbcf29474bc368de" + }, + "disc_count": 1, + "disc_number": 1, + "format": "iso", + "implant_md5": "340b7dc15b9c74b8576b81c3b33fc3f2", + "mtime": 1636012560, + "path": "Server-Gluster/x86_64/iso/Gluster-2.3-DP-1-20211104.t.4-Server-x86_64-dvd1.iso", + "size": 419840, + "subvariant": "Server-Gluster", + "type": "dvd", + "volume_id": "Gluster-2.3 DP-1 Server.x86_64" + }, + { + "arch": "x86_64", + "bootable": false, + "checksums": { + "md5": "a7977d67f6522bce7fb04c0818a3c744", + "sha1": "a7d65673b2eb477016f9e09f321935bace545515", + "sha256": "ad9cfc9be59cba96763dcca5d1b5759127d2f7920055b663dbcf29474bc368de" + }, + "disc_count": 1, + "disc_number": 1, + "format": "iso", + "implant_md5": "340b7dc15b9c74b8576b81c3b33fc3f2", + "mtime": 1636012560, + "path": "Server-Gluster/x86_64/iso/Gluster-2.3-DP-1-20211104.t.4-Server-x86_64-dvd1.iso", + "size": 419840, + "subvariant": "Server-Gluster", + "type": "dvd", + "volume_id": "Gluster-2.3 DP-1 Server.x86_64" + } + ] + } + } + } +} diff --git a/tests/test_test_phase.py b/tests/test_test_phase.py index 4e82d051..1b6f1ad1 100644 --- a/tests/test_test_phase.py +++ b/tests/test_test_phase.py @@ -4,7 +4,7 @@ import mock import os import pungi.phases.test as test_phase -from tests.helpers import DummyCompose, PungiTestCase, touch +from tests.helpers import DummyCompose, PungiTestCase, touch, FIXTURE_DIR try: import dnf # noqa: F401 @@ -305,3 +305,31 @@ class TestCheckImageSanity(PungiTestCase): test_phase.check_image_sanity(compose) self.assertEqual(compose.log_warning.call_args_list, []) + + +class TestImageMetadataValidation(PungiTestCase): + def test_valid_metadata(self): + compose = mock.Mock() + compose.im.images = {"Server": mock.Mock()} + compose.paths.compose.topdir = lambda: os.path.join( + FIXTURE_DIR, "basic-metadata" + ) + + test_phase.check_image_metadata(compose) + + def test_missing_metadata(self): + compose = mock.Mock() + compose.im.images = {} + compose.paths.compose.topdir = lambda: self.topdir + + test_phase.check_image_metadata(compose) + + def test_invalid_metadata(self): + compose = mock.Mock() + compose.im.images = {"Server": mock.Mock()} + compose.paths.compose.topdir = lambda: os.path.join( + FIXTURE_DIR, "invalid-image-metadata" + ) + + with self.assertRaises(RuntimeError): + test_phase.check_image_metadata(compose) From 94ffa1c5c64445612fcd994da3237b216a8f6ce6 Mon Sep 17 00:00:00 2001 From: Ken Dreyer Date: Mon, 1 Nov 2021 16:08:30 -0400 Subject: [PATCH 081/137] default "with_jigdo" to False Fedora has not composed with jigdo in a long time. Disable it by default. Signed-off-by: Ken Dreyer Merges: https://pagure.io/pungi/pull-request/1561 Fixes: https://pagure.io/pungi/issue/1560 --- doc/configuration.rst | 2 +- doc/examples.rst | 1 - pungi/checks.py | 4 ++-- pungi/phases/createiso.py | 2 +- tests/data/dummy-pungi.conf | 2 -- tests/test_createiso_phase.py | 23 +++++++++++------------ tests/test_extra_isos_phase.py | 8 ++++---- 7 files changed, 19 insertions(+), 23 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index f52257fb..f698821f 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1240,7 +1240,7 @@ Options Format: ``[(variant_uid_regex, {arch|*: bool})]`` -**create_jigdo** = True +**create_jigdo** = False (*bool*) -- controls the creation of jigdo from ISO **create_optional_isos** = False diff --git a/doc/examples.rst b/doc/examples.rst index 956383da..701a5f5a 100644 --- a/doc/examples.rst +++ b/doc/examples.rst @@ -83,7 +83,6 @@ This is a shortened configuration for Fedora Radhide compose as of 2019-10-14. # CREATEISO iso_hfs_ppc64le_compatible = False - create_jigdo = False # BUILDINSTALL buildinstall_method = 'lorax' diff --git a/pungi/checks.py b/pungi/checks.py index 66f852b0..b1ecf61b 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -53,7 +53,7 @@ from . import util def is_jigdo_needed(conf): - return conf.get("create_jigdo", True) + return conf.get("create_jigdo", False) def is_isohybrid_needed(conf): @@ -609,7 +609,7 @@ def make_schema(): "runroot_ssh_init_template": {"type": "string"}, "runroot_ssh_install_packages_template": {"type": "string"}, "runroot_ssh_run_template": {"type": "string"}, - "create_jigdo": {"type": "boolean", "default": True}, + "create_jigdo": {"type": "boolean", "default": False}, "check_deps": {"type": "boolean", "default": True}, "require_all_comps_packages": {"type": "boolean", "default": False}, "bootable": { diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py index 706ba577..895c588c 100644 --- a/pungi/phases/createiso.py +++ b/pungi/phases/createiso.py @@ -518,7 +518,7 @@ def add_iso_to_metadata( def run_createiso_command( - num, compose, bootable, arch, cmd, mounts, log_file, with_jigdo=True + num, compose, bootable, arch, cmd, mounts, log_file, with_jigdo=False ): packages = [ "coreutils", diff --git a/tests/data/dummy-pungi.conf b/tests/data/dummy-pungi.conf index 5225f34d..d27423d4 100644 --- a/tests/data/dummy-pungi.conf +++ b/tests/data/dummy-pungi.conf @@ -109,5 +109,3 @@ extra_isos = { 'filename': 'extra-{filename}', }] } - -create_jigdo = False diff --git a/tests/test_createiso_phase.py b/tests/test_createiso_phase.py index 180760a5..937f2137 100644 --- a/tests/test_createiso_phase.py +++ b/tests/test_createiso_phase.py @@ -120,8 +120,8 @@ class CreateisoPhaseTest(helpers.PungiTestCase): graft_points="dummy-graft-points", arch="x86_64", supported=True, - jigdo_dir="%s/compose/Server/x86_64/jigdo" % self.topdir, - os_tree="%s/compose/Server/x86_64/os" % self.topdir, + jigdo_dir=None, + os_tree=None, hfs_compat=True, use_xorrisofs=False, ) @@ -246,8 +246,8 @@ class CreateisoPhaseTest(helpers.PungiTestCase): arch="x86_64", buildinstall_method="lorax", supported=True, - jigdo_dir="%s/compose/Server/x86_64/jigdo" % self.topdir, - os_tree="%s/compose/Server/x86_64/os" % self.topdir, + jigdo_dir=None, + os_tree=None, hfs_compat=True, use_xorrisofs=False, ), @@ -258,8 +258,8 @@ class CreateisoPhaseTest(helpers.PungiTestCase): graft_points="dummy-graft-points", arch="src", supported=True, - jigdo_dir="%s/compose/Server/source/jigdo" % self.topdir, - os_tree="%s/compose/Server/source/tree" % self.topdir, + jigdo_dir=None, + os_tree=None, hfs_compat=True, use_xorrisofs=False, ), @@ -390,8 +390,8 @@ class CreateisoPhaseTest(helpers.PungiTestCase): graft_points="dummy-graft-points", arch="src", supported=True, - jigdo_dir="%s/compose/Server/source/jigdo" % self.topdir, - os_tree="%s/compose/Server/source/tree" % self.topdir, + jigdo_dir=None, + os_tree=None, hfs_compat=True, use_xorrisofs=False, ) @@ -497,8 +497,8 @@ class CreateisoPhaseTest(helpers.PungiTestCase): graft_points="dummy-graft-points", arch="x86_64", supported=True, - jigdo_dir="%s/compose/Server/x86_64/jigdo" % self.topdir, - os_tree="%s/compose/Server/x86_64/os" % self.topdir, + jigdo_dir=None, + os_tree=None, hfs_compat=False, use_xorrisofs=False, ) @@ -580,7 +580,7 @@ class CreateisoThreadTest(helpers.PungiTestCase): cmd["cmd"], channel=None, mounts=[self.topdir], - packages=["coreutils", "genisoimage", "isomd5sum", "jigdo"], + packages=["coreutils", "genisoimage", "isomd5sum"], use_shell=True, weight=None, ) @@ -750,7 +750,6 @@ class CreateisoThreadTest(helpers.PungiTestCase): "coreutils", "genisoimage", "isomd5sum", - "jigdo", "lorax", "which", ], diff --git a/tests/test_extra_isos_phase.py b/tests/test_extra_isos_phase.py index 7b7c7346..9dcd75e1 100644 --- a/tests/test_extra_isos_phase.py +++ b/tests/test_extra_isos_phase.py @@ -148,7 +148,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase): log_file=os.path.join( self.topdir, "logs/x86_64/extraiso-my.iso.x86_64.log" ), - with_jigdo=True, + with_jigdo=False, ) ], ) @@ -298,7 +298,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase): log_file=os.path.join( self.topdir, "logs/x86_64/extraiso-my.iso.x86_64.log" ), - with_jigdo=True, + with_jigdo=False, ) ], ) @@ -374,7 +374,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase): log_file=os.path.join( self.topdir, "logs/x86_64/extraiso-my.iso.x86_64.log" ), - with_jigdo=True, + with_jigdo=False, ) ], ) @@ -445,7 +445,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase): log_file=os.path.join( self.topdir, "logs/src/extraiso-my.iso.src.log" ), - with_jigdo=True, + with_jigdo=False, ) ], ) From 80bd2543474a7cf6d28256c22323cbefeeaf28d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 4 Nov 2021 10:17:02 +0100 Subject: [PATCH 082/137] Check dependencies after config validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This way the checks can rely on default values from the config. Signed-off-by: Lubomír Sedlář --- pungi/checks.py | 4 ++-- pungi/scripts/pungi_koji.py | 5 +++-- tests/test_checks.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pungi/checks.py b/pungi/checks.py index b1ecf61b..329c94fc 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -53,7 +53,7 @@ from . import util def is_jigdo_needed(conf): - return conf.get("create_jigdo", False) + return conf.get("create_jigdo") def is_isohybrid_needed(conf): @@ -93,7 +93,7 @@ def is_xorrisofs_needed(conf): def is_createrepo_c_needed(conf): - return conf.get("createrepo_c", True) + return conf.get("createrepo_c") # The first element in the tuple is package name expected to have the diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index e393220d..b8669d8d 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -265,8 +265,6 @@ def main(): # check if all requirements are met import pungi.checks - if not pungi.checks.check(conf): - sys.exit(1) pungi.checks.check_umask(logger) if not pungi.checks.check_skip_phases( logger, opts.skip_phase + conf.get("skip_phases", []), opts.just_phase @@ -297,6 +295,9 @@ def main(): fail_to_start("Config validation failed", errors=errors) sys.exit(1) + if not pungi.checks.check(conf): + sys.exit(1) + if opts.target_dir: compose_dir = Compose.get_compose_dir( opts.target_dir, conf, compose_type=compose_type, compose_label=opts.label diff --git a/tests/test_checks.py b/tests/test_checks.py index 788f727a..9d53b119 100644 --- a/tests/test_checks.py +++ b/tests/test_checks.py @@ -147,7 +147,7 @@ class CheckDependenciesTestCase(unittest.TestCase): with mock.patch("sys.stdout", new_callable=StringIO) as out: with mock.patch("os.path.exists") as exists: exists.side_effect = self.dont_find(["/usr/bin/createrepo_c"]) - result = checks.check({}) + result = checks.check({"createrepo_c": True}) self.assertIn("createrepo_c", out.getvalue()) self.assertFalse(result) From 1d654522bef012ded28ea0951aee299b097e1135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 4 Nov 2021 10:21:07 +0100 Subject: [PATCH 083/137] Remove with_jigdo argument MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It was checked in a condition together with the configuration value, and only ever explicitly used with the same value. Signed-off-by: Lubomír Sedlář --- pungi/phases/createiso.py | 6 ++---- pungi/phases/extra_isos.py | 1 - tests/test_extra_isos_phase.py | 5 ----- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py index 895c588c..d73ec056 100644 --- a/pungi/phases/createiso.py +++ b/pungi/phases/createiso.py @@ -517,15 +517,13 @@ def add_iso_to_metadata( return img -def run_createiso_command( - num, compose, bootable, arch, cmd, mounts, log_file, with_jigdo=False -): +def run_createiso_command(num, compose, bootable, arch, cmd, mounts, log_file): packages = [ "coreutils", "xorriso" if compose.conf.get("createiso_use_xorrisofs") else "genisoimage", "isomd5sum", ] - if with_jigdo and compose.conf["create_jigdo"]: + if compose.conf["create_jigdo"]: packages.append("jigdo") if bootable: extra_packages = { diff --git a/pungi/phases/extra_isos.py b/pungi/phases/extra_isos.py index 31139159..dd0c65fb 100644 --- a/pungi/phases/extra_isos.py +++ b/pungi/phases/extra_isos.py @@ -164,7 +164,6 @@ class ExtraIsosThread(WorkerThread): log_file=compose.paths.log.log_file( arch, "extraiso-%s" % os.path.basename(iso_path) ), - with_jigdo=compose.conf["create_jigdo"], ) img = add_iso_to_metadata( diff --git a/tests/test_extra_isos_phase.py b/tests/test_extra_isos_phase.py index 9dcd75e1..3e51c266 100644 --- a/tests/test_extra_isos_phase.py +++ b/tests/test_extra_isos_phase.py @@ -148,7 +148,6 @@ class ExtraIsosThreadTest(helpers.PungiTestCase): log_file=os.path.join( self.topdir, "logs/x86_64/extraiso-my.iso.x86_64.log" ), - with_jigdo=False, ) ], ) @@ -224,7 +223,6 @@ class ExtraIsosThreadTest(helpers.PungiTestCase): log_file=os.path.join( self.topdir, "logs/x86_64/extraiso-my.iso.x86_64.log" ), - with_jigdo=False, ) ], ) @@ -298,7 +296,6 @@ class ExtraIsosThreadTest(helpers.PungiTestCase): log_file=os.path.join( self.topdir, "logs/x86_64/extraiso-my.iso.x86_64.log" ), - with_jigdo=False, ) ], ) @@ -374,7 +371,6 @@ class ExtraIsosThreadTest(helpers.PungiTestCase): log_file=os.path.join( self.topdir, "logs/x86_64/extraiso-my.iso.x86_64.log" ), - with_jigdo=False, ) ], ) @@ -445,7 +441,6 @@ class ExtraIsosThreadTest(helpers.PungiTestCase): log_file=os.path.join( self.topdir, "logs/src/extraiso-my.iso.src.log" ), - with_jigdo=False, ) ], ) From 9bae86a51e7df0ffecd46f35300f8677cbcaf72d Mon Sep 17 00:00:00 2001 From: Ken Dreyer Date: Thu, 4 Nov 2021 10:55:54 -0400 Subject: [PATCH 084/137] doc: make dnf "backend" settings easier to discover Mention the corresponding "gather" or "repoclosure" backend settings in the documentation for each setting. Signed-off-by: Ken Dreyer --- doc/configuration.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/configuration.rst b/doc/configuration.rst index f698821f..65226de2 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -182,6 +182,8 @@ Options Please note that when ``dnf`` is used, the build dependencies check is skipped. On Python 3, only ``dnf`` backend is available. + See also: the ``gather_backend`` setting for Pungi's gather phase. + **cts_url** (*str*) -- URL to Compose Tracking Service. If defined, Pungi will add the compose to Compose Tracking Service and ge the compose ID from it. @@ -781,6 +783,9 @@ Options ``python-multilib`` library. Please refer to ``multilib`` option to see the differences. + See also: the ``repoclosure_backend`` setting for Pungi's repoclosure + phase. + **multilib** (*list*) -- mapping of variant regexes and arches to list of multilib methods From 33d7290d780ab976f6d7a448778be534fdec5145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 10 Nov 2021 10:51:43 +0100 Subject: [PATCH 085/137] gather: Stop requiring all variants/arches in JSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The JSON source file should not require a mapping for all variants/architectures. When something is specified, it should be included. Signed-off-by: Lubomír Sedlář --- pungi/phases/gather/sources/source_json.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pungi/phases/gather/sources/source_json.py b/pungi/phases/gather/sources/source_json.py index 2be88eb0..819ba25a 100644 --- a/pungi/phases/gather/sources/source_json.py +++ b/pungi/phases/gather/sources/source_json.py @@ -48,12 +48,14 @@ class GatherSourceJson(pungi.phases.gather.source.GatherSourceBase): if variant is None: # get all packages for all variants for variant_uid in mapping: - for pkg_name, pkg_arches in mapping[variant_uid][arch].items(): + for pkg_name, pkg_arches in mapping[variant_uid].get(arch, {}).items(): for pkg_arch in pkg_arches: packages.add((pkg_name, pkg_arch)) else: # get packages for a particular variant - for pkg_name, pkg_arches in mapping[variant.uid][arch].items(): + for pkg_name, pkg_arches in ( + mapping.get(variant.uid, {}).get(arch, {}).items() + ): for pkg_arch in pkg_arches: packages.add((pkg_name, pkg_arch)) return packages, set() From b652119d54faea7bf25a9c12f7178fb76d16193b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 10 Nov 2021 10:59:16 +0100 Subject: [PATCH 086/137] gather: Load JSON mapping relative to config dir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JIRA: RHELCMP-7195 Signed-off-by: Lubomír Sedlář --- doc/configuration.rst | 3 ++- pungi/phases/gather/sources/source_json.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 65226de2..3c9f66d1 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -911,7 +911,8 @@ Options **gather_source_mapping** (*str*) -- JSON mapping with initial packages for the compose. The value should be a path to JSON file with following mapping: ``{variant: {arch: - {rpm_name: [rpm_arch|None]}}}``. + {rpm_name: [rpm_arch|None]}}}``. Relative paths are interpreted relative to + the location of main config file. **gather_profiler** = False (*bool*) -- When set to ``True`` the gather tool will produce additional diff --git a/pungi/phases/gather/sources/source_json.py b/pungi/phases/gather/sources/source_json.py index 819ba25a..b336f9b5 100644 --- a/pungi/phases/gather/sources/source_json.py +++ b/pungi/phases/gather/sources/source_json.py @@ -32,6 +32,7 @@ set([(rpm_name, rpm_arch or None)]) import json +import os import pungi.phases.gather.source @@ -41,7 +42,7 @@ class GatherSourceJson(pungi.phases.gather.source.GatherSourceBase): json_path = self.compose.conf.get("gather_source_mapping") if not json_path: return set(), set() - with open(json_path, "r") as f: + with open(os.path.join(self.compose.config_dir, json_path), "r") as f: mapping = json.load(f) packages = set() From cfb9882269713997843224c06496220b92e11da6 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Thu, 11 Nov 2021 11:55:56 +0800 Subject: [PATCH 087/137] 4.3.2 release JIRA: RHELCMP-7182 Signed-off-by: Haibo Lin --- doc/conf.py | 4 ++-- pungi.spec | 16 +++++++++++++++- setup.py | 2 +- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 3b34418a..9bc4aa8f 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -51,9 +51,9 @@ copyright = u'2016, Red Hat, Inc.' # built documents. # # The short X.Y version. -version = '4.2' +version = '4.3' # The full version, including alpha/beta/rc tags. -release = '4.3.1' +release = '4.3.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pungi.spec b/pungi.spec index bcd6d86a..8e22fccd 100644 --- a/pungi.spec +++ b/pungi.spec @@ -1,5 +1,5 @@ Name: pungi -Version: 4.3.1 +Version: 4.3.2 Release: 1%{?dist} Summary: Distribution compose tool @@ -111,6 +111,20 @@ pytest cd tests && ./test_compose.sh %changelog +* Thu Nov 11 2021 Haibo Lin - 4.3.2-1 +- gather: Load JSON mapping relative to config dir (lsedlar) +- gather: Stop requiring all variants/arches in JSON (lsedlar) +- doc: make dnf "backend" settings easier to discover (kdreyer) +- Remove with_jigdo argument (lsedlar) +- Check dependencies after config validation (lsedlar) +- default "with_jigdo" to False (kdreyer) +- Stop trying to validate non-existent metadata (lsedlar) +- test images for metadata deserialization error (fdipretre) +- repoclosure: Use --forcearch for dnf repoclosure (lsedlar) +- extra_isos: Allow reusing old images (lsedlar) +- createiso: Allow reusing old images (lsedlar) +- Remove default runroot channel (lsedlar) + * Mon Oct 25 2021 Ozan Unsal - 4.3.1-1 - Correct irc network name & add matrix room (dan.cermak) - Add missing mock to osbs tests (lsedlar) diff --git a/setup.py b/setup.py index 37212b92..9ad49aed 100755 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ packages = sorted(packages) setup( name="pungi", - version="4.3.1", + version="4.3.2", description="Distribution compose tool", url="https://pagure.io/pungi", author="Dennis Gilmore", From f681956cf139411c963a16189e247c4c6b97e623 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Fri, 12 Nov 2021 17:01:46 +0800 Subject: [PATCH 088/137] Fix tests for python 2.6 It failed to build RHEL 6 package as logging.NullHandler does not exist in python 2.6 JIRA: RHELCMP-7188 Signed-off-by: Haibo Lin --- tests/test_createiso_phase.py | 2 +- tests/test_extra_isos_phase.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_createiso_phase.py b/tests/test_createiso_phase.py index 937f2137..02ccd8be 100644 --- a/tests/test_createiso_phase.py +++ b/tests/test_createiso_phase.py @@ -1329,7 +1329,7 @@ class CreateisoTryReusePhaseTest(helpers.PungiTestCase): super(CreateisoTryReusePhaseTest, self).setUp() self.logger = logging.getLogger() self.logger.setLevel(logging.DEBUG) - self.logger.addHandler(logging.NullHandler()) + self.logger.addHandler(logging.StreamHandler(os.devnull)) def test_disabled(self): compose = helpers.DummyCompose(self.topdir, {"createiso_allow_reuse": False}) diff --git a/tests/test_extra_isos_phase.py b/tests/test_extra_isos_phase.py index 3e51c266..fab8aee6 100644 --- a/tests/test_extra_isos_phase.py +++ b/tests/test_extra_isos_phase.py @@ -1064,7 +1064,7 @@ class ExtraisoTryReusePhaseTest(helpers.PungiTestCase): super(ExtraisoTryReusePhaseTest, self).setUp() self.logger = logging.getLogger() self.logger.setLevel(logging.DEBUG) - self.logger.addHandler(logging.NullHandler()) + self.logger.addHandler(logging.StreamHandler(os.devnull)) def test_disabled(self): compose = helpers.DummyCompose(self.topdir, {"extraiso_allow_reuse": False}) From 5e6248e3e0222af2f4c0cf564aad584fabf093f8 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Mon, 15 Nov 2021 14:38:53 +0800 Subject: [PATCH 089/137] Generate images.json for extra_isos phase JIRA: RHELCMP-7241 Signed-off-by: Haibo Lin --- pungi/scripts/pungi_koji.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index b8669d8d..97db2263 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -553,6 +553,7 @@ def run_compose( buildinstall_phase.skip() and ostree_installer_phase.skip() and createiso_phase.skip() + and extra_isos_phase.skip() and liveimages_phase.skip() and livemedia_phase.skip() and image_build_phase.skip() From 20c2e59218a87f1dfe8496334352b47e4c0eec39 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Fri, 26 Nov 2021 12:03:03 +0800 Subject: [PATCH 090/137] Pass compose parameter for debugging git issue With this param, get_dir_from_scm will try to copy the tmp git dir to compose target dir when error occurs. This does not fix the issue but it would be helpful for debugging when it occurs again. JIRA: RHELCMP-7244 Signed-off-by: Haibo Lin --- pungi/phases/createrepo.py | 3 ++- tests/test_createrepophase.py | 30 ++++++++++++++++++++++++------ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index c299169f..3df030a4 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -81,6 +81,7 @@ class CreaterepoPhase(PhaseBase): get_dir_from_scm( self.compose.conf["createrepo_extra_modulemd"][variant.uid], self.compose.paths.work.tmp_dir(variant=variant, create_dir=False), + compose=self.compose, ) self.pool.queue_put((self.compose, None, variant, "srpm")) @@ -363,7 +364,7 @@ def get_productids_from_scm(compose): tmp_dir = compose.mkdtemp(prefix="pungi_") try: - get_dir_from_scm(product_id, tmp_dir) + get_dir_from_scm(product_id, tmp_dir, compose=compose) except OSError as e: if e.errno == errno.ENOENT and product_id_allow_missing: compose.log_warning("No product IDs in %s" % product_id) diff --git a/tests/test_createrepophase.py b/tests/test_createrepophase.py index 45c2c25c..aecff998 100644 --- a/tests/test_createrepophase.py +++ b/tests/test_createrepophase.py @@ -141,7 +141,13 @@ class TestCreaterepoPhase(PungiTestCase): self.assertEqual( get_dir_from_scm.call_args_list, - [mock.call(scm, os.path.join(compose.topdir, "work/global/tmp-Server"))], + [ + mock.call( + scm, + os.path.join(compose.topdir, "work/global/tmp-Server"), + compose=compose, + ) + ], ) @@ -1333,7 +1339,7 @@ class TestCreateVariantRepo(PungiTestCase): class TestGetProductIds(PungiTestCase): def mock_get(self, filenames): - def _mock_get(scm, dest): + def _mock_get(scm, dest, compose=None): for filename in filenames: touch(os.path.join(dest, filename)) @@ -1379,7 +1385,10 @@ class TestGetProductIds(PungiTestCase): get_productids_from_scm(self.compose) - self.assertEqual(get_dir_from_scm.call_args_list, [mock.call(cfg, mock.ANY)]) + self.assertEqual( + get_dir_from_scm.call_args_list, + [mock.call(cfg, mock.ANY, compose=self.compose)], + ) self.assertProductIds( { "Client": ["amd64"], @@ -1400,7 +1409,10 @@ class TestGetProductIds(PungiTestCase): get_productids_from_scm(self.compose) - self.assertEqual(get_dir_from_scm.call_args_list, [mock.call(cfg, mock.ANY)]) + self.assertEqual( + get_dir_from_scm.call_args_list, + [mock.call(cfg, mock.ANY, compose=self.compose)], + ) self.assertProductIds({"Server": ["amd64", "x86_64"]}) @mock.patch("pungi.phases.createrepo.get_dir_from_scm") @@ -1414,7 +1426,10 @@ class TestGetProductIds(PungiTestCase): with self.assertRaises(RuntimeError) as ctx: get_productids_from_scm(self.compose) - self.assertEqual(get_dir_from_scm.call_args_list, [mock.call(cfg, mock.ANY)]) + self.assertEqual( + get_dir_from_scm.call_args_list, + [mock.call(cfg, mock.ANY, compose=self.compose)], + ) self.assertRegex( str(ctx.exception), r"No product certificate found \(arch: amd64, variant: (Everything|Client)\)", # noqa: E501 @@ -1438,5 +1453,8 @@ class TestGetProductIds(PungiTestCase): with self.assertRaises(RuntimeError) as ctx: get_productids_from_scm(self.compose) - self.assertEqual(get_dir_from_scm.call_args_list, [mock.call(cfg, mock.ANY)]) + self.assertEqual( + get_dir_from_scm.call_args_list, + [mock.call(cfg, mock.ANY, compose=self.compose)], + ) self.assertRegex(str(ctx.exception), "Multiple product certificates found.+") From 260b3fce8d16a126bc176827498fdf16e380a441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 15 Dec 2021 07:53:09 +0100 Subject: [PATCH 091/137] compose: Make sure temporary dirs are world readable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the temporary directory is created with 0700, other programs (potentially on another host) will have problems reading it. Signed-off-by: Lubomír Sedlář JIRA: RHELCMP-7635 --- pungi/compose.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pungi/compose.py b/pungi/compose.py index f02df9d3..47066807 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -607,7 +607,9 @@ class Compose(kobo.log.LoggingBase): /work/{global,}/tmp[-]/ """ path = os.path.join(self.paths.work.tmp_dir(arch=arch, variant=variant)) - return tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=path) + tmpdir = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=path) + os.chmod(tmpdir, 0o755) + return tmpdir def dump_containers_metadata(self): """Create a file with container metadata if there are any containers.""" From 894cce6a5a2d964bc33e978170e65ce7e30fe103 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Tue, 4 Jan 2022 16:11:05 +0800 Subject: [PATCH 092/137] Ignore osbs/osbuild config when reusing iso images JIRA: RHELCMP-7562 Signed-off-by: Haibo Lin --- pungi/phases/createiso.py | 2 ++ pungi/phases/extra_isos.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py index d73ec056..340edc9b 100644 --- a/pungi/phases/createiso.py +++ b/pungi/phases/createiso.py @@ -195,6 +195,8 @@ class CreateisoPhase(PhaseLoggerMixin, PhaseBase): "pkgset_koji_module_builds", ] ) + # Skip irrelevant options + config_whitelist.update(["osbs", "osbuild"]) if opt in config_whitelist: continue diff --git a/pungi/phases/extra_isos.py b/pungi/phases/extra_isos.py index dd0c65fb..56c5f28f 100644 --- a/pungi/phases/extra_isos.py +++ b/pungi/phases/extra_isos.py @@ -216,6 +216,8 @@ class ExtraIsosThread(WorkerThread): "pkgset_koji_module_builds", ] ) + # Skip irrelevant options + config_whitelist.update(["osbs", "osbuild"]) if opt in config_whitelist: continue From 42f668d96911e9bcac4c7e24c9fd9b0d5731e238 Mon Sep 17 00:00:00 2001 From: Ozan Unsal Date: Tue, 4 Jan 2022 12:00:24 +0100 Subject: [PATCH 093/137] buildinstall: Add ability to install extra packages in runroot Resolves: https://pagure.io/pungi/issue/1461 Merges: https://pagure.io/pungi/pull-request/1580 JIRA: RHELCMP-2911 Signed-off-by: Ozan Unsal --- doc/configuration.rst | 12 ++++++++++++ pungi/checks.py | 4 ++++ pungi/phases/buildinstall.py | 4 +++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 3c9f66d1..4aad6dba 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -672,6 +672,11 @@ Options **buildinstall_allow_reuse** = False (*bool*) -- When set to ``True``, *Pungi* will try to reuse buildinstall results from old compose specified by ``--old-composes``. +**buildinstall_packages** + (list) – Additional packages to be installed in the runroot environment + where lorax will run to create installer. Format: ``[(variant_uid_regex, + {arch|*: [package_globs]})]``. + Example ------- @@ -706,6 +711,13 @@ Example }) ] + # Additional packages to be installed in the Koji runroot environment where + # lorax will run. + buildinstall_packages = [ + ('^Simple$', { + '*': ['dummy-package'], + }) + ] .. note:: diff --git a/pungi/checks.py b/pungi/checks.py index 329c94fc..30114110 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -785,6 +785,10 @@ def make_schema(): "buildinstall_kickstart": {"$ref": "#/definitions/str_or_scm_dict"}, "buildinstall_use_guestmount": {"type": "boolean", "default": True}, "buildinstall_skip": _variant_arch_mapping({"type": "boolean"}), + "buildinstall_packages": { + "$ref": "#/definitions/package_mapping", + "default": [], + }, "global_ksurl": {"type": "url"}, "global_version": {"type": "string"}, "global_target": {"type": "string"}, diff --git a/pungi/phases/buildinstall.py b/pungi/phases/buildinstall.py index 9f34423c..f175bec1 100644 --- a/pungi/phases/buildinstall.py +++ b/pungi/phases/buildinstall.py @@ -809,7 +809,9 @@ class BuildinstallThread(WorkerThread): chown_paths.append(_get_log_dir(compose, variant, arch)) elif buildinstall_method == "buildinstall": packages += ["anaconda"] - + packages += get_arch_variant_data( + compose.conf, "buildinstall_packages", arch, variant + ) if self._reuse_old_buildinstall_result( compose, arch, variant, cmd, pkgset_phase ): From fe986d68b96e7b074a52bc365ca489afa22a09be Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 15 Dec 2021 10:13:23 +0100 Subject: [PATCH 094/137] Add module obsoletes feature JIRA: MODULAR-113 Merges: https://pagure.io/pungi/pull-request/1578 Signed-off-by: Filip Valder --- .gitignore | 2 ++ doc/examples.rst | 10 +++++- pungi/checks.py | 1 + pungi/compose.py | 4 +++ pungi/module_util.py | 26 +++++++++++++-- pungi/paths.py | 10 ++++++ pungi/phases/createrepo.py | 5 ++- pungi/phases/gather/__init__.py | 8 ++++- pungi/phases/init.py | 59 +++++++++++++++++++++++++-------- pungi/phases/pkgset/common.py | 9 ++++- tests/test_initphase.py | 14 +++++--- 11 files changed, 124 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 101ccee0..6c20baf6 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,5 @@ htmlcov/ .idea/ .tox .venv +.kdev4/ +pungi.kdev4 diff --git a/doc/examples.rst b/doc/examples.rst index 701a5f5a..0370fa69 100644 --- a/doc/examples.rst +++ b/doc/examples.rst @@ -30,9 +30,17 @@ This is a shortened configuration for Fedora Radhide compose as of 2019-10-14. module_defaults_dir = { 'scm': 'git', 'repo': 'https://pagure.io/releng/fedora-module-defaults.git', - 'branch': 'master', + 'branch': 'main', 'dir': '.' } + # Optional module obsoletes configuration which is merged + # into the module index and gets resolved + module_obsoletes_dir = { + 'scm': 'git', + 'repo': 'https://pagure.io/releng/fedora-module-defaults.git', + 'branch': 'main', + 'dir': 'obsoletes' + } variants_file='variants-fedora.xml' sigkeys = ['12C944D0'] diff --git a/pungi/checks.py b/pungi/checks.py index 30114110..81c11699 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -736,6 +736,7 @@ def make_schema(): "patternProperties": {".+": {"$ref": "#/definitions/strings"}}, "additionalProperties": False, }, + "module_obsoletes_dir": {"$ref": "#/definitions/str_or_scm_dict"}, "create_optional_isos": {"type": "boolean", "default": False}, "symlink_isos_to": {"type": "string"}, "dogpile_cache_backend": {"type": "string"}, diff --git a/pungi/compose.py b/pungi/compose.py index 47066807..ce0fdb74 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -377,6 +377,10 @@ class Compose(kobo.log.LoggingBase): def has_module_defaults(self): return bool(self.conf.get("module_defaults_dir", False)) + @property + def has_module_obsoletes(self): + return bool(self.conf.get("module_obsoletes_dir", False)) + @property def config_dir(self): return os.path.dirname(self.conf._open_file or "") diff --git a/pungi/module_util.py b/pungi/module_util.py index ab29a67c..94a0ec5a 100644 --- a/pungi/module_util.py +++ b/pungi/module_util.py @@ -25,9 +25,10 @@ except (ImportError, ValueError): Modulemd = None -def iter_module_defaults(path): +def iter_module_defaults_or_obsoletes(path, obsoletes=False): """Given a path to a directory with yaml files, yield each module default in there as a pair (module_name, ModuleDefaults instance). + The same happens for module obsoletes if the obsoletes switch is True. """ # It is really tempting to merge all the module indexes into a single one # and work with it. However that does not allow for detecting conflicting @@ -41,7 +42,10 @@ def iter_module_defaults(path): index = Modulemd.ModuleIndex() index.update_from_file(file, strict=False) for module_name in index.get_module_names(): - yield module_name, index.get_module(module_name).get_defaults() + if obsoletes: + yield module_name, index.get_module(module_name).get_obsoletes() + else: + yield module_name, index.get_module(module_name).get_defaults() def collect_module_defaults( @@ -69,3 +73,21 @@ def collect_module_defaults( mod_index.add_defaults(defaults) return mod_index + + +def collect_module_obsoletes(obsoletes_dir, modules_to_load, mod_index=None): + """Load module obsoletes into index. + + This works in a similar fashion as collect_module_defaults except the overrides_dir + feature. + """ + mod_index = mod_index or Modulemd.ModuleIndex() + + for module_name, obsoletes in iter_module_defaults_or_obsoletes( + obsoletes_dir, obsoletes=True + ): + for obsolete in obsoletes: + if not modules_to_load or module_name in modules_to_load: + mod_index.add_obsoletes(obsoletes) + + return mod_index diff --git a/pungi/paths.py b/pungi/paths.py index 6049b667..aff17a93 100644 --- a/pungi/paths.py +++ b/pungi/paths.py @@ -509,6 +509,16 @@ class WorkPaths(object): makedirs(path) return path + def module_obsoletes_dir(self, create_dir=True): + """ + Example: + work/global/module_obsoletes + """ + path = os.path.join(self.topdir(create_dir=create_dir), "module_obsoletes") + if create_dir: + makedirs(path) + return path + def pkgset_file_cache(self, pkgset_name): """ Returns the path to file in which the cached version of diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index 3df030a4..c170b382 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -29,7 +29,7 @@ import productmd.rpms from kobo.shortcuts import relative_path, run from kobo.threads import ThreadPool, WorkerThread -from ..module_util import Modulemd, collect_module_defaults +from ..module_util import Modulemd, collect_module_defaults, collect_module_obsoletes from ..util import ( get_arch_variant_data, read_single_module_stream_from_file, @@ -266,6 +266,9 @@ def create_variant_repo( defaults_dir, module_names, mod_index, overrides_dir=overrides_dir ) + obsoletes_dir = compose.paths.work.module_obsoletes_dir() + collect_module_obsoletes(obsoletes_dir, module_names, mod_index) + # Add extra modulemd files if variant.uid in compose.conf.get("createrepo_extra_modulemd", {}): compose.log_debug("Adding extra modulemd for %s.%s", variant.uid, arch) diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 00af2ff1..09b57287 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -33,7 +33,11 @@ except ImportError: import pungi.wrappers.kojiwrapper from pungi.arch import get_compatible_arches, split_name_arch from pungi.compose import get_ordered_variant_uids -from pungi.module_util import Modulemd, collect_module_defaults +from pungi.module_util import ( + Modulemd, + collect_module_defaults, + collect_module_obsoletes, +) from pungi.phases.base import PhaseBase from pungi.phases.createrepo import add_modular_metadata from pungi.util import get_arch_data, get_arch_variant_data, get_variant_data, makedirs @@ -698,6 +702,8 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None): collect_module_defaults( defaults_dir, module_names, mod_index, overrides_dir=overrides_dir ) + obsoletes_dir = compose.paths.work.module_obsoletes_dir() + collect_module_obsoletes(obsoletes_dir, module_names, mod_index) log_file = compose.paths.log.log_file( arch, "lookaside_repo_modules_%s" % (variant.uid) diff --git a/pungi/phases/init.py b/pungi/phases/init.py index f0590128..30c8a742 100644 --- a/pungi/phases/init.py +++ b/pungi/phases/init.py @@ -24,7 +24,7 @@ from kobo.threads import run_in_threads from pungi.phases.base import PhaseBase from pungi.phases.gather import write_prepopulate_file from pungi.util import temp_dir -from pungi.module_util import iter_module_defaults +from pungi.module_util import iter_module_defaults_or_obsoletes from pungi.wrappers.comps import CompsWrapper from pungi.wrappers.createrepo import CreaterepoWrapper from pungi.wrappers.scm import get_dir_from_scm, get_file_from_scm @@ -68,10 +68,18 @@ class InitPhase(PhaseBase): # download module defaults if self.compose.has_module_defaults: write_module_defaults(self.compose) - validate_module_defaults( + validate_module_defaults_or_obsoletes( self.compose.paths.work.module_defaults_dir(create_dir=False) ) + # download module obsoletes + if self.compose.has_module_obsoletes: + write_module_obsoletes(self.compose) + validate_module_defaults_or_obsoletes( + self.compose.paths.work.module_obsoletes_dir(create_dir=False), + obsoletes=True, + ) + # write prepopulate file write_prepopulate_file(self.compose) @@ -218,28 +226,53 @@ def write_module_defaults(compose): ) -def validate_module_defaults(path): +def write_module_obsoletes(compose): + scm_dict = compose.conf["module_obsoletes_dir"] + if isinstance(scm_dict, dict): + if scm_dict["scm"] == "file": + scm_dict["dir"] = os.path.join(compose.config_dir, scm_dict["dir"]) + else: + scm_dict = os.path.join(compose.config_dir, scm_dict) + + with temp_dir(prefix="moduleobsoletes_") as tmp_dir: + get_dir_from_scm(scm_dict, tmp_dir, compose=compose) + compose.log_debug("Writing module obsoletes") + shutil.copytree( + tmp_dir, + compose.paths.work.module_obsoletes_dir(create_dir=False), + ignore=shutil.ignore_patterns(".git"), + ) + + +def validate_module_defaults_or_obsoletes(path, obsoletes=False): """Make sure there are no conflicting defaults. Each module name can only - have one default stream. + have one default stream or module obsolete. - :param str path: directory with cloned module defaults + :param str path: directory with cloned module defaults/obsoletes """ - seen_defaults = collections.defaultdict(set) + seen = collections.defaultdict(set) + mmd_type = "obsoletes" if obsoletes else "defaults" - for module_name, defaults in iter_module_defaults(path): - seen_defaults[module_name].add(defaults.get_default_stream()) + for module_name, defaults_or_obsoletes in iter_module_defaults_or_obsoletes( + path, obsoletes + ): + if obsoletes: + for obsolete in defaults_or_obsoletes: + seen[obsolete.props.module_name].add(obsolete) + else: + seen[module_name].add(defaults_or_obsoletes.get_default_stream()) errors = [] - for module_name, defaults in seen_defaults.items(): - if len(defaults) > 1: + for module_name, defaults_or_obsoletes in seen.items(): + if len(defaults_or_obsoletes) > 1: errors.append( - "Module %s has multiple defaults: %s" - % (module_name, ", ".join(sorted(defaults))) + "Module %s has multiple %s: %s" + % (module_name, mmd_type, ", ".join(sorted(defaults_or_obsoletes))) ) if errors: raise RuntimeError( - "There are duplicated module defaults:\n%s" % "\n".join(errors) + "There are duplicated module %s:\n%s" % (mmd_type, "\n".join(errors)) ) diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py index f2a38457..14dad789 100644 --- a/pungi/phases/pkgset/common.py +++ b/pungi/phases/pkgset/common.py @@ -28,7 +28,11 @@ from pungi.util import ( PartialFuncWorkerThread, PartialFuncThreadPool, ) -from pungi.module_util import Modulemd, collect_module_defaults +from pungi.module_util import ( + Modulemd, + collect_module_defaults, + collect_module_obsoletes, +) from pungi.phases.createrepo import add_modular_metadata @@ -159,6 +163,9 @@ def _create_arch_repo(worker_thread, args, task_num): mod_index = collect_module_defaults( compose.paths.work.module_defaults_dir(), names, overrides_dir=overrides_dir ) + mod_index = collect_module_obsoletes( + compose.paths.work.module_obsoletes_dir(), names, mod_index + ) for x in mmd: mod_index.add_module_stream(x) add_modular_metadata( diff --git a/tests/test_initphase.py b/tests/test_initphase.py index afd2601e..f6fad433 100644 --- a/tests/test_initphase.py +++ b/tests/test_initphase.py @@ -24,7 +24,7 @@ from tests.helpers import ( @mock.patch("pungi.phases.init.run_in_threads", new=fake_run_in_threads) @mock.patch("pungi.phases.init.validate_comps") -@mock.patch("pungi.phases.init.validate_module_defaults") +@mock.patch("pungi.phases.init.validate_module_defaults_or_obsoletes") @mock.patch("pungi.phases.init.write_module_defaults") @mock.patch("pungi.phases.init.write_global_comps") @mock.patch("pungi.phases.init.write_arch_comps") @@ -46,6 +46,7 @@ class TestInitPhase(PungiTestCase): compose = DummyCompose(self.topdir, {}) compose.has_comps = True compose.has_module_defaults = False + compose.has_module_obsoletes = False compose.setup_optional() phase = init.InitPhase(compose) phase.run() @@ -100,6 +101,7 @@ class TestInitPhase(PungiTestCase): compose = DummyCompose(self.topdir, {}) compose.has_comps = True compose.has_module_defaults = False + compose.has_module_obsoletes = False compose.variants["Everything"].groups = [] compose.variants["Everything"].modules = [] phase = init.InitPhase(compose) @@ -156,6 +158,7 @@ class TestInitPhase(PungiTestCase): compose = DummyCompose(self.topdir, {}) compose.has_comps = False compose.has_module_defaults = False + compose.has_module_obsoletes = False phase = init.InitPhase(compose) phase.run() @@ -182,6 +185,7 @@ class TestInitPhase(PungiTestCase): compose = DummyCompose(self.topdir, {}) compose.has_comps = False compose.has_module_defaults = True + compose.has_module_obsoletes = False phase = init.InitPhase(compose) phase.run() @@ -620,13 +624,13 @@ class TestValidateModuleDefaults(PungiTestCase): def test_valid_files(self): self._write_defaults({"httpd": ["1"], "python": ["3.6"]}) - init.validate_module_defaults(self.topdir) + init.validate_module_defaults_or_obsoletes(self.topdir) def test_duplicated_stream(self): self._write_defaults({"httpd": ["1"], "python": ["3.6", "3.5"]}) with self.assertRaises(RuntimeError) as ctx: - init.validate_module_defaults(self.topdir) + init.validate_module_defaults_or_obsoletes(self.topdir) self.assertIn( "Module python has multiple defaults: 3.5, 3.6", str(ctx.exception) @@ -636,7 +640,7 @@ class TestValidateModuleDefaults(PungiTestCase): self._write_defaults({"httpd": ["1", "2"], "python": ["3.6", "3.5"]}) with self.assertRaises(RuntimeError) as ctx: - init.validate_module_defaults(self.topdir) + init.validate_module_defaults_or_obsoletes(self.topdir) self.assertIn("Module httpd has multiple defaults: 1, 2", str(ctx.exception)) self.assertIn( @@ -661,7 +665,7 @@ class TestValidateModuleDefaults(PungiTestCase): ), ) - init.validate_module_defaults(self.topdir) + init.validate_module_defaults_or_obsoletes(self.topdir) @mock.patch("pungi.phases.init.CompsWrapper") From 32221e8f363d26cedbb5c0fa50f0ee970c25d769 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 9 Oct 2020 11:46:15 +0200 Subject: [PATCH 095/137] hybrid: Explicitly pull in debugsource packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This should cover case where we there's a build like this: foo-1-1.src.rpm foo-sub-1-1.noarch.rpm foo-debugsource-1-1.x86_64.rpm The compose contains the noarch package, and should also have the debugsource package. The original code only checked for foo-sub-debugsource though. JIRA: RHELCMP-7628 Signed-off-by: Lubomír Sedlář --- pungi/phases/gather/methods/method_hybrid.py | 7 +++- tests/test_gather_method_hybrid.py | 43 +++++++++++++++----- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/pungi/phases/gather/methods/method_hybrid.py b/pungi/phases/gather/methods/method_hybrid.py index 3aa60cbd..32fa85e8 100644 --- a/pungi/phases/gather/methods/method_hybrid.py +++ b/pungi/phases/gather/methods/method_hybrid.py @@ -349,8 +349,11 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): # There are two ways how the debuginfo package can be named. We # want to get them all. - for pattern in ["%s-debuginfo", "%s-debugsource"]: - debuginfo_name = pattern % pkg.name + source_name = kobo.rpmlib.parse_nvra(pkg.rpm_sourcerpm)["name"] + for debuginfo_name in [ + "%s-debuginfo" % pkg.name, + "%s-debugsource" % source_name, + ]: debuginfo = self._get_debuginfo(debuginfo_name, pkg_arch) for dbg in debuginfo: # For each debuginfo package that matches on name and diff --git a/tests/test_gather_method_hybrid.py b/tests/test_gather_method_hybrid.py index bc0a283b..b053f229 100644 --- a/tests/test_gather_method_hybrid.py +++ b/tests/test_gather_method_hybrid.py @@ -391,7 +391,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): ), } po.return_value = ([("p-1-1", "x86_64", frozenset())], ["m1"]) - self.phase.packages = {"p-1-1.x86_64": mock.Mock()} + self.phase.packages = {"p-1-1.x86_64": mock.Mock(rpm_sourcerpm="p-1-1.src.rpm")} res = self.phase.run_solver( self.compose.variants["Server"], @@ -431,7 +431,9 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): ) def test_with_comps(self, run, gc, po, wc): - self.phase.packages = {"pkg-1.0-1.x86_64": mock.Mock()} + self.phase.packages = { + "pkg-1.0-1.x86_64": mock.Mock(rpm_sourcerpm="pkg-1.0-1.src.rpm") + } self.phase.debuginfo = {"x86_64": {}} po.return_value = ([("pkg-1.0-1", "x86_64", frozenset())], []) res = self.phase.run_solver( @@ -473,11 +475,23 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): ) def test_with_comps_with_debuginfo(self, run, gc, po, wc): - dbg1 = NamedMock(name="pkg-debuginfo", arch="x86_64", sourcerpm="pkg.src.rpm") - dbg2 = NamedMock(name="pkg-debuginfo", arch="x86_64", sourcerpm="x.src.rpm") + # dbg1 and dbg2 mocks both package from Kobo (with sourcerpm) and from + # createrepo_c (with rpm_sourcerpm) + dbg1 = NamedMock( + name="pkg-debuginfo", + arch="x86_64", + sourcerpm="pkg-1.0-1.src.rpm", + rpm_sourcerpm="pkg-1.0-1.src.rpm", + ) + dbg2 = NamedMock( + name="pkg-debuginfo", + arch="x86_64", + sourcerpm="pkg-1.0-2.src.rpm", + rpm_sourcerpm="pkg-1.0-2.src.rpm", + ) self.phase.packages = { "pkg-1.0-1.x86_64": NamedMock( - name="pkg", arch="x86_64", rpm_sourcerpm="pkg.src.rpm" + name="pkg", arch="x86_64", rpm_sourcerpm="pkg-1.0-1.src.rpm" ), "pkg-debuginfo-1.0-1.x86_64": dbg1, "pkg-debuginfo-1.0-2.x86_64": dbg2, @@ -558,8 +572,8 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): ] po.side_effect = [([("pkg-1.0-1", "x86_64", frozenset())], []), (final, [])] self.phase.packages = { - "pkg-1.0-1.x86_64": mock.Mock(), - "pkg-en-1.0-1.noarch": mock.Mock(), + "pkg-1.0-1.x86_64": mock.Mock(rpm_sourcerpm="pkg-1.0-1.src.rpm"), + "pkg-en-1.0-1.noarch": mock.Mock(rpm_sourcerpm="pkg-1.0-1.src.rpm"), } res = self.phase.run_solver( @@ -628,9 +642,15 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): cr.Metadata.return_value.keys.return_value = [] self.phase.package_maps = { "x86_64": { - "pkg-devel-1.0-1.x86_64": NamedMock(name="pkg-devel"), - "pkg-devel-1.0-1.i686": NamedMock(name="pkg-devel"), - "foo-1.0-1.x86_64": NamedMock(name="foo"), + "pkg-devel-1.0-1.x86_64": NamedMock( + name="pkg-devel", rpm_sourcerpm="pkg-1.0-1.src.rpm" + ), + "pkg-devel-1.0-1.i686": NamedMock( + name="pkg-devel", rpm_sourcerpm="pkg-1.0-1.src.rpm" + ), + "foo-1.0-1.x86_64": NamedMock( + name="foo", rpm_sourcerpm="foo-1.0-1.src.rpm" + ), } } self.phase.packages = self.phase.package_maps["x86_64"] @@ -718,6 +738,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): release="1", arch="x86_64", provides=[("/usr/lib/libfoo.1.so.1", None, None)], + rpm_sourcerpm="foo-1.0-1.src.rpm", ), "def": NamedMock( name="foo", @@ -726,6 +747,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): release="1", arch="i686", provides=[("/usr/lib/libfoo.1.so.1", None, None)], + rpm_sourcerpm="foo-1.0-1.src.rpm", ), "ghi": NamedMock( name="pkg-devel", @@ -734,6 +756,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): release="1", arch="x86_64", provides=[], + rpm_sourcerpm="pkg-devel-1.0-1.src.rpm", ), } cr.Metadata.return_value.keys.return_value = packages.keys() From 52c9816755e80d142efa4122af90696fbece7687 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Sat, 8 Jan 2022 22:44:01 +0800 Subject: [PATCH 096/137] 4.3.3 release JIRA: RHELCMP-7691 Signed-off-by: Haibo Lin --- doc/conf.py | 2 +- pungi.spec | 12 +++++++++++- setup.py | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 9bc4aa8f..ec267f8a 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -53,7 +53,7 @@ copyright = u'2016, Red Hat, Inc.' # The short X.Y version. version = '4.3' # The full version, including alpha/beta/rc tags. -release = '4.3.2' +release = '4.3.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pungi.spec b/pungi.spec index 8e22fccd..8882c347 100644 --- a/pungi.spec +++ b/pungi.spec @@ -1,5 +1,5 @@ Name: pungi -Version: 4.3.2 +Version: 4.3.3 Release: 1%{?dist} Summary: Distribution compose tool @@ -111,6 +111,16 @@ pytest cd tests && ./test_compose.sh %changelog +* Sat Jan 08 2022 Haibo Lin - 4.3.3-1 +- hybrid: Explicitly pull in debugsource packages (lsedlar) +- Add module obsoletes feature (fvalder) +- buildinstall: Add ability to install extra packages in runroot (ounsal) +- Ignore osbs/osbuild config when reusing iso images (hlin) +- compose: Make sure temporary dirs are world readable (lsedlar) +- Pass compose parameter for debugging git issue (hlin) +- Generate images.json for extra_isos phase (hlin) +- Fix tests for python 2.6 (hlin) + * Thu Nov 11 2021 Haibo Lin - 4.3.2-1 - gather: Load JSON mapping relative to config dir (lsedlar) - gather: Stop requiring all variants/arches in JSON (lsedlar) diff --git a/setup.py b/setup.py index 9ad49aed..3e057d0e 100755 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ packages = sorted(packages) setup( name="pungi", - version="4.3.2", + version="4.3.3", description="Distribution compose tool", url="https://pagure.io/pungi", author="Dennis Gilmore", From 330ba9b9c4b0222a19d7f964e4124176cdea2a6c Mon Sep 17 00:00:00 2001 From: Ozan Unsal Date: Wed, 19 Jan 2022 17:06:32 +0100 Subject: [PATCH 097/137] Do not clone the same repository multiple times, re-use already cloned repository Clone the directory to the compose tmp directory Update the test_scm in order to create real Compose object. Mock objects are not allowed to create/delete files for preventing multiple clones JIRA: RHELCMP-5250 Signed-off-by: Ozan Unsal --- pungi/wrappers/scm.py | 69 ++++++++++++++++++++++++++++++++++++------- tests/test_scm.py | 23 +++++++++++---- 2 files changed, 75 insertions(+), 17 deletions(-) diff --git a/pungi/wrappers/scm.py b/pungi/wrappers/scm.py index 5c4b37fb..6d79d86f 100644 --- a/pungi/wrappers/scm.py +++ b/pungi/wrappers/scm.py @@ -19,7 +19,9 @@ from __future__ import absolute_import import os import shutil import glob +import threading import six +import tempfile from six.moves import shlex_quote from six.moves.urllib.request import urlretrieve from fnmatch import fnmatch @@ -29,6 +31,8 @@ from kobo.shortcuts import run, force_list from pungi.util import explode_rpm_package, makedirs, copy_all, temp_dir, retry from .kojiwrapper import KojiWrapper +scm_lock = threading.Lock() + class ScmBase(kobo.log.LoggingBase): def __init__(self, logger=None, command=None, compose=None): @@ -372,10 +376,33 @@ def get_file_from_scm(scm_dict, target_path, compose=None): scm = _get_wrapper(scm_type, logger=logger, command=command, compose=compose) files_copied = [] - for i in force_list(scm_file): - with temp_dir(prefix="scm_checkout_") as tmp_dir: - scm.export_file(scm_repo, i, scm_branch=scm_branch, target_dir=tmp_dir) - files_copied += copy_all(tmp_dir, target_path) + branch = scm_branch if scm_branch else "master" + delete_after_flag = False + with scm_lock: + if compose and scm_repo: + repo = scm_repo.rsplit("/")[-1] + tmp_dir = compose.paths.work.tmp_dir() + tmp_dir = os.path.join(tmp_dir, repo, branch) + else: + tmp_dir = tempfile.mkdtemp(prefix="scm_checkout_") + delete_after_flag = True + + if not os.path.isdir(tmp_dir): + makedirs(tmp_dir) + + for i in force_list(scm_file): + # Check the files which are included with subdirectories + check_file = os.path.join(tmp_dir, i[i.rfind("/") + 1 :]) + if ( + type(scm_dict) is not dict + or command is not None + or not compose + or not os.path.isfile(check_file) + ): + scm.export_file(scm_repo, i, scm_branch=scm_branch, target_dir=tmp_dir) + files_copied += copy_all(tmp_dir, target_path) + if delete_after_flag: + shutil.rmtree(tmp_dir) return files_copied @@ -459,14 +486,34 @@ def get_dir_from_scm(scm_dict, target_path, compose=None): logger = compose._logger if compose else None scm = _get_wrapper(scm_type, logger=logger, command=command, compose=compose) + branch = scm_branch if scm_branch else "master" + delete_after_flag = False - with temp_dir(prefix="scm_checkout_") as tmp_dir: - scm.export_dir(scm_repo, scm_dir, scm_branch=scm_branch, target_dir=tmp_dir) + with scm_lock: + if compose and scm_repo: + repo = scm_repo.rsplit("/")[-1] + tmp_dir = compose.paths.work.tmp_dir() + tmp_dir = os.path.join(tmp_dir, repo, branch) + else: + tmp_dir = tempfile.mkdtemp(prefix="scm_checkout_") + delete_after_flag = True + + if not os.path.isdir(tmp_dir): + makedirs(tmp_dir) + scm.export_dir(scm_repo, scm_dir, scm_branch=scm_branch, target_dir=tmp_dir) + elif ( + type(scm_dict) is not dict + or command is not None + or not scm_repo + or not compose + ): + scm.export_dir(scm_repo, scm_dir, scm_branch=scm_branch, target_dir=tmp_dir) files_copied = copy_all(tmp_dir, target_path) - # Make sure the directory has permissions set to 755. This is a workaround - # for a problem where sometimes the directory will be 700 and it will not - # be accessible via httpd. - os.chmod(target_path, 0o755) - + # Make sure the directory has permissions set to 755. This is a workaround + # for a problem where sometimes the directory will be 700 and it will not + # be accessible via httpd. + os.chmod(target_path, 0o755) + if delete_after_flag: + shutil.rmtree(tmp_dir) return files_copied diff --git a/tests/test_scm.py b/tests/test_scm.py index f6307967..e45c32f4 100644 --- a/tests/test_scm.py +++ b/tests/test_scm.py @@ -16,14 +16,17 @@ import six from pungi.wrappers import scm from tests.helpers import touch from kobo.shortcuts import run +from pungi.compose import Compose class SCMBaseTest(unittest.TestCase): def setUp(self): self.destdir = tempfile.mkdtemp() + self.tmp_dir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.destdir) + shutil.rmtree(self.tmp_dir) def assertStructure(self, returned, expected): # Check we returned the correct files @@ -143,7 +146,8 @@ class GitSCMTestCase(SCMBaseTest): @mock.patch("pungi.wrappers.scm.run") def test_get_file_function(self, run): - compose = mock.Mock(conf={}) + with mock.patch("pungi.compose.ComposeInfo"): + compose = Compose({}, self.tmp_dir) def process(cmd, workdir=None, **kwargs): touch(os.path.join(workdir, "some_file.txt")) @@ -338,6 +342,8 @@ class GitSCMTestCaseReal(SCMBaseTest): shutil.rmtree(self.gitRepositoryLocation) def test_get_file_function(self): + with mock.patch("pungi.compose.ComposeInfo"): + compose = Compose({}, self.tmp_dir) sourceFileLocation = random.choice(list(self.files.keys())) sourceFilename = os.path.basename(sourceFileLocation) destinationFileLocation = os.path.join(self.destdir, "other_file.txt") @@ -348,7 +354,7 @@ class GitSCMTestCaseReal(SCMBaseTest): "file": sourceFilename, }, os.path.join(self.destdir, destinationFileLocation), - compose=self.compose, + compose=compose, ) self.assertEqual(destinationFileActualLocation, destinationFileLocation) self.assertTrue(os.path.isfile(destinationFileActualLocation)) @@ -361,6 +367,8 @@ class GitSCMTestCaseReal(SCMBaseTest): self.assertEqual(sourceFileContent, destinationFileContent) def test_get_file_function_with_overwrite(self): + with mock.patch("pungi.compose.ComposeInfo"): + compose = Compose({}, self.tmp_dir) sourceFileLocation = random.choice(list(self.files.keys())) sourceFilename = os.path.basename(sourceFileLocation) destinationFileLocation = os.path.join(self.destdir, "other_file.txt") @@ -375,7 +383,7 @@ class GitSCMTestCaseReal(SCMBaseTest): "file": sourceFilename, }, os.path.join(self.destdir, destinationFileLocation), - compose=self.compose, + compose=compose, overwrite=True, ) self.assertEqual(destinationFileActualLocation, destinationFileLocation) @@ -603,7 +611,8 @@ class KojiSCMTestCase(SCMBaseTest): @mock.patch("pungi.wrappers.scm.KojiWrapper") def test_doesnt_get_dirs(self, KW, dl): - compose = mock.Mock(conf={"koji_profile": "koji"}) + with mock.patch("pungi.compose.ComposeInfo"): + compose = Compose({"koji_profile": "koji"}, self.tmp_dir) with self.assertRaises(RuntimeError) as ctx: scm.get_dir_from_scm( @@ -628,7 +637,8 @@ class KojiSCMTestCase(SCMBaseTest): @mock.patch("pungi.wrappers.scm.KojiWrapper") def test_get_from_build(self, KW, dl): - compose = mock.Mock(conf={"koji_profile": "koji"}) + with mock.patch("pungi.compose.ComposeInfo"): + compose = Compose({"koji_profile": "koji"}, self.tmp_dir) def download(src, dst): touch(dst) @@ -659,7 +669,8 @@ class KojiSCMTestCase(SCMBaseTest): @mock.patch("pungi.wrappers.scm.KojiWrapper") def test_get_from_latest_build(self, KW, dl): - compose = mock.Mock(conf={"koji_profile": "koji"}) + with mock.patch("pungi.compose.ComposeInfo"): + compose = Compose({"koji_profile": "koji"}, self.tmp_dir) def download(src, dst): touch(dst) From 38810b3f13861aa36f832d9e8f7ac14276d181a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20P=C3=ADsa=C5=99?= Date: Fri, 4 Feb 2022 11:03:34 +0100 Subject: [PATCH 098/137] modules: Correct a typo in loading obsoletes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pungi failed: TypeError: argument obsoletes: Expected Modulemd.Obsoletes, but got list Frame collect_module_obsoletes in /usr/lib/python3.10/site-packages/pungi/module_util.py at line 91 84 mod_index = mod_index or Modulemd.ModuleIndex() 85 86 for module_name, obsoletes in iter_module_defaults_or_obsoletes( 87 obsoletes_dir, obsoletes=True 88 ): 89 for obsolete in obsoletes: 90 if not modules_to_load or module_name in modules_to_load: --> 91 mod_index.add_obsoletes(obsoletes) 92 93 return mod_index mod_index = module_name = 'perl' modules_to_load = {'perl-Date-Manip', 'subversion', 'sway', 'nginx', 'perl-YAML', 'ghc', 'perl-App-cpanminus', 'perl-XML-Parser', 'varnish', 'nodejs', 'cri-o', 'perl-DBD-Pg', 'perl-DBI', 'perl', 'swig', 'perl-FCGI', 'p obsolete = obsoletes = [] obsoletes_dir = '/mnt/koji/compose/rawhide/Fedora-Rawhide-20220203.n.1/work/global/module_obsoletes' This patches fixes the typo in add_obsoletes() argument. https://pagure.io/releng/failed-composes/issue/3058 Signed-off-by: Petr Písař --- pungi/module_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pungi/module_util.py b/pungi/module_util.py index 94a0ec5a..a25cb36d 100644 --- a/pungi/module_util.py +++ b/pungi/module_util.py @@ -88,6 +88,6 @@ def collect_module_obsoletes(obsoletes_dir, modules_to_load, mod_index=None): ): for obsolete in obsoletes: if not modules_to_load or module_name in modules_to_load: - mod_index.add_obsoletes(obsoletes) + mod_index.add_obsoletes(obsolete) return mod_index From aabf8faea07e19ed5d8c34c22a278ec7be7248e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Tue, 1 Feb 2022 12:26:49 +0100 Subject: [PATCH 099/137] profiler: Respect provided output stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lubomír Sedlář --- pungi/profiler.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pungi/profiler.py b/pungi/profiler.py index cb7488b3..3ffef8b9 100644 --- a/pungi/profiler.py +++ b/pungi/profiler.py @@ -69,10 +69,8 @@ class Profiler(object): @classmethod def print_results(cls, stream=sys.stdout): - print("Profiling results:", file=sys.stdout) + print("Profiling results:", file=stream) results = cls._data.items() results = sorted(results, key=lambda x: x[1]["time"], reverse=True) for name, data in results: - print( - " %6.2f %5d %s" % (data["time"], data["calls"], name), file=sys.stdout - ) + print(" %6.2f %5d %s" % (data["time"], data["calls"], name), file=stream) From 6c280f2c46a3e758d2f09a039ebd600bd0a23da4 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Tue, 22 Feb 2022 09:57:01 +0800 Subject: [PATCH 100/137] Filter out environment groups unmatch given arch JIRA: RHELCMP-7926 Signed-off-by: Haibo Lin --- pungi/scripts/comps_filter.py | 2 +- pungi/wrappers/comps.py | 14 ++++++++++---- tests/data/dummy-comps.xml | 2 +- tests/test_comps_wrapper.py | 8 ++++---- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/pungi/scripts/comps_filter.py b/pungi/scripts/comps_filter.py index 21ebcfff..8c07cee4 100644 --- a/pungi/scripts/comps_filter.py +++ b/pungi/scripts/comps_filter.py @@ -96,7 +96,7 @@ def main(): f.filter_environments(opts.arch, opts.variant, opts.arch_only_environments) if not opts.no_cleanup: - f.cleanup(opts.keep_empty_group, opts.lookaside_group) + f.cleanup(opts.arch, opts.keep_empty_group, opts.lookaside_group) if opts.remove_categories: f.remove_categories() diff --git a/pungi/wrappers/comps.py b/pungi/wrappers/comps.py index c9194407..589330b2 100644 --- a/pungi/wrappers/comps.py +++ b/pungi/wrappers/comps.py @@ -177,9 +177,9 @@ class CompsFilter(object): for i in self.tree.xpath("//*[@xml:lang]"): i.getparent().remove(i) - def filter_environment_groups(self, lookaside_groups=[]): + def filter_environment_groups(self, arch, lookaside_groups=[]): """ - Remove undefined groups from environments. + Remove undefined groups or groups not matching given arch from environments. """ all_groups = self.tree.xpath("/comps/group/id/text()") + lookaside_groups for environment in self.tree.xpath("/comps/environment"): @@ -187,6 +187,12 @@ class CompsFilter(object): if group.text not in all_groups: group.getparent().remove(group) + for group in environment.xpath("grouplist/groupid[@arch]"): + value = group.attrib.get("arch") + values = [v for v in re.split(r"[, ]+", value) if v] + if arch not in values: + group.getparent().remove(group) + def remove_empty_environments(self): """ Remove all environments without groups. @@ -212,7 +218,7 @@ class CompsFilter(object): ) file_obj.write(b"\n") - def cleanup(self, keep_groups=[], lookaside_groups=[]): + def cleanup(self, arch, keep_groups=[], lookaside_groups=[]): """ Remove empty groups, categories and environment from the comps file. Groups given in ``keep_groups`` will be preserved even if empty. @@ -223,7 +229,7 @@ class CompsFilter(object): self.remove_empty_groups(keep_groups) self.filter_category_groups() self.remove_empty_categories() - self.filter_environment_groups(lookaside_groups) + self.filter_environment_groups(arch, lookaside_groups) self.remove_empty_environments() diff --git a/tests/data/dummy-comps.xml b/tests/data/dummy-comps.xml index 72ed7738..3e73366b 100644 --- a/tests/data/dummy-comps.xml +++ b/tests/data/dummy-comps.xml @@ -118,7 +118,7 @@ 10 core - standard + standard basic-desktop diff --git a/tests/test_comps_wrapper.py b/tests/test_comps_wrapper.py index 47d323bf..cee03dcb 100644 --- a/tests/test_comps_wrapper.py +++ b/tests/test_comps_wrapper.py @@ -196,22 +196,22 @@ class CompsFilterTest(unittest.TestCase): self.assertOutput(os.path.join(FIXTURE_DIR, "comps-removed-environments.xml")) def test_cleanup(self): - self.filter.cleanup() + self.filter.cleanup("ppc64le") self.assertOutput(os.path.join(FIXTURE_DIR, "comps-cleanup.xml")) def test_cleanup_after_filter(self): self.filter.filter_packages("ppc64le", None) - self.filter.cleanup() + self.filter.cleanup("ppc64le") self.assertOutput(os.path.join(FIXTURE_DIR, "comps-cleanup-filter.xml")) def test_cleanup_after_filter_keep_group(self): self.filter.filter_packages("ppc64le", None) - self.filter.cleanup(["standard"]) + self.filter.cleanup("ppc64le", ["standard"]) self.assertOutput(os.path.join(FIXTURE_DIR, "comps-cleanup-keep.xml")) def test_cleanup_all(self): self.filter.filter_packages("ppc64le", None) self.filter.filter_groups("ppc64le", None) self.filter.filter_environments("ppc64le", None) - self.filter.cleanup() + self.filter.cleanup("ppc64le") self.assertOutput(os.path.join(FIXTURE_DIR, "comps-cleanup-all.xml")) From ecb164604259e309179a3c523f5dc5fc8e593718 Mon Sep 17 00:00:00 2001 From: Ozan Unsal Date: Mon, 28 Feb 2022 14:54:21 +0100 Subject: [PATCH 101/137] Fix the wrong working directory for the progress_notification script Jira: RHELCMP-7901 Signed-off-by: Ozan Unsal --- doc/messaging.rst | 5 +++-- pungi/notifier.py | 3 --- tests/test_notifier.py | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/doc/messaging.rst b/doc/messaging.rst index 291206f2..c94ef40b 100644 --- a/doc/messaging.rst +++ b/doc/messaging.rst @@ -12,8 +12,9 @@ happened. A JSON-encoded object will be passed to standard input to provide more information about the event. At the very least, the object will contain a ``compose_id`` key. -The script is invoked in compose directory and can read other information -there. +The notification script inherits working directory from the parent process and it +can be called from the same directory ``pungi-koji`` is called from. The working directory +is listed at the start of main log. Currently these messages are sent: diff --git a/pungi/notifier.py b/pungi/notifier.py index 5eed865c..bef2ae63 100644 --- a/pungi/notifier.py +++ b/pungi/notifier.py @@ -81,9 +81,6 @@ class PungiNotifier(object): self._update_args(kwargs) - if self.compose: - workdir = self.compose.paths.compose.topdir() - with self.lock: for cmd in self.cmds: self._run_script(cmd, msg, workdir, kwargs) diff --git a/tests/test_notifier.py b/tests/test_notifier.py index 445b6ff3..914d6314 100644 --- a/tests/test_notifier.py +++ b/tests/test_notifier.py @@ -73,7 +73,7 @@ class TestNotifier(unittest.TestCase): stdin_data=json.dumps(data), can_fail=True, return_stdout=False, - workdir=self.compose.paths.compose.topdir.return_value, + workdir=None, universal_newlines=True, show_cmd=True, logfile=self.logfile, From 0e82663327f5991f8d8840628d733c4f74e47309 Mon Sep 17 00:00:00 2001 From: Ozan Unsal Date: Thu, 10 Mar 2022 15:17:47 +0100 Subject: [PATCH 102/137] Update the default greedy_method value in doc JIRA: RHELCMP-6308 Signed-off-by: Ozan Unsal --- doc/configuration.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 4aad6dba..464f7d8b 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -760,7 +760,7 @@ Options (*bool*) -- When set to ``True``, *Pungi* will try to reuse gather results from old compose specified by ``--old-composes``. -**greedy_method** +**greedy_method** = none (*str*) -- This option controls how package requirements are satisfied in case a particular ``Requires`` has multiple candidates. @@ -781,7 +781,7 @@ Options pulled in. * With ``greedy_method = "all"`` all three packages will be pulled in. - * With ``greedy_method = "build" ``pkg-b-provider-1`` and + * With ``greedy_method = "build"`` ``pkg-b-provider-1`` and ``pkg-b-provider-2`` will be pulled in. **gather_backend** From b805ce3d129420beb39957a08ef307a8e27c0084 Mon Sep 17 00:00:00 2001 From: Ken Dreyer Date: Thu, 17 Mar 2022 11:04:02 -0400 Subject: [PATCH 103/137] osbs: only handle archives of type "image" Prior to this change, if a container image used Cachito with OSBS, then OSBS would store additional "remote-sources" files in the Koji archives for the build. Pungi cannot parse the metadata for these archive entries, so it would crash in add_metadata(): File "pungi/phases/osbs.py", line 81, in process self.worker(compose, variant, config) File "pungi/phases/osbs.py", line 141, in worker nvr, archive_ids = add_metadata(variant, task_id, compose, scratch) File "pungi/phases/osbs.py", line 447, in add_metadata arch = archive["extra"]["image"]["arch"] KeyError: 'image' Tell Koji to only return container image archives, and ignore these remote-source archives. Signed-off-by: Ken Dreyer --- pungi/phases/osbs.py | 2 +- tests/test_osbs_phase.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pungi/phases/osbs.py b/pungi/phases/osbs.py index 9db4371e..e5ddcb60 100644 --- a/pungi/phases/osbs.py +++ b/pungi/phases/osbs.py @@ -422,7 +422,7 @@ def add_metadata(variant, task_id, compose, is_scratch): else: build_id = int(result["koji_builds"][0]) buildinfo = koji.koji_proxy.getBuild(build_id) - archives = koji.koji_proxy.listArchives(build_id) + archives = koji.koji_proxy.listArchives(build_id, type="image") nvr = "%(name)s-%(version)s-%(release)s" % buildinfo diff --git a/tests/test_osbs_phase.py b/tests/test_osbs_phase.py index 5ad64d83..9a45dfea 100644 --- a/tests/test_osbs_phase.py +++ b/tests/test_osbs_phase.py @@ -235,7 +235,7 @@ class OSBSThreadTest(helpers.PungiTestCase): expect_calls.extend( [ mock.call.koji_proxy.getBuild(54321), - mock.call.koji_proxy.listArchives(54321), + mock.call.koji_proxy.listArchives(54321, type="image"), mock.call.koji_proxy.listRPMs(imageID=1436049), ] ) From 903ab076baab2aa724710a9a29c5ff0c9a669f5f Mon Sep 17 00:00:00 2001 From: Ken Dreyer Date: Mon, 21 Mar 2022 14:50:36 -0400 Subject: [PATCH 104/137] doc: improve osbs_registries explanation Explain the use-case for this setting, and use the active voice to explain what actions Pungi performs relative to other tools. Signed-off-by: Ken Dreyer --- doc/configuration.rst | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 464f7d8b..a199d956 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1876,15 +1876,22 @@ they are not scratch builds). ``gpgkey`` can be specified to enable gpgcheck in repo files for variants. **osbs_registries** - (*dict*) -- It is possible to configure extra information about where to - push the image (unless it is a scratch build). For each finished build, - Pungi will try to match NVR against a key in this mapping (using shell-style - globbing) and take the corresponding value and collect them across all built - images. The data will be saved into ``logs/global/osbs-registries.json`` as - a mapping from Koji NVR to the registry data. The same data is also sent to - the message bus on ``osbs-request-push`` topic once the compose finishes - successfully. Handling the message and performing the actual push is outside - of scope for Pungi. + (*dict*) -- Use this optional setting to emit ``osbs-request-push`` + messages for each non-scratch container build. These messages can guide + other tools how to push the images to other registries. For example, an + external tool might trigger on these messages and copy the images from + OSBS's registry to a staging or production registry. + + For each completed container build, Pungi will try to match the NVR against + a key in ``osbs_registries`` mapping (using shell-style globbing) and take + the corresponding value and collect them across all built images. Pungi + will save this data into ``logs/global/osbs-registries.json``, mapping each + Koji NVR to the registry data. Pungi will also send this data to the + message bus on the ``osbs-request-push`` topic once the compose finishes + successfully. + + Pungi simply logs the mapped data and emits the messages. It does not + handle the messages or push images. A separate tool must do that. Example config From d55770898c024e3c965c9290d9ba741ab41d34a6 Mon Sep 17 00:00:00 2001 From: Christopher O'Brien Date: Thu, 3 Mar 2022 13:54:12 -0500 Subject: [PATCH 105/137] nomacboot option for livemedia koji tasks Merges: https://pagure.io/pungi/pull-request/1591 Signed-off-by: Christopher O'Brien --- .gitignore | 1 + doc/configuration.rst | 1 + pungi/checks.py | 1 + pungi/phases/livemedia_phase.py | 1 + pungi/wrappers/kojiwrapper.py | 3 +++ tests/test_livemediaphase.py | 11 +++++++++++ 6 files changed, 18 insertions(+) diff --git a/.gitignore b/.gitignore index 6c20baf6..fe291c82 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ tests/data/repo-krb5-lookaside tests/_composes htmlcov/ .coverage +.eggs .idea/ .tox .venv diff --git a/doc/configuration.rst b/doc/configuration.rst index a199d956..4600e69b 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1451,6 +1451,7 @@ Live Media Settings * ``repo`` (*str|[str]*) -- repos specified by URL or variant UID * ``title`` (*str*) * ``install_tree_from`` (*str*) -- variant to take install tree from + * ``nomacboot`` (*bool*) Image Build Settings diff --git a/pungi/checks.py b/pungi/checks.py index 81c11699..a08b5638 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -981,6 +981,7 @@ def make_schema(): "arches": {"$ref": "#/definitions/list_of_strings"}, "failable": {"$ref": "#/definitions/list_of_strings"}, "release": {"$ref": "#/definitions/optional_string"}, + "nomacboot": {"type": "boolean"}, }, "required": ["name", "kickstart"], "additionalProperties": False, diff --git a/pungi/phases/livemedia_phase.py b/pungi/phases/livemedia_phase.py index 9796418a..f28c68a3 100644 --- a/pungi/phases/livemedia_phase.py +++ b/pungi/phases/livemedia_phase.py @@ -71,6 +71,7 @@ class LiveMediaPhase(PhaseLoggerMixin, ImageConfigMixin, ConfigGuardedPhase): "ksurl": self.get_ksurl(image_conf), "ksversion": image_conf.get("ksversion"), "scratch": image_conf.get("scratch", False), + "nomacboot": image_conf.get("nomacboot", False), "release": self.get_release(image_conf), "skip_tag": image_conf.get("skip_tag"), "name": name, diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py index 140b2bfc..b11f5e6f 100644 --- a/pungi/wrappers/kojiwrapper.py +++ b/pungi/wrappers/kojiwrapper.py @@ -391,6 +391,9 @@ class KojiWrapper(object): if "can_fail" in options: cmd.append("--can-fail=%s" % ",".join(options["can_fail"])) + if options.get("nomacboot"): + cmd.append("--nomacboot") + if wait: cmd.append("--wait") diff --git a/tests/test_livemediaphase.py b/tests/test_livemediaphase.py index 7a2b878f..7a406716 100644 --- a/tests/test_livemediaphase.py +++ b/tests/test_livemediaphase.py @@ -60,6 +60,7 @@ class TestLiveMediaPhase(PungiTestCase): "version": "Rawhide", "subvariant": "Server", "failable_arches": [], + "nomacboot": False, }, ) ) @@ -116,6 +117,7 @@ class TestLiveMediaPhase(PungiTestCase): "version": "Rawhide", "subvariant": "Server", "failable_arches": ["amd64", "x86_64"], + "nomacboot": False, }, ) ) @@ -178,6 +180,7 @@ class TestLiveMediaPhase(PungiTestCase): "version": "Rawhide", "subvariant": "Server", "failable_arches": [], + "nomacboot": False, }, ) ), @@ -201,6 +204,7 @@ class TestLiveMediaPhase(PungiTestCase): "version": "Rawhide", "subvariant": "Server", "failable_arches": [], + "nomacboot": False, }, ) ), @@ -224,6 +228,7 @@ class TestLiveMediaPhase(PungiTestCase): "version": "25", "subvariant": "Server", "failable_arches": [], + "nomacboot": False, }, ) ), @@ -286,6 +291,7 @@ class TestLiveMediaPhase(PungiTestCase): "version": "Rawhide", "subvariant": "Server", "failable_arches": [], + "nomacboot": False, }, ) ), @@ -309,6 +315,7 @@ class TestLiveMediaPhase(PungiTestCase): "version": "Rawhide", "subvariant": "Server", "failable_arches": [], + "nomacboot": False, }, ) ), @@ -332,6 +339,7 @@ class TestLiveMediaPhase(PungiTestCase): "version": "25", "subvariant": "Server", "failable_arches": [], + "nomacboot": False, }, ) ), @@ -423,6 +431,7 @@ class TestLiveMediaPhase(PungiTestCase): "install_tree_from": "Server-optional", "subvariant": "Something", "failable": ["*"], + "nomacboot": True, } ] } @@ -436,6 +445,7 @@ class TestLiveMediaPhase(PungiTestCase): phase.run() self.assertTrue(phase.pool.add.called) + self.assertEqual( phase.pool.queue_put.call_args_list, [ @@ -464,6 +474,7 @@ class TestLiveMediaPhase(PungiTestCase): "version": "25", "subvariant": "Something", "failable_arches": ["x86_64"], + "nomacboot": True, }, ) ) From bebbefe46e815ab805b262adf01bc1fa0a5cd67b Mon Sep 17 00:00:00 2001 From: Ondrej Nosek Date: Wed, 30 Mar 2022 22:34:37 +0200 Subject: [PATCH 106/137] Variants file in config can contain path rcm-metadata configs contain definition of variants file. It can be in form of SCM or file path. Before the fix, only variants file's basename was consireded. Now the path can be written. Example: variants_file = "comps/variants-rcmtools-2.0-rhel-8.xml" JIRA: RHELCMP-8705 Signed-off-by: Ondrej Nosek --- pungi/compose.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pungi/compose.py b/pungi/compose.py index ce0fdb74..e3ed352c 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -408,7 +408,7 @@ class Compose(kobo.log.LoggingBase): ) else: file_name = os.path.basename(scm_dict) - scm_dict = os.path.join(self.config_dir, os.path.basename(scm_dict)) + scm_dict = os.path.join(self.config_dir, scm_dict) self.log_debug("Writing variants file: %s", variants_file) tmp_dir = self.mkdtemp(prefix="variants_file_") From f8c7ad28e40a86176d385b3405a2f84798ffd77e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Tue, 29 Mar 2022 09:50:59 +0200 Subject: [PATCH 107/137] kojiwrapper: Add retries to login call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gssapi_login call is not retried automatically by Koji yet (see koji#3170). Let's try to work around that by retrying in the calling code. JIRA: RHELCMP-8700 Signed-off-by: Lubomír Sedlář --- pungi/wrappers/kojiwrapper.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py index b11f5e6f..d53da5d9 100644 --- a/pungi/wrappers/kojiwrapper.py +++ b/pungi/wrappers/kojiwrapper.py @@ -65,6 +65,9 @@ class KojiWrapper(object): self.koji_module.config.server, session_opts ) + # This retry should be removed once https://pagure.io/koji/issue/3170 is + # fixed and released. + @util.retry(wait_on=(xmlrpclib.ProtocolError, koji.GenericError)) def login(self): """Authenticate to the hub.""" auth_type = self.koji_module.config.authtype From 707a2c8d10ebb89c8acfddf01bc9e78c446c2f4f Mon Sep 17 00:00:00 2001 From: Ondrej Nosek Date: Sat, 2 Apr 2022 00:01:34 +0200 Subject: [PATCH 108/137] 4.3.4 release JIRA: RHELCMP-8627 Signed-off-by: Ondrej Nosek --- doc/conf.py | 2 +- pungi.spec | 16 +++++++++++++++- setup.py | 2 +- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index ec267f8a..5e153c73 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -53,7 +53,7 @@ copyright = u'2016, Red Hat, Inc.' # The short X.Y version. version = '4.3' # The full version, including alpha/beta/rc tags. -release = '4.3.3' +release = '4.3.4' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pungi.spec b/pungi.spec index 8882c347..0304519b 100644 --- a/pungi.spec +++ b/pungi.spec @@ -1,5 +1,5 @@ Name: pungi -Version: 4.3.3 +Version: 4.3.4 Release: 1%{?dist} Summary: Distribution compose tool @@ -111,6 +111,20 @@ pytest cd tests && ./test_compose.sh %changelog +* Fri Apr 01 2022 Ondřej Nosek - 4.3.4-1 +- kojiwrapper: Add retries to login call (lsedlar) +- Variants file in config can contain path (onosek) +- nomacboot option for livemedia koji tasks (cobrien) +- doc: improve osbs_registries explanation (kdreyer) +- osbs: only handle archives of type "image" (kdreyer) +- Update the default greedy_method value in doc (ounsal) +- Fix the wrong working directory for the progress_notification script (ounsal) +- Filter out environment groups unmatch given arch (hlin) +- profiler: Respect provided output stream (lsedlar) +- modules: Correct a typo in loading obsoletes (ppisar) +- Do not clone the same repository multiple times, re-use already cloned + repository (ounsal) + * Sat Jan 08 2022 Haibo Lin - 4.3.3-1 - hybrid: Explicitly pull in debugsource packages (lsedlar) - Add module obsoletes feature (fvalder) diff --git a/setup.py b/setup.py index 3e057d0e..51cf9230 100755 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ packages = sorted(packages) setup( name="pungi", - version="4.3.3", + version="4.3.4", description="Distribution compose tool", url="https://pagure.io/pungi", author="Dennis Gilmore", From e490764985f8683acfc205008535e89386c14656 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Thu, 7 Apr 2022 13:47:12 +0800 Subject: [PATCH 109/137] Involve bandit JIRA: RHELCMP-8562 Signed-off-by: Haibo Lin --- tox.ini | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 4ba977cf..ae1cf711 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = flake8, black, py27, py3 +envlist = bandit, flake8, black, py27, py3 [testenv:flake8] deps = @@ -8,6 +8,14 @@ whitelist_externals = sh commands = sh -c "flake8 pungi pungi_utils setup.py tests/*py" +[testenv:bandit] +basepython = python3 +skip_install = true +deps = bandit +commands = + bandit -r -ll pungi pungi_utils +ignore_outcome = True + [testenv:black] basepython = python3 whitelist_externals = sh From c5cdd498ac988c6dbb2ab156531e37314d128408 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Wed, 13 Apr 2022 16:19:26 +0800 Subject: [PATCH 110/137] Revert "Do not clone the same repository multiple times, re-use already cloned repository" This reverts commit 330ba9b9c4b0222a19d7f964e4124176cdea2a6c. As of RHELCMP-8874, revert this patch as a quick fix. Signed-off-by: Haibo Lin --- pungi/wrappers/scm.py | 69 +++++++------------------------------------ tests/test_scm.py | 23 ++++----------- 2 files changed, 17 insertions(+), 75 deletions(-) diff --git a/pungi/wrappers/scm.py b/pungi/wrappers/scm.py index 6d79d86f..5c4b37fb 100644 --- a/pungi/wrappers/scm.py +++ b/pungi/wrappers/scm.py @@ -19,9 +19,7 @@ from __future__ import absolute_import import os import shutil import glob -import threading import six -import tempfile from six.moves import shlex_quote from six.moves.urllib.request import urlretrieve from fnmatch import fnmatch @@ -31,8 +29,6 @@ from kobo.shortcuts import run, force_list from pungi.util import explode_rpm_package, makedirs, copy_all, temp_dir, retry from .kojiwrapper import KojiWrapper -scm_lock = threading.Lock() - class ScmBase(kobo.log.LoggingBase): def __init__(self, logger=None, command=None, compose=None): @@ -376,33 +372,10 @@ def get_file_from_scm(scm_dict, target_path, compose=None): scm = _get_wrapper(scm_type, logger=logger, command=command, compose=compose) files_copied = [] - branch = scm_branch if scm_branch else "master" - delete_after_flag = False - with scm_lock: - if compose and scm_repo: - repo = scm_repo.rsplit("/")[-1] - tmp_dir = compose.paths.work.tmp_dir() - tmp_dir = os.path.join(tmp_dir, repo, branch) - else: - tmp_dir = tempfile.mkdtemp(prefix="scm_checkout_") - delete_after_flag = True - - if not os.path.isdir(tmp_dir): - makedirs(tmp_dir) - - for i in force_list(scm_file): - # Check the files which are included with subdirectories - check_file = os.path.join(tmp_dir, i[i.rfind("/") + 1 :]) - if ( - type(scm_dict) is not dict - or command is not None - or not compose - or not os.path.isfile(check_file) - ): - scm.export_file(scm_repo, i, scm_branch=scm_branch, target_dir=tmp_dir) - files_copied += copy_all(tmp_dir, target_path) - if delete_after_flag: - shutil.rmtree(tmp_dir) + for i in force_list(scm_file): + with temp_dir(prefix="scm_checkout_") as tmp_dir: + scm.export_file(scm_repo, i, scm_branch=scm_branch, target_dir=tmp_dir) + files_copied += copy_all(tmp_dir, target_path) return files_copied @@ -486,34 +459,14 @@ def get_dir_from_scm(scm_dict, target_path, compose=None): logger = compose._logger if compose else None scm = _get_wrapper(scm_type, logger=logger, command=command, compose=compose) - branch = scm_branch if scm_branch else "master" - delete_after_flag = False - with scm_lock: - if compose and scm_repo: - repo = scm_repo.rsplit("/")[-1] - tmp_dir = compose.paths.work.tmp_dir() - tmp_dir = os.path.join(tmp_dir, repo, branch) - else: - tmp_dir = tempfile.mkdtemp(prefix="scm_checkout_") - delete_after_flag = True - - if not os.path.isdir(tmp_dir): - makedirs(tmp_dir) - scm.export_dir(scm_repo, scm_dir, scm_branch=scm_branch, target_dir=tmp_dir) - elif ( - type(scm_dict) is not dict - or command is not None - or not scm_repo - or not compose - ): - scm.export_dir(scm_repo, scm_dir, scm_branch=scm_branch, target_dir=tmp_dir) + with temp_dir(prefix="scm_checkout_") as tmp_dir: + scm.export_dir(scm_repo, scm_dir, scm_branch=scm_branch, target_dir=tmp_dir) files_copied = copy_all(tmp_dir, target_path) - # Make sure the directory has permissions set to 755. This is a workaround - # for a problem where sometimes the directory will be 700 and it will not - # be accessible via httpd. - os.chmod(target_path, 0o755) - if delete_after_flag: - shutil.rmtree(tmp_dir) + # Make sure the directory has permissions set to 755. This is a workaround + # for a problem where sometimes the directory will be 700 and it will not + # be accessible via httpd. + os.chmod(target_path, 0o755) + return files_copied diff --git a/tests/test_scm.py b/tests/test_scm.py index e45c32f4..f6307967 100644 --- a/tests/test_scm.py +++ b/tests/test_scm.py @@ -16,17 +16,14 @@ import six from pungi.wrappers import scm from tests.helpers import touch from kobo.shortcuts import run -from pungi.compose import Compose class SCMBaseTest(unittest.TestCase): def setUp(self): self.destdir = tempfile.mkdtemp() - self.tmp_dir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.destdir) - shutil.rmtree(self.tmp_dir) def assertStructure(self, returned, expected): # Check we returned the correct files @@ -146,8 +143,7 @@ class GitSCMTestCase(SCMBaseTest): @mock.patch("pungi.wrappers.scm.run") def test_get_file_function(self, run): - with mock.patch("pungi.compose.ComposeInfo"): - compose = Compose({}, self.tmp_dir) + compose = mock.Mock(conf={}) def process(cmd, workdir=None, **kwargs): touch(os.path.join(workdir, "some_file.txt")) @@ -342,8 +338,6 @@ class GitSCMTestCaseReal(SCMBaseTest): shutil.rmtree(self.gitRepositoryLocation) def test_get_file_function(self): - with mock.patch("pungi.compose.ComposeInfo"): - compose = Compose({}, self.tmp_dir) sourceFileLocation = random.choice(list(self.files.keys())) sourceFilename = os.path.basename(sourceFileLocation) destinationFileLocation = os.path.join(self.destdir, "other_file.txt") @@ -354,7 +348,7 @@ class GitSCMTestCaseReal(SCMBaseTest): "file": sourceFilename, }, os.path.join(self.destdir, destinationFileLocation), - compose=compose, + compose=self.compose, ) self.assertEqual(destinationFileActualLocation, destinationFileLocation) self.assertTrue(os.path.isfile(destinationFileActualLocation)) @@ -367,8 +361,6 @@ class GitSCMTestCaseReal(SCMBaseTest): self.assertEqual(sourceFileContent, destinationFileContent) def test_get_file_function_with_overwrite(self): - with mock.patch("pungi.compose.ComposeInfo"): - compose = Compose({}, self.tmp_dir) sourceFileLocation = random.choice(list(self.files.keys())) sourceFilename = os.path.basename(sourceFileLocation) destinationFileLocation = os.path.join(self.destdir, "other_file.txt") @@ -383,7 +375,7 @@ class GitSCMTestCaseReal(SCMBaseTest): "file": sourceFilename, }, os.path.join(self.destdir, destinationFileLocation), - compose=compose, + compose=self.compose, overwrite=True, ) self.assertEqual(destinationFileActualLocation, destinationFileLocation) @@ -611,8 +603,7 @@ class KojiSCMTestCase(SCMBaseTest): @mock.patch("pungi.wrappers.scm.KojiWrapper") def test_doesnt_get_dirs(self, KW, dl): - with mock.patch("pungi.compose.ComposeInfo"): - compose = Compose({"koji_profile": "koji"}, self.tmp_dir) + compose = mock.Mock(conf={"koji_profile": "koji"}) with self.assertRaises(RuntimeError) as ctx: scm.get_dir_from_scm( @@ -637,8 +628,7 @@ class KojiSCMTestCase(SCMBaseTest): @mock.patch("pungi.wrappers.scm.KojiWrapper") def test_get_from_build(self, KW, dl): - with mock.patch("pungi.compose.ComposeInfo"): - compose = Compose({"koji_profile": "koji"}, self.tmp_dir) + compose = mock.Mock(conf={"koji_profile": "koji"}) def download(src, dst): touch(dst) @@ -669,8 +659,7 @@ class KojiSCMTestCase(SCMBaseTest): @mock.patch("pungi.wrappers.scm.KojiWrapper") def test_get_from_latest_build(self, KW, dl): - with mock.patch("pungi.compose.ComposeInfo"): - compose = Compose({"koji_profile": "koji"}, self.tmp_dir) + compose = mock.Mock(conf={"koji_profile": "koji"}) def download(src, dst): touch(dst) From e8d79e92698c2d63fc3a9c776dfa07dd647ef15c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Tue, 19 Apr 2022 12:12:06 +0200 Subject: [PATCH 111/137] Restrict jsonschema version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There's a new major version released on PyPI, and it doesn't seem to work with Pungi yet. Until code is updated to be compatible, let's ensure tox won't try to install it. Signed-off-by: Lubomír Sedlář --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ee73d0c3..4eba1ca4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ dict.sorted dogpile.cache fedmsg funcsigs -jsonschema +jsonschema < 4.0.0 kobo koji lxml From 80957f5205f871b60f893ca034ffc3607003c92a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Tue, 26 Apr 2022 08:02:16 +0200 Subject: [PATCH 112/137] kojiwrapper: Ignore warnings before task id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When looking for task ID in output of koji runroot command, do not check just the first line. Instead look for first line that contains just a number. Most of the time, this should really be the first line. But if koji client decides to print any warnings, this patch should skip that. JIRA: RHELCMP-8944 Signed-off-by: Lubomír Sedlář --- pungi/wrappers/kojiwrapper.py | 15 ++++++++++----- tests/test_koji_wrapper.py | 24 ++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py index d53da5d9..b1f2459d 100644 --- a/pungi/wrappers/kojiwrapper.py +++ b/pungi/wrappers/kojiwrapper.py @@ -291,16 +291,21 @@ class KojiWrapper(object): universal_newlines=True, ) - first_line = output.splitlines()[0] - match = re.search(r"^(\d+)$", first_line) - if not match: + # Look for first line that contains only a number. This is the ID of + # the new task. Usually this should be the first line, but there may be + # warnings before it. + for line in output.splitlines(): + match = re.search(r"^(\d+)$", line) + if match: + task_id = int(match.groups()[0]) + break + + if not task_id: raise RuntimeError( "Could not find task ID in output. Command '%s' returned '%s'." % (" ".join(command), output) ) - task_id = int(match.groups()[0]) - self.save_task_id(task_id) retcode, output = self._wait_for_task(task_id, logfile=log_file) diff --git a/tests/test_koji_wrapper.py b/tests/test_koji_wrapper.py index 2a7c2df9..4b943596 100644 --- a/tests/test_koji_wrapper.py +++ b/tests/test_koji_wrapper.py @@ -668,6 +668,30 @@ class RunrootKojiWrapperTest(KojiWrapperBaseTestCase): ], ) + @mock.patch("pungi.wrappers.kojiwrapper.run") + def test_run_runroot_cmd_with_warnings_before_task_id(self, run): + cmd = ["koji", "runroot", "--task-id"] + run.return_value = (0, "DeprecatioNWarning: whatever\n1234\n") + output = "Output ..." + self.koji._wait_for_task = mock.Mock(return_value=(0, output)) + + result = self.koji.run_runroot_cmd(cmd) + self.assertDictEqual(result, {"retcode": 0, "output": output, "task_id": 1234}) + self.assertEqual( + run.call_args_list, + [ + mock.call( + cmd, + can_fail=True, + env={"FOO": "BAR", "PYTHONUNBUFFERED": "1"}, + buffer_size=-1, + logfile=None, + show_cmd=True, + universal_newlines=True, + ) + ], + ) + @mock.patch("shutil.rmtree") @mock.patch("tempfile.mkdtemp") @mock.patch("pungi.wrappers.kojiwrapper.run") From c4aa45beab9f0b0945db3c1c5b777844009c3534 Mon Sep 17 00:00:00 2001 From: Lingyan Zhuang Date: Sat, 7 May 2022 19:58:19 +0800 Subject: [PATCH 113/137] Add skip_branding to ostree_installer. Fixes: #1594 Merges: https://pagure.io/pungi/pull-request/1609 Signed-off-by: Lingyan Zhuang --- doc/configuration.rst | 2 ++ pungi/checks.py | 1 + pungi/phases/ostree_installer.py | 1 + 3 files changed, 4 insertions(+) diff --git a/doc/configuration.rst b/doc/configuration.rst index 4600e69b..4a7f8986 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1786,6 +1786,8 @@ an OSTree repository. This always runs in Koji as a ``runroot`` task. with the optional key: * ``extra_runroot_pkgs`` -- (*[str]*) + * ``skip_branding`` -- (*bool*) Stops lorax to install packages with branding. + Defaults to ``False``. **ostree_installer_overwrite** = False (*bool*) -- by default if a variant including OSTree installer also creates diff --git a/pungi/checks.py b/pungi/checks.py index a08b5638..bc5d908f 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -1078,6 +1078,7 @@ def make_schema(): "template_repo": {"type": "string"}, "template_branch": {"type": "string"}, "extra_runroot_pkgs": {"$ref": "#/definitions/list_of_strings"}, + "skip_branding": {"type": "boolean"}, }, "additionalProperties": False, } diff --git a/pungi/phases/ostree_installer.py b/pungi/phases/ostree_installer.py index 3424ea8c..8e9a1f6e 100644 --- a/pungi/phases/ostree_installer.py +++ b/pungi/phases/ostree_installer.py @@ -272,6 +272,7 @@ class OstreeInstallerThread(WorkerThread): rootfs_size=config.get("rootfs_size"), is_final=compose.supported, log_dir=self.logdir, + skip_branding=config.get("skip_branding"), ) cmd = "rm -rf %s && %s" % ( shlex_quote(output_dir), From 895b3982d7c835a3fe4d4bd8d21de2fc6a95abac Mon Sep 17 00:00:00 2001 From: Ozan Unsal Date: Tue, 24 May 2022 13:12:06 +0200 Subject: [PATCH 114/137] Update the cts_keytab field in order to get the hostname of the server - This change is required for the following issue. Authentication is required for importing composes to the CTS and finding generic keytabs in different servers. JIRA: RHELCMP-8930 Signed-off-by: Ozan Unsal --- pungi/compose.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pungi/compose.py b/pungi/compose.py index e3ed352c..e289a7a2 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -24,6 +24,7 @@ import time import tempfile import shutil import json +import socket import kobo.log import kobo.tback @@ -98,6 +99,8 @@ def get_compose_info( authentication = get_authentication(conf) if cts_keytab: environ_copy = dict(os.environ) + if "$HOSTNAME" in cts_keytab: + cts_keytab = cts_keytab.replace("$HOSTNAME", socket.gethostname()) os.environ["KRB5_CLIENT_KTNAME"] = cts_keytab try: From ca185aaea81ae426b71e963f941bf685c7e5b70b Mon Sep 17 00:00:00 2001 From: Marek Kulik Date: Wed, 27 Apr 2022 15:31:14 +0200 Subject: [PATCH 115/137] Fix module defaults and obsoletes validation - Remove validation for modules obsoletes We can have multiple obsoletes for one module - Add unit tests to cover basic scenarios for modules defaults && obsoletes - Add additional check for invalid yaml file in Defaults. Previously, empty list of default would be returned when invalid yaml is present in Defaults directory. - Using MergeIndex for Obsoletes only (for now). https://pagure.io/pungi/issue/1592 Signed-off-by: Marek Kulik --- pungi/module_util.py | 57 +++++++--- pungi/phases/createrepo.py | 2 +- pungi/phases/gather/__init__.py | 2 +- pungi/phases/init.py | 48 ++++---- tests/test_initphase.py | 51 ++++++++- tests/test_module_util.py | 192 ++++++++++++++++++++++++++++++++ tests/test_pkgset_common.py | 43 +++++-- 7 files changed, 339 insertions(+), 56 deletions(-) create mode 100644 tests/test_module_util.py diff --git a/pungi/module_util.py b/pungi/module_util.py index a25cb36d..ba97590f 100644 --- a/pungi/module_util.py +++ b/pungi/module_util.py @@ -25,10 +25,9 @@ except (ImportError, ValueError): Modulemd = None -def iter_module_defaults_or_obsoletes(path, obsoletes=False): +def iter_module_defaults(path): """Given a path to a directory with yaml files, yield each module default in there as a pair (module_name, ModuleDefaults instance). - The same happens for module obsoletes if the obsoletes switch is True. """ # It is really tempting to merge all the module indexes into a single one # and work with it. However that does not allow for detecting conflicting @@ -42,10 +41,31 @@ def iter_module_defaults_or_obsoletes(path, obsoletes=False): index = Modulemd.ModuleIndex() index.update_from_file(file, strict=False) for module_name in index.get_module_names(): - if obsoletes: - yield module_name, index.get_module(module_name).get_obsoletes() - else: - yield module_name, index.get_module(module_name).get_defaults() + yield module_name, index.get_module(module_name).get_defaults() + + +def get_module_obsoletes_idx(path, mod_list): + """Given a path to a directory with yaml files, return Index with + merged all obsoletes. + """ + + merger = Modulemd.ModuleIndexMerger.new() + md_idxs = [] + + # associate_index does NOT copy it's argument (nor increases a + # reference counter on the object). It only stores a pointer. + for file in glob.glob(os.path.join(path, "*.yaml")): + index = Modulemd.ModuleIndex() + index.update_from_file(file, strict=False) + mod_name = index.get_module_names()[0] + + if mod_name and (mod_name in mod_list or not mod_list): + md_idxs.append(index) + merger.associate_index(md_idxs[-1], 0) + + merged_idx = merger.resolve() + + return merged_idx def collect_module_defaults( @@ -78,16 +98,21 @@ def collect_module_defaults( def collect_module_obsoletes(obsoletes_dir, modules_to_load, mod_index=None): """Load module obsoletes into index. - This works in a similar fashion as collect_module_defaults except the overrides_dir - feature. + This works in a similar fashion as collect_module_defaults except it + merges indexes together instead of adding them during iteration. + + Additionally if modules_to_load is not empty returned Index will include + only obsoletes for those modules. """ - mod_index = mod_index or Modulemd.ModuleIndex() - for module_name, obsoletes in iter_module_defaults_or_obsoletes( - obsoletes_dir, obsoletes=True - ): - for obsolete in obsoletes: - if not modules_to_load or module_name in modules_to_load: - mod_index.add_obsoletes(obsolete) + obsoletes_index = get_module_obsoletes_idx(obsoletes_dir, modules_to_load) - return mod_index + # Merge Obsoletes with Modules Index. + if mod_index: + merger = Modulemd.ModuleIndexMerger.new() + merger.associate_index(mod_index, 0) + merger.associate_index(obsoletes_index, 0) + merged_idx = merger.resolve() + obsoletes_index = merged_idx + + return obsoletes_index diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index c170b382..d784f023 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -267,7 +267,7 @@ def create_variant_repo( ) obsoletes_dir = compose.paths.work.module_obsoletes_dir() - collect_module_obsoletes(obsoletes_dir, module_names, mod_index) + mod_index = collect_module_obsoletes(obsoletes_dir, module_names, mod_index) # Add extra modulemd files if variant.uid in compose.conf.get("createrepo_extra_modulemd", {}): diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 09b57287..0474411b 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -703,7 +703,7 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None): defaults_dir, module_names, mod_index, overrides_dir=overrides_dir ) obsoletes_dir = compose.paths.work.module_obsoletes_dir() - collect_module_obsoletes(obsoletes_dir, module_names, mod_index) + mod_index = collect_module_obsoletes(obsoletes_dir, module_names, mod_index) log_file = compose.paths.log.log_file( arch, "lookaside_repo_modules_%s" % (variant.uid) diff --git a/pungi/phases/init.py b/pungi/phases/init.py index 30c8a742..a78c0dc7 100644 --- a/pungi/phases/init.py +++ b/pungi/phases/init.py @@ -16,6 +16,7 @@ import collections import os +import glob import shutil from kobo.shortcuts import run @@ -24,7 +25,7 @@ from kobo.threads import run_in_threads from pungi.phases.base import PhaseBase from pungi.phases.gather import write_prepopulate_file from pungi.util import temp_dir -from pungi.module_util import iter_module_defaults_or_obsoletes +from pungi.module_util import iter_module_defaults from pungi.wrappers.comps import CompsWrapper from pungi.wrappers.createrepo import CreaterepoWrapper from pungi.wrappers.scm import get_dir_from_scm, get_file_from_scm @@ -68,17 +69,13 @@ class InitPhase(PhaseBase): # download module defaults if self.compose.has_module_defaults: write_module_defaults(self.compose) - validate_module_defaults_or_obsoletes( + validate_module_defaults( self.compose.paths.work.module_defaults_dir(create_dir=False) ) # download module obsoletes if self.compose.has_module_obsoletes: write_module_obsoletes(self.compose) - validate_module_defaults_or_obsoletes( - self.compose.paths.work.module_obsoletes_dir(create_dir=False), - obsoletes=True, - ) # write prepopulate file write_prepopulate_file(self.compose) @@ -244,37 +241,38 @@ def write_module_obsoletes(compose): ) -def validate_module_defaults_or_obsoletes(path, obsoletes=False): - """Make sure there are no conflicting defaults. Each module name can only - have one default stream or module obsolete. +def validate_module_defaults(path): + """Make sure there are no conflicting defaults and every default can be loaded. + Each module name can onlyhave one default stream. - :param str path: directory with cloned module defaults/obsoletes + :param str path: directory with cloned module defaults """ - seen = collections.defaultdict(set) - mmd_type = "obsoletes" if obsoletes else "defaults" - for module_name, defaults_or_obsoletes in iter_module_defaults_or_obsoletes( - path, obsoletes - ): - if obsoletes: - for obsolete in defaults_or_obsoletes: - seen[obsolete.props.module_name].add(obsolete) - else: - seen[module_name].add(defaults_or_obsoletes.get_default_stream()) + defaults_num = len(glob.glob(os.path.join(path, "*.yaml"))) + + seen_defaults = collections.defaultdict(set) + + for module_name, defaults in iter_module_defaults(path): + seen_defaults[module_name].add(defaults.get_default_stream()) errors = [] - for module_name, defaults_or_obsoletes in seen.items(): - if len(defaults_or_obsoletes) > 1: + for module_name, defaults in seen_defaults.items(): + if len(defaults) > 1: errors.append( - "Module %s has multiple %s: %s" - % (module_name, mmd_type, ", ".join(sorted(defaults_or_obsoletes))) + "Module %s has multiple defaults: %s" + % (module_name, ", ".join(sorted(defaults))) ) if errors: raise RuntimeError( - "There are duplicated module %s:\n%s" % (mmd_type, "\n".join(errors)) + "There are duplicated module defaults:\n%s" % "\n".join(errors) ) + # Make sure all defaults are valid otherwise update_from_defaults_directory + # will return empty object + if defaults_num != len(seen_defaults): + raise RuntimeError("Defaults contains not valid default file") + def validate_comps(path): """Check that there are whitespace issues in comps.""" diff --git a/tests/test_initphase.py b/tests/test_initphase.py index f6fad433..1fb80c48 100644 --- a/tests/test_initphase.py +++ b/tests/test_initphase.py @@ -24,7 +24,8 @@ from tests.helpers import ( @mock.patch("pungi.phases.init.run_in_threads", new=fake_run_in_threads) @mock.patch("pungi.phases.init.validate_comps") -@mock.patch("pungi.phases.init.validate_module_defaults_or_obsoletes") +@mock.patch("pungi.phases.init.validate_module_defaults") +@mock.patch("pungi.phases.init.write_module_obsoletes") @mock.patch("pungi.phases.init.write_module_defaults") @mock.patch("pungi.phases.init.write_global_comps") @mock.patch("pungi.phases.init.write_arch_comps") @@ -40,6 +41,7 @@ class TestInitPhase(PungiTestCase): write_arch, write_global, write_defaults, + write_obsoletes, validate_defaults, validate_comps, ): @@ -85,6 +87,7 @@ class TestInitPhase(PungiTestCase): ], ) self.assertEqual(write_defaults.call_args_list, []) + self.assertEqual(write_obsoletes.call_args_list, []) self.assertEqual(validate_defaults.call_args_list, []) def test_run_with_preserve( @@ -95,6 +98,7 @@ class TestInitPhase(PungiTestCase): write_arch, write_global, write_defaults, + write_obsoletes, validate_defaults, validate_comps, ): @@ -142,6 +146,7 @@ class TestInitPhase(PungiTestCase): ], ) self.assertEqual(write_defaults.call_args_list, []) + self.assertEqual(write_obsoletes.call_args_list, []) self.assertEqual(validate_defaults.call_args_list, []) def test_run_without_comps( @@ -152,6 +157,7 @@ class TestInitPhase(PungiTestCase): write_arch, write_global, write_defaults, + write_obsoletes, validate_defaults, validate_comps, ): @@ -169,6 +175,7 @@ class TestInitPhase(PungiTestCase): self.assertEqual(create_comps.mock_calls, []) self.assertEqual(write_variant.mock_calls, []) self.assertEqual(write_defaults.call_args_list, []) + self.assertEqual(write_obsoletes.call_args_list, []) self.assertEqual(validate_defaults.call_args_list, []) def test_with_module_defaults( @@ -179,6 +186,7 @@ class TestInitPhase(PungiTestCase): write_arch, write_global, write_defaults, + write_obsoletes, validate_defaults, validate_comps, ): @@ -196,11 +204,41 @@ class TestInitPhase(PungiTestCase): self.assertEqual(create_comps.mock_calls, []) self.assertEqual(write_variant.mock_calls, []) self.assertEqual(write_defaults.call_args_list, [mock.call(compose)]) + self.assertEqual(write_obsoletes.call_args_list, []) self.assertEqual( validate_defaults.call_args_list, [mock.call(compose.paths.work.module_defaults_dir())], ) + def test_with_module_obsoletes( + self, + write_prepopulate, + write_variant, + create_comps, + write_arch, + write_global, + write_defaults, + write_obsoletes, + validate_defaults, + validate_comps, + ): + compose = DummyCompose(self.topdir, {}) + compose.has_comps = False + compose.has_module_defaults = False + compose.has_module_obsoletes = True + phase = init.InitPhase(compose) + phase.run() + + self.assertEqual(write_global.mock_calls, []) + self.assertEqual(validate_comps.call_args_list, []) + self.assertEqual(write_prepopulate.mock_calls, [mock.call(compose)]) + self.assertEqual(write_arch.mock_calls, []) + self.assertEqual(create_comps.mock_calls, []) + self.assertEqual(write_variant.mock_calls, []) + self.assertEqual(write_defaults.call_args_list, []) + self.assertEqual(write_obsoletes.call_args_list, [mock.call(compose)]) + self.assertEqual(validate_defaults.call_args_list, []) + class TestWriteArchComps(PungiTestCase): @mock.patch("pungi.phases.init.run") @@ -624,13 +662,13 @@ class TestValidateModuleDefaults(PungiTestCase): def test_valid_files(self): self._write_defaults({"httpd": ["1"], "python": ["3.6"]}) - init.validate_module_defaults_or_obsoletes(self.topdir) + init.validate_module_defaults(self.topdir) def test_duplicated_stream(self): self._write_defaults({"httpd": ["1"], "python": ["3.6", "3.5"]}) with self.assertRaises(RuntimeError) as ctx: - init.validate_module_defaults_or_obsoletes(self.topdir) + init.validate_module_defaults(self.topdir) self.assertIn( "Module python has multiple defaults: 3.5, 3.6", str(ctx.exception) @@ -640,7 +678,7 @@ class TestValidateModuleDefaults(PungiTestCase): self._write_defaults({"httpd": ["1", "2"], "python": ["3.6", "3.5"]}) with self.assertRaises(RuntimeError) as ctx: - init.validate_module_defaults_or_obsoletes(self.topdir) + init.validate_module_defaults(self.topdir) self.assertIn("Module httpd has multiple defaults: 1, 2", str(ctx.exception)) self.assertIn( @@ -665,7 +703,10 @@ class TestValidateModuleDefaults(PungiTestCase): ), ) - init.validate_module_defaults_or_obsoletes(self.topdir) + with self.assertRaises(RuntimeError) as ctx: + init.validate_module_defaults(self.topdir) + + self.assertIn("Defaults contains not valid default file", str(ctx.exception)) @mock.patch("pungi.phases.init.CompsWrapper") diff --git a/tests/test_module_util.py b/tests/test_module_util.py new file mode 100644 index 00000000..0d083af0 --- /dev/null +++ b/tests/test_module_util.py @@ -0,0 +1,192 @@ +import os + +try: + import unittest2 as unittest +except ImportError: + import unittest + +from parameterized import parameterized +from pungi import module_util +from pungi.module_util import Modulemd + +from tests import helpers + + +@unittest.skipUnless(Modulemd, "Skipped test, no module support.") +class TestModuleUtil(helpers.PungiTestCase): + def _get_stream(self, mod_name, stream_name): + stream = Modulemd.ModuleStream.new( + Modulemd.ModuleStreamVersionEnum.TWO, mod_name, stream_name + ) + stream.props.version = 42 + stream.props.context = "deadbeef" + stream.props.arch = "x86_64" + + return stream + + def _write_obsoletes(self, defs): + for mod_name, stream, obsoleted_by in defs: + mod_index = Modulemd.ModuleIndex.new() + mmdobs = Modulemd.Obsoletes.new(1, 10993435, mod_name, stream, "testmsg") + mmdobs.set_obsoleted_by(obsoleted_by[0], obsoleted_by[1]) + mod_index.add_obsoletes(mmdobs) + filename = "%s:%s.yaml" % (mod_name, stream) + with open(os.path.join(self.topdir, filename), "w") as f: + f.write(mod_index.dump_to_string()) + + def _write_defaults(self, defs): + for mod_name, streams in defs.items(): + for stream in streams: + mod_index = Modulemd.ModuleIndex.new() + mmddef = Modulemd.DefaultsV1.new(mod_name) + mmddef.set_default_stream(stream) + mod_index.add_defaults(mmddef) + filename = "%s-%s.yaml" % (mod_name, stream) + with open(os.path.join(self.topdir, filename), "w") as f: + f.write(mod_index.dump_to_string()) + + @parameterized.expand( + [ + ( + "MULTIPLE", + [ + ("httpd", "1.22.1", ("httpd-new", "3.0")), + ("httpd", "10.4", ("httpd", "11.1.22")), + ], + ), + ( + "NORMAL", + [ + ("gdb", "2.8", ("gdb", "3.0")), + ("nginx", "12.7", ("nginx-nightly", "13.3")), + ], + ), + ] + ) + def test_merged_module_obsoletes_idx(self, test_name, data): + self._write_obsoletes(data) + + mod_index = module_util.get_module_obsoletes_idx(self.topdir, []) + + if test_name == "MULTIPLE": + # Multiple obsoletes are allowed + mod = mod_index.get_module("httpd") + self.assertEqual(len(mod.get_obsoletes()), 2) + else: + mod = mod_index.get_module("gdb") + self.assertEqual(len(mod.get_obsoletes()), 1) + mod_obsolete = mod.get_obsoletes() + self.assertIsNotNone(mod_obsolete) + self.assertEqual(mod_obsolete[0].get_obsoleted_by_module_stream(), "3.0") + + def test_collect_module_defaults_with_index(self): + stream = self._get_stream("httpd", "1") + mod_index = Modulemd.ModuleIndex() + mod_index.add_module_stream(stream) + + defaults_data = {"httpd": ["1.44.2"], "python": ["3.6", "3.5"]} + self._write_defaults(defaults_data) + + mod_index = module_util.collect_module_defaults( + self.topdir, defaults_data.keys(), mod_index + ) + + for module_name in defaults_data.keys(): + mod = mod_index.get_module(module_name) + self.assertIsNotNone(mod) + + mod_defaults = mod.get_defaults() + self.assertIsNotNone(mod_defaults) + + if module_name == "httpd": + self.assertEqual(mod_defaults.get_default_stream(), "1.44.2") + else: + # Can't have multiple defaults for one stream + self.assertEqual(mod_defaults.get_default_stream(), None) + + def test_handles_non_defaults_file_without_validation(self): + self._write_defaults({"httpd": ["1"], "python": ["3.6"]}) + helpers.touch( + os.path.join(self.topdir, "boom.yaml"), + "\n".join( + [ + "document: modulemd", + "version: 2", + "data:", + " summary: dummy module", + " description: dummy module", + " license:", + " module: [GPL]", + " content: [GPL]", + ] + ), + ) + + idx = module_util.collect_module_defaults(self.topdir) + + self.assertEqual(len(idx.get_module_names()), 0) + + @parameterized.expand([(False, ["httpd"]), (False, ["python"])]) + def test_collect_module_obsoletes(self, no_index, mod_list): + if not no_index: + stream = self._get_stream(mod_list[0], "1.22.1") + mod_index = Modulemd.ModuleIndex() + mod_index.add_module_stream(stream) + else: + mod_index = None + + data = [ + ("httpd", "1.22.1", ("httpd-new", "3.0")), + ("httpd", "10.4", ("httpd", "11.1.22")), + ] + self._write_obsoletes(data) + + mod_index = module_util.collect_module_obsoletes( + self.topdir, mod_list, mod_index + ) + + # Obsoletes should not me merged without corresponding module + # if module list is present + if "python" in mod_list: + mod = mod_index.get_module("httpd") + self.assertIsNone(mod) + else: + mod = mod_index.get_module("httpd") + + # No modules + if "httpd" not in mod_list: + self.assertIsNone(mod.get_obsoletes()) + else: + self.assertIsNotNone(mod) + obsoletes_from_orig = mod.get_newest_active_obsoletes("1.22.1", None) + + self.assertEqual( + obsoletes_from_orig.get_obsoleted_by_module_name(), "httpd-new" + ) + + def test_collect_module_obsoletes_without_modlist(self): + stream = self._get_stream("nginx", "1.22.1") + mod_index = Modulemd.ModuleIndex() + mod_index.add_module_stream(stream) + + data = [ + ("httpd", "1.22.1", ("httpd-new", "3.0")), + ("nginx", "10.4", ("nginx", "11.1.22")), + ("nginx", "11.1.22", ("nginx", "66")), + ] + self._write_obsoletes(data) + + mod_index = module_util.collect_module_obsoletes(self.topdir, [], mod_index) + + # All obsoletes are merged into main Index when filter is empty + self.assertEqual(len(mod_index.get_module_names()), 2) + + mod = mod_index.get_module("httpd") + self.assertIsNotNone(mod) + + self.assertEqual(len(mod.get_obsoletes()), 1) + + mod = mod_index.get_module("nginx") + self.assertIsNotNone(mod) + + self.assertEqual(len(mod.get_obsoletes()), 2) diff --git a/tests/test_pkgset_common.py b/tests/test_pkgset_common.py index 455c7101..4e4ae9fe 100755 --- a/tests/test_pkgset_common.py +++ b/tests/test_pkgset_common.py @@ -96,24 +96,51 @@ class TestMaterializedPkgsetCreate(helpers.PungiTestCase): @helpers.unittest.skipUnless(Modulemd, "Skipping tests, no module support") @mock.patch("pungi.phases.pkgset.common.collect_module_defaults") + @mock.patch("pungi.phases.pkgset.common.collect_module_obsoletes") @mock.patch("pungi.phases.pkgset.common.add_modular_metadata") - def test_run_with_modulemd(self, amm, cmd, mock_run): - mmd = {"x86_64": [mock.Mock()]} + def test_run_with_modulemd(self, amm, cmo, cmd, mock_run): + # Test Index for cmo + mod_index = Modulemd.ModuleIndex.new() + mmdobs = Modulemd.Obsoletes.new( + 1, 10993435, "mod_name", "mod_stream", "testmsg" + ) + mmdobs.set_obsoleted_by("mod_name", "mod_name_2") + mod_index.add_obsoletes(mmdobs) + cmo.return_value = mod_index + + mmd = { + "x86_64": [ + Modulemd.ModuleStream.new( + Modulemd.ModuleStreamVersionEnum.TWO, "mod_name", "stream_name" + ) + ] + } common.MaterializedPackageSet.create( self.compose, self.pkgset, self.prefix, mmd=mmd ) cmd.assert_called_once_with( os.path.join(self.topdir, "work/global/module_defaults"), - set(x.get_module_name.return_value for x in mmd["x86_64"]), + {"mod_name"}, overrides_dir=None, ) - amm.assert_called_once_with( - mock.ANY, - os.path.join(self.topdir, "work/x86_64/repo/foo"), - cmd.return_value, + + cmo.assert_called_once() + cmd.assert_called_once() + amm.assert_called_once() + + self.assertEqual( + amm.mock_calls[0].args[1], os.path.join(self.topdir, "work/x86_64/repo/foo") + ) + self.assertIsInstance(amm.mock_calls[0].args[2], Modulemd.ModuleIndex) + self.assertIsNotNone(amm.mock_calls[0].args[2].get_module("mod_name")) + # Check if proper Index is used by add_modular_metadata + self.assertIsNotNone( + amm.mock_calls[0].args[2].get_module("mod_name").get_obsoletes() + ) + self.assertEqual( + amm.mock_calls[0].args[3], os.path.join(self.topdir, "logs/x86_64/arch_repo_modulemd.foo.x86_64.log"), ) - cmd.return_value.add_module_stream.assert_called_once_with(mmd["x86_64"][0]) class TestCreateArchRepos(helpers.PungiTestCase): From d7aebfc7f9360d198a8d9af7512ebb85435c1482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 15 Jun 2022 12:27:04 +0200 Subject: [PATCH 116/137] 4.3.5 release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JIRA: RHELCMP-9389 Signed-off-by: Lubomír Sedlář --- doc/conf.py | 2 +- pungi.spec | 13 ++++++++++++- setup.py | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 5e153c73..ad9ea0d1 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -53,7 +53,7 @@ copyright = u'2016, Red Hat, Inc.' # The short X.Y version. version = '4.3' # The full version, including alpha/beta/rc tags. -release = '4.3.4' +release = '4.3.5' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pungi.spec b/pungi.spec index 0304519b..dcab6b6d 100644 --- a/pungi.spec +++ b/pungi.spec @@ -1,5 +1,5 @@ Name: pungi -Version: 4.3.4 +Version: 4.3.5 Release: 1%{?dist} Summary: Distribution compose tool @@ -111,6 +111,17 @@ pytest cd tests && ./test_compose.sh %changelog +* Wed Jun 15 2022 Lubomír Sedlář - 4.3.5-1 +- Fix module defaults and obsoletes validation (mkulik) +- Update the cts_keytab field in order to get the hostname of the server + (ounsal) +- Add skip_branding to ostree_installer. (lzhuang) +- kojiwrapper: Ignore warnings before task id (lsedlar) +- Restrict jsonschema version (lsedlar) +- Revert "Do not clone the same repository multiple times, re-use already + cloned repository" (hlin) +- Involve bandit (hlin) + * Fri Apr 01 2022 Ondřej Nosek - 4.3.4-1 - kojiwrapper: Add retries to login call (lsedlar) - Variants file in config can contain path (onosek) diff --git a/setup.py b/setup.py index 51cf9230..41e21f25 100755 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ packages = sorted(packages) setup( name="pungi", - version="4.3.4", + version="4.3.5", description="Distribution compose tool", url="https://pagure.io/pungi", author="Dennis Gilmore", From 960c85efdec62e1d6d83c4c3436ea179771b37a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Mon, 27 Jun 2022 09:45:23 +0200 Subject: [PATCH 117/137] extra_isos: Fix detection of changed packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Checking start of the line is not sufficient for extra_isos that have the variants in separate directories. Signed-off-by: Lubomír Sedlář --- pungi/phases/createiso.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py index 340edc9b..e237cf75 100644 --- a/pungi/phases/createiso.py +++ b/pungi/phases/createiso.py @@ -387,11 +387,18 @@ def read_packages(graft_points): which can change checksum despite data being the same. """ with open(graft_points) as f: - return set(line.split("=", 1)[0] for line in f if line.startswith("Packages/")) + return set( + line.split("=", 1)[0] + for line in f + if line.startswith("Packages/") or "/Packages/" in line + ) def compare_packages(old_graft_points, new_graft_points): - """Read packages from the two files and compare them.""" + """Read packages from the two files and compare them. + + :returns bool: True if there are differences, False otherwise + """ old_files = read_packages(old_graft_points) new_files = read_packages(new_graft_points) return old_files != new_files From da336f75f8c2d3068dc1b0a7814e75ef834651bd Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Fri, 1 Jul 2022 10:14:29 +0800 Subject: [PATCH 118/137] Avoid crash when loading pickle file failed The pickle files are used for reusing results from old compose and the failure should not block the compose process. JIRA: RHELCMP-9494 Signed-off-by: Haibo Lin --- pungi/phases/buildinstall.py | 13 ++++++++++--- pungi/phases/gather/__init__.py | 13 ++++++++++--- pungi/phases/pkgset/sources/source_koji.py | 13 +++++++++---- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/pungi/phases/buildinstall.py b/pungi/phases/buildinstall.py index f175bec1..6240730f 100644 --- a/pungi/phases/buildinstall.py +++ b/pungi/phases/buildinstall.py @@ -669,9 +669,16 @@ class BuildinstallThread(WorkerThread): return None compose.log_info("Loading old BUILDINSTALL phase metadata: %s", old_metadata) - with open(old_metadata, "rb") as f: - old_result = pickle.load(f) - return old_result + try: + with open(old_metadata, "rb") as f: + old_result = pickle.load(f) + return old_result + except Exception as e: + compose.log_debug( + "Failed to load old BUILDINSTALL phase metadata %s : %s" + % (old_metadata, str(e)) + ) + return None def _reuse_old_buildinstall_result(self, compose, arch, variant, cmd, pkgset_phase): """ diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 0474411b..5d2b3c40 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -193,9 +193,16 @@ def load_old_gather_result(compose, arch, variant): return None compose.log_info("Loading old GATHER phase results: %s", old_gather_result) - with open(old_gather_result, "rb") as f: - old_result = pickle.load(f) - return old_result + try: + with open(old_gather_result, "rb") as f: + old_result = pickle.load(f) + return old_result + except Exception as e: + compose.log_debug( + "Failed to load old GATHER phase results %s : %s" + % (old_gather_result, str(e)) + ) + return None def reuse_old_gather_packages(compose, arch, variant, package_sets, methods): diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 1219677c..90a81019 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -819,11 +819,16 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): compose.paths.work.pkgset_file_cache(compose_tag) ) if old_cache_path: - pkgset.set_old_file_cache( - pungi.phases.pkgset.pkgsets.KojiPackageSet.load_old_file_cache( - old_cache_path + try: + pkgset.set_old_file_cache( + pungi.phases.pkgset.pkgsets.KojiPackageSet.load_old_file_cache( + old_cache_path + ) + ) + except Exception as e: + compose.log_debug( + "Failed to load old cache file %s : %s" % (old_cache_path, str(e)) ) - ) is_traditional = compose_tag in compose.conf.get("pkgset_koji_tag", []) should_inherit = inherit if is_traditional else inherit_modules From b27301641afb87e853020509484f3fea6fe92300 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Thu, 7 Jul 2022 13:36:35 +0800 Subject: [PATCH 119/137] Log time taken of each phase Signed-off-by: Haibo Lin --- pungi/phases/base.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pungi/phases/base.py b/pungi/phases/base.py index 917dce26..bf306f1a 100644 --- a/pungi/phases/base.py +++ b/pungi/phases/base.py @@ -14,6 +14,8 @@ # along with this program; if not, see . import logging +import math +import time from pungi import util @@ -58,6 +60,7 @@ class PhaseBase(object): self.compose.log_warning("[SKIP ] %s" % self.msg) self.finished = True return + self._start_time = time.time() self.compose.log_info("[BEGIN] %s" % self.msg) self.compose.notifier.send("phase-start", phase_name=self.name) self.run() @@ -108,6 +111,13 @@ class PhaseBase(object): self.pool.stop() self.finished = True self.compose.log_info("[DONE ] %s" % self.msg) + + if hasattr(self, "_start_time"): + self.compose.log_info( + "PHASE %s took %d seconds" + % (self.name.upper(), math.ceil(time.time() - self._start_time)) + ) + if self.used_patterns is not None: # We only want to report this if the config was actually queried. self.report_unused_patterns() From 19cb013fec107739ba1f5cc537e6f746dfb4252b Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Thu, 14 Jul 2022 10:59:34 +0800 Subject: [PATCH 120/137] Print more logs for git_ls_remote e.output probably contains the root cause of git ls-remote failure. JIRA: RHELCMP-9598 JIRA: RHELCMP-9599 Signed-off-by: Haibo Lin --- pungi/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pungi/util.py b/pungi/util.py index 06b657ad..b744f572 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -292,7 +292,8 @@ def resolve_git_ref(repourl, ref): _, output = git_ls_remote(repourl, ref) except RuntimeError as e: raise GitUrlResolveError( - "ref does not exist in remote repo %s with the error %s" % (repourl, e) + "ref does not exist in remote repo %s with the error %s %s" + % (repourl, e, e.output) ) lines = [] From b0b494fff025ae35a5881297075475684adafee5 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Fri, 15 Jul 2022 13:17:46 +0800 Subject: [PATCH 121/137] Convert _ssh_run output to str for python3 This is for fixing "a bytes-like object is required, not 'str'" issue in runroot task. JIRA: RHELCMP-9224 Signed-off-by: Haibo Lin --- pungi/runroot.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pungi/runroot.py b/pungi/runroot.py index 95912e90..eb7e2dc4 100644 --- a/pungi/runroot.py +++ b/pungi/runroot.py @@ -15,6 +15,7 @@ import os import re +import six from six.moves import shlex_quote import kobo.log from kobo.shortcuts import run @@ -149,7 +150,11 @@ class Runroot(kobo.log.LoggingBase): """ formatted_cmd = command.format(**fmt_dict) if fmt_dict else command ssh_cmd = ["ssh", "-oBatchMode=yes", "-n", "-l", user, hostname, formatted_cmd] - return run(ssh_cmd, show_cmd=True, logfile=log_file)[1] + output = run(ssh_cmd, show_cmd=True, logfile=log_file)[1] + if six.PY3 and isinstance(output, bytes): + return output.decode() + else: + return output def _log_file(self, base, suffix): return base.replace(".log", "." + suffix + ".log") From ea8020473d0b8f3fc38f3d9fc0bc0b7dcab0f1ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Budai?= Date: Tue, 9 Aug 2022 17:37:16 +0200 Subject: [PATCH 122/137] doc: fix osbuild's image_types field name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's actually image_types, not just image_type. See https://pagure.io/fork/obudai/pungi/blob/master/f/pungi/checks.py#_1160 Signed-off-by: Ondřej Budai --- doc/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 4a7f8986..697784b8 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1592,7 +1592,7 @@ OSBuild Composer for building images * ``name`` -- name of the Koji package * ``distro`` -- image for which distribution should be build TODO examples - * ``image_type`` -- a list of image types to build (e.g. ``qcow2``) + * ``image_types`` -- a list of image types to build (e.g. ``qcow2``) Optional keys: From 778dcfa587570396fea959f9445f02ccf2adcc71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 10 Aug 2022 08:31:02 +0200 Subject: [PATCH 123/137] Fix black complaint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lubomír Sedlář --- pungi/media_split.py | 4 ++-- pungi/phases/gather/__init__.py | 3 ++- pungi/scripts/pungi.py | 6 +++--- tests/test_createiso_phase.py | 16 ++++++++-------- tests/test_patch_iso.py | 2 +- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/pungi/media_split.py b/pungi/media_split.py index 74be8fab..708ebd5d 100644 --- a/pungi/media_split.py +++ b/pungi/media_split.py @@ -20,8 +20,8 @@ import os SIZE_UNITS = { "b": 1, "k": 1024, - "M": 1024 ** 2, - "G": 1024 ** 3, + "M": 1024**2, + "G": 1024**3, } diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 5d2b3c40..72843571 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -541,7 +541,8 @@ def write_packages(compose, arch, variant, pkg_map, path_prefix): def trim_packages(compose, arch, variant, pkg_map, parent_pkgs=None, remove_pkgs=None): - """Remove parent variant's packages from pkg_map <-- it gets modified in this function + """Remove parent variant's packages from pkg_map <-- it gets modified in + this function There are three cases where changes may happen: diff --git a/pungi/scripts/pungi.py b/pungi/scripts/pungi.py index 59b96ddc..9c307eaf 100644 --- a/pungi/scripts/pungi.py +++ b/pungi/scripts/pungi.py @@ -476,14 +476,14 @@ def main(): else: mypungi.downloadSRPMs() - print("RPM size: %s MiB" % (mypungi.size_packages() / 1024 ** 2)) + print("RPM size: %s MiB" % (mypungi.size_packages() / 1024**2)) if not opts.nodebuginfo: print( "DEBUGINFO size: %s MiB" - % (mypungi.size_debuginfo() / 1024 ** 2) + % (mypungi.size_debuginfo() / 1024**2) ) if not opts.nosource: - print("SRPM size: %s MiB" % (mypungi.size_srpms() / 1024 ** 2)) + print("SRPM size: %s MiB" % (mypungi.size_srpms() / 1024**2)) # Furthermore (but without the yumlock...) if not opts.sourceisos: diff --git a/tests/test_createiso_phase.py b/tests/test_createiso_phase.py index 02ccd8be..fd3c42ec 100644 --- a/tests/test_createiso_phase.py +++ b/tests/test_createiso_phase.py @@ -1105,8 +1105,8 @@ class SplitIsoTest(helpers.PungiTestCase): os.path.join(self.topdir, "compose/Server/x86_64/os/n/media.repo") ) - M = 1024 ** 2 - G = 1024 ** 3 + M = 1024**2 + G = 1024**3 with mock.patch( "os.path.getsize", @@ -1157,8 +1157,8 @@ class SplitIsoTest(helpers.PungiTestCase): os.path.join(self.topdir, "compose/Server/x86_64/os/n/media.repo") ) - M = 1024 ** 2 - G = 1024 ** 3 + M = 1024**2 + G = 1024**3 with mock.patch( "os.path.getsize", @@ -1209,7 +1209,7 @@ class SplitIsoTest(helpers.PungiTestCase): os.path.join(self.topdir, "compose/Server/x86_64/os/Packages/x/pad.rpm") ) - M = 1024 ** 2 + M = 1024**2 # treeinfo has size 0, spacer leaves 11M of free space, so with 10M # reserve the padding package should be on second disk @@ -1233,7 +1233,7 @@ class SplitIsoTest(helpers.PungiTestCase): ) def test_can_customize_reserve(self): - compose = helpers.DummyCompose(self.topdir, {"split_iso_reserve": 1024 ** 2}) + compose = helpers.DummyCompose(self.topdir, {"split_iso_reserve": 1024**2}) helpers.touch( os.path.join(self.topdir, "compose/Server/x86_64/os/.treeinfo"), TREEINFO ) @@ -1244,7 +1244,7 @@ class SplitIsoTest(helpers.PungiTestCase): os.path.join(self.topdir, "compose/Server/x86_64/os/Packages/x/pad.rpm") ) - M = 1024 ** 2 + M = 1024**2 with mock.patch( "os.path.getsize", DummySize({"spacer": 4688465664, "pad": 5 * M}) @@ -1265,7 +1265,7 @@ class SplitIsoTest(helpers.PungiTestCase): os.path.join(self.topdir, "compose/Server/x86_64/os/Packages/x/pad.rpm") ) - M = 1024 ** 2 + M = 1024**2 with mock.patch( "os.path.getsize", DummySize({"spacer": 4688465664, "pad": 5 * M}) diff --git a/tests/test_patch_iso.py b/tests/test_patch_iso.py index 55abf12b..9fe8d7b4 100644 --- a/tests/test_patch_iso.py +++ b/tests/test_patch_iso.py @@ -61,7 +61,7 @@ class EqualsAny(object): return True def __repr__(self): - return u"ANYTHING" + return "ANYTHING" ANYTHING = EqualsAny() From 0abf937b0eeafa96e2cbdce0c47c350a88789d5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Tue, 9 Aug 2022 15:10:38 +0200 Subject: [PATCH 124/137] Fix compatibility with jsonschema >= 4.0.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fedora Rawhide (to be 37) packages jsonschema 4.9.0 at the moment, so we can no longer get by with limiting the requirements. This patch makes the validation work with both old and new version. Fixes: rhbz#2113607 Signed-off-by: Lubomír Sedlář --- pungi/checks.py | 12 +++++++++++- requirements.txt | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pungi/checks.py b/pungi/checks.py index bc5d908f..6d4465ed 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -229,7 +229,6 @@ def validate(config, offline=False, schema=None): ) validator = DefaultValidator( schema, - {"array": (tuple, list), "regex": six.string_types, "url": six.string_types}, ) errors = [] warnings = [] @@ -445,6 +444,16 @@ def _extend_with_default_and_alias(validator_class, offline=False): context=all_errors, ) + def is_array(checker, instance): + return isinstance(instance, (tuple, list)) + + def is_string_type(checker, instance): + return isinstance(instance, six.string_types) + + type_checker = validator_class.TYPE_CHECKER.redefine_many( + {"array": is_array, "regex": is_string_type, "url": is_string_type} + ) + return jsonschema.validators.extend( validator_class, { @@ -455,6 +464,7 @@ def _extend_with_default_and_alias(validator_class, offline=False): "additionalProperties": _validate_additional_properties, "anyOf": _validate_any_of, }, + type_checker=type_checker, ) diff --git a/requirements.txt b/requirements.txt index 4eba1ca4..ee73d0c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ dict.sorted dogpile.cache fedmsg funcsigs -jsonschema < 4.0.0 +jsonschema kobo koji lxml From 13ea8e58347170d8cc4856975ceb9f8714dfc9e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Mon, 6 Jun 2022 12:33:17 +0200 Subject: [PATCH 125/137] Create DVDs with xorriso MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use a different approach for building DVDs when xorriso is enabled. The default of using genisoimage is not changed at all. When the config option is set to use xorriso, the actual execution is different between bootable and non-bootable images. The non-bootable images are still created by running xorrisofs (which is a compatibility tool with same UI as genisoimage). Since the image is not bootable, there should be no problems with boot options. For bootable images, Pungi will instead take the boot.iso generated by Lorax, and use xorriso to inject all the extra files into the image. The shell script that used to invoke all the commands to build the ISO now runs the `xorriso` command in interactive mode and feeds another file into it. The new file contains the xorriso commands to add the required files to the image. Signed-off-by: Lubomír Sedlář --- pungi/createiso.py | 33 +++++++++++++++++++++++++++++++-- pungi/phases/createiso.py | 12 +++++++----- pungi/phases/extra_isos.py | 11 ++++++----- tests/test_createiso_phase.py | 8 ++++++++ 4 files changed, 52 insertions(+), 12 deletions(-) diff --git a/pungi/createiso.py b/pungi/createiso.py index 4e1a5336..4d80678c 100644 --- a/pungi/createiso.py +++ b/pungi/createiso.py @@ -15,6 +15,7 @@ CreateIsoOpts = namedtuple( "CreateIsoOpts", [ "buildinstall_method", + "boot_iso", "arch", "output_dir", "jigdo_dir", @@ -26,6 +27,7 @@ CreateIsoOpts = namedtuple( "hfs_compat", "use_xorrisofs", "iso_level", + "script_dir", ], ) CreateIsoOpts.__new__.__defaults__ = (None,) * len(CreateIsoOpts._fields) @@ -116,6 +118,27 @@ def make_jigdo(f, opts): emit(f, cmd) +def write_xorriso_commands(opts): + script = os.path.join(opts.script_dir, "xorriso-%s.txt" % id(opts)) + with open(script, "w") as f: + emit(f, "-indev %s" % opts.boot_iso) + emit(f, "-outdev %s" % os.path.join(opts.output_dir, opts.iso_name)) + emit(f, "-boot_image any replay") + emit(f, "-volid %s" % opts.volid) + + with open(opts.graft_points) as gp: + for line in gp: + iso_path, fs_path = line.strip().split("=", 1) + emit(f, "-map %s %s" % (fs_path, iso_path)) + + if opts.arch == "ppc64le": + # This is needed for the image to be bootable. + emit(f, "-as mkisofs -U --") + + emit(f, "-end") + return script + + def write_script(opts, f): if bool(opts.jigdo_dir) != bool(opts.os_tree): raise RuntimeError("jigdo_dir must be used together with os_tree") @@ -123,8 +146,14 @@ def write_script(opts, f): emit(f, "#!/bin/bash") emit(f, "set -ex") emit(f, "cd %s" % opts.output_dir) - make_image(f, opts) - run_isohybrid(f, opts) + + if opts.use_xorrisofs and opts.buildinstall_method: + script = write_xorriso_commands(opts) + emit(f, "xorriso -dialog on <%s" % script) + else: + make_image(f, opts) + run_isohybrid(f, opts) + implant_md5(f, opts) make_manifest(f, opts) if opts.jigdo_dir: diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py index e237cf75..722cfb04 100644 --- a/pungi/phases/createiso.py +++ b/pungi/phases/createiso.py @@ -343,7 +343,10 @@ class CreateisoPhase(PhaseLoggerMixin, PhaseBase): if bootable: opts = opts._replace( - buildinstall_method=self.compose.conf["buildinstall_method"] + buildinstall_method=self.compose.conf[ + "buildinstall_method" + ], + boot_iso=os.path.join(os_tree, "images", "boot.iso"), ) if self.compose.conf["create_jigdo"]: @@ -355,10 +358,9 @@ class CreateisoPhase(PhaseLoggerMixin, PhaseBase): # Reuse was successful, go to next ISO continue - script_file = os.path.join( - self.compose.paths.work.tmp_dir(arch, variant), - "createiso-%s.sh" % filename, - ) + script_dir = self.compose.paths.work.tmp_dir(arch, variant) + opts = opts._replace(script_dir=script_dir) + script_file = os.path.join(script_dir, "createiso-%s.sh" % filename) with open(script_file, "w") as f: createiso.write_script(opts, f) cmd["cmd"] = ["bash", script_file] diff --git a/pungi/phases/extra_isos.py b/pungi/phases/extra_isos.py index 56c5f28f..45d06b63 100644 --- a/pungi/phases/extra_isos.py +++ b/pungi/phases/extra_isos.py @@ -132,14 +132,15 @@ class ExtraIsosThread(WorkerThread): use_xorrisofs=compose.conf.get("createiso_use_xorrisofs"), iso_level=compose.conf.get("iso_level"), ) + os_tree = compose.paths.compose.os_tree(arch, variant) if compose.conf["create_jigdo"]: jigdo_dir = compose.paths.compose.jigdo_dir(arch, variant) - os_tree = compose.paths.compose.os_tree(arch, variant) opts = opts._replace(jigdo_dir=jigdo_dir, os_tree=os_tree) if bootable: opts = opts._replace( - buildinstall_method=compose.conf["buildinstall_method"] + buildinstall_method=compose.conf["buildinstall_method"], + boot_iso=os.path.join(os_tree, "images", "boot.iso"), ) # Check if it can be reused. @@ -148,9 +149,9 @@ class ExtraIsosThread(WorkerThread): config_hash = hash.hexdigest() if not self.try_reuse(compose, variant, arch, config_hash, opts): - script_file = os.path.join( - compose.paths.work.tmp_dir(arch, variant), "extraiso-%s.sh" % filename - ) + script_dir = compose.paths.work.tmp_dir(arch, variant) + opts = opts._replace(script_dir=script_dir) + script_file = os.path.join(script_dir, "extraiso-%s.sh" % filename) with open(script_file, "w") as f: createiso.write_script(opts, f) diff --git a/tests/test_createiso_phase.py b/tests/test_createiso_phase.py index fd3c42ec..538cfb2e 100644 --- a/tests/test_createiso_phase.py +++ b/tests/test_createiso_phase.py @@ -124,6 +124,7 @@ class CreateisoPhaseTest(helpers.PungiTestCase): os_tree=None, hfs_compat=True, use_xorrisofs=False, + script_dir="%s/work/x86_64/tmp-Server" % self.topdir, ) ], ) @@ -240,6 +241,9 @@ class CreateisoPhaseTest(helpers.PungiTestCase): [ CreateIsoOpts( output_dir="%s/compose/Server/x86_64/iso" % self.topdir, + boot_iso=( + "%s/compose/Server/x86_64/os/images/boot.iso" % self.topdir + ), iso_name="image-name", volid="test-1.0 Server.x86_64", graft_points="dummy-graft-points", @@ -250,6 +254,7 @@ class CreateisoPhaseTest(helpers.PungiTestCase): os_tree=None, hfs_compat=True, use_xorrisofs=False, + script_dir="%s/work/x86_64/tmp-Server" % self.topdir, ), CreateIsoOpts( output_dir="%s/compose/Server/source/iso" % self.topdir, @@ -262,6 +267,7 @@ class CreateisoPhaseTest(helpers.PungiTestCase): os_tree=None, hfs_compat=True, use_xorrisofs=False, + script_dir="%s/work/src/tmp-Server" % self.topdir, ), ], ) @@ -394,6 +400,7 @@ class CreateisoPhaseTest(helpers.PungiTestCase): os_tree=None, hfs_compat=True, use_xorrisofs=False, + script_dir="%s/work/src/tmp-Server" % self.topdir, ) ], ) @@ -501,6 +508,7 @@ class CreateisoPhaseTest(helpers.PungiTestCase): os_tree=None, hfs_compat=False, use_xorrisofs=False, + script_dir="%s/work/x86_64/tmp-Server" % self.topdir, ) ], ) From 11fa3425077625e4999bfbac1b0c942972b3054a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Tue, 16 Aug 2022 14:25:43 +0200 Subject: [PATCH 126/137] createiso: Make ISO level more granular MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make it possible to set the level separately for each variant and architecture. JIRA: RHELCMP-9341 Signed-off-by: Lubomír Sedlář --- doc/configuration.rst | 4 +++- pungi/checks.py | 6 ++++-- pungi/phases/createiso.py | 14 +++++++++++++- pungi/phases/extra_isos.py | 3 ++- tests/data/dummy-pungi.conf | 6 ++++++ tests/test_createiso_phase.py | 35 +++++++++++++++++++++++++++++++++++ 6 files changed, 63 insertions(+), 5 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 697784b8..4bf64ade 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1286,7 +1286,9 @@ Options suffix (using multiples of 1024). **iso_level** - (*int*) [optional] -- Set the ISO9660 conformance level. Valid numbers are 1 to 4. + (*int|list*) [optional] -- Set the ISO9660 conformance level. This is + either a global single value (a number from 1 to 4), or a variant/arch + mapping. **split_iso_reserve** = 10MiB (*int|str*) -- how much free space should be left on each disk. The format diff --git a/pungi/checks.py b/pungi/checks.py index 6d4465ed..14ad674e 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -760,8 +760,10 @@ def make_schema(): "createiso_break_hardlinks": {"type": "boolean", "default": False}, "createiso_use_xorrisofs": {"type": "boolean", "default": False}, "iso_level": { - "type": "number", - "enum": [1, 2, 3, 4], + "anyOf": [ + {"type": "number", "enum": [1, 2, 3, 4]}, + _variant_arch_mapping({"type": "number", "enum": [1, 2, 3, 4]}), + ], }, "iso_hfs_ppc64le_compatible": {"type": "boolean", "default": True}, "multilib": _variant_arch_mapping( diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py index 722cfb04..481d38fb 100644 --- a/pungi/phases/createiso.py +++ b/pungi/phases/createiso.py @@ -338,7 +338,7 @@ class CreateisoPhase(PhaseLoggerMixin, PhaseBase): supported=self.compose.supported, hfs_compat=self.compose.conf["iso_hfs_ppc64le_compatible"], use_xorrisofs=self.compose.conf.get("createiso_use_xorrisofs"), - iso_level=self.compose.conf.get("iso_level"), + iso_level=get_iso_level_config(self.compose, variant, arch), ) if bootable: @@ -821,3 +821,15 @@ class OldFileLinker(object): """Clean up all files created by this instance.""" for f in self.linked_files: os.unlink(f) + + +def get_iso_level_config(compose, variant, arch): + """ + Get configured ISO level for this variant and architecture. + """ + level = compose.conf.get("iso_level") + if isinstance(level, list): + level = None + for c in get_arch_variant_data(compose.conf, "iso_level", arch, variant): + level = c + return level diff --git a/pungi/phases/extra_isos.py b/pungi/phases/extra_isos.py index 45d06b63..2f1d9964 100644 --- a/pungi/phases/extra_isos.py +++ b/pungi/phases/extra_isos.py @@ -32,6 +32,7 @@ from pungi.phases.createiso import ( load_and_tweak_treeinfo, compare_packages, OldFileLinker, + get_iso_level_config, ) from pungi.util import ( failable, @@ -130,7 +131,7 @@ class ExtraIsosThread(WorkerThread): supported=compose.supported, hfs_compat=compose.conf["iso_hfs_ppc64le_compatible"], use_xorrisofs=compose.conf.get("createiso_use_xorrisofs"), - iso_level=compose.conf.get("iso_level"), + iso_level=get_iso_level_config(compose, variant, arch), ) os_tree = compose.paths.compose.os_tree(arch, variant) if compose.conf["create_jigdo"]: diff --git a/tests/data/dummy-pungi.conf b/tests/data/dummy-pungi.conf index d27423d4..f7e56388 100644 --- a/tests/data/dummy-pungi.conf +++ b/tests/data/dummy-pungi.conf @@ -109,3 +109,9 @@ extra_isos = { 'filename': 'extra-{filename}', }] } + +iso_level = [ + (".*", { + "src": 3, + }), +] diff --git a/tests/test_createiso_phase.py b/tests/test_createiso_phase.py index 538cfb2e..f71a45c0 100644 --- a/tests/test_createiso_phase.py +++ b/tests/test_createiso_phase.py @@ -1554,3 +1554,38 @@ class CreateisoPerformReusePhaseTest(helpers.PungiTestCase): mock.call.abort(), ], ) + + +class ComposeConfGetIsoLevelTest(helpers.PungiTestCase): + def test_global_config(self): + compose = helpers.DummyCompose(self.topdir, {"iso_level": 3}) + + self.assertEqual( + createiso.get_iso_level_config( + compose, compose.variants["Server"], "x86_64" + ), + 3, + ) + + def test_src_only_config(self): + compose = helpers.DummyCompose( + self.topdir, + {"iso_level": [(".*", {"src": 4})]}, + ) + + self.assertEqual( + createiso.get_iso_level_config(compose, compose.variants["Server"], "src"), + 4, + ) + + def test_no_match(self): + compose = helpers.DummyCompose( + self.topdir, + {"iso_level": [("^Server$", {"*": 4})]}, + ) + + self.assertIsNone( + createiso.get_iso_level_config( + compose, compose.variants["Client"], "x86_64" + ), + ) From 603c61a0333df56c0a0970c33b833b9c5193227f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Ravier?= Date: Fri, 19 Aug 2022 23:21:36 +0200 Subject: [PATCH 127/137] ostree: Add unified core mode for compose in rpm-ostree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rpm-ostree is moving to unified core composes and this is now working for Silverblue & Kinoite. This is untested for IoT but they should move to os-build with Fedora 37. See: https://github.com/coreos/rpm-ostree/issues/729 Merges: https://pagure.io/pungi/pull-request/1626 Signed-off-by: Timothée Ravier --- doc/configuration.rst | 2 ++ doc/examples.rst | 2 ++ pungi/checks.py | 2 ++ pungi/ostree/__init__.py | 5 +++++ pungi/ostree/tree.py | 4 ++++ pungi/phases/ostree.py | 1 + tests/test_ostree_phase.py | 1 + tests/test_ostree_script.py | 16 ++++++++++++++++ 8 files changed, 33 insertions(+) diff --git a/doc/configuration.rst b/doc/configuration.rst index 4bf64ade..8602a520 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1707,6 +1707,8 @@ repository with a new commit. * ``force_new_commit`` -- (*bool*) Do not use rpm-ostree's built-in change detection. Defaults to ``False``. + * ``unified_core`` -- (*bool*) Use rpm-ostree in unified core mode for composes. + Defaults to ``False``. * ``version`` -- (*str*) Version string to be added as versioning metadata. If this option is set to ``!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN``, a value will be generated automatically as ``$VERSION.$RELEASE``. diff --git a/doc/examples.rst b/doc/examples.rst index 0370fa69..aef18e34 100644 --- a/doc/examples.rst +++ b/doc/examples.rst @@ -332,6 +332,8 @@ This is a shortened configuration for Fedora Radhide compose as of 2019-10-14. "tag_ref": False, # Don't use change detection in ostree. "force_new_commit": True, + # Use unified core mode for rpm-ostree composes + "unified_core": True, # This is the location for the repo where new commit will be # created. Note that this is outside of the compose dir. "ostree_repo": "/mnt/koji/compose/ostree/repo/", diff --git a/pungi/checks.py b/pungi/checks.py index 14ad674e..0f339556 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -1027,6 +1027,7 @@ def make_schema(): }, "update_summary": {"type": "boolean"}, "force_new_commit": {"type": "boolean"}, + "unified_core": {"type": "boolean"}, "version": {"type": "string"}, "config_branch": {"type": "string"}, "tag_ref": {"type": "boolean"}, @@ -1061,6 +1062,7 @@ def make_schema(): "failable": {"$ref": "#/definitions/list_of_strings"}, "update_summary": {"type": "boolean"}, "force_new_commit": {"type": "boolean"}, + "unified_core": {"type": "boolean"}, "version": {"type": "string"}, "config_branch": {"type": "string"}, "tag_ref": {"type": "boolean"}, diff --git a/pungi/ostree/__init__.py b/pungi/ostree/__init__.py index 03a02a73..49162692 100644 --- a/pungi/ostree/__init__.py +++ b/pungi/ostree/__init__.py @@ -65,6 +65,11 @@ def main(args=None): action="store_true", help="do not use rpm-ostree's built-in change detection", ) + treep.add_argument( + "--unified-core", + action="store_true", + help="use unified core mode in rpm-ostree", + ) installerp = subparser.add_parser( "installer", help="Create an OSTree installer image" diff --git a/pungi/ostree/tree.py b/pungi/ostree/tree.py index a2ee379d..1ba138b3 100644 --- a/pungi/ostree/tree.py +++ b/pungi/ostree/tree.py @@ -43,6 +43,9 @@ class Tree(OSTree): # because something went wrong. "--touch-if-changed=%s.stamp" % self.commitid_file, ] + if self.unified_core: + # See https://github.com/coreos/rpm-ostree/issues/729 + cmd.append("--unified-core") if self.version: # Add versioning metadata cmd.append("--add-metadata-string=version=%s" % self.version) @@ -121,6 +124,7 @@ class Tree(OSTree): self.extra_config = self.args.extra_config self.ostree_ref = self.args.ostree_ref self.force_new_commit = self.args.force_new_commit + self.unified_core = self.args.unified_core if self.extra_config or self.ostree_ref: if self.extra_config: diff --git a/pungi/phases/ostree.py b/pungi/phases/ostree.py index 2fcfce6c..cbfcd76e 100644 --- a/pungi/phases/ostree.py +++ b/pungi/phases/ostree.py @@ -165,6 +165,7 @@ class OSTreeThread(WorkerThread): ("update-summary", config.get("update_summary", False)), ("ostree-ref", config.get("ostree_ref")), ("force-new-commit", config.get("force_new_commit", False)), + ("unified-core", config.get("unified_core", False)), ] ) packages = ["pungi", "ostree", "rpm-ostree"] diff --git a/tests/test_ostree_phase.py b/tests/test_ostree_phase.py index b214a127..40c99076 100644 --- a/tests/test_ostree_phase.py +++ b/tests/test_ostree_phase.py @@ -325,6 +325,7 @@ class OSTreeThreadTest(helpers.PungiTestCase): "ostree-ref": None, "force-new-commit": False, "version": None, + "unified-core": False, }, channel=None, mounts=[self.topdir, self.repo], diff --git a/tests/test_ostree_script.py b/tests/test_ostree_script.py index c32ce466..b6710e57 100644 --- a/tests/test_ostree_script.py +++ b/tests/test_ostree_script.py @@ -238,6 +238,22 @@ class OstreeTreeScriptTest(helpers.PungiTestCase): self.assertCorrectCall(run, extra_args=["--force-nocache"]) + @mock.patch("kobo.shortcuts.run") + def test_unified_core(self, run): + helpers.touch(os.path.join(self.repo, "initialized")) + + ostree.main( + [ + "tree", + "--repo=%s" % self.repo, + "--log-dir=%s" % os.path.join(self.topdir, "logs", "Atomic"), + "--treefile=%s/fedora-atomic-docker-host.json" % self.topdir, + "--unified-core", + ] + ) + + self.assertCorrectCall(run, extra_args=["--unified-core"]) + @mock.patch("kobo.shortcuts.run") def test_extra_config_with_extra_repos(self, run): configdir = os.path.join(self.topdir, "config") From 779793386c07ae5145810ecc59472ef61fa90999 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Budai?= Date: Wed, 10 Aug 2022 10:28:56 +0200 Subject: [PATCH 128/137] osbuild: add support for building ostree artifacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to start building Fedora IoT images with osbuild, we need to be able to pass ostree options from pungi to the koji's osbuildImage task. This commit adds support for it via new configuration options: ostree_url, ostree_url and ostree_parent. A test was added to cover these new options and they are were also added into the documentation. JIRA: COMPOSER-1702 Merges: https://pagure.io/pungi/pull-request/1624 Signed-off-by: Ondřej Budai --- doc/configuration.rst | 5 ++ pungi/checks.py | 3 + pungi/phases/osbuild.py | 11 +++ tests/test_osbuild_phase.py | 134 +++++++++++++++++++++++++++++++++++- 4 files changed, 152 insertions(+), 1 deletion(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 8602a520..d91d37d8 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1610,6 +1610,11 @@ OSBuild Composer for building images * ``arches`` -- list of architectures for which to build the image. By default, the variant arches are used. This option can only restrict it, not add a new one. + * ``ostree_url`` -- URL of the repository that's used to fetch the parent + commit from. + * ``ostree_ref`` -- name of the ostree branch + * ``ostree_parent`` -- commit hash or a a branch-like reference to the + parent commit. .. note:: There is initial support for having this task as failable without aborting diff --git a/pungi/checks.py b/pungi/checks.py index 0f339556..80d4e506 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -1177,6 +1177,9 @@ def make_schema(): "repo": {"$ref": "#/definitions/list_of_strings"}, "failable": {"$ref": "#/definitions/list_of_strings"}, "subvariant": {"type": "string"}, + "ostree_url": {"type": "string"}, + "ostree_ref": {"type": "string"}, + "ostree_parent": {"type": "string"}, }, "required": ["name", "distro", "image_types"], "additionalProperties": False, diff --git a/pungi/phases/osbuild.py b/pungi/phases/osbuild.py index 89719050..9215acfa 100644 --- a/pungi/phases/osbuild.py +++ b/pungi/phases/osbuild.py @@ -113,8 +113,19 @@ class RunOSBuildThread(WorkerThread): koji = kojiwrapper.KojiWrapper(compose) koji.login() + ostree = {} + if config.get("ostree_url"): + ostree["url"] = config["ostree_url"] + if config.get("ostree_ref"): + ostree["ref"] = config["ostree_ref"] + if config.get("ostree_parent"): + ostree["parent"] = config["ostree_parent"] + # Start task opts = {"repo": repo} + if ostree: + opts["ostree"] = ostree + if release: opts["release"] = release task_id = koji.koji_proxy.osbuildImage( diff --git a/tests/test_osbuild_phase.py b/tests/test_osbuild_phase.py index 3dbf6fe5..b3f5078a 100644 --- a/tests/test_osbuild_phase.py +++ b/tests/test_osbuild_phase.py @@ -178,7 +178,6 @@ class RunOSBuildThreadTest(helpers.PungiTestCase): # Verify two Koji instances were created. self.assertEqual(len(KojiWrapper.call_args), 2) - print(koji.mock_calls) # Verify correct calls to Koji self.assertEqual( koji.mock_calls, @@ -248,6 +247,139 @@ class RunOSBuildThreadTest(helpers.PungiTestCase): ], ) + @mock.patch("pungi.util.get_file_size", new=lambda fp: 65536) + @mock.patch("pungi.util.get_mtime", new=lambda fp: 1024) + @mock.patch("pungi.phases.osbuild.Linker") + @mock.patch("pungi.phases.osbuild.kojiwrapper.KojiWrapper") + def test_process_ostree(self, KojiWrapper, Linker): + cfg = { + "name": "test-image", + "distro": "rhel-8", + "image_types": ["edge-raw-disk"], + "ostree_url": "http://edge.example.com/repo", + "ostree_ref": "test/iot", + "ostree_parent": "test/iot-parent", + } + build_id = 5678 + koji = KojiWrapper.return_value + koji.watch_task.side_effect = self.make_fake_watch(0) + koji.koji_proxy.osbuildImage.return_value = 1234 + koji.koji_proxy.getTaskResult.return_value = { + "composer": {"server": "https://composer.osbuild.org", "id": ""}, + "koji": {"build": build_id}, + } + koji.koji_proxy.getBuild.return_value = { + "build_id": build_id, + "name": "test-image", + "version": "1", + "release": "1", + } + koji.koji_proxy.listArchives.return_value = [ + { + "extra": {"image": {"arch": "aarch64"}}, + "filename": "image.aarch64.raw.xz", + "type_name": "raw-xz", + }, + { + "extra": {"image": {"arch": "x86_64"}}, + "filename": "image.x86_64.raw.xz", + "type_name": "raw-xz", + }, + ] + koji.koji_module.pathinfo = orig_koji.pathinfo + + self.t.process( + ( + self.compose, + self.compose.variants["Everything"], + cfg, + ["aarch64", "x86_64"], + "1", # version + "15", # release + "image-target", + [self.topdir + "/compose/Everything/$arch/os"], + ["x86_64"], + ), + 1, + ) + + # Verify two Koji instances were created. + self.assertEqual(len(KojiWrapper.call_args), 2) + # Verify correct calls to Koji + self.assertEqual( + koji.mock_calls, + [ + mock.call.login(), + mock.call.koji_proxy.osbuildImage( + "test-image", + "1", + "rhel-8", + ["edge-raw-disk"], + "image-target", + ["aarch64", "x86_64"], + opts={ + "release": "15", + "repo": [self.topdir + "/compose/Everything/$arch/os"], + "ostree": { + "url": "http://edge.example.com/repo", + "ref": "test/iot", + "parent": "test/iot-parent", + }, + }, + ), + mock.call.save_task_id(1234), + mock.call.watch_task(1234, mock.ANY), + mock.call.koji_proxy.getTaskResult(1234), + mock.call.koji_proxy.getBuild(build_id), + mock.call.koji_proxy.listArchives(buildID=build_id), + ], + ) + + # Assert there are 2 images added to manifest and the arguments are sane + self.assertEqual( + self.compose.im.add.call_args_list, + [ + mock.call(arch="aarch64", variant="Everything", image=mock.ANY), + mock.call(arch="x86_64", variant="Everything", image=mock.ANY), + ], + ) + for call in self.compose.im.add.call_args_list: + _, kwargs = call + image = kwargs["image"] + self.assertEqual(kwargs["variant"], "Everything") + self.assertIn(kwargs["arch"], ("aarch64", "x86_64")) + self.assertEqual(kwargs["arch"], image.arch) + self.assertEqual( + "Everything/%(arch)s/images/image.%(arch)s.raw.xz" + % {"arch": image.arch}, + image.path, + ) + self.assertEqual("raw.xz", image.format) + self.assertEqual("raw-xz", image.type) + self.assertEqual("Everything", image.subvariant) + + self.assertTrue( + os.path.isdir(self.topdir + "/compose/Everything/aarch64/images") + ) + self.assertTrue( + os.path.isdir(self.topdir + "/compose/Everything/x86_64/images") + ) + + self.assertEqual( + Linker.return_value.mock_calls, + [ + mock.call.link( + "/mnt/koji/packages/test-image/1/1/images/image.%(arch)s.raw.xz" + % {"arch": arch}, + self.topdir + + "/compose/Everything/%(arch)s/images/image.%(arch)s.raw.xz" + % {"arch": arch}, + link_type="hardlink-or-copy", + ) + for arch in ["aarch64", "x86_64"] + ], + ) + @mock.patch("pungi.util.get_file_size", new=lambda fp: 65536) @mock.patch("pungi.util.get_mtime", new=lambda fp: 1024) @mock.patch("pungi.phases.osbuild.Linker") From 8aba2363e234e239a470437266202e3f3e8608ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 25 Aug 2022 10:59:15 +0200 Subject: [PATCH 129/137] pkgset: Report better error when module is missing an arch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pungi expects each module to be built for all architectures by default. Unless the module is filtered out, missing metadata for a particular arch would cause it to crash with a incomprehensible error message. This should make it a little better. Relates: https://pagure.io/releng/failed-composes/issue/3889 Signed-off-by: Lubomír Sedlář --- pungi/phases/pkgset/sources/source_koji.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 90a81019..26ec770e 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -264,8 +264,14 @@ def _add_module_to_variant( compose.log_debug("Module %s is filtered from %s.%s", nsvc, variant, arch) continue + filename = "modulemd.%s.txt" % arch + if filename not in mmds: + raise RuntimeError( + "Module %s does not have metadata for arch %s and is not filtered " + "out via filter_modules option." % (nsvc, arch) + ) mod_stream = read_single_module_stream_from_file( - mmds["modulemd.%s.txt" % arch], compose, arch, build + mmds[filename], compose, arch, build ) if mod_stream: added = True From 146b88e1e92a27e64c90c811be855f99f247f4f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 26 Aug 2022 11:13:43 +0200 Subject: [PATCH 130/137] 4.3.6 release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JIRA: RHELCMP-9914 Signed-off-by: Lubomír Sedlář --- doc/conf.py | 2 +- pungi.spec | 17 ++++++++++++++++- setup.py | 2 +- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index ad9ea0d1..d5cd50b5 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -53,7 +53,7 @@ copyright = u'2016, Red Hat, Inc.' # The short X.Y version. version = '4.3' # The full version, including alpha/beta/rc tags. -release = '4.3.5' +release = '4.3.6' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pungi.spec b/pungi.spec index dcab6b6d..b1a39b27 100644 --- a/pungi.spec +++ b/pungi.spec @@ -1,5 +1,5 @@ Name: pungi -Version: 4.3.5 +Version: 4.3.6 Release: 1%{?dist} Summary: Distribution compose tool @@ -111,6 +111,21 @@ pytest cd tests && ./test_compose.sh %changelog +* Fri Aug 26 2022 Lubomír Sedlář - 4.3.6-1 +- pkgset: Report better error when module is missing an arch (lsedlar) +- osbuild: add support for building ostree artifacts (ondrej) +- ostree: Add unified core mode for compose in rpm-ostree (tim) +- createiso: Make ISO level more granular (lsedlar) +- Create DVDs with xorriso (lsedlar) +- Fix compatibility with jsonschema >= 4.0.0 (lsedlar) +- Fix black complaint (lsedlar) +- doc: fix osbuild's image_types field name (ondrej) +- Convert _ssh_run output to str for python3 (hlin) +- Print more logs for git_ls_remote (hlin) +- Log time taken of each phase (hlin) +- Avoid crash when loading pickle file failed (hlin) +- extra_isos: Fix detection of changed packages (lsedlar) + * Wed Jun 15 2022 Lubomír Sedlář - 4.3.5-1 - Fix module defaults and obsoletes validation (mkulik) - Update the cts_keytab field in order to get the hostname of the server diff --git a/setup.py b/setup.py index 41e21f25..1e0ae1e3 100755 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ packages = sorted(packages) setup( name="pungi", - version="4.3.5", + version="4.3.6", description="Distribution compose tool", url="https://pagure.io/pungi", author="Dennis Gilmore", From c7121f9378ece850dbb7eedc148eb377b01a7343 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Mon, 22 Aug 2022 14:13:07 +0200 Subject: [PATCH 131/137] profiler: Flush stdout before printing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apparently redirecting stderr to the same pipe as stdout does not guarantee that the data will not be mangled together. Flushing stdout before the profiler data is printed should ensure that it does not end up in the middle of some RPM path. Fixes: https://pagure.io/pungi/issue/1627 Signed-off-by: Lubomír Sedlář --- pungi/profiler.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pungi/profiler.py b/pungi/profiler.py index 3ffef8b9..ed05c54f 100644 --- a/pungi/profiler.py +++ b/pungi/profiler.py @@ -69,6 +69,11 @@ class Profiler(object): @classmethod def print_results(cls, stream=sys.stdout): + # Ensure all data that was printed to stdout was already flushed. If + # the caller is redirecting stderr to stdout, and there's buffered + # data, we may end up in a situation where the stderr output printed + # below ends up mixed with the stdout lines. + sys.stdout.flush() print("Profiling results:", file=stream) results = cls._data.items() results = sorted(results, key=lambda x: x[1]["time"], reverse=True) From 57ea640916bc75ff58e67c15e92d07bf46840fae Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Mon, 29 Aug 2022 17:06:11 +0800 Subject: [PATCH 132/137] Add Jenkinsfile for CI JIRA: RHELCMP-9800 Signed-off-by: Haibo Lin --- tests/Jenkinsfile | 59 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 tests/Jenkinsfile diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile new file mode 100644 index 00000000..04826e8c --- /dev/null +++ b/tests/Jenkinsfile @@ -0,0 +1,59 @@ +def DUFFY_SESSION_ID + +pipeline { + agent { + label 'cico-workspace' + } + + parameters { + string(name: 'REPO', defaultValue: '', description: 'Git repo URL where the pull request from') + string(name: 'BRANCH', defaultValue: '', description: 'Git branch where the pull request from') + } + + stages { + stage('CI') { + steps { + script { + if (params.REPO == "" || params.BRANCH == "") { + error "Please supply both params (REPO and BRANCH)" + } + try { + echo "Requesting duffy node ..." + def session_str = sh returnStdout: true, script: "set +x; duffy client --url https://duffy.ci.centos.org/api/v1 --auth-name fedora-infra --auth-key $CICO_API_KEY request-session pool=virt-ec2-t2-centos-9s-x86_64,quantity=1" + def session = readJSON text: session_str + DUFFY_SESSION_ID= session.session.id + def hostname = session.session.nodes[0].hostname + echo "duffy session id: $DUFFY_SESSION_ID hostname: $hostname" + def remote_dir = "/tmp/$JENKINS_AGENT_NAME" + echo "remote_dir: $remote_dir" + writeFile file: 'job.sh', text: """ +set -xe +dnf install -y git podman +git config --global user.email "jenkins@localhost" +git config --global user.name "jenkins" +cd $remote_dir +git clone https://pagure.io/pungi.git -b master +cd pungi +git remote rm proposed || true +git remote add proposed "$params.REPO" +git fetch proposed +git checkout origin/master +git merge --no-ff "proposed/$params.BRANCH" -m "Merge PR" +podman run --rm -v .:/src:Z quay.io/exd-guild-compose/pungi-test tox -r -e flake8,black,py3,bandit +podman run --rm -v .:/src:Z quay.io/exd-guild-compose/pungi-test-py2 tox -r -e py27 + """ + sh "cat job.sh" + sh "ssh -o StrictHostKeyChecking=no root@$hostname mkdir $remote_dir" + sh "scp job.sh root@$hostname:$remote_dir" + sh "ssh root@$hostname sh $remote_dir/job.sh" + } finally { + if (DUFFY_SESSION_ID) { + echo "Release duffy node ..." + sh "set +x; duffy client --url https://duffy.ci.centos.org/api/v1 --auth-name fedora-infra --auth-key $CICO_API_KEY retire-session $DUFFY_SESSION_ID > /dev/null" + } + } + } + } + } + } +} From 805a1083a22c0ff1061fafd35a1f0dda951fa55a Mon Sep 17 00:00:00 2001 From: Tomas Hozza Date: Thu, 1 Sep 2022 16:41:17 +0200 Subject: [PATCH 133/137] osbuild: accept only a single image type in the configuration Modify the osbuild configuration schema to accept only an array with a single value as the `image_types`, in addition to a single string. The single string was supported by the schema also before, but this fact was not mentioned in the documentation, nor it was supported by the `koji-osbuild` plugin of version lower than `9`. Update the documentation accordingly. Add unit test for invalid configuration containing more than one image type. Signed-off-by: Tomas Hozza --- doc/configuration.rst | 4 +++- pungi/checks.py | 16 +++++++++++++++- tests/test_osbuild_phase.py | 19 +++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index d91d37d8..5d9947b1 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1594,7 +1594,9 @@ OSBuild Composer for building images * ``name`` -- name of the Koji package * ``distro`` -- image for which distribution should be build TODO examples - * ``image_types`` -- a list of image types to build (e.g. ``qcow2``) + * ``image_types`` -- a list with a single image type string or just a + string representing the image type to build (e.g. ``qcow2``). In any + case, only a single image type can be provided as an argument. Optional keys: diff --git a/pungi/checks.py b/pungi/checks.py index 80d4e506..a014c6be 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -1171,7 +1171,21 @@ def make_schema(): "version": {"type": "string"}, "distro": {"type": "string"}, "target": {"type": "string"}, - "image_types": {"$ref": "#/definitions/strings"}, + # Only a single image_type can be specified + # https://github.com/osbuild/koji-osbuild/commit/c7252650814f82281ee57b598cb2ad970b580451 + # https://github.com/osbuild/koji-osbuild/commit/f21a2de39b145eb94f3d49cb4d8775a33ba56752 + "image_types": { + "oneOf": [ + { + "type": "array", + "items": {"type": "string"}, + "description": "Deprecated variant", + "minItems": 1, + "maxItems": 1, + }, + {"type": "string"}, + ] + }, "arches": {"$ref": "#/definitions/list_of_strings"}, "release": {"type": "string"}, "repo": {"$ref": "#/definitions/list_of_strings"}, diff --git a/tests/test_osbuild_phase.py b/tests/test_osbuild_phase.py index b3f5078a..b740a82a 100644 --- a/tests/test_osbuild_phase.py +++ b/tests/test_osbuild_phase.py @@ -8,6 +8,7 @@ import koji as orig_koji from tests import helpers from pungi.phases import osbuild +from pungi.checks import validate class OSBuildPhaseTest(helpers.PungiTestCase): @@ -105,6 +106,24 @@ class OSBuildPhaseTest(helpers.PungiTestCase): phase = osbuild.OSBuildPhase(compose) self.assertTrue(phase.skip()) + def test_fail_multiple_image_types(self): + cfg = { + "name": "test-image", + "distro": "rhel-8", + # more than one image type is not allowed + "image_types": ["qcow2", "rhel-ec2"], + } + compose = helpers.DummyCompose( + self.topdir, + { + "osbuild": {"^Everything$": [cfg]}, + "osbuild_target": "image-target", + "osbuild_version": "1", + "osbuild_release": "2", + }, + ) + self.assertNotEqual(validate(compose.conf), ([], [])) + class RunOSBuildThreadTest(helpers.PungiTestCase): def setUp(self): From 57739c238fedb71406c464661eb05ae929ba774e Mon Sep 17 00:00:00 2001 From: Tomas Hozza Date: Mon, 5 Sep 2022 15:05:50 +0200 Subject: [PATCH 134/137] osbuild: support specifying upload_options Since version 9, the `koji-osbuild` plugin supports specifying upload options as part of a Koji build. This enables one to upload the built image directly to the cloud environment as part of the image build in Koji. Extend the configuration schema with `upload_options`. Extend the documentation and describe valid `upload_options` values. Add a unit test testing a scenario when `upload_options` are specified. Signed-off-by: Tomas Hozza --- doc/configuration.rst | 39 ++++++++++++ pungi/checks.py | 80 ++++++++++++++++++++++++ pungi/phases/osbuild.py | 4 ++ tests/test_osbuild_phase.py | 119 ++++++++++++++++++++++++++++++++++++ 4 files changed, 242 insertions(+) diff --git a/doc/configuration.rst b/doc/configuration.rst index 5d9947b1..907f72f4 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1617,6 +1617,45 @@ OSBuild Composer for building images * ``ostree_ref`` -- name of the ostree branch * ``ostree_parent`` -- commit hash or a a branch-like reference to the parent commit. + * ``upload_options`` -- a dictionary with upload options specific to the + target cloud environment. If provided, the image will be uploaded to the + cloud environment, in addition to the Koji server. One can't combine + arbitrary image types with arbitrary upload options. + The dictionary keys differ based on the target cloud environment. The + following keys are supported: + + * **AWS EC2 upload options** -- upload to Amazon Web Services. + + * ``region`` -- AWS region to upload the image to + * ``share_with_accounts`` -- list of AWS account IDs to share the image + with + * ``snapshot_name`` -- Snapshot name of the uploaded EC2 image + (optional) + + * **AWS S3 upload options** -- upload to Amazon Web Services S3. + + * ``region`` -- AWS region to upload the image to + + * **Azure upload options** -- upload to Microsoft Azure. + + * ``tenant_id`` -- Azure tenant ID to upload the image to + * ``subscription_id`` -- Azure subscription ID to upload the image to + * ``resource_group`` -- Azure resource group to upload the image to + * ``location`` -- Azure location to upload the image to + * ``image_name`` -- Image name of the uploaded Azure image (optional) + + * **GCP upload options** -- upload to Google Cloud Platform. + + * ``region`` -- GCP region to upload the image to + * ``bucket`` -- GCP bucket to upload the image to + * ``share_with_accounts`` -- list of GCP accounts to share the image + with + * ``image_name`` -- Image name of the uploaded GCP image (optional) + + * **Container upload options** -- upload to a container registry. + + * ``name`` -- name of the container image (optional) + * ``tag`` -- container tag to upload the image to (optional) .. note:: There is initial support for having this task as failable without aborting diff --git a/pungi/checks.py b/pungi/checks.py index a014c6be..a7f29606 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -1194,6 +1194,86 @@ def make_schema(): "ostree_url": {"type": "string"}, "ostree_ref": {"type": "string"}, "ostree_parent": {"type": "string"}, + "upload_options": { + "oneOf": [ + # AWSEC2UploadOptions + { + "type": "object", + "additionalProperties": False, + "required": [ + "region", + "share_with_accounts", + ], + "properties": { + "region": { + "type": "string", + }, + "snapshot_name": { + "type": "string", + }, + "share_with_accounts": { + "type": "array", + "items": {"type": "string"}, + }, + }, + }, + # AWSS3UploadOptions + { + "type": "object", + "additionalProperties": False, + "required": ["region"], + "properties": { + "region": {"type": "string"} + }, + }, + # AzureUploadOptions + { + "type": "object", + "additionalProperties": False, + "required": [ + "tenant_id", + "subscription_id", + "resource_group", + "location", + ], + "properties": { + "tenant_id": {"type": "string"}, + "subscription_id": {"type": "string"}, + "resource_group": {"type": "string"}, + "location": {"type": "string"}, + "image_name": { + "type": "string", + }, + }, + }, + # GCPUploadOptions + { + "type": "object", + "additionalProperties": False, + "required": ["region", "bucket"], + "properties": { + "region": {"type": "string"}, + "bucket": {"type": "string"}, + "image_name": { + "type": "string", + }, + "share_with_accounts": { + "type": "array", + "items": {"type": "string"}, + }, + }, + }, + # ContainerUploadOptions + { + "type": "object", + "additionalProperties": False, + "properties": { + "name": {"type": "string"}, + "tag": {"type": "string"}, + }, + }, + ] + }, }, "required": ["name", "distro", "image_types"], "additionalProperties": False, diff --git a/pungi/phases/osbuild.py b/pungi/phases/osbuild.py index 9215acfa..6e52e9c5 100644 --- a/pungi/phases/osbuild.py +++ b/pungi/phases/osbuild.py @@ -126,6 +126,10 @@ class RunOSBuildThread(WorkerThread): if ostree: opts["ostree"] = ostree + upload_options = config.get("upload_options") + if upload_options: + opts["upload_options"] = upload_options + if release: opts["release"] = release task_id = koji.koji_proxy.osbuildImage( diff --git a/tests/test_osbuild_phase.py b/tests/test_osbuild_phase.py index b740a82a..c53f7529 100644 --- a/tests/test_osbuild_phase.py +++ b/tests/test_osbuild_phase.py @@ -399,6 +399,125 @@ class RunOSBuildThreadTest(helpers.PungiTestCase): ], ) + @mock.patch("pungi.util.get_file_size", new=lambda fp: 65536) + @mock.patch("pungi.util.get_mtime", new=lambda fp: 1024) + @mock.patch("pungi.phases.osbuild.Linker") + @mock.patch("pungi.phases.osbuild.kojiwrapper.KojiWrapper") + def test_process_upload_options(self, KojiWrapper, Linker): + cfg = { + "name": "test-image", + "distro": "rhel-8", + "image_types": ["rhel-ec2"], + "upload_options": { + "region": "us-east-1", + "share_with_accounts": ["123456789012"], + }, + } + build_id = 5678 + koji = KojiWrapper.return_value + koji.watch_task.side_effect = self.make_fake_watch(0) + koji.koji_proxy.osbuildImage.return_value = 1234 + koji.koji_proxy.getTaskResult.return_value = { + "composer": {"server": "https://composer.osbuild.org", "id": ""}, + "koji": {"build": build_id}, + } + koji.koji_proxy.getBuild.return_value = { + "build_id": build_id, + "name": "test-image", + "version": "1", + "release": "1", + } + koji.koji_proxy.listArchives.return_value = [ + { + "extra": {"image": {"arch": "x86_64"}}, + "filename": "image.raw.xz", + "type_name": "raw-xz", + } + ] + koji.koji_module.pathinfo = orig_koji.pathinfo + + self.t.process( + ( + self.compose, + self.compose.variants["Everything"], + cfg, + ["x86_64"], + "1", # version + "15", # release + "image-target", + [self.topdir + "/compose/Everything/$arch/os"], + ["x86_64"], + ), + 1, + ) + + # Verify two Koji instances were created. + self.assertEqual(len(KojiWrapper.call_args), 2) + # Verify correct calls to Koji + self.assertEqual( + koji.mock_calls, + [ + mock.call.login(), + mock.call.koji_proxy.osbuildImage( + "test-image", + "1", + "rhel-8", + ["rhel-ec2"], + "image-target", + ["x86_64"], + opts={ + "release": "15", + "repo": [self.topdir + "/compose/Everything/$arch/os"], + "upload_options": { + "region": "us-east-1", + "share_with_accounts": ["123456789012"], + }, + }, + ), + mock.call.save_task_id(1234), + mock.call.watch_task(1234, mock.ANY), + mock.call.koji_proxy.getTaskResult(1234), + mock.call.koji_proxy.getBuild(build_id), + mock.call.koji_proxy.listArchives(buildID=build_id), + ], + ) + + # Assert there is one image added to manifest and the arguments are sane + self.assertEqual( + self.compose.im.add.call_args_list, + [ + mock.call(arch="x86_64", variant="Everything", image=mock.ANY), + ], + ) + for call in self.compose.im.add.call_args_list: + _, kwargs = call + image = kwargs["image"] + self.assertEqual(kwargs["variant"], "Everything") + self.assertIn(kwargs["arch"], ("x86_64")) + self.assertEqual(kwargs["arch"], image.arch) + self.assertEqual( + "Everything/x86_64/images/image.raw.xz", + image.path, + ) + self.assertEqual("raw.xz", image.format) + self.assertEqual("raw-xz", image.type) + self.assertEqual("Everything", image.subvariant) + + self.assertTrue( + os.path.isdir(self.topdir + "/compose/Everything/x86_64/images") + ) + + self.assertEqual( + Linker.return_value.mock_calls, + [ + mock.call.link( + "/mnt/koji/packages/test-image/1/1/images/image.raw.xz", + self.topdir + "/compose/Everything/x86_64/images/image.raw.xz", + link_type="hardlink-or-copy", + ) + ], + ) + @mock.patch("pungi.util.get_file_size", new=lambda fp: 65536) @mock.patch("pungi.util.get_mtime", new=lambda fp: 1024) @mock.patch("pungi.phases.osbuild.Linker") From fa967f79b5fa0eb42374185a7a0c513ef6eefc24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Tue, 13 Sep 2022 11:54:23 +0200 Subject: [PATCH 135/137] Ignore existing kerberos ticket for CTS auth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When there is an existing kerberos ticket, it gets precedence over the environment variable with path to a keytab. That is not expected and the user ticket can possibly lack permissions in CTS to be able to run the compose successfully. This patch fixes that by setting KRB5CCNAME to a fresh path. That way there will not be any valid ticket, since the credentials cache does not exist yet. JIRA: RHELCMP-9742 Signed-off-by: Lubomír Sedlář --- pungi/compose.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pungi/compose.py b/pungi/compose.py index e289a7a2..88ef7f6f 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -102,6 +102,7 @@ def get_compose_info( if "$HOSTNAME" in cts_keytab: cts_keytab = cts_keytab.replace("$HOSTNAME", socket.gethostname()) os.environ["KRB5_CLIENT_KTNAME"] = cts_keytab + os.environ["KRB5CCNAME"] = "DIR:%s" % tempfile.mkdtemp() try: # Create compose in CTS and get the reserved compose ID. @@ -116,6 +117,7 @@ def get_compose_info( rv.raise_for_status() finally: if cts_keytab: + shutil.rmtree(os.environ["KRB5CCNAME"].split(":", 1)[1]) os.environ.clear() os.environ.update(environ_copy) From 8cd19605bdea7fbc626e981f7dbbc89898160268 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Fri, 28 Oct 2022 18:42:29 +0800 Subject: [PATCH 136/137] Retry failed cts requests JIRA: RHELCMP-10033 Signed-off-by: Haibo Lin --- pungi/compose.py | 22 +++++++++++++--------- tests/test_compose.py | 28 ++++++++++++++++++++++++---- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/pungi/compose.py b/pungi/compose.py index 88ef7f6f..ab62bbfa 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -28,6 +28,8 @@ import socket import kobo.log import kobo.tback +import requests +from requests.exceptions import RequestException from productmd.composeinfo import ComposeInfo from productmd.images import Images from dogpile.cache import make_region @@ -42,6 +44,7 @@ from pungi.util import ( get_arch_variant_data, get_format_substs, get_variant_data, + retry, translate_path_raw, ) from pungi.metadata import compose_to_composeinfo @@ -54,6 +57,14 @@ except ImportError: SUPPORTED_MILESTONES = ["RC", "Update", "SecurityFix"] +@retry(wait_on=RequestException) +def retry_request(method, url, data=None, auth=None): + request_method = getattr(requests, method) + rv = request_method(url, json=data, auth=auth) + rv.raise_for_status() + return rv + + def get_compose_info( conf, compose_type="production", @@ -86,10 +97,6 @@ def get_compose_info( cts_url = conf.get("cts_url", None) if cts_url: - # Import requests and requests-kerberos here so it is not needed - # if running without Compose Tracking Service. - import requests - # Requests-kerberos cannot accept custom keytab, we need to use # environment variable for this. But we need to change environment # only temporarily just for this single requests.post. @@ -113,8 +120,7 @@ def get_compose_info( "parent_compose_ids": parent_compose_ids, "respin_of": respin_of, } - rv = requests.post(url, json=data, auth=authentication) - rv.raise_for_status() + rv = retry_request("post", url, data=data, auth=authentication) finally: if cts_keytab: shutil.rmtree(os.environ["KRB5CCNAME"].split(":", 1)[1]) @@ -156,8 +162,6 @@ def write_compose_info(compose_dir, ci): def update_compose_url(compose_id, compose_dir, conf): - import requests - authentication = get_authentication(conf) cts_url = conf.get("cts_url", None) if cts_url: @@ -168,7 +172,7 @@ def update_compose_url(compose_id, compose_dir, conf): "action": "set_url", "compose_url": compose_url, } - return requests.patch(url, json=data, auth=authentication) + return retry_request("patch", url, data=data, auth=authentication) def get_compose_dir( diff --git a/tests/test_compose.py b/tests/test_compose.py index 94940909..44748520 100644 --- a/tests/test_compose.py +++ b/tests/test_compose.py @@ -13,7 +13,9 @@ import tempfile import shutil import json -from pungi.compose import Compose +from requests.exceptions import HTTPError + +from pungi.compose import Compose, retry_request class ConfigWrapper(dict): @@ -608,8 +610,9 @@ class ComposeTestCase(unittest.TestCase): ci_json = json.loads(ci.dumps()) self.assertEqual(ci_json, self.ci_json) + @mock.patch("pungi.compose.requests") @mock.patch("time.strftime", new=lambda fmt, time: "20200526") - def test_get_compose_info_cts(self): + def test_get_compose_info_cts(self, mocked_requests): conf = ConfigWrapper( release_name="Test", release_version="1.0", @@ -626,7 +629,6 @@ class ComposeTestCase(unittest.TestCase): ci_copy["header"]["version"] = "1.2" mocked_response = mock.MagicMock() mocked_response.text = json.dumps(self.ci_json) - mocked_requests = mock.MagicMock() mocked_requests.post.return_value = mocked_response mocked_requests_kerberos = mock.MagicMock() @@ -637,7 +639,6 @@ class ComposeTestCase(unittest.TestCase): # `import`. with mock.patch.dict( "sys.modules", - requests=mocked_requests, requests_kerberos=mocked_requests_kerberos, ): ci = Compose.get_compose_info(conf, respin_of="Fedora-Rawhide-20200517.n.1") @@ -807,3 +808,22 @@ class TracebackTest(unittest.TestCase): def test_with_detail(self): self.compose.traceback("extra-info") self.assertTraceback("traceback-extra-info") + + +class RetryRequestTest(unittest.TestCase): + @mock.patch("pungi.compose.requests") + def test_retry_timeout(self, mocked_requests): + mocked_requests.post.side_effect = [ + HTTPError("Gateway Timeout", response=mock.Mock(status_code=504)), + mock.Mock(status_code=200), + ] + url = "http://locahost/api/1/composes/" + rv = retry_request("post", url) + self.assertEqual( + mocked_requests.mock_calls, + [ + mock.call.post(url, json=None, auth=None), + mock.call.post(url, json=None, auth=None), + ], + ) + self.assertEqual(rv.status_code, 200) From 479849042f118508a7d4ee27a7444c002ac9e119 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 3 Nov 2022 11:05:53 +0100 Subject: [PATCH 137/137] init: Filter comps for modular variants with tags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Modular variants can either be specified by a list of modules, or by a list of Koji tags. In terms of comps preprocessing there should not be any difference between the two. Resolves: https://pagure.io/pungi/issue/1640 Signed-off-by: Lubomír Sedlář --- pungi/phases/init.py | 12 +++++++++--- tests/helpers.py | 1 + tests/test_initphase.py | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/pungi/phases/init.py b/pungi/phases/init.py index a78c0dc7..a99bc595 100644 --- a/pungi/phases/init.py +++ b/pungi/phases/init.py @@ -165,12 +165,18 @@ def write_variant_comps(compose, arch, variant): run(cmd) comps = CompsWrapper(comps_file) - if variant.groups or variant.modules is not None or variant.type != "variant": - # Filter groups if the variant has some, or it's a modular variant, or - # is not a base variant. + # Filter groups if the variant has some, or it's a modular variant, or + # is not a base variant. + if ( + variant.groups + or variant.modules is not None + or variant.modular_koji_tags is not None + or variant.type != "variant" + ): unmatched = comps.filter_groups(variant.groups) for grp in unmatched: compose.log_warning(UNMATCHED_GROUP_MSG % (variant.uid, arch, grp)) + contains_all = not variant.groups and not variant.environments if compose.conf["comps_filter_environments"] and not contains_all: # We only want to filter environments if it's enabled by configuration diff --git a/tests/helpers.py b/tests/helpers.py index 7aa7452d..e221b839 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -79,6 +79,7 @@ class MockVariant(mock.Mock): self.variants = {} self.pkgsets = set() self.modules = None + self.modular_koji_tags = None self.name = name self.nsvc_to_pkgset = defaultdict(lambda: mock.Mock(rpms_by_arch={})) diff --git a/tests/test_initphase.py b/tests/test_initphase.py index 1fb80c48..2ddb82ca 100644 --- a/tests/test_initphase.py +++ b/tests/test_initphase.py @@ -497,6 +497,45 @@ class TestWriteVariantComps(PungiTestCase): ) self.assertEqual(comps.write_comps.mock_calls, [mock.call()]) + @mock.patch("pungi.phases.init.run") + @mock.patch("pungi.phases.init.CompsWrapper") + def test_run_filter_for_modular_koji_tags(self, CompsWrapper, run): + compose = DummyCompose(self.topdir, {}) + variant = compose.variants["Server"] + variant.groups = [] + variant.modular_koji_tags = ["f38-modular"] + comps = CompsWrapper.return_value + comps.filter_groups.return_value = [] + + init.write_variant_comps(compose, "x86_64", variant) + + self.assertEqual( + run.mock_calls, + [ + mock.call( + [ + "comps_filter", + "--arch=x86_64", + "--keep-empty-group=conflicts", + "--keep-empty-group=conflicts-server", + "--variant=Server", + "--output=%s/work/x86_64/comps/comps-Server.x86_64.xml" + % self.topdir, + self.topdir + "/work/global/comps/comps-global.xml", + ] + ) + ], + ) + self.assertEqual( + CompsWrapper.call_args_list, + [mock.call(self.topdir + "/work/x86_64/comps/comps-Server.x86_64.xml")], + ) + self.assertEqual(comps.filter_groups.call_args_list, [mock.call([])]) + self.assertEqual( + comps.filter_environments.mock_calls, [mock.call(variant.environments)] + ) + self.assertEqual(comps.write_comps.mock_calls, [mock.call()]) + @mock.patch("pungi.phases.init.run") @mock.patch("pungi.phases.init.CompsWrapper") def test_run_report_unmatched(self, CompsWrapper, run):