From 25f480ad4ff28f8310d1c463ff38a52d9f42c573 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Mon, 2 Dec 2019 15:20:40 +0100 Subject: [PATCH] New upstream release 4.1.41 --- ...oading-overrides-for-module-defaults.patch | 144 ------------------ 1278.patch | 98 ------------ 1281.patch | 28 ---- 1292.patch | 26 ---- 1295.patch | 43 ------ 1303.patch | 61 -------- pungi.spec | 27 +++- sources | 2 +- 8 files changed, 23 insertions(+), 406 deletions(-) delete mode 100644 0001-Allow-loading-overrides-for-module-defaults.patch delete mode 100644 1278.patch delete mode 100644 1281.patch delete mode 100644 1292.patch delete mode 100644 1295.patch delete mode 100644 1303.patch diff --git a/0001-Allow-loading-overrides-for-module-defaults.patch b/0001-Allow-loading-overrides-for-module-defaults.patch deleted file mode 100644 index 9f91fdea..00000000 --- a/0001-Allow-loading-overrides-for-module-defaults.patch +++ /dev/null @@ -1,144 +0,0 @@ -From 2b112d53f7ea443dd11be584269f559d7d911227 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= -Date: Wed, 18 Sep 2019 14:47:30 +0200 -Subject: [PATCH] Allow loading overrides for module defaults -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -This patch adds a new config option. This is expected to be a name of -subdirectory in the repo with module defaults. If supplied, overrides -from that location are loaded every time defaults are loaded. - -This raises the minimal required version of libmodulemd to 2.8.0 - -JIRA: COMPOSE-3828 -Signed-off-by: Lubomír Sedlář ---- - pungi/checks.py | 1 + - pungi/phases/createrepo.py | 5 ++++- - pungi/phases/gather/__init__.py | 5 ++++- - pungi/phases/pkgset/common.py | 3 ++- - pungi/util.py | 19 +++++++++++++++---- - tests/test_pkgset_common.py | 1 + - 6 files changed, 27 insertions(+), 7 deletions(-) - -diff --git a/pungi/checks.py b/pungi/checks.py -index 17ff8b50..7e9f65a7 100644 ---- a/pungi/checks.py -+++ b/pungi/checks.py -@@ -746,6 +746,7 @@ def make_schema(): - "default": True - }, - "module_defaults_dir": {"$ref": "#/definitions/str_or_scm_dict"}, -+ "module_defaults_override_dir": {"type": "string"}, - - "pkgset_repos": { - "type": "object", -diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py -index 64553594..b2c67ad7 100644 ---- a/pungi/phases/createrepo.py -+++ b/pungi/phases/createrepo.py -@@ -211,7 +211,10 @@ def create_variant_repo(compose, arch, variant, pkg_type, pkgset, modules_metada - - module_names = set(mod_index.get_module_names()) - defaults_dir = compose.paths.work.module_defaults_dir() -- collect_module_defaults(defaults_dir, module_names, mod_index) -+ overrides_dir = compose.conf.get("module_defaults_override_dir") -+ collect_module_defaults( -+ defaults_dir, module_names, mod_index, overrides_dir=overrides_dir -+ ) - - log_file = compose.paths.log.log_file(arch, "modifyrepo-modules-%s" % variant) - add_modular_metadata(repo, repo_dir, mod_index, log_file) -diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py -index afba72f6..736c10ec 100644 ---- a/pungi/phases/gather/__init__.py -+++ b/pungi/phases/gather/__init__.py -@@ -382,7 +382,10 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None): - - module_names = set(mod_index.get_module_names()) - defaults_dir = compose.paths.work.module_defaults_dir() -- collect_module_defaults(defaults_dir, module_names, mod_index) -+ overrides_dir = compose.conf.get("module_defaults_override_dir") -+ collect_module_defaults( -+ defaults_dir, module_names, mod_index, overrides_dir=overrides_dir -+ ) - - log_file = compose.paths.log.log_file( - arch, "lookaside_repo_modules_%s" % (variant.uid) -diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py -index 6ab86e53..d7e06cfc 100644 ---- a/pungi/phases/pkgset/common.py -+++ b/pungi/phases/pkgset/common.py -@@ -155,8 +155,9 @@ def _create_arch_repo(worker_thread, args, task_num): - # Add modulemd to the repo for all modules in all variants on this architecture. - if Modulemd and mmd: - names = set(x.get_module_name() for x in mmd) -+ overrides_dir = compose.conf.get("module_defaults_override_dir") - mod_index = collect_module_defaults( -- compose.paths.work.module_defaults_dir(), names -+ compose.paths.work.module_defaults_dir(), names, overrides_dir=overrides_dir - ) - for x in mmd: - mod_index.add_module_stream(x) -diff --git a/pungi/util.py b/pungi/util.py -index c2c9a948..bd3e022c 100644 ---- a/pungi/util.py -+++ b/pungi/util.py -@@ -934,9 +934,10 @@ def iter_module_defaults(path): - # and work with it. However that does not allow for detecting conflicting - # defaults. That should not happen in practice, but better safe than sorry. - # Once libmodulemd can report the error, this code can be simplifed by a -- # lot. It's implemented in -+ # lot. It was implemented in - # https://github.com/fedora-modularity/libmodulemd/commit/3087e4a5c38a331041fec9b6b8f1a372f9ffe64d -- # and released in 2.6.0 -+ # and released in 2.6.0, but 2.8.0 added the need to merge overrides and -+ # that breaks this use case again. - for file in glob.glob(os.path.join(path, "*.yaml")): - index = Modulemd.ModuleIndex() - index.update_from_file(file, strict=False) -@@ -944,7 +945,9 @@ def iter_module_defaults(path): - yield module_name, index.get_module(module_name).get_defaults() - - --def collect_module_defaults(defaults_dir, modules_to_load=None, mod_index=None): -+def collect_module_defaults( -+ defaults_dir, modules_to_load=None, mod_index=None, overrides_dir=None -+): - """Load module defaults into index. - - If `modules_to_load` is passed in, it should be a set of module names. Only -@@ -954,7 +957,15 @@ def collect_module_defaults(defaults_dir, modules_to_load=None, mod_index=None): - not, a new ModuleIndex will be created and returned - """ - mod_index = mod_index or Modulemd.ModuleIndex() -- for module_name, defaults in iter_module_defaults(defaults_dir): -+ -+ temp_index = Modulemd.ModuleIndex.new() -+ temp_index.update_from_defaults_directory( -+ defaults_dir, overrides_path=overrides_dir, strict=False -+ ) -+ -+ for module_name in temp_index.get_module_names(): -+ defaults = temp_index.get_module(module_name).get_defaults() -+ - if not modules_to_load or module_name in modules_to_load: - mod_index.add_defaults(defaults) - -diff --git a/tests/test_pkgset_common.py b/tests/test_pkgset_common.py -index 64366a42..d1e147fe 100755 ---- a/tests/test_pkgset_common.py -+++ b/tests/test_pkgset_common.py -@@ -105,6 +105,7 @@ class TestMaterializedPkgsetCreate(helpers.PungiTestCase): - cmd.assert_called_once_with( - os.path.join(self.topdir, "work/global/module_defaults"), - set(x.get_module_name.return_value for x in mmd["x86_64"]), -+ overrides_dir=None, - ) - amm.assert_called_once_with( - mock.ANY, --- -2.21.0 - diff --git a/1278.patch b/1278.patch deleted file mode 100644 index 91e490cb..00000000 --- a/1278.patch +++ /dev/null @@ -1,98 +0,0 @@ -From 72bf795bd45e15af86e1a4f96cfbe0d40d40a8cc Mon Sep 17 00:00:00 2001 -From: Owen W. Taylor -Date: Oct 10 2019 19:13:38 +0000 -Subject: ostree-install: allow configuring additional depenencies for runroot - - -A lorax template used for the ostree-installer might need an additional -package dependency (e.g., flatpak to embed a flatpak repository) - add -a config key 'extra_runroot_pkgs' to the ostree installer configuration -to allow supplementing the set of packages installed into the runroot. - -Signed-off-by: Owen W. Taylor - ---- - -diff --git a/doc/configuration.rst b/doc/configuration.rst -index 2c8a5a3..2ce9216 100644 ---- a/doc/configuration.rst -+++ b/doc/configuration.rst -@@ -1527,6 +1527,11 @@ an OSTree repository. This always runs in Koji as a ``runroot`` task. - ``template_repo`` needs to point to a Git repository from which to take the - templates. - -+ If the templates need to run with additional dependencies, that can be configured -+ with the optional key: -+ -+ * ``extra_runroot_pkgs`` -- (*[str]*) -+ - **ostree_installer_overwrite** = False - (*bool*) -- by default if a variant including OSTree installer also creates - regular installer images in buildinstall phase, there will be conflicts (as -diff --git a/pungi/checks.py b/pungi/checks.py -index a20d0f7..513728d 100644 ---- a/pungi/checks.py -+++ b/pungi/checks.py -@@ -1132,6 +1132,7 @@ def make_schema(): - "rootfs_size": {"type": "string"}, - "template_repo": {"type": "string"}, - "template_branch": {"type": "string"}, -+ "extra_runroot_pkgs": {"$ref": "#/definitions/list_of_strings"}, - }, - "additionalProperties": False, - }), -diff --git a/pungi/phases/ostree_installer.py b/pungi/phases/ostree_installer.py -index 995ac96..46c774b 100644 ---- a/pungi/phases/ostree_installer.py -+++ b/pungi/phases/ostree_installer.py -@@ -195,6 +195,8 @@ class OstreeInstallerThread(WorkerThread): - ' '.join([shlex_quote(x) for x in lorax_cmd])) - - packages = ['pungi', 'lorax', 'ostree'] -+ packages += config.get('extra_runroot_pkgs', []) -+ - log_file = os.path.join(self.logdir, 'runroot.log') - - runroot = Runroot(compose) -diff --git a/tests/test_ostree_installer_phase.py b/tests/test_ostree_installer_phase.py -index 0cce4b8..846e23a 100644 ---- a/tests/test_ostree_installer_phase.py -+++ b/tests/test_ostree_installer_phase.py -@@ -126,7 +126,8 @@ class OstreeThreadTest(helpers.PungiTestCase): - self.assertEqual(compose.im.add.mock_calls, - [mock.call('Everything', 'x86_64', image)]) - -- def assertRunrootCall(self, koji, sources, release, isfinal=False, extra=[], weight=None): -+ def assertRunrootCall(self, koji, sources, release, isfinal=False, extra=[], -+ extra_pkgs=[], weight=None): - lorax_cmd = [ - 'lorax', - '--product=Fedora', -@@ -156,7 +157,7 @@ class OstreeThreadTest(helpers.PungiTestCase): - [mock.call('rrt', 'x86_64', - 'rm -rf %s && %s' % (outdir, ' '.join(lorax_cmd)), - channel=None, mounts=[self.topdir], -- packages=['pungi', 'lorax', 'ostree'], -+ packages=['pungi', 'lorax', 'ostree'] + extra_pkgs, - task_id=True, use_shell=True, weight=weight, - chown_paths=[outdir])]) - self.assertEqual(koji.run_runroot_cmd.call_args_list, -@@ -418,6 +419,7 @@ class OstreeThreadTest(helpers.PungiTestCase): - 'add_arch_template': ['other_file.txt'], - 'template_repo': 'git://example.com/templates.git', - 'template_branch': 'f24', -+ 'extra_runroot_pkgs': ['templatedep'], - } - koji = KojiWrapper.return_value - koji.run_runroot_cmd.return_value = { -@@ -445,7 +447,8 @@ class OstreeThreadTest(helpers.PungiTestCase): - isfinal=True, - extra=['--add-template=%s/some_file.txt' % templ_dir, - '--add-arch-template=%s/other_file.txt' % templ_dir, -- '--logfile=%s/%s/lorax.log' % (self.topdir, LOG_PATH)]) -+ '--logfile=%s/%s/lorax.log' % (self.topdir, LOG_PATH)], -+ extra_pkgs=['templatedep']) - self.assertIsoLinked(link, get_file_size, get_mtime, final_iso_path) - self.assertImageAdded(self.compose, ImageCls, iso) - self.assertAllCopied(copy_all) - diff --git a/1281.patch b/1281.patch deleted file mode 100644 index 35cd963c..00000000 --- a/1281.patch +++ /dev/null @@ -1,28 +0,0 @@ -From 8bbf9ac13e8e54ec6f51828436744448058db2db Mon Sep 17 00:00:00 2001 -From: Lubomír Sedlář -Date: Oct 17 2019 07:17:36 +0000 -Subject: pkgset: Only reuse valid old repo - - -Instead of just checking that the repo directory exists, make sure the -repodata subdirectory is in there. If it's missing, then createrepo_c -has nothing to use anyway, and it may help avoid issues. - -Signed-off-by: Lubomír Sedlář - ---- - -diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py -index 6e53cfc..135c7f8 100644 ---- a/pungi/phases/pkgset/common.py -+++ b/pungi/phases/pkgset/common.py -@@ -77,7 +77,7 @@ def get_create_global_repo_cmd(compose, path_prefix, repo_dir_global, pkgset): - repo_dir, os.path.abspath(compose.topdir).rstrip("/") + "/" - ) - old_repo_dir = os.path.join(old_compose_path, rel_path) -- if os.path.isdir(old_repo_dir): -+ if os.path.isdir(os.path.join(old_repo_dir, "repodata")): - compose.log_info("Using old repodata from: %s", old_repo_dir) - update_md_path = old_repo_dir - - diff --git a/1292.patch b/1292.patch deleted file mode 100644 index e0a0b691..00000000 --- a/1292.patch +++ /dev/null @@ -1,26 +0,0 @@ -From cc0e7af866ec260126ec11aee68353bad590493f Mon Sep 17 00:00:00 2001 -From: Lubomír Sedlář -Date: Nov 08 2019 14:24:15 +0000 -Subject: Get message from LogRecord with a method - - -Relates: https://pagure.io/releng/failed-composes/issue/469 -Signed-off-by: Lubomír Sedlář - ---- - -diff --git a/pungi/compose.py b/pungi/compose.py -index ade5df2..84f2212 100644 ---- a/pungi/compose.py -+++ b/pungi/compose.py -@@ -137,7 +137,8 @@ class Compose(kobo.log.LoggingBase): - - class ExcludingArchLogFilter(logging.Filter): - def filter(self, record): -- if 'Populating package set for arch:' in record.message or \ -+ message = record.getMessage() -+ if 'Populating package set for arch:' in message or \ - (record.funcName and record.funcName == 'is_excluded'): - return True - else: - diff --git a/1295.patch b/1295.patch deleted file mode 100644 index 1e185eb5..00000000 --- a/1295.patch +++ /dev/null @@ -1,43 +0,0 @@ -From d34b0d79002f127fbea4407b17a157dcdaaf045e Mon Sep 17 00:00:00 2001 -From: Lubomír Sedlář -Date: Nov 11 2019 08:30:37 +0000 -Subject: pkgset: Fix running without any koji tags - - -All places in the code assume the option to not be required except for -this one line. - -An obsolete comment is removed as well. - -Relates: https://pagure.io/releng/failed-composes/issue/477 -Signed-off-by: Lubomír Sedlář - ---- - -diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py -index 187e6f1..c573ad8 100644 ---- a/pungi/phases/pkgset/sources/source_koji.py -+++ b/pungi/phases/pkgset/sources/source_koji.py -@@ -588,6 +588,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): - - tag_to_mmd = {} - -+ pkgset_koji_tags = force_list(compose.conf.get("pkgset_koji_tag", [])) -+ - for variant in compose.all_variants.values(): - variant_tags[variant] = [] - -@@ -617,11 +619,9 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): - if variant_tag not in compose_tags: - compose_tags.append(variant_tag) - -- # TODO check if this works for Fedora Modular variant -- variant_tags[variant].extend(force_list(compose.conf["pkgset_koji_tag"])) -+ variant_tags[variant].extend(pkgset_koji_tags) - - # Add global tag(s) if supplied. -- pkgset_koji_tags = force_list(compose.conf.get("pkgset_koji_tag", [])) - compose_tags.extend(pkgset_koji_tags) - - inherit = compose.conf["pkgset_koji_inherit"] - diff --git a/1303.patch b/1303.patch deleted file mode 100644 index 922ca911..00000000 --- a/1303.patch +++ /dev/null @@ -1,61 +0,0 @@ -From 7f35ac622a7389a3b2077b245d7f322918b03515 Mon Sep 17 00:00:00 2001 -From: Lubomír Sedlář -Date: Nov 25 2019 13:28:34 +0000 -Subject: gather: Collect and re-raise errors from gather method - - -When there is an exception in gathering (such as after seeing unsigned -packages in deps method), the exception was lost and the compose -continued to run until it tried to access the result and crashed on -KeyError. - -Relates: https://pagure.io/releng/failed-composes/issue/587 -JIRA: COMPOSE-3986 -Signed-off-by: Lubomír Sedlář - ---- - -diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py -index 444a8c2..9d54f15 100644 ---- a/pungi/phases/gather/__init__.py -+++ b/pungi/phases/gather/__init__.py -@@ -443,6 +443,7 @@ def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltr - continue - threads_list = [] - que = Queue() -+ errors = Queue() - for arch in variant.arches: - fulltree_excludes = set() - if exclude_fulltree: -@@ -454,11 +455,17 @@ def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltr - # there. - _update_lookaside_config(compose, variant, arch, result, package_sets) - -+ def worker(que, errors, arch, *args, **kwargs): -+ try: -+ que.put((arch, gather_packages(*args, **kwargs))) -+ except Exception as exc: -+ errors.put(exc) -+ - # Run gather_packages() in parallel with multi threads and store - # its return value in a Queue() for later use. - t = threading.Thread( -- target=lambda q, arch, *args, **kwargs: q.put((arch, gather_packages(*args, **kwargs))), -- args=(que, arch, compose, arch, variant, package_sets), -+ target=worker, -+ args=(que, errors, arch, compose, arch, variant, package_sets), - kwargs={'fulltree_excludes': fulltree_excludes}, - ) - threads_list.append(t) -@@ -467,6 +474,10 @@ def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltr - for t in threads_list: - t.join() - -+ while not errors.empty(): -+ exc = errors.get() -+ raise exc -+ - while not que.empty(): - arch, pkg_map = que.get() - result.setdefault(arch, {})[variant.uid] = pkg_map - diff --git a/pungi.spec b/pungi.spec index 94c69146..9811094d 100644 --- a/pungi.spec +++ b/pungi.spec @@ -1,16 +1,13 @@ %{?python_enable_dependency_generator} Name: pungi -Version: 4.1.40 -Release: 4%{?dist} +Version: 4.1.41 +Release: 1%{?dist} Summary: Distribution compose tool License: GPLv2 URL: https://pagure.io/pungi Source0: https://pagure.io/releases/%{name}/%{name}-%{version}.tar.bz2 -Patch1: https://pagure.io/pungi/pull-request/1292.patch -Patch2: https://pagure.io/pungi/pull-request/1295.patch -Patch3: https://pagure.io/pungi/pull-request/1303.patch BuildRequires: python3-nose BuildRequires: python3-mock @@ -144,6 +141,26 @@ nosetests-3 --exe %{_bindir}/%{name}-wait-for-signed-ostree-handler %changelog +* Mon Dec 02 2019 Lubomír Sedlář - 4.1.41-1 +- repoclosure: Parse all fus logs (lsedlar) +- runroot: Log different commands to different files (lsedlar) +- buildinstall: Log message for boot config change (hlin) +- livemedia: Remove title option (hlin) +- Make sure repoclosure cache cleaned when running as root (hlin) +- Prefix lookaside cache dir with compose id (hlin) +- Prefix repoclosure cache dir with compose id (hlin) +- hybrid: Download remote files when getting platform (lsedlar) +- gather: Use fresh cache for each sequence of fus runs (lsedlar) +- tests: Add test for compose logger setup (hlin) +- Do not write images.json if all related phases skipped (hlin) +- gather: Do not write metadata if gather phase skipped (hlin) +- Write global metadata about extra files (lsedlar) +- scm: Add backend for downloading archives from Koji (lsedlar) +- Pass compose to SCM wrappers (lsedlar) +- Remove misleading warning about size of the ISO (hlin) +- Retry watching koji tasks on server outage (hlin) +- config-dump: Allow overwritting config via -e option (hlin) + * Tue Nov 26 2019 Lubomír Sedlář - 4.1.40-4 - Propagate errors from depsolving methods diff --git a/sources b/sources index 699d985a..ec1d22bd 100644 --- a/sources +++ b/sources @@ -1 +1 @@ -SHA512 (pungi-4.1.40.tar.bz2) = 0f1a0445f6a995b541e44afdd7a81bdfecbcb71212a35af26763bd8bfbc1e76b59d565080d5d88f46b4ad907e65b3a78ff153dfb3a5609f5c19fa0752fe1fad4 +SHA512 (pungi-4.1.41.tar.bz2) = d2f1263c209f3f2a1e95571fdd6a1327852e480339d64a03b3b985bd235ebac05b63538ce3d8518a7174a25a8b2210007d0490aa1d07d092fe5d49cb40c9fd83