New upstream release 4.1.41

0001-Allow-loading-overrides-for-module-defaults.patch

@@ -1,144 +0,0 @@
-From 2b112d53f7ea443dd11be584269f559d7d911227 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
-Date: Wed, 18 Sep 2019 14:47:30 +0200
-Subject: [PATCH] Allow loading overrides for module defaults
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-This patch adds a new config option. This is expected to be a name of
-subdirectory in the repo with module defaults. If supplied, overrides
-from that location are loaded every time defaults are loaded.
-
-This raises the minimal required version of libmodulemd to 2.8.0
-
-JIRA: COMPOSE-3828
-Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
----
- pungi/checks.py                 |  1 +
- pungi/phases/createrepo.py      |  5 ++++-
- pungi/phases/gather/__init__.py |  5 ++++-
- pungi/phases/pkgset/common.py   |  3 ++-
- pungi/util.py                   | 19 +++++++++++++++----
- tests/test_pkgset_common.py     |  1 +
- 6 files changed, 27 insertions(+), 7 deletions(-)
-
-diff --git a/pungi/checks.py b/pungi/checks.py
-index 17ff8b50..7e9f65a7 100644
---- a/pungi/checks.py
-+++ b/pungi/checks.py
-@@ -746,6 +746,7 @@ def make_schema():
-                 "default": True
-             },
-             "module_defaults_dir": {"$ref": "#/definitions/str_or_scm_dict"},
-+            "module_defaults_override_dir": {"type": "string"},
- 
-             "pkgset_repos": {
-                 "type": "object",
-diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py
-index 64553594..b2c67ad7 100644
---- a/pungi/phases/createrepo.py
-+++ b/pungi/phases/createrepo.py
-@@ -211,7 +211,10 @@ def create_variant_repo(compose, arch, variant, pkg_type, pkgset, modules_metada
- 
-         module_names = set(mod_index.get_module_names())
-         defaults_dir = compose.paths.work.module_defaults_dir()
--        collect_module_defaults(defaults_dir, module_names, mod_index)
-+        overrides_dir = compose.conf.get("module_defaults_override_dir")
-+        collect_module_defaults(
-+            defaults_dir, module_names, mod_index, overrides_dir=overrides_dir
-+        )
- 
-         log_file = compose.paths.log.log_file(arch, "modifyrepo-modules-%s" % variant)
-         add_modular_metadata(repo, repo_dir, mod_index, log_file)
-diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py
-index afba72f6..736c10ec 100644
---- a/pungi/phases/gather/__init__.py
-+++ b/pungi/phases/gather/__init__.py
-@@ -382,7 +382,10 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None):
- 
-         module_names = set(mod_index.get_module_names())
-         defaults_dir = compose.paths.work.module_defaults_dir()
--        collect_module_defaults(defaults_dir, module_names, mod_index)
-+        overrides_dir = compose.conf.get("module_defaults_override_dir")
-+        collect_module_defaults(
-+            defaults_dir, module_names, mod_index, overrides_dir=overrides_dir
-+        )
- 
-         log_file = compose.paths.log.log_file(
-             arch, "lookaside_repo_modules_%s" % (variant.uid)
-diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py
-index 6ab86e53..d7e06cfc 100644
---- a/pungi/phases/pkgset/common.py
-+++ b/pungi/phases/pkgset/common.py
-@@ -155,8 +155,9 @@ def _create_arch_repo(worker_thread, args, task_num):
-     # Add modulemd to the repo for all modules in all variants on this architecture.
-     if Modulemd and mmd:
-         names = set(x.get_module_name() for x in mmd)
-+        overrides_dir = compose.conf.get("module_defaults_override_dir")
-         mod_index = collect_module_defaults(
--            compose.paths.work.module_defaults_dir(), names
-+            compose.paths.work.module_defaults_dir(), names, overrides_dir=overrides_dir
-         )
-         for x in mmd:
-             mod_index.add_module_stream(x)
-diff --git a/pungi/util.py b/pungi/util.py
-index c2c9a948..bd3e022c 100644
---- a/pungi/util.py
-+++ b/pungi/util.py
-@@ -934,9 +934,10 @@ def iter_module_defaults(path):
-     # and work with it. However that does not allow for detecting conflicting
-     # defaults. That should not happen in practice, but better safe than sorry.
-     # Once libmodulemd can report the error, this code can be simplifed by a
--    # lot. It's implemented in
-+    # lot. It was implemented in
-     # https://github.com/fedora-modularity/libmodulemd/commit/3087e4a5c38a331041fec9b6b8f1a372f9ffe64d
--    # and released in 2.6.0
-+    # and released in 2.6.0, but 2.8.0 added the need to merge overrides and
-+    # that breaks this use case again.
-     for file in glob.glob(os.path.join(path, "*.yaml")):
-         index = Modulemd.ModuleIndex()
-         index.update_from_file(file, strict=False)
-@@ -944,7 +945,9 @@ def iter_module_defaults(path):
-             yield module_name, index.get_module(module_name).get_defaults()
- 
- 
--def collect_module_defaults(defaults_dir, modules_to_load=None, mod_index=None):
-+def collect_module_defaults(
-+    defaults_dir, modules_to_load=None, mod_index=None, overrides_dir=None
-+):
-     """Load module defaults into index.
- 
-     If `modules_to_load` is passed in, it should be a set of module names. Only
-@@ -954,7 +957,15 @@ def collect_module_defaults(defaults_dir, modules_to_load=None, mod_index=None):
-     not, a new ModuleIndex will be created and returned
-     """
-     mod_index = mod_index or Modulemd.ModuleIndex()
--    for module_name, defaults in iter_module_defaults(defaults_dir):
-+
-+    temp_index = Modulemd.ModuleIndex.new()
-+    temp_index.update_from_defaults_directory(
-+        defaults_dir, overrides_path=overrides_dir, strict=False
-+    )
-+
-+    for module_name in temp_index.get_module_names():
-+        defaults = temp_index.get_module(module_name).get_defaults()
-+
-         if not modules_to_load or module_name in modules_to_load:
-             mod_index.add_defaults(defaults)
- 
-diff --git a/tests/test_pkgset_common.py b/tests/test_pkgset_common.py
-index 64366a42..d1e147fe 100755
---- a/tests/test_pkgset_common.py
-+++ b/tests/test_pkgset_common.py
-@@ -105,6 +105,7 @@ class TestMaterializedPkgsetCreate(helpers.PungiTestCase):
-         cmd.assert_called_once_with(
-             os.path.join(self.topdir, "work/global/module_defaults"),
-             set(x.get_module_name.return_value for x in mmd["x86_64"]),
-+            overrides_dir=None,
-         )
-         amm.assert_called_once_with(
-             mock.ANY,
--- 
-2.21.0
-

1278.patch

@@ -1,98 +0,0 @@
-From 72bf795bd45e15af86e1a4f96cfbe0d40d40a8cc Mon Sep 17 00:00:00 2001
-From: Owen W. Taylor <otaylor@fishsoup.net>
-Date: Oct 10 2019 19:13:38 +0000
-Subject: ostree-install: allow configuring additional depenencies for runroot
-
-
-A lorax template used for the ostree-installer might need an additional
-package dependency (e.g., flatpak to embed a flatpak repository) - add
-a config key 'extra_runroot_pkgs' to the ostree installer configuration
-to allow supplementing the set of packages installed into the runroot.
-
-Signed-off-by: Owen W. Taylor <otaylor@fishsoup.net>
-
----
-
-diff --git a/doc/configuration.rst b/doc/configuration.rst
-index 2c8a5a3..2ce9216 100644
---- a/doc/configuration.rst
-+++ b/doc/configuration.rst
-@@ -1527,6 +1527,11 @@ an OSTree repository. This always runs in Koji as a ``runroot`` task.
-     ``template_repo`` needs to point to a Git repository from which to take the
-     templates.
- 
-+    If the templates need to run with additional dependencies, that can be configured
-+    with the optional key:
-+
-+    * ``extra_runroot_pkgs`` -- (*[str]*)
-+
- **ostree_installer_overwrite** = False
-     (*bool*) -- by default if a variant including OSTree installer also creates
-     regular installer images in buildinstall phase, there will be conflicts (as
-diff --git a/pungi/checks.py b/pungi/checks.py
-index a20d0f7..513728d 100644
---- a/pungi/checks.py
-+++ b/pungi/checks.py
-@@ -1132,6 +1132,7 @@ def make_schema():
-                     "rootfs_size": {"type": "string"},
-                     "template_repo": {"type": "string"},
-                     "template_branch": {"type": "string"},
-+                    "extra_runroot_pkgs": {"$ref": "#/definitions/list_of_strings"},
-                 },
-                 "additionalProperties": False,
-             }),
-diff --git a/pungi/phases/ostree_installer.py b/pungi/phases/ostree_installer.py
-index 995ac96..46c774b 100644
---- a/pungi/phases/ostree_installer.py
-+++ b/pungi/phases/ostree_installer.py
-@@ -195,6 +195,8 @@ class OstreeInstallerThread(WorkerThread):
-                                    ' '.join([shlex_quote(x) for x in lorax_cmd]))
- 
-         packages = ['pungi', 'lorax', 'ostree']
-+        packages += config.get('extra_runroot_pkgs', [])
-+
-         log_file = os.path.join(self.logdir, 'runroot.log')
- 
-         runroot = Runroot(compose)
-diff --git a/tests/test_ostree_installer_phase.py b/tests/test_ostree_installer_phase.py
-index 0cce4b8..846e23a 100644
---- a/tests/test_ostree_installer_phase.py
-+++ b/tests/test_ostree_installer_phase.py
-@@ -126,7 +126,8 @@ class OstreeThreadTest(helpers.PungiTestCase):
-         self.assertEqual(compose.im.add.mock_calls,
-                          [mock.call('Everything', 'x86_64', image)])
- 
--    def assertRunrootCall(self, koji, sources, release, isfinal=False, extra=[], weight=None):
-+    def assertRunrootCall(self, koji, sources, release, isfinal=False, extra=[],
-+                          extra_pkgs=[], weight=None):
-         lorax_cmd = [
-             'lorax',
-             '--product=Fedora',
-@@ -156,7 +157,7 @@ class OstreeThreadTest(helpers.PungiTestCase):
-                          [mock.call('rrt', 'x86_64',
-                                     'rm -rf %s && %s' % (outdir, ' '.join(lorax_cmd)),
-                                     channel=None, mounts=[self.topdir],
--                                    packages=['pungi', 'lorax', 'ostree'],
-+                                    packages=['pungi', 'lorax', 'ostree'] + extra_pkgs,
-                                     task_id=True, use_shell=True, weight=weight,
-                                     chown_paths=[outdir])])
-         self.assertEqual(koji.run_runroot_cmd.call_args_list,
-@@ -418,6 +419,7 @@ class OstreeThreadTest(helpers.PungiTestCase):
-             'add_arch_template': ['other_file.txt'],
-             'template_repo': 'git://example.com/templates.git',
-             'template_branch': 'f24',
-+            'extra_runroot_pkgs': ['templatedep'],
-         }
-         koji = KojiWrapper.return_value
-         koji.run_runroot_cmd.return_value = {
-@@ -445,7 +447,8 @@ class OstreeThreadTest(helpers.PungiTestCase):
-                                isfinal=True,
-                                extra=['--add-template=%s/some_file.txt' % templ_dir,
-                                       '--add-arch-template=%s/other_file.txt' % templ_dir,
--                                      '--logfile=%s/%s/lorax.log' % (self.topdir, LOG_PATH)])
-+                                      '--logfile=%s/%s/lorax.log' % (self.topdir, LOG_PATH)],
-+                               extra_pkgs=['templatedep'])
-         self.assertIsoLinked(link, get_file_size, get_mtime, final_iso_path)
-         self.assertImageAdded(self.compose, ImageCls, iso)
-         self.assertAllCopied(copy_all)
-

1281.patch

@@ -1,28 +0,0 @@
-From 8bbf9ac13e8e54ec6f51828436744448058db2db Mon Sep 17 00:00:00 2001
-From: Lubomír Sedlář <lsedlar@redhat.com>
-Date: Oct 17 2019 07:17:36 +0000
-Subject: pkgset: Only reuse valid old repo
-
-
-Instead of just checking that the repo directory exists, make sure the
-repodata subdirectory is in there. If it's missing, then createrepo_c
-has nothing to use anyway, and it may help avoid issues.
-
-Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
-
----
-
-diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py
-index 6e53cfc..135c7f8 100644
---- a/pungi/phases/pkgset/common.py
-+++ b/pungi/phases/pkgset/common.py
-@@ -77,7 +77,7 @@ def get_create_global_repo_cmd(compose, path_prefix, repo_dir_global, pkgset):
-                 repo_dir, os.path.abspath(compose.topdir).rstrip("/") + "/"
-             )
-             old_repo_dir = os.path.join(old_compose_path, rel_path)
--            if os.path.isdir(old_repo_dir):
-+            if os.path.isdir(os.path.join(old_repo_dir, "repodata")):
-                 compose.log_info("Using old repodata from: %s", old_repo_dir)
-                 update_md_path = old_repo_dir
- 
-

1292.patch

@@ -1,26 +0,0 @@
-From cc0e7af866ec260126ec11aee68353bad590493f Mon Sep 17 00:00:00 2001
-From: Lubomír Sedlář <lsedlar@redhat.com>
-Date: Nov 08 2019 14:24:15 +0000
-Subject: Get message from LogRecord with a method
-
-
-Relates: https://pagure.io/releng/failed-composes/issue/469
-Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
-
----
-
-diff --git a/pungi/compose.py b/pungi/compose.py
-index ade5df2..84f2212 100644
---- a/pungi/compose.py
-+++ b/pungi/compose.py
-@@ -137,7 +137,8 @@ class Compose(kobo.log.LoggingBase):
- 
-             class ExcludingArchLogFilter(logging.Filter):
-                 def filter(self, record):
--                    if 'Populating package set for arch:' in record.message or \
-+                    message = record.getMessage()
-+                    if 'Populating package set for arch:' in message or \
-                             (record.funcName and record.funcName == 'is_excluded'):
-                         return True
-                     else:
-

1295.patch

@@ -1,43 +0,0 @@
-From d34b0d79002f127fbea4407b17a157dcdaaf045e Mon Sep 17 00:00:00 2001
-From: Lubomír Sedlář <lsedlar@redhat.com>
-Date: Nov 11 2019 08:30:37 +0000
-Subject: pkgset: Fix running without any koji tags
-
-
-All places in the code assume the option to not be required except for
-this one line.
-
-An obsolete comment is removed as well.
-
-Relates: https://pagure.io/releng/failed-composes/issue/477
-Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
-
----
-
-diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py
-index 187e6f1..c573ad8 100644
---- a/pungi/phases/pkgset/sources/source_koji.py
-+++ b/pungi/phases/pkgset/sources/source_koji.py
-@@ -588,6 +588,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
- 
-     tag_to_mmd = {}
- 
-+    pkgset_koji_tags = force_list(compose.conf.get("pkgset_koji_tag", []))
-+
-     for variant in compose.all_variants.values():
-         variant_tags[variant] = []
- 
-@@ -617,11 +619,9 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
-             if variant_tag not in compose_tags:
-                 compose_tags.append(variant_tag)
- 
--        # TODO check if this works for Fedora Modular variant
--        variant_tags[variant].extend(force_list(compose.conf["pkgset_koji_tag"]))
-+        variant_tags[variant].extend(pkgset_koji_tags)
- 
-     # Add global tag(s) if supplied.
--    pkgset_koji_tags = force_list(compose.conf.get("pkgset_koji_tag", []))
-     compose_tags.extend(pkgset_koji_tags)
- 
-     inherit = compose.conf["pkgset_koji_inherit"]
-

1303.patch

@@ -1,61 +0,0 @@
-From 7f35ac622a7389a3b2077b245d7f322918b03515 Mon Sep 17 00:00:00 2001
-From: Lubomír Sedlář <lsedlar@redhat.com>
-Date: Nov 25 2019 13:28:34 +0000
-Subject: gather: Collect and re-raise errors from gather method
-
-
-When there is an exception in gathering (such as after seeing unsigned
-packages in deps method), the exception was lost and the compose
-continued to run until it tried to access the result and crashed on
-KeyError.
-
-Relates: https://pagure.io/releng/failed-composes/issue/587
-JIRA: COMPOSE-3986
-Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
-
----
-
-diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py
-index 444a8c2..9d54f15 100644
---- a/pungi/phases/gather/__init__.py
-+++ b/pungi/phases/gather/__init__.py
-@@ -443,6 +443,7 @@ def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltr
-             continue
-         threads_list = []
-         que = Queue()
-+        errors = Queue()
-         for arch in variant.arches:
-             fulltree_excludes = set()
-             if exclude_fulltree:
-@@ -454,11 +455,17 @@ def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltr
-             # there.
-             _update_lookaside_config(compose, variant, arch, result, package_sets)
- 
-+            def worker(que, errors, arch, *args, **kwargs):
-+                try:
-+                    que.put((arch, gather_packages(*args, **kwargs)))
-+                except Exception as exc:
-+                    errors.put(exc)
-+
-             # Run gather_packages() in parallel with multi threads and store
-             # its return value in a Queue() for later use.
-             t = threading.Thread(
--                target=lambda q, arch, *args, **kwargs: q.put((arch, gather_packages(*args, **kwargs))),
--                args=(que, arch, compose, arch, variant, package_sets),
-+                target=worker,
-+                args=(que, errors, arch, compose, arch, variant, package_sets),
-                 kwargs={'fulltree_excludes': fulltree_excludes},
-             )
-             threads_list.append(t)
-@@ -467,6 +474,10 @@ def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltr
-         for t in threads_list:
-             t.join()
- 
-+        while not errors.empty():
-+            exc = errors.get()
-+            raise exc
-+
-         while not que.empty():
-             arch, pkg_map = que.get()
-             result.setdefault(arch, {})[variant.uid] = pkg_map
-

pungi.spec

@@ -1,16 +1,13 @@
 %{?python_enable_dependency_generator}
 
 Name:           pungi
-Version:        4.1.40
+Version:        4.1.41
-Release:        4%{?dist}
+Release:        1%{?dist}
 Summary:        Distribution compose tool
 
 License:        GPLv2
 URL:            https://pagure.io/pungi
 Source0:        https://pagure.io/releases/%{name}/%{name}-%{version}.tar.bz2
-Patch1:         https://pagure.io/pungi/pull-request/1292.patch
-Patch2:         https://pagure.io/pungi/pull-request/1295.patch
-Patch3:         https://pagure.io/pungi/pull-request/1303.patch
 
 BuildRequires:  python3-nose
 BuildRequires:  python3-mock
@@ -144,6 +141,26 @@ nosetests-3 --exe
 %{_bindir}/%{name}-wait-for-signed-ostree-handler
 
 %changelog
+* Mon Dec 02 2019 Lubomír Sedlář <lsedlar@redhat.com> - 4.1.41-1
+- repoclosure: Parse all fus logs (lsedlar)
+- runroot: Log different commands to different files (lsedlar)
+- buildinstall: Log message for boot config change (hlin)
+- livemedia: Remove title option (hlin)
+- Make sure repoclosure cache cleaned when running as root (hlin)
+- Prefix lookaside cache dir with compose id (hlin)
+- Prefix repoclosure cache dir with compose id (hlin)
+- hybrid: Download remote files when getting platform (lsedlar)
+- gather: Use fresh cache for each sequence of fus runs (lsedlar)
+- tests: Add test for compose logger setup (hlin)
+- Do not write images.json if all related phases skipped (hlin)
+- gather: Do not write metadata if gather phase skipped (hlin)
+- Write global metadata about extra files (lsedlar)
+- scm: Add backend for downloading archives from Koji (lsedlar)
+- Pass compose to SCM wrappers (lsedlar)
+- Remove misleading warning about size of the ISO (hlin)
+- Retry watching koji tasks on server outage (hlin)
+- config-dump: Allow overwritting config via -e option (hlin)
+
 * Tue Nov 26 2019 Lubomír Sedlář <lsedlar@redhat.com> - 4.1.40-4
 - Propagate errors from depsolving methods
 

sources

@@ -1 +1 @@
-SHA512 (pungi-4.1.40.tar.bz2) = 0f1a0445f6a995b541e44afdd7a81bdfecbcb71212a35af26763bd8bfbc1e76b59d565080d5d88f46b4ad907e65b3a78ff153dfb3a5609f5c19fa0752fe1fad4
+SHA512 (pungi-4.1.41.tar.bz2) = d2f1263c209f3f2a1e95571fdd6a1327852e480339d64a03b3b985bd235ebac05b63538ce3d8518a7174a25a8b2210007d0490aa1d07d092fe5d49cb40c9fd83