almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

New upstream release 4.1.41

Browse Source
This commit is contained in:
Lubomír Sedlář 2019-12-02 15:20:40 +01:00
parent 980ae723bc
commit 25f480ad4f
8 changed files with 23 additions and 406 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
0001-Allow-loading-overrides-for-module-defaults.patch
View File

@ -1,144 +0,0 @@
From 2b112d53f7ea443dd11be584269f559d7d911227 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
Date: Wed, 18 Sep 2019 14:47:30 +0200
Subject: [PATCH] Allow loading overrides for module defaults
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This patch adds a new config option. This is expected to be a name of
subdirectory in the repo with module defaults. If supplied, overrides
from that location are loaded every time defaults are loaded.
This raises the minimal required version of libmodulemd to 2.8.0
JIRA: COMPOSE-3828
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
pungi/checks.py | 1 +
pungi/phases/createrepo.py | 5 ++++-
pungi/phases/gather/__init__.py | 5 ++++-
pungi/phases/pkgset/common.py | 3 ++-
pungi/util.py | 19 +++++++++++++++----
tests/test_pkgset_common.py | 1 +
6 files changed, 27 insertions(+), 7 deletions(-)
diff --git a/pungi/checks.py b/pungi/checks.py
index 17ff8b50..7e9f65a7 100644
--- a/pungi/checks.py
+++ b/pungi/checks.py
@@ -746,6 +746,7 @@ def make_schema():
"default": True
},
"module_defaults_dir": {"$ref": "#/definitions/str_or_scm_dict"},
+ "module_defaults_override_dir": {"type": "string"},
"pkgset_repos": {
"type": "object",
diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py
index 64553594..b2c67ad7 100644
--- a/pungi/phases/createrepo.py
+++ b/pungi/phases/createrepo.py
@@ -211,7 +211,10 @@ def create_variant_repo(compose, arch, variant, pkg_type, pkgset, modules_metada
module_names = set(mod_index.get_module_names())
defaults_dir = compose.paths.work.module_defaults_dir()
- collect_module_defaults(defaults_dir, module_names, mod_index)
+ overrides_dir = compose.conf.get("module_defaults_override_dir")
+ collect_module_defaults(
+ defaults_dir, module_names, mod_index, overrides_dir=overrides_dir
+ )
log_file = compose.paths.log.log_file(arch, "modifyrepo-modules-%s" % variant)
add_modular_metadata(repo, repo_dir, mod_index, log_file)
diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py
index afba72f6..736c10ec 100644
--- a/pungi/phases/gather/__init__.py
+++ b/pungi/phases/gather/__init__.py
@@ -382,7 +382,10 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None):
module_names = set(mod_index.get_module_names())
defaults_dir = compose.paths.work.module_defaults_dir()
- collect_module_defaults(defaults_dir, module_names, mod_index)
+ overrides_dir = compose.conf.get("module_defaults_override_dir")
+ collect_module_defaults(
+ defaults_dir, module_names, mod_index, overrides_dir=overrides_dir
+ )
log_file = compose.paths.log.log_file(
arch, "lookaside_repo_modules_%s" % (variant.uid)
diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py
index 6ab86e53..d7e06cfc 100644
--- a/pungi/phases/pkgset/common.py
+++ b/pungi/phases/pkgset/common.py
@@ -155,8 +155,9 @@ def _create_arch_repo(worker_thread, args, task_num):
# Add modulemd to the repo for all modules in all variants on this architecture.
if Modulemd and mmd:
names = set(x.get_module_name() for x in mmd)
+ overrides_dir = compose.conf.get("module_defaults_override_dir")
mod_index = collect_module_defaults(
- compose.paths.work.module_defaults_dir(), names
+ compose.paths.work.module_defaults_dir(), names, overrides_dir=overrides_dir
)
for x in mmd:
mod_index.add_module_stream(x)
diff --git a/pungi/util.py b/pungi/util.py
index c2c9a948..bd3e022c 100644
--- a/pungi/util.py
+++ b/pungi/util.py
@@ -934,9 +934,10 @@ def iter_module_defaults(path):
# and work with it. However that does not allow for detecting conflicting
# defaults. That should not happen in practice, but better safe than sorry.
# Once libmodulemd can report the error, this code can be simplifed by a
- # lot. It's implemented in
+ # lot. It was implemented in
# https://github.com/fedora-modularity/libmodulemd/commit/3087e4a5c38a331041fec9b6b8f1a372f9ffe64d
- # and released in 2.6.0
+ # and released in 2.6.0, but 2.8.0 added the need to merge overrides and
+ # that breaks this use case again.
for file in glob.glob(os.path.join(path, "*.yaml")):
index = Modulemd.ModuleIndex()
index.update_from_file(file, strict=False)
@@ -944,7 +945,9 @@ def iter_module_defaults(path):
yield module_name, index.get_module(module_name).get_defaults()
-def collect_module_defaults(defaults_dir, modules_to_load=None, mod_index=None):
+def collect_module_defaults(
+ defaults_dir, modules_to_load=None, mod_index=None, overrides_dir=None
+):
"""Load module defaults into index.
If `modules_to_load` is passed in, it should be a set of module names. Only
@@ -954,7 +957,15 @@ def collect_module_defaults(defaults_dir, modules_to_load=None, mod_index=None):
not, a new ModuleIndex will be created and returned
"""
mod_index = mod_index or Modulemd.ModuleIndex()
- for module_name, defaults in iter_module_defaults(defaults_dir):
+
+ temp_index = Modulemd.ModuleIndex.new()
+ temp_index.update_from_defaults_directory(
+ defaults_dir, overrides_path=overrides_dir, strict=False
+ )
+
+ for module_name in temp_index.get_module_names():
+ defaults = temp_index.get_module(module_name).get_defaults()
+
if not modules_to_load or module_name in modules_to_load:
mod_index.add_defaults(defaults)
diff --git a/tests/test_pkgset_common.py b/tests/test_pkgset_common.py
index 64366a42..d1e147fe 100755
--- a/tests/test_pkgset_common.py
+++ b/tests/test_pkgset_common.py
@@ -105,6 +105,7 @@ class TestMaterializedPkgsetCreate(helpers.PungiTestCase):
cmd.assert_called_once_with(
os.path.join(self.topdir, "work/global/module_defaults"),
set(x.get_module_name.return_value for x in mmd["x86_64"]),
+ overrides_dir=None,
)
amm.assert_called_once_with(
mock.ANY,
--
2.21.0

98
1278.patch
View File

@ -1,98 +0,0 @@
From 72bf795bd45e15af86e1a4f96cfbe0d40d40a8cc Mon Sep 17 00:00:00 2001
From: Owen W. Taylor <otaylor@fishsoup.net>
Date: Oct 10 2019 19:13:38 +0000
Subject: ostree-install: allow configuring additional depenencies for runroot
A lorax template used for the ostree-installer might need an additional
package dependency (e.g., flatpak to embed a flatpak repository) - add
a config key 'extra_runroot_pkgs' to the ostree installer configuration
to allow supplementing the set of packages installed into the runroot.
Signed-off-by: Owen W. Taylor <otaylor@fishsoup.net>
---
diff --git a/doc/configuration.rst b/doc/configuration.rst
index 2c8a5a3..2ce9216 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -1527,6 +1527,11 @@ an OSTree repository. This always runs in Koji as a ``runroot`` task.
``template_repo`` needs to point to a Git repository from which to take the
templates.
+ If the templates need to run with additional dependencies, that can be configured
+ with the optional key:
+
+ * ``extra_runroot_pkgs`` -- (*[str]*)
+
**ostree_installer_overwrite** = False
(*bool*) -- by default if a variant including OSTree installer also creates
regular installer images in buildinstall phase, there will be conflicts (as
diff --git a/pungi/checks.py b/pungi/checks.py
index a20d0f7..513728d 100644
--- a/pungi/checks.py
+++ b/pungi/checks.py
@@ -1132,6 +1132,7 @@ def make_schema():
"rootfs_size": {"type": "string"},
"template_repo": {"type": "string"},
"template_branch": {"type": "string"},
+ "extra_runroot_pkgs": {"$ref": "#/definitions/list_of_strings"},
},
"additionalProperties": False,
}),
diff --git a/pungi/phases/ostree_installer.py b/pungi/phases/ostree_installer.py
index 995ac96..46c774b 100644
--- a/pungi/phases/ostree_installer.py
+++ b/pungi/phases/ostree_installer.py
@@ -195,6 +195,8 @@ class OstreeInstallerThread(WorkerThread):
' '.join([shlex_quote(x) for x in lorax_cmd]))
packages = ['pungi', 'lorax', 'ostree']
+ packages += config.get('extra_runroot_pkgs', [])
+
log_file = os.path.join(self.logdir, 'runroot.log')
runroot = Runroot(compose)
diff --git a/tests/test_ostree_installer_phase.py b/tests/test_ostree_installer_phase.py
index 0cce4b8..846e23a 100644
--- a/tests/test_ostree_installer_phase.py
+++ b/tests/test_ostree_installer_phase.py
@@ -126,7 +126,8 @@ class OstreeThreadTest(helpers.PungiTestCase):
self.assertEqual(compose.im.add.mock_calls,
[mock.call('Everything', 'x86_64', image)])
- def assertRunrootCall(self, koji, sources, release, isfinal=False, extra=[], weight=None):
+ def assertRunrootCall(self, koji, sources, release, isfinal=False, extra=[],
+ extra_pkgs=[], weight=None):
lorax_cmd = [
'lorax',
'--product=Fedora',
@@ -156,7 +157,7 @@ class OstreeThreadTest(helpers.PungiTestCase):
[mock.call('rrt', 'x86_64',
'rm -rf %s && %s' % (outdir, ' '.join(lorax_cmd)),
channel=None, mounts=[self.topdir],
- packages=['pungi', 'lorax', 'ostree'],
+ packages=['pungi', 'lorax', 'ostree'] + extra_pkgs,
task_id=True, use_shell=True, weight=weight,
chown_paths=[outdir])])
self.assertEqual(koji.run_runroot_cmd.call_args_list,
@@ -418,6 +419,7 @@ class OstreeThreadTest(helpers.PungiTestCase):
'add_arch_template': ['other_file.txt'],
'template_repo': 'git://example.com/templates.git',
'template_branch': 'f24',
+ 'extra_runroot_pkgs': ['templatedep'],
}
koji = KojiWrapper.return_value
koji.run_runroot_cmd.return_value = {
@@ -445,7 +447,8 @@ class OstreeThreadTest(helpers.PungiTestCase):
isfinal=True,
extra=['--add-template=%s/some_file.txt' % templ_dir,
'--add-arch-template=%s/other_file.txt' % templ_dir,
- '--logfile=%s/%s/lorax.log' % (self.topdir, LOG_PATH)])
+ '--logfile=%s/%s/lorax.log' % (self.topdir, LOG_PATH)],
+ extra_pkgs=['templatedep'])
self.assertIsoLinked(link, get_file_size, get_mtime, final_iso_path)
self.assertImageAdded(self.compose, ImageCls, iso)
self.assertAllCopied(copy_all)

28
1281.patch
View File

@ -1,28 +0,0 @@
From 8bbf9ac13e8e54ec6f51828436744448058db2db Mon Sep 17 00:00:00 2001
From: Lubomír Sedlář <lsedlar@redhat.com>
Date: Oct 17 2019 07:17:36 +0000
Subject: pkgset: Only reuse valid old repo
Instead of just checking that the repo directory exists, make sure the
repodata subdirectory is in there. If it's missing, then createrepo_c
has nothing to use anyway, and it may help avoid issues.
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py
index 6e53cfc..135c7f8 100644
--- a/pungi/phases/pkgset/common.py
+++ b/pungi/phases/pkgset/common.py
@@ -77,7 +77,7 @@ def get_create_global_repo_cmd(compose, path_prefix, repo_dir_global, pkgset):
repo_dir, os.path.abspath(compose.topdir).rstrip("/") + "/"
)
old_repo_dir = os.path.join(old_compose_path, rel_path)
- if os.path.isdir(old_repo_dir):
+ if os.path.isdir(os.path.join(old_repo_dir, "repodata")):
compose.log_info("Using old repodata from: %s", old_repo_dir)
update_md_path = old_repo_dir

26
1292.patch
View File

@ -1,26 +0,0 @@
From cc0e7af866ec260126ec11aee68353bad590493f Mon Sep 17 00:00:00 2001
From: Lubomír Sedlář <lsedlar@redhat.com>
Date: Nov 08 2019 14:24:15 +0000
Subject: Get message from LogRecord with a method
Relates: https://pagure.io/releng/failed-composes/issue/469
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
diff --git a/pungi/compose.py b/pungi/compose.py
index ade5df2..84f2212 100644
--- a/pungi/compose.py
+++ b/pungi/compose.py
@@ -137,7 +137,8 @@ class Compose(kobo.log.LoggingBase):
class ExcludingArchLogFilter(logging.Filter):
def filter(self, record):
- if 'Populating package set for arch:' in record.message or \
+ message = record.getMessage()
+ if 'Populating package set for arch:' in message or \
(record.funcName and record.funcName == 'is_excluded'):
return True
else:

43
1295.patch
View File

@ -1,43 +0,0 @@
From d34b0d79002f127fbea4407b17a157dcdaaf045e Mon Sep 17 00:00:00 2001
From: Lubomír Sedlář <lsedlar@redhat.com>
Date: Nov 11 2019 08:30:37 +0000
Subject: pkgset: Fix running without any koji tags
All places in the code assume the option to not be required except for
this one line.
An obsolete comment is removed as well.
Relates: https://pagure.io/releng/failed-composes/issue/477
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py
index 187e6f1..c573ad8 100644
--- a/pungi/phases/pkgset/sources/source_koji.py
+++ b/pungi/phases/pkgset/sources/source_koji.py
@@ -588,6 +588,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
tag_to_mmd = {}
+ pkgset_koji_tags = force_list(compose.conf.get("pkgset_koji_tag", []))
+
for variant in compose.all_variants.values():
variant_tags[variant] = []
@@ -617,11 +619,9 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
if variant_tag not in compose_tags:
compose_tags.append(variant_tag)
- # TODO check if this works for Fedora Modular variant
- variant_tags[variant].extend(force_list(compose.conf["pkgset_koji_tag"]))
+ variant_tags[variant].extend(pkgset_koji_tags)
# Add global tag(s) if supplied.
- pkgset_koji_tags = force_list(compose.conf.get("pkgset_koji_tag", []))
compose_tags.extend(pkgset_koji_tags)
inherit = compose.conf["pkgset_koji_inherit"]

61
1303.patch
View File

@ -1,61 +0,0 @@
From 7f35ac622a7389a3b2077b245d7f322918b03515 Mon Sep 17 00:00:00 2001
From: Lubomír Sedlář <lsedlar@redhat.com>
Date: Nov 25 2019 13:28:34 +0000
Subject: gather: Collect and re-raise errors from gather method
When there is an exception in gathering (such as after seeing unsigned
packages in deps method), the exception was lost and the compose
continued to run until it tried to access the result and crashed on
KeyError.
Relates: https://pagure.io/releng/failed-composes/issue/587
JIRA: COMPOSE-3986
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py
index 444a8c2..9d54f15 100644
--- a/pungi/phases/gather/__init__.py
+++ b/pungi/phases/gather/__init__.py
@@ -443,6 +443,7 @@ def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltr
continue
threads_list = []
que = Queue()
+ errors = Queue()
for arch in variant.arches:
fulltree_excludes = set()
if exclude_fulltree:
@@ -454,11 +455,17 @@ def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltr
# there.
_update_lookaside_config(compose, variant, arch, result, package_sets)
+ def worker(que, errors, arch, *args, **kwargs):
+ try:
+ que.put((arch, gather_packages(*args, **kwargs)))
+ except Exception as exc:
+ errors.put(exc)
+
# Run gather_packages() in parallel with multi threads and store
# its return value in a Queue() for later use.
t = threading.Thread(
- target=lambda q, arch, *args, **kwargs: q.put((arch, gather_packages(*args, **kwargs))),
- args=(que, arch, compose, arch, variant, package_sets),
+ target=worker,
+ args=(que, errors, arch, compose, arch, variant, package_sets),
kwargs={'fulltree_excludes': fulltree_excludes},
)
threads_list.append(t)
@@ -467,6 +474,10 @@ def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltr
for t in threads_list:
t.join()
+ while not errors.empty():
+ exc = errors.get()
+ raise exc
+
while not que.empty():
arch, pkg_map = que.get()
result.setdefault(arch, {})[variant.uid] = pkg_map

27
pungi.spec
View File

@ -1,16 +1,13 @@
%{?python_enable_dependency_generator}
Name: pungi
Version: 4.1.40
Release: 4%{?dist}
Version: 4.1.41
Release: 1%{?dist}
Summary: Distribution compose tool
License: GPLv2
URL: https://pagure.io/pungi
Source0: https://pagure.io/releases/%{name}/%{name}-%{version}.tar.bz2
Patch1: https://pagure.io/pungi/pull-request/1292.patch
Patch2: https://pagure.io/pungi/pull-request/1295.patch
Patch3: https://pagure.io/pungi/pull-request/1303.patch
BuildRequires: python3-nose
BuildRequires: python3-mock
@ -144,6 +141,26 @@ nosetests-3 --exe
%{_bindir}/%{name}-wait-for-signed-ostree-handler
%changelog
* Mon Dec 02 2019 Lubomír Sedlář <lsedlar@redhat.com> - 4.1.41-1
- repoclosure: Parse all fus logs (lsedlar)
- runroot: Log different commands to different files (lsedlar)
- buildinstall: Log message for boot config change (hlin)
- livemedia: Remove title option (hlin)
- Make sure repoclosure cache cleaned when running as root (hlin)
- Prefix lookaside cache dir with compose id (hlin)
- Prefix repoclosure cache dir with compose id (hlin)
- hybrid: Download remote files when getting platform (lsedlar)
- gather: Use fresh cache for each sequence of fus runs (lsedlar)
- tests: Add test for compose logger setup (hlin)
- Do not write images.json if all related phases skipped (hlin)
- gather: Do not write metadata if gather phase skipped (hlin)
- Write global metadata about extra files (lsedlar)
- scm: Add backend for downloading archives from Koji (lsedlar)
- Pass compose to SCM wrappers (lsedlar)
- Remove misleading warning about size of the ISO (hlin)
- Retry watching koji tasks on server outage (hlin)
- config-dump: Allow overwritting config via -e option (hlin)
* Tue Nov 26 2019 Lubomír Sedlář <lsedlar@redhat.com> - 4.1.40-4
- Propagate errors from depsolving methods

2
sources
View File

@ -1 +1 @@
SHA512 (pungi-4.1.40.tar.bz2) = 0f1a0445f6a995b541e44afdd7a81bdfecbcb71212a35af26763bd8bfbc1e76b59d565080d5d88f46b4ad907e65b3a78ff153dfb3a5609f5c19fa0752fe1fad4
SHA512 (pungi-4.1.41.tar.bz2) = d2f1263c209f3f2a1e95571fdd6a1327852e480339d64a03b3b985bd235ebac05b63538ce3d8518a7174a25a8b2210007d0490aa1d07d092fe5d49cb40c9fd83