diff --git a/1183.patch b/1183.patch deleted file mode 100644 index 9257f07d..00000000 --- a/1183.patch +++ /dev/null @@ -1,65 +0,0 @@ -From b0f0579a9e8d31d616ef29e5118ab9bbec392baf Mon Sep 17 00:00:00 2001 -From: Lubomír Sedlář -Date: May 07 2019 11:44:57 +0000 -Subject: util: Resolve ref if duplicate branches are present - - -If the repo contains the same name under multiple directories, make the -resolving work by filtering only to refs/heads and refs/tags. - -Fixes: https://pagure.io/pungi/issue/1180 -Signed-off-by: Lubomír Sedlář - ---- - -diff --git a/pungi/util.py b/pungi/util.py -index 86f35d8..d5a9eed 100644 ---- a/pungi/util.py -+++ b/pungi/util.py -@@ -263,7 +263,10 @@ def resolve_git_ref(repourl, ref): - - _, output = git_ls_remote(repourl, ref) - -- lines = [line for line in output.split('\n') if line] -+ lines = [] -+ for line in output.split("\n"): -+ if line and ("refs/heads/" in line or "refs/tags/" in line or "HEAD" in line): -+ lines.append(line) - if len(lines) == 0: - # Branch does not exist in remote repo - raise GitUrlResolveError( -@@ -272,7 +275,7 @@ def resolve_git_ref(repourl, ref): - if len(lines) != 1: - # This should never happen. HEAD can not match multiple commits in a - # single repo, and there can not be a repo without a HEAD. -- raise GitUrlResolveError("Failed to resolve %s", repourl) -+ raise GitUrlResolveError("Failed to resolve %r in %s" % (ref, repourl)) - - return lines[0].split()[0] - -diff --git a/tests/test_util.py b/tests/test_util.py -index 5a65df2..820d1cc 100644 ---- a/tests/test_util.py -+++ b/tests/test_util.py -@@ -48,6 +48,20 @@ class TestGitRefResolver(unittest.TestCase): - self.assertEqual(ref, "a" * 40) - - @mock.patch('pungi.util.run') -+ def test_resolve_ref_multiple_matches(self, run): -+ run.return_value = ( -+ 0, "CAFEBABE\trefs/heads/master\nBABECAFE\trefs/remotes/origin/master" -+ ) -+ -+ ref = util.resolve_git_ref("https://git.example.com/repo.git", "master") -+ -+ self.assertEqual(ref, "CAFEBABE") -+ run.assert_called_once_with( -+ ["git", "ls-remote", "https://git.example.com/repo.git", "master"], -+ universal_newlines=True, -+ ) -+ -+ @mock.patch('pungi.util.run') - def test_resolve_missing_spec(self, run): - url = util.resolve_git_url('https://git.example.com/repo.git') - - diff --git a/1184.patch b/1184.patch deleted file mode 100644 index 9908583a..00000000 --- a/1184.patch +++ /dev/null @@ -1,32 +0,0 @@ -From 9517df44c73576905041c0ad2d245d9286a574fe Mon Sep 17 00:00:00 2001 -From: Lubomír Sedlář -Date: May 07 2019 08:35:49 +0000 -Subject: config: Fix getting default branch in SCM dict - - -If user configures branch as explicit None, we want to default to HEAD -(which is most likely refs/heads/master in git). - -The original code was getting branch as None, which lead to wrong -resolver being used and the repo url being used as branch. - -Fixes: https://pagure.io/pungi/issue/1181 -Signed-off-by: Lubomír Sedlář - ---- - -diff --git a/pungi/checks.py b/pungi/checks.py -index 0df15dd..2f9478d 100644 ---- a/pungi/checks.py -+++ b/pungi/checks.py -@@ -305,7 +305,8 @@ def _extend_with_default_and_alias(validator_class, offline=False): - and "repo" in instance[property] - ): - instance[property]["branch"] = resolver( -- instance[property]["repo"], instance[property].get("branch", "HEAD") -+ instance[property]["repo"], -+ instance[property].get("branch") or "HEAD", - ) - - for error in _hook_errors(properties, instance, schema): - diff --git a/1186.patch b/1186.patch deleted file mode 100644 index a575ad82..00000000 --- a/1186.patch +++ /dev/null @@ -1,25 +0,0 @@ -From c1a03c259bca87dad12477692f1cfce15c770d92 Mon Sep 17 00:00:00 2001 -From: Lubomír Sedlář -Date: May 09 2019 06:01:13 +0000 -Subject: Switch locale to C.UTF-8 - - -Relates: https://pagure.io/dusty/failed-composes/issue/1642 -Signed-off-by: Lubomír Sedlář - ---- - -diff --git a/bin/pungi-koji b/bin/pungi-koji -index e48686e..049eeaa 100755 ---- a/bin/pungi-koji -+++ b/bin/pungi-koji -@@ -27,7 +27,7 @@ from pungi import get_full_version, util - - - # force C locales --locale.setlocale(locale.LC_ALL, "C") -+locale.setlocale(locale.LC_ALL, "C.UTF-8") - - - COMPOSE = None - diff --git a/1189.patch b/1189.patch deleted file mode 100644 index fd281cd7..00000000 --- a/1189.patch +++ /dev/null @@ -1,30 +0,0 @@ -From dc69281025cbc95ef85f378c839030ca4d8235b4 Mon Sep 17 00:00:00 2001 -From: Lubomír Sedlář -Date: May 13 2019 08:25:00 +0000 -Subject: Fall back to C locale if UTF8 version does not exist - - -This can happen on RHEL 6 or 7. All current Fedora version have it. - -Signed-off-by: Lubomír Sedlář - ---- - -diff --git a/bin/pungi-koji b/bin/pungi-koji -index 049eeaa..38e2fcf 100755 ---- a/bin/pungi-koji -+++ b/bin/pungi-koji -@@ -27,7 +27,11 @@ from pungi import get_full_version, util - - - # force C locales --locale.setlocale(locale.LC_ALL, "C.UTF-8") -+try: -+ locale.setlocale(locale.LC_ALL, "C.UTF-8") -+except locale.Error: -+ # RHEL < 8 does not have C.UTF-8 locale... -+ locale.setlocale(locale.LC_ALL, "C") - - - COMPOSE = None - diff --git a/1190.patch b/1190.patch deleted file mode 100644 index 4456c34d..00000000 --- a/1190.patch +++ /dev/null @@ -1,269 +0,0 @@ -From a451996476f7dd3f244e3705aed87c89af63b9f8 Mon Sep 17 00:00:00 2001 -From: Lubomír Sedlář -Date: May 13 2019 11:12:54 +0000 -Subject: gather: Introduce module source again - - -This reverts commit ac15f211351b96a41b4e715c5599d928df394679. - -It is still needed if nodeps gather method is used. It simply returns -all packages listed in all modules. - -Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1708661 -Signed-off-by: Lubomír Sedlář - ---- - -diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py -index 58f5919..8a20ceb 100644 ---- a/pungi/phases/gather/__init__.py -+++ b/pungi/phases/gather/__init__.py -@@ -202,7 +202,7 @@ def gather_packages(compose, arch, variant, package_sets, fulltree_excludes=None - - else: - -- for source_name in ('comps', 'json'): -+ for source_name in ("module", "comps", "json"): - - packages, groups, filter_packages = get_variant_packages(compose, arch, variant, - source_name, package_sets) -@@ -724,7 +724,7 @@ def get_packages_to_gather(compose, arch=None, variant=None, include_arch=True, - """ - packages = set([]) - groups = set([]) -- for source_name in ('comps', 'json'): -+ for source_name in ("module", "comps", "json"): - GatherSource = get_gather_source(source_name) - src = GatherSource(compose) - -diff --git a/pungi/phases/gather/sources/source_module.py b/pungi/phases/gather/sources/source_module.py -new file mode 100644 -index 0000000..2dc818e ---- /dev/null -+++ b/pungi/phases/gather/sources/source_module.py -@@ -0,0 +1,54 @@ -+# -*- coding: utf-8 -*- -+ -+ -+# This program is free software; you can redistribute it and/or modify -+# it under the terms of the GNU General Public License as published by -+# the Free Software Foundation; version 2 of the License. -+# -+# This program is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+# GNU Library General Public License for more details. -+# -+# You should have received a copy of the GNU General Public License -+# along with this program; if not, see . -+ -+ -+""" -+Get a package list based on modulemd metadata loaded in pkgset phase. Each -+modulemd file contains a list of exact RPM NEVRAs that should be include, so -+just go over all modules in a given variant and join all lists together. -+""" -+ -+ -+import pungi.arch -+import pungi.phases.gather.source -+ -+ -+class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase): -+ enabled = True -+ -+ def __call__(self, arch, variant): -+ groups = set() -+ packages = set() -+ -+ # Check if this variant contains some modules -+ if variant is None or variant.modules is None: -+ return packages, groups -+ -+ compatible_arches = pungi.arch.get_compatible_arches(arch, multilib=True) -+ -+ for nsvc, mmd in variant.arch_mmds[arch].items(): -+ available_rpms = sum( -+ ( -+ variant.nsvc_to_pkgset[nsvc].rpms_by_arch.get(a, []) -+ for a in compatible_arches -+ ), -+ [], -+ ) -+ to_include = set(mmd.get_rpm_artifacts().get()) -+ for rpm_obj in available_rpms: -+ if rpm_obj.nevra in to_include: -+ packages.add((rpm_obj, None)) -+ -+ return packages, groups -diff --git a/tests/helpers.py b/tests/helpers.py -index 7287f04..4c2e24f 100644 ---- a/tests/helpers.py -+++ b/tests/helpers.py -@@ -83,7 +83,7 @@ class MockVariant(mock.Mock): - def get_modular_koji_tags(self, arch=None, types=None): - return [] - -- def add_fake_module(self, nsvc, rpm_nvrs=None, with_artifacts=False): -+ def add_fake_module(self, nsvc, rpm_nvrs=None, with_artifacts=False, mmd_arch=None): - if not Modulemd: - # No support for modules - return -@@ -116,6 +116,8 @@ class MockVariant(mock.Mock): - self.modules = [] - self.modules.append(":".join([name, stream, version])) - self.mmds.append(mmd) -+ if mmd_arch: -+ self.arch_mmds.setdefault(mmd_arch, {})[mmd.dup_nsvc()] = mmd - return mmd - - -diff --git a/tests/test_gather_phase.py b/tests/test_gather_phase.py -index 122e30c..c71599a 100644 ---- a/tests/test_gather_phase.py -+++ b/tests/test_gather_phase.py -@@ -644,10 +644,11 @@ class TestGatherPackages(helpers.PungiTestCase): - {'rpm': [], 'srpm': [], 'debuginfo': []} - ) - self.assertEqual(get_gather_method.call_args_list, -- [mock.call(compose.conf['gather_method'])] * 2) -+ [mock.call(compose.conf['gather_method'])] * 3) - self.assertEqual( - get_variant_packages.call_args_list, - [ -+ mock.call(compose, 'x86_64', compose.variants['Server'], 'module', pkg_set), - mock.call(compose, 'x86_64', compose.variants['Server'], 'comps', pkg_set), - mock.call(compose, 'x86_64', compose.variants['Server'], 'json', pkg_set), - ], -@@ -656,7 +657,7 @@ class TestGatherPackages(helpers.PungiTestCase): - get_gather_method.return_value.return_value.call_args_list, - [mock.call('x86_64', compose.variants['Server'], packages, groups, - filters, set(), set(), pkg_set, fulltree_excludes=set(), -- prepopulate=set())] * 2 -+ prepopulate=set())] * 3 - ) - - @mock.patch('pungi.phases.gather.get_variant_packages') -@@ -687,10 +688,11 @@ class TestGatherPackages(helpers.PungiTestCase): - {'rpm': [], 'srpm': [], 'debuginfo': []} - ) - self.assertEqual(get_gather_method.call_args_list, -- [mock.call(compose.conf['gather_method'])] * 2) -+ [mock.call(compose.conf['gather_method'])] * 3) - self.assertEqual( - get_variant_packages.call_args_list, - [ -+ mock.call(compose, 'x86_64', compose.variants['Server'], 'module', pkg_set), - mock.call(compose, 'x86_64', compose.variants['Server'], 'comps', pkg_set), - mock.call(compose, 'x86_64', compose.variants['Server'], 'json', pkg_set), - ], -@@ -699,7 +701,7 @@ class TestGatherPackages(helpers.PungiTestCase): - get_gather_method.return_value.return_value.call_args_list, - [mock.call('x86_64', compose.variants['Server'], packages, groups, - filters, set(['white']), set(['black']), pkg_set, -- fulltree_excludes=set(), prepopulate=set())] * 2 -+ fulltree_excludes=set(), prepopulate=set())] * 3 - ) - - @mock.patch('pungi.phases.gather.get_variant_packages') -@@ -710,12 +712,12 @@ class TestGatherPackages(helpers.PungiTestCase): - compose = helpers.DummyCompose(self.topdir, { - 'multilib_whitelist': {'*': ['white']}, - 'multilib_blacklist': {'*': ['black']}, -- 'gather_method': {'^Server$': {'comps': 'deps', 'json': 'deps'}}, -+ 'gather_method': {'^Server$': {'comps': 'deps', 'module': 'nodeps', 'json': 'deps'}}, - }) - pkg_set = mock.Mock() - gather.gather_packages(compose, 'x86_64', compose.variants['Server'], pkg_set), - self.assertEqual(get_gather_method.call_args_list, -- [mock.call('deps'), mock.call('deps')]) -+ [mock.call('nodeps'), mock.call('deps'), mock.call('deps')]) - - @mock.patch("pungi.phases.gather.get_variant_packages") - @mock.patch("pungi.phases.gather.get_gather_method") -diff --git a/tests/test_gather_source_module.py b/tests/test_gather_source_module.py -new file mode 100644 -index 0000000..400cc9c ---- /dev/null -+++ b/tests/test_gather_source_module.py -@@ -0,0 +1,79 @@ -+# -*- coding: utf-8 -*- -+ -+try: -+ import unittest2 as unittest -+except ImportError: -+ import unittest -+ -+import mock -+import os -+import sys -+ -+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) -+ -+from pungi.phases.gather.sources.source_module import GatherSourceModule -+from tests import helpers -+from pungi import Modulemd -+ -+ -+@unittest.skipUnless(Modulemd is not None, "Skipped test, no module support.") -+class TestGatherSourceModule(helpers.PungiTestCase): -+ def setUp(self): -+ super(TestGatherSourceModule, self).setUp() -+ self.compose = helpers.DummyCompose(self.topdir, {}) -+ -+ def _add_pkg(self, arch): -+ mock_rpm = mock.Mock( -+ version="1.0.0", -+ release="1", -+ epoch=0, -+ excludearch=None, -+ exclusivearch=None, -+ sourcerpm="pkg-1.0.0-1", -+ nevra="pkg-0:1.0.0-1.%s" % arch, -+ arch=arch, -+ ) -+ mock_rpm.name = "pkg" -+ self.compose.variants["Server"].nsvc_to_pkgset[ -+ "testmodule:master:1:2017" -+ ].rpms_by_arch[arch] = [mock_rpm] -+ -+ def test_without_modules(self): -+ source = GatherSourceModule(self.compose) -+ packages, groups = source("x86_64", self.compose.variants["Server"]) -+ self.assertItemsEqual(packages, []) -+ self.assertItemsEqual(groups, []) -+ -+ def test_include_two_packages(self): -+ self.compose.variants["Server"].add_fake_module( -+ "testmodule:master:1:2017", -+ rpm_nvrs=["pkg-0:1.0.0-1.x86_64", "pkg-0:1.0.0-1.i686"], -+ with_artifacts=True, -+ mmd_arch="x86_64", -+ ) -+ -+ self._add_pkg("x86_64") -+ self._add_pkg("i686") -+ -+ source = GatherSourceModule(self.compose) -+ packages, groups = source("x86_64", self.compose.variants["Server"]) -+ self.assertItemsEqual( -+ [(rpm[0].nevra, rpm[1]) for rpm in packages], -+ [("pkg-0:1.0.0-1.x86_64", None), ("pkg-0:1.0.0-1.i686", None)], -+ ) -+ self.assertItemsEqual(groups, []) -+ -+ def test_does_not_include_unlisted(self): -+ self.compose.variants["Server"].add_fake_module( -+ "testmodule:master:1:2017", -+ rpm_nvrs=[], -+ with_artifacts=True, -+ mmd_arch="x86_64", -+ ) -+ -+ self._add_pkg("x86_64") -+ -+ source = GatherSourceModule(self.compose) -+ packages, groups = source("x86_64", self.compose.variants["Server"]) -+ self.assertItemsEqual(packages, []) -+ self.assertItemsEqual(groups, []) - diff --git a/1199.patch b/1199.patch deleted file mode 100644 index f0f09e44..00000000 --- a/1199.patch +++ /dev/null @@ -1,63 +0,0 @@ -From 33471c38bb859130e5843e896b07af2811b9b0a3 Mon Sep 17 00:00:00 2001 -From: Lubomír Sedlář -Date: May 23 2019 11:51:15 +0000 -Subject: pkgset: Ignore modules without metadata in Koji - - -This is fairly similar to a package only being built for particular -arches. - -Fixes: https://pagure.io/pungi/issue/1198 -Signed-off-by: Lubomír Sedlář - ---- - -diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py -index c406b5a..815576d 100644 ---- a/pungi/phases/pkgset/sources/source_koji.py -+++ b/pungi/phases/pkgset/sources/source_koji.py -@@ -239,12 +239,24 @@ def _add_module_to_variant(koji_wrapper, variant, build, add_to_variant_modules= - pass - mmds[filename] = Modulemd.Module.new_from_file(file_path) - -+ if len(mmds) <= 1: -+ # There was only one modulemd file. This means the build is rather old -+ # and final modulemd files were not uploaded. Such modules are no -+ # longer supported and should be rebuilt. Let's skip it. -+ return -+ - source_mmd = mmds["modulemd.txt"] - nsvc = source_mmd.dup_nsvc() - - variant.mmds.append(source_mmd) - for arch in variant.arches: -- variant.arch_mmds.setdefault(arch, {})[nsvc] = mmds["modulemd.%s.txt" % arch] -+ try: -+ variant.arch_mmds.setdefault(arch, {})[nsvc] = mmds["modulemd.%s.txt" % arch] -+ except KeyError: -+ # There is no modulemd for this arch. This could mean an arch was -+ # added to the compose after the module was built. We don't want to -+ # process this, let's skip this module. -+ pass - - if add_to_variant_modules: - variant.modules.append(nsvc) -@@ -270,6 +282,8 @@ def _get_modules_from_koji(compose, koji_wrapper, event, variant, variant_tags): - koji_modules = get_koji_modules(compose, koji_wrapper, event, module["name"]) - for koji_module in koji_modules: - mmd = _add_module_to_variant(koji_wrapper, variant, koji_module) -+ if not mmd: -+ continue - - tag = koji_module["tag"] - nsvc = mmd.dup_nsvc() -@@ -447,6 +461,8 @@ def _get_modules_from_koji_tags(compose, koji_wrapper, event_id, variant, varian - variant_tags[variant].append(module_tag) - - mmd = _add_module_to_variant(koji_wrapper, variant, build, True) -+ if not mmd: -+ continue - - # Store mapping module-uid --> koji_tag into variant. - # This is needed in createrepo phase where metadata is exposed by producmd - diff --git a/pungi.spec b/pungi.spec index 0ea1e020..e2b74992 100644 --- a/pungi.spec +++ b/pungi.spec @@ -1,19 +1,13 @@ %{?python_enable_dependency_generator} Name: pungi -Version: 4.1.36 -Release: 5%{?dist} +Version: 4.1.37 +Release: 1%{?dist} Summary: Distribution compose tool License: GPLv2 URL: https://pagure.io/pungi Source0: https://pagure.io/releases/%{name}/%{name}-%{version}.tar.bz2 -Patch0: https://pagure.io/pungi/pull-request/1183.patch -Patch1: https://pagure.io/pungi/pull-request/1184.patch -Patch2: https://pagure.io/pungi/pull-request/1186.patch -Patch3: https://pagure.io/pungi/pull-request/1189.patch -Patch4: https://pagure.io/pungi/pull-request/1190.patch -Patch5: https://pagure.io/pungi/pull-request/1199.patch BuildRequires: python3-nose BuildRequires: python3-mock @@ -208,6 +202,30 @@ nosetests-3 --exe %{_bindir}/%{name}-wait-for-signed-ostree-handler %changelog +* Mon May 27 2019 Lubomír Sedlář - 4.1.37-1 +- config-dump: Allow dumping config for multi compose (lsedlar) +- runroot: Remove useless argument output_path (lsedlar) +- buildinstall: Change owner of lorax logs (lsedlar) +- kojiwrapper: Allow changing mode of multiple files (lsedlar) +- buildinstall: Create toplevel directory on compose host (lsedlar) +- arch_utils: add Hygon Dhyana CPU support (fanjinke) +- metadata: Include empty directories in metadata (lsedlar) +- gather: Relax validations on variant_as_lookaside (lsedlar) +- tests: Use correct Python interpreter (lsedlar) +- tests: Ignore warnings when running validation script (lsedlar) +- Remove invalid escape sequences (lsedlar) +- Fix issues in OpenSSH Runroot method found by real tests. (jkaluza) +- buildinstall: Copy files in thread (lsedlar) +- init: Create comps repos in parallel (lsedlar) +- pkgset: Fix whitelist for modules (lsedlar) +- pkgset: Fix filtering excluded modular packages (lsedlar) +- pkgset: Do not overwrite version in module (lsedlar) +- pkgset: Treat modular version as number for sorting (lsedlar) +- Use absolute path for hardlink (lsedlar) +- createiso: Run hardlink on staged content (jdisnard) +- comps-wrapper: Emit attributes sorted (lsedlar) +- patch-iso supports multiple graft directories (jkonecny) + * Thu May 23 2019 Lubomír Sedlář - 4.1.36-5 - Ignore modules without modulemd in Koji diff --git a/sources b/sources index c0ccdacb..a47501d8 100644 --- a/sources +++ b/sources @@ -1 +1 @@ -SHA512 (pungi-4.1.36.tar.bz2) = 95131adc65c7ca0fc4dc1df71522b1ed158331e5f23edf3f6b39f4c70773dd005473882b7f5251e67a3aae10cc430b71d06bf5af4693a4d9ed2802e2f76b27cf +SHA512 (pungi-4.1.37.tar.bz2) = 4d8b60db6838ba1392e11488da9dc3929b1015809aacd2804d39d17165a3ad73817c42b7eb4b7976a072154df794a525db611752ef6ddf0d00e2f2b1a76735e8