diff --git a/1185.patch b/1185.patch deleted file mode 100644 index b2a958e5..00000000 --- a/1185.patch +++ /dev/null @@ -1,141 +0,0 @@ -From c961fdc99e96a3c341273876b621d78733c70862 Mon Sep 17 00:00:00 2001 -From: Lubomír Sedlář -Date: May 07 2019 12:31:53 +0000 -Subject: kojiwrapper: Decode output of runroot as UTF-8 - - -Starting with Python 3.6, it is possible to specify encoding for output -of a `subprocess.Popen`. Try to use it to decode as UTF-8. Particularly -output of ostree runroot tasks is using unicode characters. - -This should be safe: anything that is valid ASCII is also valid UTF-8. - -Relates: https://pagure.io/dusty/failed-composes/issue/1642 -Signed-off-by: Lubomír Sedlář - ---- - -diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py -index 3cd9ac9..48534ff 100644 ---- a/pungi/wrappers/kojiwrapper.py -+++ b/pungi/wrappers/kojiwrapper.py -@@ -19,6 +19,7 @@ import re - import time - import threading - import contextlib -+import sys - - import koji - from kobo.shortcuts import run -@@ -146,8 +147,16 @@ class KojiWrapper(object): - """ - task_id = None - with self.get_koji_cmd_env() as env: -- retcode, output = run(command, can_fail=True, logfile=log_file, -- show_cmd=True, env=env, universal_newlines=True) -+ extra_kwarg = {"encoding": "utf-8"} if sys.version_info >= (3, 6) else {} -+ retcode, output = run( -+ command, -+ can_fail=True, -+ logfile=log_file, -+ show_cmd=True, -+ env=env, -+ universal_newlines=True, -+ **extra_kwarg -+ ) - if "--task-id" in command: - first_line = output.splitlines()[0] - if re.match(r'^\d+$', first_line): -diff --git a/tests/test_koji_wrapper.py b/tests/test_koji_wrapper.py -index 34d8b3a..39d405d 100644 ---- a/tests/test_koji_wrapper.py -+++ b/tests/test_koji_wrapper.py -@@ -405,6 +405,21 @@ class LiveImageKojiWrapperTest(KojiWrapperBaseTestCase): - - - class RunrootKojiWrapperTest(KojiWrapperBaseTestCase): -+ -+ def kwargs(self, **kwargs): -+ defaults= { -+ "can_fail": True, -+ "env": None, -+ "logfile": None, -+ "show_cmd": True, -+ "universal_newlines": True, -+ } -+ if sys.version_info >= (3, 6): -+ defaults["encoding"] = "utf-8" -+ -+ defaults.update(kwargs) -+ return defaults -+ - def test_get_cmd_minimal(self): - cmd = self.koji.get_runroot_cmd('tgt', 's390x', 'date', use_shell=False, task_id=False) - self.assertEqual(len(cmd), 7) -@@ -457,11 +472,7 @@ class RunrootKojiWrapperTest(KojiWrapperBaseTestCase): - - result = self.koji.run_runroot_cmd(cmd) - self.assertDictEqual(result, {'retcode': 0, 'output': output, 'task_id': None}) -- self.assertEqual( -- run.call_args_list, -- [mock.call(cmd, can_fail=True, env=None, logfile=None, show_cmd=True, -- universal_newlines=True)] -- ) -+ self.assertEqual(run.call_args_list, [mock.call(cmd, **self.kwargs())]) - - @mock.patch('pungi.wrappers.kojiwrapper.run') - def test_run_runroot_cmd_with_task_id(self, run): -@@ -471,11 +482,7 @@ class RunrootKojiWrapperTest(KojiWrapperBaseTestCase): - - result = self.koji.run_runroot_cmd(cmd) - self.assertDictEqual(result, {'retcode': 0, 'output': output, 'task_id': 1234}) -- self.assertEqual( -- run.call_args_list, -- [mock.call(cmd, can_fail=True, env=None, logfile=None, show_cmd=True, -- universal_newlines=True)] -- ) -+ self.assertEqual(run.call_args_list, [mock.call(cmd, **self.kwargs())]) - - @mock.patch('pungi.wrappers.kojiwrapper.run') - def test_run_runroot_cmd_with_task_id_and_fail(self, run): -@@ -485,11 +492,7 @@ class RunrootKojiWrapperTest(KojiWrapperBaseTestCase): - - result = self.koji.run_runroot_cmd(cmd) - self.assertDictEqual(result, {'retcode': 1, 'output': output, 'task_id': None}) -- self.assertEqual( -- run.call_args_list, -- [mock.call(cmd, can_fail=True, env=None, logfile=None, show_cmd=True, -- universal_newlines=True)] -- ) -+ self.assertEqual(run.call_args_list, [mock.call(cmd, **self.kwargs())]) - - @mock.patch('pungi.wrappers.kojiwrapper.run') - def test_run_runroot_cmd_with_task_id_and_fail_but_emit_id(self, run): -@@ -499,11 +502,7 @@ class RunrootKojiWrapperTest(KojiWrapperBaseTestCase): - - result = self.koji.run_runroot_cmd(cmd) - self.assertDictEqual(result, {'retcode': 1, 'output': output, 'task_id': 12345}) -- self.assertEqual( -- run.call_args_list, -- [mock.call(cmd, can_fail=True, env=None, logfile=None, show_cmd=True, -- universal_newlines=True)] -- ) -+ self.assertEqual(run.call_args_list, [mock.call(cmd, **self.kwargs())]) - - @mock.patch.dict('os.environ', {'FOO': 'BAR'}, clear=True) - @mock.patch('shutil.rmtree') -@@ -521,8 +520,11 @@ class RunrootKojiWrapperTest(KojiWrapperBaseTestCase): - self.assertDictEqual(result, {'retcode': 0, 'output': output, 'task_id': None}) - self.assertEqual( - run.call_args_list, -- [mock.call(cmd, can_fail=True, env={'KRB5CCNAME': 'DIR:/tmp/foo', 'FOO': 'BAR'}, -- logfile=None, show_cmd=True, universal_newlines=True)] -+ [ -+ mock.call( -+ cmd, **self.kwargs(env={'KRB5CCNAME': 'DIR:/tmp/foo', 'FOO': 'BAR'}) -+ ), -+ ], - ) - - - diff --git a/1186.patch b/1186.patch new file mode 100644 index 00000000..a575ad82 --- /dev/null +++ b/1186.patch @@ -0,0 +1,25 @@ +From c1a03c259bca87dad12477692f1cfce15c770d92 Mon Sep 17 00:00:00 2001 +From: Lubomír Sedlář +Date: May 09 2019 06:01:13 +0000 +Subject: Switch locale to C.UTF-8 + + +Relates: https://pagure.io/dusty/failed-composes/issue/1642 +Signed-off-by: Lubomír Sedlář + +--- + +diff --git a/bin/pungi-koji b/bin/pungi-koji +index e48686e..049eeaa 100755 +--- a/bin/pungi-koji ++++ b/bin/pungi-koji +@@ -27,7 +27,7 @@ from pungi import get_full_version, util + + + # force C locales +-locale.setlocale(locale.LC_ALL, "C") ++locale.setlocale(locale.LC_ALL, "C.UTF-8") + + + COMPOSE = None + diff --git a/1189.patch b/1189.patch new file mode 100644 index 00000000..fd281cd7 --- /dev/null +++ b/1189.patch @@ -0,0 +1,30 @@ +From dc69281025cbc95ef85f378c839030ca4d8235b4 Mon Sep 17 00:00:00 2001 +From: Lubomír Sedlář +Date: May 13 2019 08:25:00 +0000 +Subject: Fall back to C locale if UTF8 version does not exist + + +This can happen on RHEL 6 or 7. All current Fedora version have it. + +Signed-off-by: Lubomír Sedlář + +--- + +diff --git a/bin/pungi-koji b/bin/pungi-koji +index 049eeaa..38e2fcf 100755 +--- a/bin/pungi-koji ++++ b/bin/pungi-koji +@@ -27,7 +27,11 @@ from pungi import get_full_version, util + + + # force C locales +-locale.setlocale(locale.LC_ALL, "C.UTF-8") ++try: ++ locale.setlocale(locale.LC_ALL, "C.UTF-8") ++except locale.Error: ++ # RHEL < 8 does not have C.UTF-8 locale... ++ locale.setlocale(locale.LC_ALL, "C") + + + COMPOSE = None + diff --git a/1190.patch b/1190.patch new file mode 100644 index 00000000..4456c34d --- /dev/null +++ b/1190.patch @@ -0,0 +1,269 @@ +From a451996476f7dd3f244e3705aed87c89af63b9f8 Mon Sep 17 00:00:00 2001 +From: Lubomír Sedlář +Date: May 13 2019 11:12:54 +0000 +Subject: gather: Introduce module source again + + +This reverts commit ac15f211351b96a41b4e715c5599d928df394679. + +It is still needed if nodeps gather method is used. It simply returns +all packages listed in all modules. + +Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1708661 +Signed-off-by: Lubomír Sedlář + +--- + +diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py +index 58f5919..8a20ceb 100644 +--- a/pungi/phases/gather/__init__.py ++++ b/pungi/phases/gather/__init__.py +@@ -202,7 +202,7 @@ def gather_packages(compose, arch, variant, package_sets, fulltree_excludes=None + + else: + +- for source_name in ('comps', 'json'): ++ for source_name in ("module", "comps", "json"): + + packages, groups, filter_packages = get_variant_packages(compose, arch, variant, + source_name, package_sets) +@@ -724,7 +724,7 @@ def get_packages_to_gather(compose, arch=None, variant=None, include_arch=True, + """ + packages = set([]) + groups = set([]) +- for source_name in ('comps', 'json'): ++ for source_name in ("module", "comps", "json"): + GatherSource = get_gather_source(source_name) + src = GatherSource(compose) + +diff --git a/pungi/phases/gather/sources/source_module.py b/pungi/phases/gather/sources/source_module.py +new file mode 100644 +index 0000000..2dc818e +--- /dev/null ++++ b/pungi/phases/gather/sources/source_module.py +@@ -0,0 +1,54 @@ ++# -*- coding: utf-8 -*- ++ ++ ++# This program is free software; you can redistribute it and/or modify ++# it under the terms of the GNU General Public License as published by ++# the Free Software Foundation; version 2 of the License. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU Library General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, see . ++ ++ ++""" ++Get a package list based on modulemd metadata loaded in pkgset phase. Each ++modulemd file contains a list of exact RPM NEVRAs that should be include, so ++just go over all modules in a given variant and join all lists together. ++""" ++ ++ ++import pungi.arch ++import pungi.phases.gather.source ++ ++ ++class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase): ++ enabled = True ++ ++ def __call__(self, arch, variant): ++ groups = set() ++ packages = set() ++ ++ # Check if this variant contains some modules ++ if variant is None or variant.modules is None: ++ return packages, groups ++ ++ compatible_arches = pungi.arch.get_compatible_arches(arch, multilib=True) ++ ++ for nsvc, mmd in variant.arch_mmds[arch].items(): ++ available_rpms = sum( ++ ( ++ variant.nsvc_to_pkgset[nsvc].rpms_by_arch.get(a, []) ++ for a in compatible_arches ++ ), ++ [], ++ ) ++ to_include = set(mmd.get_rpm_artifacts().get()) ++ for rpm_obj in available_rpms: ++ if rpm_obj.nevra in to_include: ++ packages.add((rpm_obj, None)) ++ ++ return packages, groups +diff --git a/tests/helpers.py b/tests/helpers.py +index 7287f04..4c2e24f 100644 +--- a/tests/helpers.py ++++ b/tests/helpers.py +@@ -83,7 +83,7 @@ class MockVariant(mock.Mock): + def get_modular_koji_tags(self, arch=None, types=None): + return [] + +- def add_fake_module(self, nsvc, rpm_nvrs=None, with_artifacts=False): ++ def add_fake_module(self, nsvc, rpm_nvrs=None, with_artifacts=False, mmd_arch=None): + if not Modulemd: + # No support for modules + return +@@ -116,6 +116,8 @@ class MockVariant(mock.Mock): + self.modules = [] + self.modules.append(":".join([name, stream, version])) + self.mmds.append(mmd) ++ if mmd_arch: ++ self.arch_mmds.setdefault(mmd_arch, {})[mmd.dup_nsvc()] = mmd + return mmd + + +diff --git a/tests/test_gather_phase.py b/tests/test_gather_phase.py +index 122e30c..c71599a 100644 +--- a/tests/test_gather_phase.py ++++ b/tests/test_gather_phase.py +@@ -644,10 +644,11 @@ class TestGatherPackages(helpers.PungiTestCase): + {'rpm': [], 'srpm': [], 'debuginfo': []} + ) + self.assertEqual(get_gather_method.call_args_list, +- [mock.call(compose.conf['gather_method'])] * 2) ++ [mock.call(compose.conf['gather_method'])] * 3) + self.assertEqual( + get_variant_packages.call_args_list, + [ ++ mock.call(compose, 'x86_64', compose.variants['Server'], 'module', pkg_set), + mock.call(compose, 'x86_64', compose.variants['Server'], 'comps', pkg_set), + mock.call(compose, 'x86_64', compose.variants['Server'], 'json', pkg_set), + ], +@@ -656,7 +657,7 @@ class TestGatherPackages(helpers.PungiTestCase): + get_gather_method.return_value.return_value.call_args_list, + [mock.call('x86_64', compose.variants['Server'], packages, groups, + filters, set(), set(), pkg_set, fulltree_excludes=set(), +- prepopulate=set())] * 2 ++ prepopulate=set())] * 3 + ) + + @mock.patch('pungi.phases.gather.get_variant_packages') +@@ -687,10 +688,11 @@ class TestGatherPackages(helpers.PungiTestCase): + {'rpm': [], 'srpm': [], 'debuginfo': []} + ) + self.assertEqual(get_gather_method.call_args_list, +- [mock.call(compose.conf['gather_method'])] * 2) ++ [mock.call(compose.conf['gather_method'])] * 3) + self.assertEqual( + get_variant_packages.call_args_list, + [ ++ mock.call(compose, 'x86_64', compose.variants['Server'], 'module', pkg_set), + mock.call(compose, 'x86_64', compose.variants['Server'], 'comps', pkg_set), + mock.call(compose, 'x86_64', compose.variants['Server'], 'json', pkg_set), + ], +@@ -699,7 +701,7 @@ class TestGatherPackages(helpers.PungiTestCase): + get_gather_method.return_value.return_value.call_args_list, + [mock.call('x86_64', compose.variants['Server'], packages, groups, + filters, set(['white']), set(['black']), pkg_set, +- fulltree_excludes=set(), prepopulate=set())] * 2 ++ fulltree_excludes=set(), prepopulate=set())] * 3 + ) + + @mock.patch('pungi.phases.gather.get_variant_packages') +@@ -710,12 +712,12 @@ class TestGatherPackages(helpers.PungiTestCase): + compose = helpers.DummyCompose(self.topdir, { + 'multilib_whitelist': {'*': ['white']}, + 'multilib_blacklist': {'*': ['black']}, +- 'gather_method': {'^Server$': {'comps': 'deps', 'json': 'deps'}}, ++ 'gather_method': {'^Server$': {'comps': 'deps', 'module': 'nodeps', 'json': 'deps'}}, + }) + pkg_set = mock.Mock() + gather.gather_packages(compose, 'x86_64', compose.variants['Server'], pkg_set), + self.assertEqual(get_gather_method.call_args_list, +- [mock.call('deps'), mock.call('deps')]) ++ [mock.call('nodeps'), mock.call('deps'), mock.call('deps')]) + + @mock.patch("pungi.phases.gather.get_variant_packages") + @mock.patch("pungi.phases.gather.get_gather_method") +diff --git a/tests/test_gather_source_module.py b/tests/test_gather_source_module.py +new file mode 100644 +index 0000000..400cc9c +--- /dev/null ++++ b/tests/test_gather_source_module.py +@@ -0,0 +1,79 @@ ++# -*- coding: utf-8 -*- ++ ++try: ++ import unittest2 as unittest ++except ImportError: ++ import unittest ++ ++import mock ++import os ++import sys ++ ++sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) ++ ++from pungi.phases.gather.sources.source_module import GatherSourceModule ++from tests import helpers ++from pungi import Modulemd ++ ++ ++@unittest.skipUnless(Modulemd is not None, "Skipped test, no module support.") ++class TestGatherSourceModule(helpers.PungiTestCase): ++ def setUp(self): ++ super(TestGatherSourceModule, self).setUp() ++ self.compose = helpers.DummyCompose(self.topdir, {}) ++ ++ def _add_pkg(self, arch): ++ mock_rpm = mock.Mock( ++ version="1.0.0", ++ release="1", ++ epoch=0, ++ excludearch=None, ++ exclusivearch=None, ++ sourcerpm="pkg-1.0.0-1", ++ nevra="pkg-0:1.0.0-1.%s" % arch, ++ arch=arch, ++ ) ++ mock_rpm.name = "pkg" ++ self.compose.variants["Server"].nsvc_to_pkgset[ ++ "testmodule:master:1:2017" ++ ].rpms_by_arch[arch] = [mock_rpm] ++ ++ def test_without_modules(self): ++ source = GatherSourceModule(self.compose) ++ packages, groups = source("x86_64", self.compose.variants["Server"]) ++ self.assertItemsEqual(packages, []) ++ self.assertItemsEqual(groups, []) ++ ++ def test_include_two_packages(self): ++ self.compose.variants["Server"].add_fake_module( ++ "testmodule:master:1:2017", ++ rpm_nvrs=["pkg-0:1.0.0-1.x86_64", "pkg-0:1.0.0-1.i686"], ++ with_artifacts=True, ++ mmd_arch="x86_64", ++ ) ++ ++ self._add_pkg("x86_64") ++ self._add_pkg("i686") ++ ++ source = GatherSourceModule(self.compose) ++ packages, groups = source("x86_64", self.compose.variants["Server"]) ++ self.assertItemsEqual( ++ [(rpm[0].nevra, rpm[1]) for rpm in packages], ++ [("pkg-0:1.0.0-1.x86_64", None), ("pkg-0:1.0.0-1.i686", None)], ++ ) ++ self.assertItemsEqual(groups, []) ++ ++ def test_does_not_include_unlisted(self): ++ self.compose.variants["Server"].add_fake_module( ++ "testmodule:master:1:2017", ++ rpm_nvrs=[], ++ with_artifacts=True, ++ mmd_arch="x86_64", ++ ) ++ ++ self._add_pkg("x86_64") ++ ++ source = GatherSourceModule(self.compose) ++ packages, groups = source("x86_64", self.compose.variants["Server"]) ++ self.assertItemsEqual(packages, []) ++ self.assertItemsEqual(groups, []) + diff --git a/pungi.spec b/pungi.spec index 5d2d8c12..90035c9b 100644 --- a/pungi.spec +++ b/pungi.spec @@ -2,7 +2,7 @@ Name: pungi Version: 4.1.36 -Release: 3%{?dist} +Release: 4%{?dist} Summary: Distribution compose tool License: GPLv2 @@ -10,7 +10,9 @@ URL: https://pagure.io/pungi Source0: https://pagure.io/releases/%{name}/%{name}-%{version}.tar.bz2 Patch0: https://pagure.io/pungi/pull-request/1183.patch Patch1: https://pagure.io/pungi/pull-request/1184.patch -Patch2: https://pagure.io/pungi/pull-request/1185.patch +Patch2: https://pagure.io/pungi/pull-request/1186.patch +Patch3: https://pagure.io/pungi/pull-request/1189.patch +Patch4: https://pagure.io/pungi/pull-request/1190.patch BuildRequires: python3-nose BuildRequires: python3-mock @@ -205,6 +207,9 @@ nosetests-3 --exe %{_bindir}/%{name}-wait-for-signed-ostree-handler %changelog +* Mon May 13 2019 Lubomír Sedlář - 4.1.36-4 +- Include more backported patches + * Tue May 07 2019 Lubomír Sedlář - 4.1.36-3 - Backport patch for decoding output as UTF-8