From fc89aac70ce0e1047d5c1c8ef607bd985c86f395 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 20 Jul 2018 09:38:44 +0200 Subject: [PATCH] Backport patches from upstream --- 0001-Fix-tests-for-DNF-3.patch | 62 -------------- ...ee-phase-and-pipelines-for-running-p.patch | 2 +- ...evert-Other-repo-for-OstreeInstaller.patch | 2 +- 0003-Revert-Ostree-can-use-pkgset-repos.patch | 2 +- ...arch-for-modules-with-dash-in-stream.patch | 84 +++++++++++++++++++ 0005-Fix-tests-for-DNF-3.patch | 80 ++++++++++++++++++ pungi.spec | 9 +- 7 files changed, 174 insertions(+), 67 deletions(-) delete mode 100644 0001-Fix-tests-for-DNF-3.patch create mode 100644 0004-Fix-Koji-search-for-modules-with-dash-in-stream.patch create mode 100644 0005-Fix-tests-for-DNF-3.patch diff --git a/0001-Fix-tests-for-DNF-3.patch b/0001-Fix-tests-for-DNF-3.patch deleted file mode 100644 index 4e32974a..00000000 --- a/0001-Fix-tests-for-DNF-3.patch +++ /dev/null @@ -1,62 +0,0 @@ -From a6c0cf38c05e67e616a424dad16194d5de36134a Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= -Date: Wed, 4 Jul 2018 09:16:51 +0200 -Subject: [PATCH] Fix tests for DNF 3 -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -The configuration for modules has changed. - -Lookaside handling has changed, and there are now test failures. This -could be a bug in DNF, or maybe Pungi is not setting things up -correctly. As a temporary workaround we can skip the tests. - -Signed-off-by: Lubomír Sedlář ---- - tests/test_gather.py | 11 ++++++++++- - 1 file changed, 10 insertions(+), 1 deletion(-) - -diff --git a/tests/test_gather.py b/tests/test_gather.py -index 25dfc399..8ccbf2f8 100644 ---- a/tests/test_gather.py -+++ b/tests/test_gather.py -@@ -22,6 +22,7 @@ os.environ['PATH'] = '%s:%s' % (BINDIR, os.environ['PATH']) - - from pungi.wrappers.pungi import PungiWrapper - try: -+ from dnf import __version__ as dnf_version - from pungi.dnf_wrapper import DnfWrapper, Conf - from pungi.gather_dnf import Gather, GatherOptions, PkgFlag - HAS_DNF = True -@@ -1810,7 +1811,10 @@ class DNFDepsolvingTestCase(DepsolvingBase, unittest.TestCase): - conf = Conf(base_arch) - conf.persistdir = persistdir - conf.cachedir = self.cachedir -- conf.modulesdir = os.path.join(persistdir, 'modules.d') -+ if int(dnf_version.split('.')[0]) < 3: -+ conf.modulesdir = os.path.join(persistdir, 'modules.d') -+ else: -+ conf.modulesdir._set(os.path.join(persistdir, 'modules.d')) - if exclude: - conf.exclude = exclude - dnf = DnfWrapper(conf) -@@ -1888,6 +1892,7 @@ class DNFDepsolvingTestCase(DepsolvingBase, unittest.TestCase): - "dummy-glibc-debuginfo-common-2.14-5.x86_64.rpm", - ]) - -+ @unittest.skipUnless(int(dnf_version.split('.')[0]) < 3, 'Lookaside is not handled correctly') - def test_firefox_selfhosting_with_krb5_lookaside(self): - super(DNFDepsolvingTestCase, self).test_firefox_selfhosting_with_krb5_lookaside() - self.assertFlags("dummy-krb5-devel-1.10-5.x86_64", [PkgFlag.lookaside]) -@@ -1910,3 +1915,7 @@ class DNFDepsolvingTestCase(DepsolvingBase, unittest.TestCase): - ]) - self.assertItemsEqual(pkg_map["debuginfo"], [ - ]) -+ -+ @unittest.skipUnless(int(dnf_version.split('.')[0]) < 3, 'Lookaside is not handled correctly') -+ def test_lookaside_empty(self): -+ super(DNFDepsolvingTestCase, self).test_lookaside_empty() --- -2.14.4 - diff --git a/0001-Revert-Move-ostree-phase-and-pipelines-for-running-p.patch b/0001-Revert-Move-ostree-phase-and-pipelines-for-running-p.patch index 2679d302..25c3b67b 100644 --- a/0001-Revert-Move-ostree-phase-and-pipelines-for-running-p.patch +++ b/0001-Revert-Move-ostree-phase-and-pipelines-for-running-p.patch @@ -1,7 +1,7 @@ From 9dc20c455df71f95a7f131ac10b7013042c5f71d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 11 Apr 2018 09:18:59 +0200 -Subject: [PATCH 1/3] Revert "Move ostree phase and pipelines for running +Subject: [PATCH 1/5] Revert "Move ostree phase and pipelines for running phases" This reverts commit 660c04368ba1abed310f121d01f0fa029eea5f11. diff --git a/0002-Revert-Other-repo-for-OstreeInstaller.patch b/0002-Revert-Other-repo-for-OstreeInstaller.patch index a78b4a48..ac7287a4 100644 --- a/0002-Revert-Other-repo-for-OstreeInstaller.patch +++ b/0002-Revert-Other-repo-for-OstreeInstaller.patch @@ -1,7 +1,7 @@ From 883a69645b92b243787312af232aeb804cf53292 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 11 Apr 2018 09:19:53 +0200 -Subject: [PATCH 2/3] Revert "Other repo for OstreeInstaller" +Subject: [PATCH 2/5] Revert "Other repo for OstreeInstaller" This reverts commit 5c081cb545715c2a912ff50fa57554e89d905868. --- diff --git a/0003-Revert-Ostree-can-use-pkgset-repos.patch b/0003-Revert-Ostree-can-use-pkgset-repos.patch index 27750d4c..6be94f00 100644 --- a/0003-Revert-Ostree-can-use-pkgset-repos.patch +++ b/0003-Revert-Ostree-can-use-pkgset-repos.patch @@ -1,7 +1,7 @@ From 079177c5bf498fe21c1f0b03d127bfa25fa09fe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 11 Apr 2018 09:20:51 +0200 -Subject: [PATCH 3/3] Revert "Ostree can use pkgset repos" +Subject: [PATCH 3/5] Revert "Ostree can use pkgset repos" This reverts commit c7cc200246300c6a3946b2e3a9f5f7693896a7d6. --- diff --git a/0004-Fix-Koji-search-for-modules-with-dash-in-stream.patch b/0004-Fix-Koji-search-for-modules-with-dash-in-stream.patch new file mode 100644 index 00000000..e5e06cd0 --- /dev/null +++ b/0004-Fix-Koji-search-for-modules-with-dash-in-stream.patch @@ -0,0 +1,84 @@ +From 97787114f6b081c1761560663dd8b3ad80305def Mon Sep 17 00:00:00 2001 +From: Jan Kaluza +Date: Wed, 18 Jul 2018 10:22:33 +0200 +Subject: [PATCH 4/5] Fix Koji search for modules with dash in stream + +MBS replaces the dash by underscore. We have to match that. + +Merges: https://pagure.io/pungi/pull-request/1005 +Signed-off-by: Jan Kaluza +--- + pungi/phases/pkgset/sources/source_koji.py | 12 +++++++++--- + tests/test_pkgset_source_koji.py | 10 +++++----- + 2 files changed, 14 insertions(+), 8 deletions(-) + +diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py +index 3381d792..8c76e1ba 100644 +--- a/pungi/phases/pkgset/sources/source_koji.py ++++ b/pungi/phases/pkgset/sources/source_koji.py +@@ -114,9 +114,15 @@ def get_koji_modules(compose, koji_wrapper, module_info_str): + + module_info = variant_dict_from_str(compose, module_info_str) + +- # we need to format the query string to koji reguirements +- query_str = "%s-%s-%s.%s" % (module_info["name"], module_info["stream"], +- module_info.get("version", "*"), module_info.get("context", "*")) ++ # We need to format the query string to koji reguirements. The ++ # transformation to NVR for use in Koji has to match what MBS is doing when ++ # importing the build. ++ query_str = "%s-%s-%s.%s" % ( ++ module_info["name"], ++ module_info["stream"].replace("-", "_"), ++ module_info.get("version", "*"), ++ module_info.get("context", "*"), ++ ) + query_str = query_str.replace('*.*', '*') + + koji_builds = koji_proxy.search(query_str, "build", "glob") +diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py +index b6a85f93..a24d2e27 100644 +--- a/tests/test_pkgset_source_koji.py ++++ b/tests/test_pkgset_source_koji.py +@@ -341,7 +341,7 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase): + self.assertEqual(pkgsets, expected) + + def test_get_koji_modules(self): +- mock_build_ids = [{'id': 1065873, 'name': 'testmodule2-master-20180406051653.96c371af'}] ++ mock_build_ids = [{'id': 1065873, 'name': 'testmodule2-master_dash-20180406051653.96c371af'}] + mock_extra = { + 'typeinfo': { + 'module': { +@@ -355,10 +355,10 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase): + 'epoch': None, + 'extra': mock_extra, + 'name': 'testmodule2', +- 'nvr': 'testmodule2-master-20180406051653.2e6f5e0a', ++ 'nvr': 'testmodule2-master_dash-20180406051653.2e6f5e0a', + 'release': '20180406051653.2e6f5e0a', + 'state': 1, +- 'version': 'master', ++ 'version': 'master_dash', + } + ] + mock_archives = [ +@@ -392,7 +392,7 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase): + self.koji_wrapper.koji_proxy.listArchives.return_value = mock_archives + self.koji_wrapper.koji_proxy.listRPMs.return_value = mock_rpms + +- module_info_str = "testmodule2:master:20180406051653:96c371af" ++ module_info_str = "testmodule2:master-dash:20180406051653:96c371af" + result = source_koji.get_koji_modules(self.compose, self.koji_wrapper, module_info_str) + + assert type(result) is list +@@ -405,7 +405,7 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase): + assert "stream" in module + assert "context" in module + +- expected_query = "testmodule2-master-20180406051653.96c371af" ++ expected_query = "testmodule2-master_dash-20180406051653.96c371af" + self.koji_wrapper.koji_proxy.search.assert_called_once_with(expected_query, "build", + "glob") + self.koji_wrapper.koji_proxy.getBuild.assert_called_once_with(mock_build_ids[0]["id"]) +-- +2.14.4 + diff --git a/0005-Fix-tests-for-DNF-3.patch b/0005-Fix-tests-for-DNF-3.patch new file mode 100644 index 00000000..b047dd78 --- /dev/null +++ b/0005-Fix-tests-for-DNF-3.patch @@ -0,0 +1,80 @@ +From 346bf883593e3d4a3531c1dce15abbc696130c6e Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= +Date: Wed, 4 Jul 2018 09:16:51 +0200 +Subject: [PATCH 5/5] Fix tests for DNF 3 +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +The configuration for modules has changed. + +Lookaside handling has changed, and there are now test failures. This +is a not a bug in DNF, so we need to fix it on our side. + +Relates: https://bugzilla.redhat.com/show_bug.cgi?id=1603123 +Signed-off-by: Lubomír Sedlář +--- + pungi/gather_dnf.py | 19 ++++++++++++++++++- + tests/test_gather.py | 6 +++++- + 2 files changed, 23 insertions(+), 2 deletions(-) + +diff --git a/pungi/gather_dnf.py b/pungi/gather_dnf.py +index 328d69c8..1d6c510a 100644 +--- a/pungi/gather_dnf.py ++++ b/pungi/gather_dnf.py +@@ -227,10 +227,27 @@ class Gather(GatherBase): + if not package_list: + return [] + ++ # The list can contain packages from lookaside and outside of ++ # lookaside. If the package is in both, we want to prefer the version ++ # from lookaside. This can be achieved by removing any package that is ++ # also in lookaside from the list. ++ lookaside_pkgs = set() ++ for pkg in package_list: ++ if pkg.repoid in self.opts.lookaside_repos: ++ lookaside_pkgs.add("{0.name}-{0.evr}".format(pkg)) ++ + if self.opts.greedy_method == "all": + return list(package_list) + +- all_pkgs = list(package_list) ++ all_pkgs = [] ++ for pkg in package_list: ++ # Remove packages that are also in lookaside ++ if ( ++ "{0.name}-{0.evr}".format(pkg) not in lookaside_pkgs ++ or pkg.repoid in self.opts.lookaside_repos ++ ): ++ all_pkgs.append(pkg) ++ + native_pkgs = self.q_native_binary_packages.filter(pkg=all_pkgs).apply() + multilib_pkgs = self.q_multilib_binary_packages.filter(pkg=all_pkgs).apply() + +diff --git a/tests/test_gather.py b/tests/test_gather.py +index 25dfc399..60085b89 100644 +--- a/tests/test_gather.py ++++ b/tests/test_gather.py +@@ -22,6 +22,7 @@ os.environ['PATH'] = '%s:%s' % (BINDIR, os.environ['PATH']) + + from pungi.wrappers.pungi import PungiWrapper + try: ++ from dnf import __version__ as dnf_version + from pungi.dnf_wrapper import DnfWrapper, Conf + from pungi.gather_dnf import Gather, GatherOptions, PkgFlag + HAS_DNF = True +@@ -1810,7 +1811,10 @@ class DNFDepsolvingTestCase(DepsolvingBase, unittest.TestCase): + conf = Conf(base_arch) + conf.persistdir = persistdir + conf.cachedir = self.cachedir +- conf.modulesdir = os.path.join(persistdir, 'modules.d') ++ if int(dnf_version.split('.')[0]) < 3: ++ conf.modulesdir = os.path.join(persistdir, 'modules.d') ++ else: ++ conf.modulesdir._set(os.path.join(persistdir, 'modules.d')) + if exclude: + conf.exclude = exclude + dnf = DnfWrapper(conf) +-- +2.14.4 + diff --git a/pungi.spec b/pungi.spec index 80fc5aa5..0b14250d 100644 --- a/pungi.spec +++ b/pungi.spec @@ -1,6 +1,6 @@ Name: pungi Version: 4.1.26 -Release: 1%{?dist} +Release: 2%{?dist} Summary: Distribution compose tool Group: Development/Tools @@ -10,7 +10,8 @@ Source0: https://pagure.io/releases/%{name}/%{name}-%{version}.tar.bz2 Patch0: 0001-Revert-Move-ostree-phase-and-pipelines-for-running-p.patch Patch1: 0002-Revert-Other-repo-for-OstreeInstaller.patch Patch2: 0003-Revert-Ostree-can-use-pkgset-repos.patch -Patch3: 0001-Fix-tests-for-DNF-3.patch +Patch3: 0004-Fix-Koji-search-for-modules-with-dash-in-stream.patch +Patch4: 0005-Fix-tests-for-DNF-3.patch BuildRequires: python3-nose BuildRequires: python3-mock @@ -180,6 +181,10 @@ nosetests-3 --exe %{_bindir}/%{name}-wait-for-signed-ostree-handler %changelog +* Fri Jul 20 2018 Lubomír Sedlář - 4.1.26-2 +- Backport patch for DNF 3 compatibility +- Fix querying Koji about modules with dash in stream + * Mon Jul 16 2018 Lubomír Sedlář - 4.1.26-1 - gather: Add a hybrid depsolver backend (lsedlar) - Always use lookasides for repoclosure (lsedlar)