almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Backport more patches

Browse Source 
- Fix unicode handling in ostree tasks
- Introduce bach module gather source
This commit is contained in:
Lubomír Sedlář 2019-05-13 13:15:51 +02:00
parent b31609f76e
commit 7a76bd0b4c
5 changed files with 331 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

141
1185.patch
View File

@ -1,141 +0,0 @@
From c961fdc99e96a3c341273876b621d78733c70862 Mon Sep 17 00:00:00 2001
From: Lubomír Sedlář <lsedlar@redhat.com>
Date: May 07 2019 12:31:53 +0000
Subject: kojiwrapper: Decode output of runroot as UTF-8
Starting with Python 3.6, it is possible to specify encoding for output
of a `subprocess.Popen`. Try to use it to decode as UTF-8. Particularly
output of ostree runroot tasks is using unicode characters.
This should be safe: anything that is valid ASCII is also valid UTF-8.
Relates: https://pagure.io/dusty/failed-composes/issue/1642
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py
index 3cd9ac9..48534ff 100644
--- a/pungi/wrappers/kojiwrapper.py
+++ b/pungi/wrappers/kojiwrapper.py
@@ -19,6 +19,7 @@ import re
import time
import threading
import contextlib
+import sys
import koji
from kobo.shortcuts import run
@@ -146,8 +147,16 @@ class KojiWrapper(object):
"""
task_id = None
with self.get_koji_cmd_env() as env:
- retcode, output = run(command, can_fail=True, logfile=log_file,
- show_cmd=True, env=env, universal_newlines=True)
+ extra_kwarg = {"encoding": "utf-8"} if sys.version_info >= (3, 6) else {}
+ retcode, output = run(
+ command,
+ can_fail=True,
+ logfile=log_file,
+ show_cmd=True,
+ env=env,
+ universal_newlines=True,
+ **extra_kwarg
+ )
if "--task-id" in command:
first_line = output.splitlines()[0]
if re.match(r'^\d+$', first_line):
diff --git a/tests/test_koji_wrapper.py b/tests/test_koji_wrapper.py
index 34d8b3a..39d405d 100644
--- a/tests/test_koji_wrapper.py
+++ b/tests/test_koji_wrapper.py
@@ -405,6 +405,21 @@ class LiveImageKojiWrapperTest(KojiWrapperBaseTestCase):
class RunrootKojiWrapperTest(KojiWrapperBaseTestCase):
+
+ def kwargs(self, **kwargs):
+ defaults= {
+ "can_fail": True,
+ "env": None,
+ "logfile": None,
+ "show_cmd": True,
+ "universal_newlines": True,
+ }
+ if sys.version_info >= (3, 6):
+ defaults["encoding"] = "utf-8"
+
+ defaults.update(kwargs)
+ return defaults
+
def test_get_cmd_minimal(self):
cmd = self.koji.get_runroot_cmd('tgt', 's390x', 'date', use_shell=False, task_id=False)
self.assertEqual(len(cmd), 7)
@@ -457,11 +472,7 @@ class RunrootKojiWrapperTest(KojiWrapperBaseTestCase):
result = self.koji.run_runroot_cmd(cmd)
self.assertDictEqual(result, {'retcode': 0, 'output': output, 'task_id': None})
- self.assertEqual(
- run.call_args_list,
- [mock.call(cmd, can_fail=True, env=None, logfile=None, show_cmd=True,
- universal_newlines=True)]
- )
+ self.assertEqual(run.call_args_list, [mock.call(cmd, **self.kwargs())])
@mock.patch('pungi.wrappers.kojiwrapper.run')
def test_run_runroot_cmd_with_task_id(self, run):
@@ -471,11 +482,7 @@ class RunrootKojiWrapperTest(KojiWrapperBaseTestCase):
result = self.koji.run_runroot_cmd(cmd)
self.assertDictEqual(result, {'retcode': 0, 'output': output, 'task_id': 1234})
- self.assertEqual(
- run.call_args_list,
- [mock.call(cmd, can_fail=True, env=None, logfile=None, show_cmd=True,
- universal_newlines=True)]
- )
+ self.assertEqual(run.call_args_list, [mock.call(cmd, **self.kwargs())])
@mock.patch('pungi.wrappers.kojiwrapper.run')
def test_run_runroot_cmd_with_task_id_and_fail(self, run):
@@ -485,11 +492,7 @@ class RunrootKojiWrapperTest(KojiWrapperBaseTestCase):
result = self.koji.run_runroot_cmd(cmd)
self.assertDictEqual(result, {'retcode': 1, 'output': output, 'task_id': None})
- self.assertEqual(
- run.call_args_list,
- [mock.call(cmd, can_fail=True, env=None, logfile=None, show_cmd=True,
- universal_newlines=True)]
- )
+ self.assertEqual(run.call_args_list, [mock.call(cmd, **self.kwargs())])
@mock.patch('pungi.wrappers.kojiwrapper.run')
def test_run_runroot_cmd_with_task_id_and_fail_but_emit_id(self, run):
@@ -499,11 +502,7 @@ class RunrootKojiWrapperTest(KojiWrapperBaseTestCase):
result = self.koji.run_runroot_cmd(cmd)
self.assertDictEqual(result, {'retcode': 1, 'output': output, 'task_id': 12345})
- self.assertEqual(
- run.call_args_list,
- [mock.call(cmd, can_fail=True, env=None, logfile=None, show_cmd=True,
- universal_newlines=True)]
- )
+ self.assertEqual(run.call_args_list, [mock.call(cmd, **self.kwargs())])
@mock.patch.dict('os.environ', {'FOO': 'BAR'}, clear=True)
@mock.patch('shutil.rmtree')
@@ -521,8 +520,11 @@ class RunrootKojiWrapperTest(KojiWrapperBaseTestCase):
self.assertDictEqual(result, {'retcode': 0, 'output': output, 'task_id': None})
self.assertEqual(
run.call_args_list,
- [mock.call(cmd, can_fail=True, env={'KRB5CCNAME': 'DIR:/tmp/foo', 'FOO': 'BAR'},
- logfile=None, show_cmd=True, universal_newlines=True)]
+ [
+ mock.call(
+ cmd, **self.kwargs(env={'KRB5CCNAME': 'DIR:/tmp/foo', 'FOO': 'BAR'})
+ ),
+ ],
)

25
1186.patch Normal file
View File

@ -0,0 +1,25 @@
From c1a03c259bca87dad12477692f1cfce15c770d92 Mon Sep 17 00:00:00 2001
From: Lubomír Sedlář <lsedlar@redhat.com>
Date: May 09 2019 06:01:13 +0000
Subject: Switch locale to C.UTF-8
Relates: https://pagure.io/dusty/failed-composes/issue/1642
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
diff --git a/bin/pungi-koji b/bin/pungi-koji
index e48686e..049eeaa 100755
--- a/bin/pungi-koji
+++ b/bin/pungi-koji
@@ -27,7 +27,7 @@ from pungi import get_full_version, util
# force C locales
-locale.setlocale(locale.LC_ALL, "C")
+locale.setlocale(locale.LC_ALL, "C.UTF-8")
COMPOSE = None

30
1189.patch Normal file
View File

@ -0,0 +1,30 @@
From dc69281025cbc95ef85f378c839030ca4d8235b4 Mon Sep 17 00:00:00 2001
From: Lubomír Sedlář <lsedlar@redhat.com>
Date: May 13 2019 08:25:00 +0000
Subject: Fall back to C locale if UTF8 version does not exist
This can happen on RHEL 6 or 7. All current Fedora version have it.
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
diff --git a/bin/pungi-koji b/bin/pungi-koji
index 049eeaa..38e2fcf 100755
--- a/bin/pungi-koji
+++ b/bin/pungi-koji
@@ -27,7 +27,11 @@ from pungi import get_full_version, util
# force C locales
-locale.setlocale(locale.LC_ALL, "C.UTF-8")
+try:
+ locale.setlocale(locale.LC_ALL, "C.UTF-8")
+except locale.Error:
+ # RHEL < 8 does not have C.UTF-8 locale...
+ locale.setlocale(locale.LC_ALL, "C")
COMPOSE = None

269
1190.patch Normal file
View File

@ -0,0 +1,269 @@
From a451996476f7dd3f244e3705aed87c89af63b9f8 Mon Sep 17 00:00:00 2001
From: Lubomír Sedlář <lsedlar@redhat.com>
Date: May 13 2019 11:12:54 +0000
Subject: gather: Introduce module source again
This reverts commit ac15f211351b96a41b4e715c5599d928df394679.
It is still needed if nodeps gather method is used. It simply returns
all packages listed in all modules.
Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1708661
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py
index 58f5919..8a20ceb 100644
--- a/pungi/phases/gather/__init__.py
+++ b/pungi/phases/gather/__init__.py
@@ -202,7 +202,7 @@ def gather_packages(compose, arch, variant, package_sets, fulltree_excludes=None
else:
- for source_name in ('comps', 'json'):
+ for source_name in ("module", "comps", "json"):
packages, groups, filter_packages = get_variant_packages(compose, arch, variant,
source_name, package_sets)
@@ -724,7 +724,7 @@ def get_packages_to_gather(compose, arch=None, variant=None, include_arch=True,
"""
packages = set([])
groups = set([])
- for source_name in ('comps', 'json'):
+ for source_name in ("module", "comps", "json"):
GatherSource = get_gather_source(source_name)
src = GatherSource(compose)
diff --git a/pungi/phases/gather/sources/source_module.py b/pungi/phases/gather/sources/source_module.py
new file mode 100644
index 0000000..2dc818e
--- /dev/null
+++ b/pungi/phases/gather/sources/source_module.py
@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+"""
+Get a package list based on modulemd metadata loaded in pkgset phase. Each
+modulemd file contains a list of exact RPM NEVRAs that should be include, so
+just go over all modules in a given variant and join all lists together.
+"""
+
+
+import pungi.arch
+import pungi.phases.gather.source
+
+
+class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase):
+ enabled = True
+
+ def __call__(self, arch, variant):
+ groups = set()
+ packages = set()
+
+ # Check if this variant contains some modules
+ if variant is None or variant.modules is None:
+ return packages, groups
+
+ compatible_arches = pungi.arch.get_compatible_arches(arch, multilib=True)
+
+ for nsvc, mmd in variant.arch_mmds[arch].items():
+ available_rpms = sum(
+ (
+ variant.nsvc_to_pkgset[nsvc].rpms_by_arch.get(a, [])
+ for a in compatible_arches
+ ),
+ [],
+ )
+ to_include = set(mmd.get_rpm_artifacts().get())
+ for rpm_obj in available_rpms:
+ if rpm_obj.nevra in to_include:
+ packages.add((rpm_obj, None))
+
+ return packages, groups
diff --git a/tests/helpers.py b/tests/helpers.py
index 7287f04..4c2e24f 100644
--- a/tests/helpers.py
+++ b/tests/helpers.py
@@ -83,7 +83,7 @@ class MockVariant(mock.Mock):
def get_modular_koji_tags(self, arch=None, types=None):
return []
- def add_fake_module(self, nsvc, rpm_nvrs=None, with_artifacts=False):
+ def add_fake_module(self, nsvc, rpm_nvrs=None, with_artifacts=False, mmd_arch=None):
if not Modulemd:
# No support for modules
return
@@ -116,6 +116,8 @@ class MockVariant(mock.Mock):
self.modules = []
self.modules.append(":".join([name, stream, version]))
self.mmds.append(mmd)
+ if mmd_arch:
+ self.arch_mmds.setdefault(mmd_arch, {})[mmd.dup_nsvc()] = mmd
return mmd
diff --git a/tests/test_gather_phase.py b/tests/test_gather_phase.py
index 122e30c..c71599a 100644
--- a/tests/test_gather_phase.py
+++ b/tests/test_gather_phase.py
@@ -644,10 +644,11 @@ class TestGatherPackages(helpers.PungiTestCase):
{'rpm': [], 'srpm': [], 'debuginfo': []}
)
self.assertEqual(get_gather_method.call_args_list,
- [mock.call(compose.conf['gather_method'])] * 2)
+ [mock.call(compose.conf['gather_method'])] * 3)
self.assertEqual(
get_variant_packages.call_args_list,
[
+ mock.call(compose, 'x86_64', compose.variants['Server'], 'module', pkg_set),
mock.call(compose, 'x86_64', compose.variants['Server'], 'comps', pkg_set),
mock.call(compose, 'x86_64', compose.variants['Server'], 'json', pkg_set),
],
@@ -656,7 +657,7 @@ class TestGatherPackages(helpers.PungiTestCase):
get_gather_method.return_value.return_value.call_args_list,
[mock.call('x86_64', compose.variants['Server'], packages, groups,
filters, set(), set(), pkg_set, fulltree_excludes=set(),
- prepopulate=set())] * 2
+ prepopulate=set())] * 3
)
@mock.patch('pungi.phases.gather.get_variant_packages')
@@ -687,10 +688,11 @@ class TestGatherPackages(helpers.PungiTestCase):
{'rpm': [], 'srpm': [], 'debuginfo': []}
)
self.assertEqual(get_gather_method.call_args_list,
- [mock.call(compose.conf['gather_method'])] * 2)
+ [mock.call(compose.conf['gather_method'])] * 3)
self.assertEqual(
get_variant_packages.call_args_list,
[
+ mock.call(compose, 'x86_64', compose.variants['Server'], 'module', pkg_set),
mock.call(compose, 'x86_64', compose.variants['Server'], 'comps', pkg_set),
mock.call(compose, 'x86_64', compose.variants['Server'], 'json', pkg_set),
],
@@ -699,7 +701,7 @@ class TestGatherPackages(helpers.PungiTestCase):
get_gather_method.return_value.return_value.call_args_list,
[mock.call('x86_64', compose.variants['Server'], packages, groups,
filters, set(['white']), set(['black']), pkg_set,
- fulltree_excludes=set(), prepopulate=set())] * 2
+ fulltree_excludes=set(), prepopulate=set())] * 3
)
@mock.patch('pungi.phases.gather.get_variant_packages')
@@ -710,12 +712,12 @@ class TestGatherPackages(helpers.PungiTestCase):
compose = helpers.DummyCompose(self.topdir, {
'multilib_whitelist': {'*': ['white']},
'multilib_blacklist': {'*': ['black']},
- 'gather_method': {'^Server$': {'comps': 'deps', 'json': 'deps'}},
+ 'gather_method': {'^Server$': {'comps': 'deps', 'module': 'nodeps', 'json': 'deps'}},
})
pkg_set = mock.Mock()
gather.gather_packages(compose, 'x86_64', compose.variants['Server'], pkg_set),
self.assertEqual(get_gather_method.call_args_list,
- [mock.call('deps'), mock.call('deps')])
+ [mock.call('nodeps'), mock.call('deps'), mock.call('deps')])
@mock.patch("pungi.phases.gather.get_variant_packages")
@mock.patch("pungi.phases.gather.get_gather_method")
diff --git a/tests/test_gather_source_module.py b/tests/test_gather_source_module.py
new file mode 100644
index 0000000..400cc9c
--- /dev/null
+++ b/tests/test_gather_source_module.py
@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+
+try:
+ import unittest2 as unittest
+except ImportError:
+ import unittest
+
+import mock
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from pungi.phases.gather.sources.source_module import GatherSourceModule
+from tests import helpers
+from pungi import Modulemd
+
+
+@unittest.skipUnless(Modulemd is not None, "Skipped test, no module support.")
+class TestGatherSourceModule(helpers.PungiTestCase):
+ def setUp(self):
+ super(TestGatherSourceModule, self).setUp()
+ self.compose = helpers.DummyCompose(self.topdir, {})
+
+ def _add_pkg(self, arch):
+ mock_rpm = mock.Mock(
+ version="1.0.0",
+ release="1",
+ epoch=0,
+ excludearch=None,
+ exclusivearch=None,
+ sourcerpm="pkg-1.0.0-1",
+ nevra="pkg-0:1.0.0-1.%s" % arch,
+ arch=arch,
+ )
+ mock_rpm.name = "pkg"
+ self.compose.variants["Server"].nsvc_to_pkgset[
+ "testmodule:master:1:2017"
+ ].rpms_by_arch[arch] = [mock_rpm]
+
+ def test_without_modules(self):
+ source = GatherSourceModule(self.compose)
+ packages, groups = source("x86_64", self.compose.variants["Server"])
+ self.assertItemsEqual(packages, [])
+ self.assertItemsEqual(groups, [])
+
+ def test_include_two_packages(self):
+ self.compose.variants["Server"].add_fake_module(
+ "testmodule:master:1:2017",
+ rpm_nvrs=["pkg-0:1.0.0-1.x86_64", "pkg-0:1.0.0-1.i686"],
+ with_artifacts=True,
+ mmd_arch="x86_64",
+ )
+
+ self._add_pkg("x86_64")
+ self._add_pkg("i686")
+
+ source = GatherSourceModule(self.compose)
+ packages, groups = source("x86_64", self.compose.variants["Server"])
+ self.assertItemsEqual(
+ [(rpm[0].nevra, rpm[1]) for rpm in packages],
+ [("pkg-0:1.0.0-1.x86_64", None), ("pkg-0:1.0.0-1.i686", None)],
+ )
+ self.assertItemsEqual(groups, [])
+
+ def test_does_not_include_unlisted(self):
+ self.compose.variants["Server"].add_fake_module(
+ "testmodule:master:1:2017",
+ rpm_nvrs=[],
+ with_artifacts=True,
+ mmd_arch="x86_64",
+ )
+
+ self._add_pkg("x86_64")
+
+ source = GatherSourceModule(self.compose)
+ packages, groups = source("x86_64", self.compose.variants["Server"])
+ self.assertItemsEqual(packages, [])
+ self.assertItemsEqual(groups, [])

9
pungi.spec
View File

@ -2,7 +2,7 @@
Name: pungi
Version: 4.1.36
Release: 3%{?dist}
Release: 4%{?dist}
Summary: Distribution compose tool
License: GPLv2
@ -10,7 +10,9 @@ URL: https://pagure.io/pungi
Source0: https://pagure.io/releases/%{name}/%{name}-%{version}.tar.bz2
Patch0: https://pagure.io/pungi/pull-request/1183.patch
Patch1: https://pagure.io/pungi/pull-request/1184.patch
Patch2: https://pagure.io/pungi/pull-request/1185.patch
Patch2: https://pagure.io/pungi/pull-request/1186.patch
Patch3: https://pagure.io/pungi/pull-request/1189.patch
Patch4: https://pagure.io/pungi/pull-request/1190.patch
BuildRequires: python3-nose
BuildRequires: python3-mock
@ -205,6 +207,9 @@ nosetests-3 --exe
%{_bindir}/%{name}-wait-for-signed-ostree-handler
%changelog
* Mon May 13 2019 Lubomír Sedlář <lsedlar@redhat.com> - 4.1.36-4
- Include more backported patches
* Tue May 07 2019 Lubomír Sedlář <lsedlar@redhat.com> - 4.1.36-3
- Backport patch for decoding output as UTF-8