diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 3a6a3992..67c8ad01 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -205,7 +205,7 @@ def gather_packages(compose, arch, variant, package_sets, fulltree_excludes=None else: - for source_name in ('comps', 'json'): + for source_name in ("module", "comps", "json"): packages, groups, filter_packages = get_variant_packages(compose, arch, variant, source_name, package_sets) @@ -727,7 +727,7 @@ def get_packages_to_gather(compose, arch=None, variant=None, include_arch=True, """ packages = set([]) groups = set([]) - for source_name in ('comps', 'json'): + for source_name in ("module", "comps", "json"): GatherSource = get_gather_source(source_name) src = GatherSource(compose) diff --git a/pungi/phases/gather/sources/source_module.py b/pungi/phases/gather/sources/source_module.py new file mode 100644 index 00000000..2dc818e8 --- /dev/null +++ b/pungi/phases/gather/sources/source_module.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + + +""" +Get a package list based on modulemd metadata loaded in pkgset phase. Each +modulemd file contains a list of exact RPM NEVRAs that should be include, so +just go over all modules in a given variant and join all lists together. +""" + + +import pungi.arch +import pungi.phases.gather.source + + +class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase): + enabled = True + + def __call__(self, arch, variant): + groups = set() + packages = set() + + # Check if this variant contains some modules + if variant is None or variant.modules is None: + return packages, groups + + compatible_arches = pungi.arch.get_compatible_arches(arch, multilib=True) + + for nsvc, mmd in variant.arch_mmds[arch].items(): + available_rpms = sum( + ( + variant.nsvc_to_pkgset[nsvc].rpms_by_arch.get(a, []) + for a in compatible_arches + ), + [], + ) + to_include = set(mmd.get_rpm_artifacts().get()) + for rpm_obj in available_rpms: + if rpm_obj.nevra in to_include: + packages.add((rpm_obj, None)) + + return packages, groups diff --git a/tests/helpers.py b/tests/helpers.py index 7287f047..4c2e24f1 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -83,7 +83,7 @@ class MockVariant(mock.Mock): def get_modular_koji_tags(self, arch=None, types=None): return [] - def add_fake_module(self, nsvc, rpm_nvrs=None, with_artifacts=False): + def add_fake_module(self, nsvc, rpm_nvrs=None, with_artifacts=False, mmd_arch=None): if not Modulemd: # No support for modules return @@ -116,6 +116,8 @@ class MockVariant(mock.Mock): self.modules = [] self.modules.append(":".join([name, stream, version])) self.mmds.append(mmd) + if mmd_arch: + self.arch_mmds.setdefault(mmd_arch, {})[mmd.dup_nsvc()] = mmd return mmd diff --git a/tests/test_gather_phase.py b/tests/test_gather_phase.py index e7c3c0b0..84ff6f21 100644 --- a/tests/test_gather_phase.py +++ b/tests/test_gather_phase.py @@ -644,10 +644,11 @@ class TestGatherPackages(helpers.PungiTestCase): {'rpm': [], 'srpm': [], 'debuginfo': []} ) self.assertEqual(get_gather_method.call_args_list, - [mock.call(compose.conf['gather_method'])] * 2) + [mock.call(compose.conf['gather_method'])] * 3) self.assertEqual( get_variant_packages.call_args_list, [ + mock.call(compose, 'x86_64', compose.variants['Server'], 'module', pkg_set), mock.call(compose, 'x86_64', compose.variants['Server'], 'comps', pkg_set), mock.call(compose, 'x86_64', compose.variants['Server'], 'json', pkg_set), ], @@ -656,7 +657,7 @@ class TestGatherPackages(helpers.PungiTestCase): get_gather_method.return_value.return_value.call_args_list, [mock.call('x86_64', compose.variants['Server'], packages, groups, filters, set(), set(), pkg_set, fulltree_excludes=set(), - prepopulate=set())] * 2 + prepopulate=set())] * 3 ) @mock.patch('pungi.phases.gather.get_variant_packages') @@ -687,10 +688,11 @@ class TestGatherPackages(helpers.PungiTestCase): {'rpm': [], 'srpm': [], 'debuginfo': []} ) self.assertEqual(get_gather_method.call_args_list, - [mock.call(compose.conf['gather_method'])] * 2) + [mock.call(compose.conf['gather_method'])] * 3) self.assertEqual( get_variant_packages.call_args_list, [ + mock.call(compose, 'x86_64', compose.variants['Server'], 'module', pkg_set), mock.call(compose, 'x86_64', compose.variants['Server'], 'comps', pkg_set), mock.call(compose, 'x86_64', compose.variants['Server'], 'json', pkg_set), ], @@ -699,7 +701,7 @@ class TestGatherPackages(helpers.PungiTestCase): get_gather_method.return_value.return_value.call_args_list, [mock.call('x86_64', compose.variants['Server'], packages, groups, filters, set(['white']), set(['black']), pkg_set, - fulltree_excludes=set(), prepopulate=set())] * 2 + fulltree_excludes=set(), prepopulate=set())] * 3 ) @mock.patch('pungi.phases.gather.get_variant_packages') @@ -710,12 +712,12 @@ class TestGatherPackages(helpers.PungiTestCase): compose = helpers.DummyCompose(self.topdir, { 'multilib_whitelist': {'*': ['white']}, 'multilib_blacklist': {'*': ['black']}, - 'gather_method': {'^Server$': {'comps': 'deps', 'json': 'deps'}}, + 'gather_method': {'^Server$': {'comps': 'deps', 'module': 'nodeps', 'json': 'deps'}}, }) pkg_set = mock.Mock() gather.gather_packages(compose, 'x86_64', compose.variants['Server'], pkg_set), self.assertEqual(get_gather_method.call_args_list, - [mock.call('deps'), mock.call('deps')]) + [mock.call('nodeps'), mock.call('deps'), mock.call('deps')]) @mock.patch("pungi.phases.gather.get_variant_packages") @mock.patch("pungi.phases.gather.get_gather_method") diff --git a/tests/test_gather_source_module.py b/tests/test_gather_source_module.py new file mode 100644 index 00000000..400cc9c5 --- /dev/null +++ b/tests/test_gather_source_module.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- + +try: + import unittest2 as unittest +except ImportError: + import unittest + +import mock +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from pungi.phases.gather.sources.source_module import GatherSourceModule +from tests import helpers +from pungi import Modulemd + + +@unittest.skipUnless(Modulemd is not None, "Skipped test, no module support.") +class TestGatherSourceModule(helpers.PungiTestCase): + def setUp(self): + super(TestGatherSourceModule, self).setUp() + self.compose = helpers.DummyCompose(self.topdir, {}) + + def _add_pkg(self, arch): + mock_rpm = mock.Mock( + version="1.0.0", + release="1", + epoch=0, + excludearch=None, + exclusivearch=None, + sourcerpm="pkg-1.0.0-1", + nevra="pkg-0:1.0.0-1.%s" % arch, + arch=arch, + ) + mock_rpm.name = "pkg" + self.compose.variants["Server"].nsvc_to_pkgset[ + "testmodule:master:1:2017" + ].rpms_by_arch[arch] = [mock_rpm] + + def test_without_modules(self): + source = GatherSourceModule(self.compose) + packages, groups = source("x86_64", self.compose.variants["Server"]) + self.assertItemsEqual(packages, []) + self.assertItemsEqual(groups, []) + + def test_include_two_packages(self): + self.compose.variants["Server"].add_fake_module( + "testmodule:master:1:2017", + rpm_nvrs=["pkg-0:1.0.0-1.x86_64", "pkg-0:1.0.0-1.i686"], + with_artifacts=True, + mmd_arch="x86_64", + ) + + self._add_pkg("x86_64") + self._add_pkg("i686") + + source = GatherSourceModule(self.compose) + packages, groups = source("x86_64", self.compose.variants["Server"]) + self.assertItemsEqual( + [(rpm[0].nevra, rpm[1]) for rpm in packages], + [("pkg-0:1.0.0-1.x86_64", None), ("pkg-0:1.0.0-1.i686", None)], + ) + self.assertItemsEqual(groups, []) + + def test_does_not_include_unlisted(self): + self.compose.variants["Server"].add_fake_module( + "testmodule:master:1:2017", + rpm_nvrs=[], + with_artifacts=True, + mmd_arch="x86_64", + ) + + self._add_pkg("x86_64") + + source = GatherSourceModule(self.compose) + packages, groups = source("x86_64", self.compose.variants["Server"]) + self.assertItemsEqual(packages, []) + self.assertItemsEqual(groups, [])