diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py index b7081ee1..6ab86e53 100644 --- a/pungi/phases/pkgset/common.py +++ b/pungi/phases/pkgset/common.py @@ -20,7 +20,6 @@ import threading from kobo.shortcuts import run, relative_path from kobo.threads import run_in_threads -import pungi.phases.pkgset.pkgsets from pungi import Modulemd from pungi.arch import get_valid_arches from pungi.wrappers.createrepo import CreaterepoWrapper @@ -35,13 +34,7 @@ def populate_arch_pkgsets(compose, path_prefix, global_pkgset): compose.log_info("Populating package set for arch: %s", arch) is_multilib = is_arch_multilib(compose.conf, arch) arches = get_valid_arches(arch, is_multilib, add_src=True) - pkgset = pungi.phases.pkgset.pkgsets.PackageSetBase( - global_pkgset.name, - compose.conf["sigkeys"], - logger=compose._logger, - arches=arches, - ) - pkgset.merge(global_pkgset, arch, arches, exclusive_noarch=exclusive_noarch) + pkgset = global_pkgset.subset(arch, arches, exclusive_noarch=exclusive_noarch) pkgset.save_file_list( compose.paths.work.package_list(arch=arch, pkgset=global_pkgset), remove_path_prefix=path_prefix, @@ -156,7 +149,7 @@ def _create_arch_repo(worker_thread, args, task_num): ) run( cmd, - logfile=compose.paths.log.log_file(arch, "arch_repo_%s" % pkgset.name), + logfile=compose.paths.log.log_file(arch, "arch_repo.%s" % pkgset.name), show_cmd=True, ) # Add modulemd to the repo for all modules in all variants on this architecture. @@ -171,7 +164,7 @@ def _create_arch_repo(worker_thread, args, task_num): repo, repo_dir, mod_index, - compose.paths.log.log_file(arch, "arch_repo_modulemd"), + compose.paths.log.log_file(arch, "arch_repo_modulemd.%s" % pkgset.name), ) compose.log_info("[DONE ] %s", msg) diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index fe264db9..a06e17a5 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -168,6 +168,14 @@ class PackageSetBase(kobo.log.LoggingBase): return self.rpms_by_arch + def subset(self, primary_arch, arch_list, exclusive_noarch=True): + """Create a subset of this package set that only includes packages compatible with""" + pkgset = PackageSetBase( + self.name, self.sigkey_ordering, logger=self._logger, arches=arch_list + ) + pkgset.merge(self, primary_arch, arch_list, exclusive_noarch=exclusive_noarch) + return pkgset + def merge(self, other, primary_arch, arch_list, exclusive_noarch=True): """ Merge ``other`` package set into this instance. diff --git a/tests/helpers.py b/tests/helpers.py index 1d6566e8..ea1b4936 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -276,3 +276,9 @@ def load_config(data={}, **kwargs): def load_bin(name): return imp.load_source('pungi_cli_fake_' + name, os.path.dirname(__file__) + "/../bin/" + name) + + +def fake_run_in_threads(func, params, threads=None): + """Like run_in_threads from Kobo, but actually runs tasks serially.""" + for num, param in enumerate(params): + func(None, param, num) diff --git a/tests/test_initphase.py b/tests/test_initphase.py index 5a238625..4783e230 100644 --- a/tests/test_initphase.py +++ b/tests/test_initphase.py @@ -15,12 +15,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from pungi import Modulemd from pungi.phases import init -from tests.helpers import DummyCompose, PungiTestCase, touch, mk_boom - - -def fake_run_in_threads(func, params, threads=None): - for num, param in enumerate(params): - func(None, param, num) +from tests.helpers import DummyCompose, PungiTestCase, touch, mk_boom, fake_run_in_threads @mock.patch("pungi.phases.init.run_in_threads", new=fake_run_in_threads) diff --git a/tests/test_pkgset_common.py b/tests/test_pkgset_common.py new file mode 100755 index 00000000..64366a42 --- /dev/null +++ b/tests/test_pkgset_common.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +import os +import sys + +import mock + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from pungi import Modulemd +from pungi.phases.pkgset import common +from tests import helpers + + +class MockCreateRepo(object): + def __init__(self, createrepo_c): + self.createrepo_c = createrepo_c + + def get_createrepo_cmd(self, path_prefix, outputdir, pkglist, **kwargs): + return (path_prefix, outputdir, pkglist) + + +@mock.patch("pungi.phases.init.run_in_threads", new=helpers.fake_run_in_threads) +@mock.patch("pungi.phases.pkgset.common.CreaterepoWrapper", new=MockCreateRepo) +@mock.patch("pungi.phases.pkgset.common.run") +class TestMaterializedPkgsetCreate(helpers.PungiTestCase): + def setUp(self): + super(TestMaterializedPkgsetCreate, self).setUp() + self.compose = helpers.DummyCompose(self.topdir, {}) + self.prefix = "/prefix" + self.pkgset = self._make_pkgset("foo") + self.subsets = {} + + def _mk_call(self, arch, name): + pkglist = "%s.%s.conf" % (arch, name) + logfile = "arch_repo.%s.%s.log" % (name, arch) + return mock.call( + ( + self.prefix, + os.path.join(self.topdir, "work", arch, "repo", name), + os.path.join(self.topdir, "work", arch, "package_list", pkglist), + ), + logfile=os.path.join(self.topdir, "logs", arch, logfile), + show_cmd=True, + ) + + def _make_pkgset(self, name): + pkgset = mock.Mock() + pkgset.name = name + + def mock_subset(primary, arch_list, exclusive_noarch): + self.subsets[primary] = mock.Mock() + return self.subsets[primary] + + pkgset.subset.side_effect = mock_subset + return pkgset + + def _mk_paths(self, name, arches): + paths = {"global": os.path.join(self.topdir, "work/global/repo", name)} + for arch in arches: + paths[arch] = os.path.join(self.topdir, "work", arch, "repo", name) + return paths + + def test_run(self, mock_run): + result = common.MaterializedPackageSet.create( + self.compose, self.pkgset, self.prefix + ) + + self.assertItemsEqual(result.package_sets.keys(), ["global", "amd64", "x86_64"]) + self.assertEqual(result["global"], self.pkgset) + self.assertEqual(result["x86_64"], self.subsets["x86_64"]) + self.assertEqual(result["amd64"], self.subsets["amd64"]) + + self.pkgset.subset.assert_any_call( + "x86_64", ["x86_64", "noarch", "src"], exclusive_noarch=True + ) + self.pkgset.subset.assert_any_call( + "amd64", ["amd64", "x86_64", "noarch", "src"], exclusive_noarch=True + ) + + for arch, pkgset in result.package_sets.items(): + pkgset.save_file_list.assed_any_call( + os.path.join( + self.topdir, "work", arch, "package_list", arch + ".foo.conf" + ), + remove_path_prefix=self.prefix, + ) + + self.assertEqual(result.paths, self._mk_paths("foo", ["amd64", "x86_64"])) + + mock_run.assert_has_calls( + [self._mk_call(arch, "foo") for arch in ["global", "amd64", "x86_64"]], + any_order=True, + ) + + @helpers.unittest.skipUnless(Modulemd, "Skipping tests, no module support") + @mock.patch("pungi.phases.pkgset.common.collect_module_defaults") + @mock.patch("pungi.phases.pkgset.common.add_modular_metadata") + def test_run_with_modulemd(self, amm, cmd, mock_run): + mmd = {"x86_64": [mock.Mock()]} + common.MaterializedPackageSet.create( + self.compose, self.pkgset, self.prefix, mmd=mmd + ) + cmd.assert_called_once_with( + os.path.join(self.topdir, "work/global/module_defaults"), + set(x.get_module_name.return_value for x in mmd["x86_64"]), + ) + amm.assert_called_once_with( + mock.ANY, + os.path.join(self.topdir, "work/x86_64/repo/foo"), + cmd.return_value, + os.path.join(self.topdir, "logs/x86_64/arch_repo_modulemd.foo.x86_64.log"), + ) + cmd.return_value.add_module_stream.assert_called_once_with(mmd["x86_64"][0])