diff --git a/bin/pungi-koji b/bin/pungi-koji index cf43ed9b..58aae35b 100755 --- a/bin/pungi-koji +++ b/bin/pungi-koji @@ -288,7 +288,7 @@ def run_compose(compose, create_latest_link=True, latest_link_status=None): # initialize all phases init_phase = pungi.phases.InitPhase(compose) pkgset_phase = pungi.phases.PkgsetPhase(compose) - buildinstall_phase = pungi.phases.BuildinstallPhase(compose) + buildinstall_phase = pungi.phases.BuildinstallPhase(compose, pkgset_phase) gather_phase = pungi.phases.GatherPhase(compose, pkgset_phase) extrafiles_phase = pungi.phases.ExtraFilesPhase(compose, pkgset_phase) createrepo_phase = pungi.phases.CreaterepoPhase(compose, pkgset_phase) diff --git a/pungi/phases/buildinstall.py b/pungi/phases/buildinstall.py index 130f0fbd..31401056 100644 --- a/pungi/phases/buildinstall.py +++ b/pungi/phases/buildinstall.py @@ -39,7 +39,7 @@ from pungi.runroot import Runroot class BuildinstallPhase(PhaseBase): name = "buildinstall" - def __init__(self, compose): + def __init__(self, compose, pkgset_phase=None): PhaseBase.__init__(self, compose) self.pool = ThreadPool(logger=self.compose._logger) # A set of (variant_uid, arch) pairs that completed successfully. This @@ -47,6 +47,7 @@ class BuildinstallPhase(PhaseBase): self.pool.finished_tasks = set() self.buildinstall_method = self.compose.conf.get("buildinstall_method") self.used_lorax = self.buildinstall_method == 'lorax' + self.pkgset_phase = pkgset_phase self.warned_skipped = False @@ -100,8 +101,10 @@ class BuildinstallPhase(PhaseBase): if self.compose.conf.get("buildinstall_topdir", None): output_dir = os.path.join(output_dir, "results") - repos = [repo_baseurl] + get_arch_variant_data(self.compose.conf, - 'lorax_extra_sources', arch, variant) + repos = repo_baseurl[:] + repos.extend( + get_arch_variant_data(self.compose.conf, "lorax_extra_sources", arch, variant) + ) if self.compose.has_comps: comps_repo = self.compose.paths.work.comps_repo(arch, variant) if final_output_dir != output_dir: @@ -131,6 +134,12 @@ class BuildinstallPhase(PhaseBase): return 'rm -rf %s && %s' % (shlex_quote(output_topdir), ' '.join([shlex_quote(x) for x in lorax_cmd])) + def get_repos(self, arch): + repos = [] + for pkgset in self.pkgset_phase.package_sets: + repos.append(pkgset.paths[arch]) + return repos + def run(self): lorax = LoraxWrapper() product = self.compose.conf["release_name"] @@ -147,9 +156,9 @@ class BuildinstallPhase(PhaseBase): output_dir = self.compose.paths.work.buildinstall_dir(arch, allow_topdir_override=True) final_output_dir = self.compose.paths.work.buildinstall_dir(arch, allow_topdir_override=False) makedirs(final_output_dir) - repo_baseurl = self.compose.paths.work.arch_repo(arch) + repo_baseurls = self.get_repos(arch) if final_output_dir != output_dir: - repo_baseurl = translate_path(self.compose, repo_baseurl) + repo_baseurls = [translate_path(self.compose, r) for r in repo_baseurls] if self.buildinstall_method == "lorax": @@ -166,8 +175,12 @@ class BuildinstallPhase(PhaseBase): volid = get_volid(self.compose, arch, variant=variant, disc_type=disc_type) commands.append( - (variant, - self._get_lorax_cmd(repo_baseurl, output_dir, variant, arch, buildarch, volid, final_output_dir)) + ( + variant, + self._get_lorax_cmd( + repo_baseurls, output_dir, variant, arch, buildarch, volid, final_output_dir + ), + ) ) elif self.buildinstall_method == "buildinstall": volid = get_volid(self.compose, arch, disc_type=disc_type) @@ -176,7 +189,7 @@ class BuildinstallPhase(PhaseBase): lorax.get_buildinstall_cmd(product, version, release, - repo_baseurl, + repo_baseurls, output_dir, is_final=self.compose.supported, buildarch=arch, diff --git a/tests/test_buildinstall.py b/tests/test_buildinstall.py index 71183475..127448a6 100644 --- a/tests/test_buildinstall.py +++ b/tests/test_buildinstall.py @@ -66,7 +66,7 @@ class TestBuildinstallPhase(PungiTestCase): get_volid.return_value = 'vol_id' loraxCls.return_value.get_lorax_cmd.return_value = ['lorax', '...'] - phase = BuildinstallPhase(compose) + phase = BuildinstallPhase(compose, self._make_pkgset_phase(["p1"])) phase.run() @@ -98,7 +98,7 @@ class TestBuildinstallPhase(PungiTestCase): get_volid.return_value = 'vol_id' loraxCls.return_value.get_lorax_cmd.return_value = ['lorax', '...'] - phase = BuildinstallPhase(compose) + phase = BuildinstallPhase(compose, self._make_pkgset_phase(["p1", "p2"])) phase.run() @@ -116,7 +116,8 @@ class TestBuildinstallPhase(PungiTestCase): self.assertItemsEqual( loraxCls.return_value.get_lorax_cmd.mock_calls, [mock.call('Test', '1', '1', - [self.topdir + '/work/x86_64/repo', + [self.topdir + "/work/x86_64/repo/p1", + self.topdir + "/work/x86_64/repo/p2", self.topdir + '/work/x86_64/comps_repo_Server'], self.topdir + '/work/x86_64/buildinstall/Server', buildarch='x86_64', is_final=True, nomacboot=True, noupgrade=True, @@ -127,7 +128,8 @@ class TestBuildinstallPhase(PungiTestCase): rootfs_size=None, log_dir=self.topdir + '/logs/x86_64/buildinstall-Server-logs'), mock.call('Test', '1', '1', - [self.topdir + '/work/amd64/repo', + [self.topdir + "/work/amd64/repo/p1", + self.topdir + "/work/amd64/repo/p2", self.topdir + '/work/amd64/comps_repo_Server'], self.topdir + '/work/amd64/buildinstall/Server', buildarch='amd64', is_final=True, nomacboot=True, noupgrade=True, @@ -138,7 +140,8 @@ class TestBuildinstallPhase(PungiTestCase): rootfs_size=None, log_dir=self.topdir + '/logs/amd64/buildinstall-Server-logs'), mock.call('Test', '1', '1', - [self.topdir + '/work/amd64/repo', + [self.topdir + "/work/amd64/repo/p1", + self.topdir + "/work/amd64/repo/p2", self.topdir + '/work/amd64/comps_repo_Client'], self.topdir + '/work/amd64/buildinstall/Client', buildarch='amd64', is_final=True, nomacboot=True, noupgrade=True, @@ -170,7 +173,7 @@ class TestBuildinstallPhase(PungiTestCase): compose.variants['Server'].is_empty = True loraxCls.return_value.get_lorax_cmd.return_value = ['lorax', '...'] - phase = BuildinstallPhase(compose) + phase = BuildinstallPhase(compose, self._make_pkgset_phase(["p1"])) phase.run() @@ -184,7 +187,7 @@ class TestBuildinstallPhase(PungiTestCase): lorax = loraxCls.return_value lorax.get_lorax_cmd.assert_has_calls( [mock.call('Test', '1', '1', - [self.topdir + '/work/amd64/repo', + [self.topdir + "/work/amd64/repo/p1", self.topdir + '/work/amd64/comps_repo_Client'], self.topdir + '/work/amd64/buildinstall/Client', buildarch='amd64', is_final=True, nomacboot=True, noupgrade=True, @@ -214,7 +217,7 @@ class TestBuildinstallPhase(PungiTestCase): get_volid.return_value = 'vol_id' - phase = BuildinstallPhase(compose) + phase = BuildinstallPhase(compose, self._make_pkgset_phase(["p1"])) phase.run() @@ -225,10 +228,10 @@ class TestBuildinstallPhase(PungiTestCase): # Obtained correct lorax commands. self.assertItemsEqual( loraxCls.return_value.get_buildinstall_cmd.mock_calls, - [mock.call('Test', '1', '1', self.topdir + '/work/x86_64/repo', + [mock.call('Test', '1', '1', [self.topdir + "/work/x86_64/repo/p1"], self.topdir + '/work/x86_64/buildinstall', buildarch='x86_64', is_final=True, volid='vol_id'), - mock.call('Test', '1', '1', self.topdir + '/work/amd64/repo', + mock.call('Test', '1', '1', [self.topdir + "/work/amd64/repo/p1"], self.topdir + '/work/amd64/buildinstall', buildarch='amd64', is_final=True, volid='vol_id')]) self.assertItemsEqual( @@ -270,7 +273,7 @@ class TestBuildinstallPhase(PungiTestCase): get_volid.return_value = 'vol_id' loraxCls.return_value.get_lorax_cmd.return_value = ['lorax', '...'] - phase = BuildinstallPhase(compose) + phase = BuildinstallPhase(compose, self._make_pkgset_phase(["p1"])) phase.run() @@ -288,7 +291,7 @@ class TestBuildinstallPhase(PungiTestCase): self.assertItemsEqual( loraxCls.return_value.get_lorax_cmd.mock_calls, [mock.call('Test', '1.2.3', '1.2.3', - [self.topdir + '/work/x86_64/repo', + [self.topdir + "/work/x86_64/repo/p1", self.topdir + '/work/x86_64/comps_repo_Server'], self.topdir + '/work/x86_64/buildinstall/Server', buildarch='x86_64', is_final=True, nomacboot=True, noupgrade=True, @@ -299,7 +302,7 @@ class TestBuildinstallPhase(PungiTestCase): rootfs_size=3, log_dir=self.topdir + '/logs/x86_64/buildinstall-Server-logs'), mock.call('Test', '1', '1', - [self.topdir + '/work/amd64/repo', + [self.topdir + "/work/amd64/repo/p1", self.topdir + '/work/amd64/comps_repo_Server'], self.topdir + '/work/amd64/buildinstall/Server', buildarch='amd64', is_final=True, nomacboot=True, noupgrade=False, @@ -310,7 +313,7 @@ class TestBuildinstallPhase(PungiTestCase): rootfs_size=None, log_dir=self.topdir + '/logs/amd64/buildinstall-Server-logs'), mock.call('Test', '1', '1', - [self.topdir + '/work/amd64/repo', + [self.topdir + "/work/amd64/repo/p1", self.topdir + '/work/amd64/comps_repo_Client'], self.topdir + '/work/amd64/buildinstall/Client', buildarch='amd64', is_final=True, nomacboot=False, noupgrade=True, @@ -347,7 +350,7 @@ class TestBuildinstallPhase(PungiTestCase): get_volid.return_value = 'vol_id' loraxCls.return_value.get_lorax_cmd.return_value = ['lorax', '...'] - phase = BuildinstallPhase(compose) + phase = BuildinstallPhase(compose, self._make_pkgset_phase(["p1"])) phase.run() @@ -365,7 +368,7 @@ class TestBuildinstallPhase(PungiTestCase): self.assertItemsEqual( loraxCls.return_value.get_lorax_cmd.mock_calls, [mock.call('Test', '1', '1', - [self.topdir + '/work/x86_64/repo', + [self.topdir + "/work/x86_64/repo/p1", self.topdir + '/work/x86_64/comps_repo_Server'], self.topdir + '/work/x86_64/buildinstall/Server', buildarch='x86_64', is_final=True, nomacboot=False, noupgrade=False, @@ -376,7 +379,7 @@ class TestBuildinstallPhase(PungiTestCase): rootfs_size=None, log_dir=self.topdir + '/logs/x86_64/buildinstall-Server-logs'), mock.call('Test', '1', '1', - [self.topdir + '/work/amd64/repo', + [self.topdir + "/work/amd64/repo/p1", self.topdir + '/work/amd64/comps_repo_Server'], self.topdir + '/work/amd64/buildinstall/Server', buildarch='amd64', is_final=True, nomacboot=True, noupgrade=False, @@ -387,7 +390,7 @@ class TestBuildinstallPhase(PungiTestCase): rootfs_size=None, log_dir=self.topdir + '/logs/amd64/buildinstall-Server-logs'), mock.call('Test', '1', '1', - [self.topdir + '/work/amd64/repo', + [self.topdir + "/work/amd64/repo/p1", self.topdir + '/work/amd64/comps_repo_Client'], self.topdir + '/work/amd64/buildinstall/Client', buildarch='amd64', is_final=True, nomacboot=True, noupgrade=False, @@ -403,6 +406,17 @@ class TestBuildinstallPhase(PungiTestCase): mock.call(compose, 'amd64', variant=compose.variants['Client'], disc_type='dvd'), mock.call(compose, 'amd64', variant=compose.variants['Server'], disc_type='dvd')]) + def _make_pkgset_phase(self, names): + pkgsets = [] + for name in names: + pkgset = mock.Mock(paths={}) + for arch in ("x86_64", "amd64"): + pkgset.paths[arch] = os.path.join( + self.topdir, "work", arch, "repo", name + ) + pkgsets.append(pkgset) + return mock.Mock(package_sets=pkgsets) + @mock.patch('pungi.phases.buildinstall.ThreadPool') @mock.patch('pungi.phases.buildinstall.LoraxWrapper') @mock.patch('pungi.phases.buildinstall.get_volid') @@ -424,7 +438,7 @@ class TestBuildinstallPhase(PungiTestCase): get_volid.return_value = 'vol_id' loraxCls.return_value.get_lorax_cmd.return_value = ['lorax', '...'] - phase = BuildinstallPhase(compose) + phase = BuildinstallPhase(compose, self._make_pkgset_phase(["p1"])) phase.run() @@ -442,7 +456,7 @@ class TestBuildinstallPhase(PungiTestCase): self.assertItemsEqual( loraxCls.return_value.get_lorax_cmd.mock_calls, [mock.call('Test', '1', '1', - ['http://localhost/work/x86_64/repo', + ["http://localhost/work/x86_64/repo/p1", 'http://localhost/work/x86_64/comps_repo_Server'], buildinstall_topdir + '/x86_64/Server/results', buildarch='x86_64', is_final=True, nomacboot=True, noupgrade=True, @@ -453,7 +467,7 @@ class TestBuildinstallPhase(PungiTestCase): rootfs_size=None, log_dir=buildinstall_topdir + '/x86_64/Server/logs'), mock.call('Test', '1', '1', - ['http://localhost/work/amd64/repo', + ["http://localhost/work/amd64/repo/p1", 'http://localhost/work/amd64/comps_repo_Server'], buildinstall_topdir + '/amd64/Server/results', buildarch='amd64', is_final=True, nomacboot=True, noupgrade=True, @@ -464,7 +478,7 @@ class TestBuildinstallPhase(PungiTestCase): rootfs_size=None, log_dir=buildinstall_topdir + '/amd64/Server/logs'), mock.call('Test', '1', '1', - ['http://localhost/work/amd64/repo', + ["http://localhost/work/amd64/repo/p1", 'http://localhost/work/amd64/comps_repo_Client'], buildinstall_topdir + '/amd64/Client/results', buildarch='amd64', is_final=True, nomacboot=True, noupgrade=True, @@ -503,7 +517,7 @@ class TestBuildinstallPhase(PungiTestCase): get_volid.return_value = 'vol_id' loraxCls.return_value.get_lorax_cmd.return_value = ['lorax', '...'] - phase = BuildinstallPhase(compose) + phase = BuildinstallPhase(compose, self._make_pkgset_phase(["p1"])) phase.run() @@ -511,7 +525,7 @@ class TestBuildinstallPhase(PungiTestCase): self.assertItemsEqual( loraxCls.return_value.get_lorax_cmd.mock_calls, [mock.call('Test', '1', '1', - [self.topdir + '/work/x86_64/repo', + [self.topdir + "/work/x86_64/repo/p1", "http://example.com/repo1", self.topdir + '/work/x86_64/comps_repo_Server'], self.topdir + '/work/x86_64/buildinstall/Server', @@ -523,7 +537,7 @@ class TestBuildinstallPhase(PungiTestCase): rootfs_size=None, log_dir=self.topdir + '/logs/x86_64/buildinstall-Server-logs'), mock.call('Test', '1', '1', - [self.topdir + '/work/amd64/repo', + [self.topdir + "/work/amd64/repo/p1", self.topdir + '/work/amd64/comps_repo_Server'], self.topdir + '/work/amd64/buildinstall/Server', buildarch='amd64', is_final=True, nomacboot=True, noupgrade=True, @@ -534,7 +548,7 @@ class TestBuildinstallPhase(PungiTestCase): rootfs_size=None, log_dir=self.topdir + '/logs/amd64/buildinstall-Server-logs'), mock.call('Test', '1', '1', - [self.topdir + '/work/amd64/repo', + [self.topdir + "/work/amd64/repo/p1", "http://example.com/repo2", "http://example.com/repo3", self.topdir + '/work/amd64/comps_repo_Client'],