almalinux
/
pungi
15
1
Fork 0
Code Issues Pull Requests Releases Activity

gather: Remove use of arch_repo 

The repo was used to speed up creating lookaside repo from a variant.
This uses a similar approach as createrepo phase: selecting the last
available package set and using that data.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2019-08-01 16:44:32 +02:00
parent 2d0ffb56ca
commit a426a83ed6
2 changed files with 20 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
pungi/phases/gather/__init__.py
View File

@ -328,7 +328,7 @@ def trim_packages(compose, arch, variant, pkg_map, parent_pkgs=None, remove_pkgs
return addon_pkgs, move_to_parent_pkgs, removed_pkgs return addon_pkgs, move_to_parent_pkgs, removed_pkgs
def _make_lookaside_repo(compose, variant, arch, pkg_map): def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None):
""" """
Create variant lookaside repo for given variant and architecture with Create variant lookaside repo for given variant and architecture with
packages from the map. If the repo repo already exists, then nothing will packages from the map. If the repo repo already exists, then nothing will
@ -360,12 +360,16 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map):
f.write('%s\n' % pkg) f.write('%s\n' % pkg)
cr = CreaterepoWrapper(compose.conf['createrepo_c']) cr = CreaterepoWrapper(compose.conf['createrepo_c'])
update_metadata = None
if package_sets:
pkgset = package_sets[-1]
update_metadata = compose.paths.work.pkgset_repo(pkgset.name, arch)
cmd = cr.get_createrepo_cmd(path_prefix, update=True, database=True, skip_stat=True, cmd = cr.get_createrepo_cmd(path_prefix, update=True, database=True, skip_stat=True,
pkglist=pkglist, pkglist=pkglist,
outputdir=repo, outputdir=repo,
baseurl="file://%s" % path_prefix, baseurl="file://%s" % path_prefix,
workers=compose.conf["createrepo_num_workers"], workers=compose.conf["createrepo_num_workers"],
update_md_path=compose.paths.work.arch_repo(arch)) update_md_path=update_metadata)
run(cmd, run(cmd,
logfile=compose.paths.log.log_file(arch, "lookaside_repo_%s" % (variant.uid)), logfile=compose.paths.log.log_file(arch, "lookaside_repo_%s" % (variant.uid)),
show_cmd=True) show_cmd=True)
@ -398,7 +402,7 @@ def _update_config(compose, variant_uid, arch, repo):
lookasides.append(('^%s$' % variant_uid, {arch: repo})) lookasides.append(('^%s$' % variant_uid, {arch: repo}))
def _update_lookaside_config(compose, variant, arch, pkg_map): def _update_lookaside_config(compose, variant, arch, pkg_map, package_sets=None):
""" """
Make sure lookaside repo for all variants that the given one depends on Make sure lookaside repo for all variants that the given one depends on
exist, and that configuration is updated to use those repos. exist, and that configuration is updated to use those repos.
@ -411,7 +415,9 @@ def _update_lookaside_config(compose, variant, arch, pkg_map):
compose.log_warning('[SKIP] Skipping lookaside from %s for %s.%s due to arch mismatch', compose.log_warning('[SKIP] Skipping lookaside from %s for %s.%s due to arch mismatch',
lookaside_variant.uid, variant.uid, arch) lookaside_variant.uid, variant.uid, arch)
continue continue
repo = _make_lookaside_repo(compose, lookaside_variant, arch, pkg_map) repo = _make_lookaside_repo(
compose, lookaside_variant, arch, pkg_map, package_sets
)
_update_config(compose, variant.uid, arch, repo) _update_config(compose, variant.uid, arch, repo)
@ -435,7 +441,7 @@ def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltr
# Get lookaside repos for this variant from other variants. Based # Get lookaside repos for this variant from other variants. Based
# on the ordering we already know that we have the packages from # on the ordering we already know that we have the packages from
# there. # there.
_update_lookaside_config(compose, variant, arch, result) _update_lookaside_config(compose, variant, arch, result, package_sets)
pkg_map = gather_packages(compose, arch, variant, package_sets, fulltree_excludes=fulltree_excludes) pkg_map = gather_packages(compose, arch, variant, package_sets, fulltree_excludes=fulltree_excludes)
result.setdefault(arch, {})[variant.uid] = pkg_map result.setdefault(arch, {})[variant.uid] = pkg_map

18
tests/test_gather_phase.py
View File

@ -1063,7 +1063,7 @@ class TestUpdateLookasideConfig(helpers.PungiTestCase):
[mock.call(self.compose, [mock.call(self.compose,
self.compose.variants['Everything'], self.compose.variants['Everything'],
'x86_64', 'x86_64',
self.pkg_map)]) self.pkg_map, None)])
self.assertEqual(mock_update_config.call_args_list, self.assertEqual(mock_update_config.call_args_list,
[mock.call(self.compose, 'Server', 'x86_64', [mock.call(self.compose, 'Server', 'x86_64',
mock_make_repo.return_value)]) mock_make_repo.return_value)])
@ -1078,6 +1078,7 @@ class TestMakeLookasideRepo(helpers.PungiTestCase):
self.arch = 'x86_64' self.arch = 'x86_64'
self.repodir = self.compose.paths.work.lookaside_repo(self.arch, self.variant, create_dir=False) self.repodir = self.compose.paths.work.lookaside_repo(self.arch, self.variant, create_dir=False)
self.pkglist = self.compose.paths.work.lookaside_package_list(self.arch, self.variant) self.pkglist = self.compose.paths.work.lookaside_package_list(self.arch, self.variant)
self.package_sets = self._make_pkgset_phase(["p1", "p2"]).package_sets
@mock.patch('pungi.phases.gather.run') @mock.patch('pungi.phases.gather.run')
def test_existing_repo(self, mock_run): def test_existing_repo(self, mock_run):
@ -1096,16 +1097,11 @@ class TestMakeLookasideRepo(helpers.PungiTestCase):
'pkg/pkg-1.0-1.src.rpm']) 'pkg/pkg-1.0-1.src.rpm'])
self.assertEqual(self.repodir, repopath) self.assertEqual(self.repodir, repopath)
print(MockCR.return_value.get_createrepo_cmd.call_args_list)
print([mock.call(path_prefix, update=True, database=True, skip_stat=True,
pkglist=self.pkglist, outputdir=repopath,
baseurl="file://%s" % path_prefix, workers=3,
update_md_path=self.compose.paths.work.arch_repo(self.arch))])
self.assertEqual(MockCR.return_value.get_createrepo_cmd.call_args_list, self.assertEqual(MockCR.return_value.get_createrepo_cmd.call_args_list,
[mock.call(path_prefix, update=True, database=True, skip_stat=True, [mock.call(path_prefix, update=True, database=True, skip_stat=True,
pkglist=self.pkglist, outputdir=repopath, pkglist=self.pkglist, outputdir=repopath,
baseurl="file://%s" % path_prefix, workers=3, baseurl="file://%s" % path_prefix, workers=3,
update_md_path=self.compose.paths.work.arch_repo(self.arch))]) update_md_path=self.compose.paths.work.pkgset_repo("p2", self.arch))])
self.assertEqual(mock_run.call_args_list, self.assertEqual(mock_run.call_args_list,
[mock.call(MockCR.return_value.get_createrepo_cmd.return_value, [mock.call(MockCR.return_value.get_createrepo_cmd.return_value,
logfile=os.path.join( logfile=os.path.join(
@ -1134,7 +1130,9 @@ class TestMakeLookasideRepo(helpers.PungiTestCase):
MockKW.return_value.koji_module.config.topdir = '/tmp/packages' MockKW.return_value.koji_module.config.topdir = '/tmp/packages'
repopath = gather._make_lookaside_repo(self.compose, self.variant, self.arch, pkg_map) repopath = gather._make_lookaside_repo(
self.compose, self.variant, self.arch, pkg_map, self.package_sets
)
self.assertCorrect(repopath, '/tmp/packages/', MockCR, mock_run) self.assertCorrect(repopath, '/tmp/packages/', MockCR, mock_run)
@ -1163,6 +1161,8 @@ class TestMakeLookasideRepo(helpers.PungiTestCase):
} }
} }
repopath = gather._make_lookaside_repo(self.compose, self.variant, self.arch, pkg_map) repopath = gather._make_lookaside_repo(
self.compose, self.variant, self.arch, pkg_map, self.package_sets
)
self.assertCorrect(repopath, dl_dir + '/download/', MockCR, mock_run) self.assertCorrect(repopath, dl_dir + '/download/', MockCR, mock_run)