From eaf58f7d4077ef1e8a96c543749bdece95a0b60e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 20 Apr 2018 09:43:33 +0200 Subject: [PATCH] gather: Use another variant as lookaside MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create a temporary repository and add it as another lookaside in the compose. JIRA: COMPOSE-2426 Signed-off-by: Lubomír Sedlář --- pungi/paths.py | 18 ++++ pungi/phases/gather/__init__.py | 83 ++++++++++++++- tests/test_gather_phase.py | 173 ++++++++++++++++++++++++++++++++ 3 files changed, 273 insertions(+), 1 deletion(-) diff --git a/pungi/paths.py b/pungi/paths.py index 49e64c3d..cfed2903 100644 --- a/pungi/paths.py +++ b/pungi/paths.py @@ -179,6 +179,17 @@ class WorkPaths(object): makedirs(path) return path + def lookaside_repo(self, arch, variant, create_dir=True): + """ + Examples: + work/x86_64/Server/lookaside_repo + """ + path = os.path.join(self.topdir(arch, create_dir=create_dir), + variant.uid, "lookaside_repo") + if create_dir: + makedirs(path) + return path + def package_list(self, arch=None, variant=None, pkg_type=None, create_dir=True): """ Examples: @@ -200,6 +211,13 @@ class WorkPaths(object): path = os.path.join(path, file_name) return path + def lookaside_package_list(self, arch, variant, create_dir=True): + """ + Examples: + work/x86_64/package_list/Server.x86_64.lookaside.conf + """ + return self.package_list(arch, variant, pkg_type='lookaside', create_dir=create_dir) + def pungi_download_dir(self, arch, create_dir=True): """ Examples: diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index a842a774..0d96f957 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -19,16 +19,20 @@ import os import shutil from kobo.rpmlib import parse_nvra +from kobo.shortcuts import run from productmd.rpms import Rpms from pungi.wrappers.scm import get_file_from_scm from .link import link_files +from ...wrappers.createrepo import CreaterepoWrapper +import pungi.wrappers.kojiwrapper from pungi import Modulemd from pungi.arch import get_compatible_arches, split_name_arch from pungi.graph import SimpleAcyclicOrientedGraph from pungi.phases.base import PhaseBase -from pungi.util import get_arch_data, get_arch_variant_data, get_variant_data +from pungi.util import (get_arch_data, get_arch_variant_data, get_variant_data, + makedirs) def get_gather_source(name): @@ -308,6 +312,77 @@ def _prepare_variant_as_lookaside(compose): return list(variant_processing_order) +def _make_lookaside_repo(compose, variant, arch, pkg_map): + """ + Create variant lookaside repo for given variant and architecture with + packages from the map. If the repo repo already exists, then nothing will + happen. This could happen if multiple variants depend on this one. + """ + repo = compose.paths.work.lookaside_repo(arch, variant, create_dir=False) + if os.path.exists(repo): + # We have already generated this, nothing to do. + return repo + + makedirs(repo) + msg = 'Generating lookaside repo from %s.%s' % (variant.uid, arch) + compose.log_info('[BEGIN] %s', msg) + + prefixes = { + 'repos': lambda: os.path.join(compose.paths.work.topdir( + arch="global"), "download") + "/", + 'koji': lambda: pungi.wrappers.kojiwrapper.KojiWrapper( + compose.conf['koji_profile']).koji_module.config.topdir.rstrip("/") + "/" + } + path_prefix = prefixes[compose.conf['pkgset_source']]() + pkglist = compose.paths.work.lookaside_package_list(arch=arch, variant=variant) + with open(pkglist, 'w') as f: + for packages in pkg_map[arch][variant.uid].values(): + for pkg in packages: + pkg = pkg['path'] + if path_prefix and pkg.startswith(path_prefix): + pkg = pkg[len(path_prefix):] + f.write('%s\n' % pkg) + + cr = CreaterepoWrapper(compose.conf['createrepo_c']) + cmd = cr.get_createrepo_cmd(path_prefix, update=True, database=True, skip_stat=True, + pkglist=pkglist, + outputdir=repo, + baseurl="file://%s" % path_prefix, + workers=compose.conf["createrepo_num_workers"], + update_md_path=compose.paths.work.arch_repo(arch)) + run(cmd, + logfile=compose.paths.log.log_file(arch, "lookaside_repo_%s" % (variant.uid)), + show_cmd=True) + compose.log_info('[DONE ] %s', msg) + + return repo + + +def _update_config(compose, variant_uid, arch, repo): + """ + Add the variant lookaside repository into the configuration. + """ + lookasides = compose.conf.setdefault('gather_lookaside_repos', []) + lookasides.append(('^%s$' % variant_uid, {arch: repo})) + + +def _update_lookaside_config(compose, variant, arch, pkg_map): + """ + Make sure lookaside repo for all variants that the given one depends on + exist, and that configuration is updated to use those repos. + """ + for dest, lookaside_variant_uid in compose.conf.get('variant_as_lookaside', []): + lookaside_variant = compose.all_variants[lookaside_variant_uid] + if dest != variant.uid: + continue + if arch not in lookaside_variant.arches: + compose.log_warning('[SKIP] Skipping lookaside from %s for %s.%s due to arch mismatch', + lookaside_variant.uid, variant.uid, arch) + continue + repo = _make_lookaside_repo(compose, lookaside_variant, arch, pkg_map) + _update_config(compose, variant.uid, arch, repo) + + def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltree=False): """Run gathering on all arches of all variants of given type. @@ -330,6 +405,12 @@ def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltr if exclude_fulltree: for pkg_name, pkg_arch in get_parent_pkgs(arch, variant, result)["srpm"]: fulltree_excludes.add(pkg_name) + + # Get lookaside repos for this variant from other variants. Based + # on the ordering we already know that we have the packages from + # there. + _update_lookaside_config(compose, variant, arch, result) + pkg_map = gather_packages(compose, arch, variant, package_sets, fulltree_excludes=fulltree_excludes) result.setdefault(arch, {})[variant.uid] = pkg_map diff --git a/tests/test_gather_phase.py b/tests/test_gather_phase.py index 1630ef33..b85af8c3 100644 --- a/tests/test_gather_phase.py +++ b/tests/test_gather_phase.py @@ -901,3 +901,176 @@ class TestGetPackagesToGather(helpers.PungiTestCase): self.assertItemsEqual(packages, ["foo", "pkg", "foo2.x86_64"]) self.assertItemsEqual(groups, ["core"]) + + +class TestUpdateConfig(unittest.TestCase): + + def test_add_to_empty(self): + compose = mock.Mock(conf={}) + gather._update_config(compose, 'Server', 'x86_64', '/tmp/foo') + self.assertEqual(compose.conf, + {'gather_lookaside_repos': [ + ('^Server$', {'x86_64': '/tmp/foo'}) + ]}) + + def test_add_to_existing(self): + compose = mock.Mock(conf={'gather_lookaside_repos': [ + ('^Server$', {'x86_64': '/tmp/bar'}), + ]}) + gather._update_config(compose, 'Server', 'x86_64', '/tmp/foo') + self.assertEqual(compose.conf, + {'gather_lookaside_repos': [ + ('^Server$', {'x86_64': '/tmp/bar'}), + ('^Server$', {'x86_64': '/tmp/foo'}) + ]}) + + +class TestUpdateLookasideConfig(helpers.PungiTestCase): + + def setUp(self): + super(TestUpdateLookasideConfig, self).setUp() + self.compose = helpers.DummyCompose(self.topdir, {}) + self.pkg_map = mock.Mock() + + @mock.patch('pungi.phases.gather._update_config') + @mock.patch('pungi.phases.gather._make_lookaside_repo') + def test_no_config(self, mock_make_repo, mock_update_config): + gather._update_lookaside_config(self.compose, self.compose.variants['Server'], + 'x86_64', self.pkg_map) + self.assertEqual(mock_make_repo.call_args_list, []) + self.assertEqual(mock_update_config.call_args_list, []) + + @mock.patch('pungi.phases.gather._update_config') + @mock.patch('pungi.phases.gather._make_lookaside_repo') + def test_no_matching_config(self, mock_make_repo, mock_update_config): + self.compose.conf['variant_as_lookaside'] = [('Everything', 'Client')] + gather._update_lookaside_config(self.compose, self.compose.variants['Server'], + 'x86_64', self.pkg_map) + self.assertEqual(mock_make_repo.call_args_list, []) + self.assertEqual(mock_update_config.call_args_list, []) + + @mock.patch('pungi.phases.gather._update_config') + @mock.patch('pungi.phases.gather._make_lookaside_repo') + def test_missing_arch(self, mock_make_repo, mock_update_config): + # Client only has amd64 + self.compose.conf['variant_as_lookaside'] = [('Server', 'Client')] + gather._update_lookaside_config(self.compose, self.compose.variants['Server'], + 'x86_64', self.pkg_map) + self.assertEqual(len(self.compose.log_warning.call_args_list), 1) + self.assertEqual(mock_make_repo.call_args_list, []) + self.assertEqual(mock_update_config.call_args_list, []) + + @mock.patch('pungi.phases.gather._update_config') + @mock.patch('pungi.phases.gather._make_lookaside_repo') + def test_match(self, mock_make_repo, mock_update_config): + self.compose.conf['variant_as_lookaside'] = [('Server', 'Everything')] + gather._update_lookaside_config(self.compose, self.compose.variants['Server'], + 'x86_64', self.pkg_map) + self.assertEqual(len(self.compose.log_warning.call_args_list), 0) + self.assertEqual(mock_make_repo.call_args_list, + [mock.call(self.compose, + self.compose.variants['Everything'], + 'x86_64', + self.pkg_map)]) + self.assertEqual(mock_update_config.call_args_list, + [mock.call(self.compose, 'Server', 'x86_64', + mock_make_repo.return_value)]) + + +class TestMakeLookasideRepo(helpers.PungiTestCase): + + def setUp(self): + super(TestMakeLookasideRepo, self).setUp() + self.compose = helpers.DummyCompose(self.topdir, {}) + self.variant = self.compose.variants['Server'] + self.arch = 'x86_64' + self.repodir = self.compose.paths.work.lookaside_repo(self.arch, self.variant, create_dir=False) + self.pkglist = self.compose.paths.work.lookaside_package_list(self.arch, self.variant) + + @mock.patch('pungi.phases.gather.run') + def test_existing_repo(self, mock_run): + helpers.touch(os.path.join(self.repodir, 'repodata', 'primary.xml')) + repopath = gather._make_lookaside_repo(self.compose, self.variant, self.arch, {}) + self.assertEqual(self.repodir, repopath) + self.assertFalse(os.path.exists(self.pkglist)) + self.assertEqual(mock_run.call_args_list, []) + + def assertCorrect(self, repopath, path_prefix, MockCR, mock_run): + with open(self.pkglist) as f: + packages = f.read().splitlines() + self.assertItemsEqual(packages, + ['pkg/pkg-1.0-1.x86_64.rpm', + 'pkg/pkg-debuginfo-1.0-1.x86_64.rpm', + 'pkg/pkg-1.0-1.src.rpm']) + + self.assertEqual(self.repodir, repopath) + print(MockCR.return_value.get_createrepo_cmd.call_args_list) + print([mock.call(path_prefix, update=True, database=True, skip_stat=True, + pkglist=self.pkglist, outputdir=repopath, + baseurl="file://%s" % path_prefix, workers=3, + update_md_path=self.compose.paths.work.arch_repo(self.arch))]) + self.assertEqual(MockCR.return_value.get_createrepo_cmd.call_args_list, + [mock.call(path_prefix, update=True, database=True, skip_stat=True, + pkglist=self.pkglist, outputdir=repopath, + baseurl="file://%s" % path_prefix, workers=3, + update_md_path=self.compose.paths.work.arch_repo(self.arch))]) + self.assertEqual(mock_run.call_args_list, + [mock.call(MockCR.return_value.get_createrepo_cmd.return_value, + logfile=os.path.join( + self.topdir, 'logs', self.arch, + 'lookaside_repo_Server.%s.log' % self.arch), + show_cmd=True)]) + + @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') + @mock.patch('pungi.phases.gather.CreaterepoWrapper') + @mock.patch('pungi.phases.gather.run') + def test_create_repo_koji_pkgset(self, mock_run, MockCR, MockKW): + self.compose.conf.update({ + 'pkgset_source': 'koji', + 'koji_profile': 'koji', + }) + + pkg_map = { + self.arch: { + self.variant.uid: { + 'rpm': [{'path': '/tmp/packages/pkg/pkg-1.0-1.x86_64.rpm'}], + 'debuginfo': [{'path': '/tmp/packages/pkg/pkg-debuginfo-1.0-1.x86_64.rpm'}], + 'srpm': [{'path': '/tmp/packages/pkg/pkg-1.0-1.src.rpm'}], + } + } + } + + MockKW.return_value.koji_module.config.topdir = '/tmp/packages' + + repopath = gather._make_lookaside_repo(self.compose, self.variant, self.arch, pkg_map) + + self.assertCorrect(repopath, '/tmp/packages/', MockCR, mock_run) + + @mock.patch('pungi.phases.gather.CreaterepoWrapper') + @mock.patch('pungi.phases.gather.run') + def test_create_repo_repos_pkgset(self, mock_run, MockCR): + self.compose.conf.update({ + 'pkgset_source': 'repos', + }) + + dl_dir = self.compose.paths.work.topdir('global') + + pkg_map = { + self.arch: { + self.variant.uid: { + 'rpm': [ + {'path': os.path.join(dl_dir, 'download/pkg/pkg-1.0-1.x86_64.rpm')} + ], + 'debuginfo': [ + {'path': os.path.join(dl_dir, 'download/pkg/pkg-debuginfo-1.0-1.x86_64.rpm')} + ], + 'srpm': [ + {'path': os.path.join(dl_dir, 'download/pkg/pkg-1.0-1.src.rpm')} + ], + } + } + } + + repopath = gather._make_lookaside_repo(self.compose, self.variant, self.arch, pkg_map) + + self.assertCorrect(repopath, dl_dir + '/download/', MockCR, mock_run)