gather: Use another variant as lookaside

Create a temporary repository and add it as another lookaside in the
compose.

JIRA: COMPOSE-2426
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2018-04-20 09:43:33 +02:00
parent ea0964eeef
commit eaf58f7d40
3 changed files with 273 additions and 1 deletions

View File

@ -179,6 +179,17 @@ class WorkPaths(object):
makedirs(path) makedirs(path)
return path return path
def lookaside_repo(self, arch, variant, create_dir=True):
"""
Examples:
work/x86_64/Server/lookaside_repo
"""
path = os.path.join(self.topdir(arch, create_dir=create_dir),
variant.uid, "lookaside_repo")
if create_dir:
makedirs(path)
return path
def package_list(self, arch=None, variant=None, pkg_type=None, create_dir=True): def package_list(self, arch=None, variant=None, pkg_type=None, create_dir=True):
""" """
Examples: Examples:
@ -200,6 +211,13 @@ class WorkPaths(object):
path = os.path.join(path, file_name) path = os.path.join(path, file_name)
return path return path
def lookaside_package_list(self, arch, variant, create_dir=True):
"""
Examples:
work/x86_64/package_list/Server.x86_64.lookaside.conf
"""
return self.package_list(arch, variant, pkg_type='lookaside', create_dir=create_dir)
def pungi_download_dir(self, arch, create_dir=True): def pungi_download_dir(self, arch, create_dir=True):
""" """
Examples: Examples:

View File

@ -19,16 +19,20 @@ import os
import shutil import shutil
from kobo.rpmlib import parse_nvra from kobo.rpmlib import parse_nvra
from kobo.shortcuts import run
from productmd.rpms import Rpms from productmd.rpms import Rpms
from pungi.wrappers.scm import get_file_from_scm from pungi.wrappers.scm import get_file_from_scm
from .link import link_files from .link import link_files
from ...wrappers.createrepo import CreaterepoWrapper
import pungi.wrappers.kojiwrapper
from pungi import Modulemd from pungi import Modulemd
from pungi.arch import get_compatible_arches, split_name_arch from pungi.arch import get_compatible_arches, split_name_arch
from pungi.graph import SimpleAcyclicOrientedGraph from pungi.graph import SimpleAcyclicOrientedGraph
from pungi.phases.base import PhaseBase from pungi.phases.base import PhaseBase
from pungi.util import get_arch_data, get_arch_variant_data, get_variant_data from pungi.util import (get_arch_data, get_arch_variant_data, get_variant_data,
makedirs)
def get_gather_source(name): def get_gather_source(name):
@ -308,6 +312,77 @@ def _prepare_variant_as_lookaside(compose):
return list(variant_processing_order) return list(variant_processing_order)
def _make_lookaside_repo(compose, variant, arch, pkg_map):
"""
Create variant lookaside repo for given variant and architecture with
packages from the map. If the repo repo already exists, then nothing will
happen. This could happen if multiple variants depend on this one.
"""
repo = compose.paths.work.lookaside_repo(arch, variant, create_dir=False)
if os.path.exists(repo):
# We have already generated this, nothing to do.
return repo
makedirs(repo)
msg = 'Generating lookaside repo from %s.%s' % (variant.uid, arch)
compose.log_info('[BEGIN] %s', msg)
prefixes = {
'repos': lambda: os.path.join(compose.paths.work.topdir(
arch="global"), "download") + "/",
'koji': lambda: pungi.wrappers.kojiwrapper.KojiWrapper(
compose.conf['koji_profile']).koji_module.config.topdir.rstrip("/") + "/"
}
path_prefix = prefixes[compose.conf['pkgset_source']]()
pkglist = compose.paths.work.lookaside_package_list(arch=arch, variant=variant)
with open(pkglist, 'w') as f:
for packages in pkg_map[arch][variant.uid].values():
for pkg in packages:
pkg = pkg['path']
if path_prefix and pkg.startswith(path_prefix):
pkg = pkg[len(path_prefix):]
f.write('%s\n' % pkg)
cr = CreaterepoWrapper(compose.conf['createrepo_c'])
cmd = cr.get_createrepo_cmd(path_prefix, update=True, database=True, skip_stat=True,
pkglist=pkglist,
outputdir=repo,
baseurl="file://%s" % path_prefix,
workers=compose.conf["createrepo_num_workers"],
update_md_path=compose.paths.work.arch_repo(arch))
run(cmd,
logfile=compose.paths.log.log_file(arch, "lookaside_repo_%s" % (variant.uid)),
show_cmd=True)
compose.log_info('[DONE ] %s', msg)
return repo
def _update_config(compose, variant_uid, arch, repo):
"""
Add the variant lookaside repository into the configuration.
"""
lookasides = compose.conf.setdefault('gather_lookaside_repos', [])
lookasides.append(('^%s$' % variant_uid, {arch: repo}))
def _update_lookaside_config(compose, variant, arch, pkg_map):
"""
Make sure lookaside repo for all variants that the given one depends on
exist, and that configuration is updated to use those repos.
"""
for dest, lookaside_variant_uid in compose.conf.get('variant_as_lookaside', []):
lookaside_variant = compose.all_variants[lookaside_variant_uid]
if dest != variant.uid:
continue
if arch not in lookaside_variant.arches:
compose.log_warning('[SKIP] Skipping lookaside from %s for %s.%s due to arch mismatch',
lookaside_variant.uid, variant.uid, arch)
continue
repo = _make_lookaside_repo(compose, lookaside_variant, arch, pkg_map)
_update_config(compose, variant.uid, arch, repo)
def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltree=False): def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltree=False):
"""Run gathering on all arches of all variants of given type. """Run gathering on all arches of all variants of given type.
@ -330,6 +405,12 @@ def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltr
if exclude_fulltree: if exclude_fulltree:
for pkg_name, pkg_arch in get_parent_pkgs(arch, variant, result)["srpm"]: for pkg_name, pkg_arch in get_parent_pkgs(arch, variant, result)["srpm"]:
fulltree_excludes.add(pkg_name) fulltree_excludes.add(pkg_name)
# Get lookaside repos for this variant from other variants. Based
# on the ordering we already know that we have the packages from
# there.
_update_lookaside_config(compose, variant, arch, result)
pkg_map = gather_packages(compose, arch, variant, package_sets, fulltree_excludes=fulltree_excludes) pkg_map = gather_packages(compose, arch, variant, package_sets, fulltree_excludes=fulltree_excludes)
result.setdefault(arch, {})[variant.uid] = pkg_map result.setdefault(arch, {})[variant.uid] = pkg_map

View File

@ -901,3 +901,176 @@ class TestGetPackagesToGather(helpers.PungiTestCase):
self.assertItemsEqual(packages, ["foo", "pkg", "foo2.x86_64"]) self.assertItemsEqual(packages, ["foo", "pkg", "foo2.x86_64"])
self.assertItemsEqual(groups, ["core"]) self.assertItemsEqual(groups, ["core"])
class TestUpdateConfig(unittest.TestCase):
def test_add_to_empty(self):
compose = mock.Mock(conf={})
gather._update_config(compose, 'Server', 'x86_64', '/tmp/foo')
self.assertEqual(compose.conf,
{'gather_lookaside_repos': [
('^Server$', {'x86_64': '/tmp/foo'})
]})
def test_add_to_existing(self):
compose = mock.Mock(conf={'gather_lookaside_repos': [
('^Server$', {'x86_64': '/tmp/bar'}),
]})
gather._update_config(compose, 'Server', 'x86_64', '/tmp/foo')
self.assertEqual(compose.conf,
{'gather_lookaside_repos': [
('^Server$', {'x86_64': '/tmp/bar'}),
('^Server$', {'x86_64': '/tmp/foo'})
]})
class TestUpdateLookasideConfig(helpers.PungiTestCase):
def setUp(self):
super(TestUpdateLookasideConfig, self).setUp()
self.compose = helpers.DummyCompose(self.topdir, {})
self.pkg_map = mock.Mock()
@mock.patch('pungi.phases.gather._update_config')
@mock.patch('pungi.phases.gather._make_lookaside_repo')
def test_no_config(self, mock_make_repo, mock_update_config):
gather._update_lookaside_config(self.compose, self.compose.variants['Server'],
'x86_64', self.pkg_map)
self.assertEqual(mock_make_repo.call_args_list, [])
self.assertEqual(mock_update_config.call_args_list, [])
@mock.patch('pungi.phases.gather._update_config')
@mock.patch('pungi.phases.gather._make_lookaside_repo')
def test_no_matching_config(self, mock_make_repo, mock_update_config):
self.compose.conf['variant_as_lookaside'] = [('Everything', 'Client')]
gather._update_lookaside_config(self.compose, self.compose.variants['Server'],
'x86_64', self.pkg_map)
self.assertEqual(mock_make_repo.call_args_list, [])
self.assertEqual(mock_update_config.call_args_list, [])
@mock.patch('pungi.phases.gather._update_config')
@mock.patch('pungi.phases.gather._make_lookaside_repo')
def test_missing_arch(self, mock_make_repo, mock_update_config):
# Client only has amd64
self.compose.conf['variant_as_lookaside'] = [('Server', 'Client')]
gather._update_lookaside_config(self.compose, self.compose.variants['Server'],
'x86_64', self.pkg_map)
self.assertEqual(len(self.compose.log_warning.call_args_list), 1)
self.assertEqual(mock_make_repo.call_args_list, [])
self.assertEqual(mock_update_config.call_args_list, [])
@mock.patch('pungi.phases.gather._update_config')
@mock.patch('pungi.phases.gather._make_lookaside_repo')
def test_match(self, mock_make_repo, mock_update_config):
self.compose.conf['variant_as_lookaside'] = [('Server', 'Everything')]
gather._update_lookaside_config(self.compose, self.compose.variants['Server'],
'x86_64', self.pkg_map)
self.assertEqual(len(self.compose.log_warning.call_args_list), 0)
self.assertEqual(mock_make_repo.call_args_list,
[mock.call(self.compose,
self.compose.variants['Everything'],
'x86_64',
self.pkg_map)])
self.assertEqual(mock_update_config.call_args_list,
[mock.call(self.compose, 'Server', 'x86_64',
mock_make_repo.return_value)])
class TestMakeLookasideRepo(helpers.PungiTestCase):
def setUp(self):
super(TestMakeLookasideRepo, self).setUp()
self.compose = helpers.DummyCompose(self.topdir, {})
self.variant = self.compose.variants['Server']
self.arch = 'x86_64'
self.repodir = self.compose.paths.work.lookaside_repo(self.arch, self.variant, create_dir=False)
self.pkglist = self.compose.paths.work.lookaside_package_list(self.arch, self.variant)
@mock.patch('pungi.phases.gather.run')
def test_existing_repo(self, mock_run):
helpers.touch(os.path.join(self.repodir, 'repodata', 'primary.xml'))
repopath = gather._make_lookaside_repo(self.compose, self.variant, self.arch, {})
self.assertEqual(self.repodir, repopath)
self.assertFalse(os.path.exists(self.pkglist))
self.assertEqual(mock_run.call_args_list, [])
def assertCorrect(self, repopath, path_prefix, MockCR, mock_run):
with open(self.pkglist) as f:
packages = f.read().splitlines()
self.assertItemsEqual(packages,
['pkg/pkg-1.0-1.x86_64.rpm',
'pkg/pkg-debuginfo-1.0-1.x86_64.rpm',
'pkg/pkg-1.0-1.src.rpm'])
self.assertEqual(self.repodir, repopath)
print(MockCR.return_value.get_createrepo_cmd.call_args_list)
print([mock.call(path_prefix, update=True, database=True, skip_stat=True,
pkglist=self.pkglist, outputdir=repopath,
baseurl="file://%s" % path_prefix, workers=3,
update_md_path=self.compose.paths.work.arch_repo(self.arch))])
self.assertEqual(MockCR.return_value.get_createrepo_cmd.call_args_list,
[mock.call(path_prefix, update=True, database=True, skip_stat=True,
pkglist=self.pkglist, outputdir=repopath,
baseurl="file://%s" % path_prefix, workers=3,
update_md_path=self.compose.paths.work.arch_repo(self.arch))])
self.assertEqual(mock_run.call_args_list,
[mock.call(MockCR.return_value.get_createrepo_cmd.return_value,
logfile=os.path.join(
self.topdir, 'logs', self.arch,
'lookaside_repo_Server.%s.log' % self.arch),
show_cmd=True)])
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
@mock.patch('pungi.phases.gather.CreaterepoWrapper')
@mock.patch('pungi.phases.gather.run')
def test_create_repo_koji_pkgset(self, mock_run, MockCR, MockKW):
self.compose.conf.update({
'pkgset_source': 'koji',
'koji_profile': 'koji',
})
pkg_map = {
self.arch: {
self.variant.uid: {
'rpm': [{'path': '/tmp/packages/pkg/pkg-1.0-1.x86_64.rpm'}],
'debuginfo': [{'path': '/tmp/packages/pkg/pkg-debuginfo-1.0-1.x86_64.rpm'}],
'srpm': [{'path': '/tmp/packages/pkg/pkg-1.0-1.src.rpm'}],
}
}
}
MockKW.return_value.koji_module.config.topdir = '/tmp/packages'
repopath = gather._make_lookaside_repo(self.compose, self.variant, self.arch, pkg_map)
self.assertCorrect(repopath, '/tmp/packages/', MockCR, mock_run)
@mock.patch('pungi.phases.gather.CreaterepoWrapper')
@mock.patch('pungi.phases.gather.run')
def test_create_repo_repos_pkgset(self, mock_run, MockCR):
self.compose.conf.update({
'pkgset_source': 'repos',
})
dl_dir = self.compose.paths.work.topdir('global')
pkg_map = {
self.arch: {
self.variant.uid: {
'rpm': [
{'path': os.path.join(dl_dir, 'download/pkg/pkg-1.0-1.x86_64.rpm')}
],
'debuginfo': [
{'path': os.path.join(dl_dir, 'download/pkg/pkg-debuginfo-1.0-1.x86_64.rpm')}
],
'srpm': [
{'path': os.path.join(dl_dir, 'download/pkg/pkg-1.0-1.src.rpm')}
],
}
}
}
repopath = gather._make_lookaside_repo(self.compose, self.variant, self.arch, pkg_map)
self.assertCorrect(repopath, dl_dir + '/download/', MockCR, mock_run)