gather: Use another variant as lookaside
Create a temporary repository and add it as another lookaside in the compose. JIRA: COMPOSE-2426 Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář
parent
ea0964eeef
commit
eaf58f7d40
@ -179,6 +179,17 @@ class WorkPaths(object):
|
||||
makedirs(path)
|
||||
return path
|
||||
|
||||
def lookaside_repo(self, arch, variant, create_dir=True):
|
||||
"""
|
||||
Examples:
|
||||
work/x86_64/Server/lookaside_repo
|
||||
"""
|
||||
path = os.path.join(self.topdir(arch, create_dir=create_dir),
|
||||
variant.uid, "lookaside_repo")
|
||||
if create_dir:
|
||||
makedirs(path)
|
||||
return path
|
||||
|
||||
def package_list(self, arch=None, variant=None, pkg_type=None, create_dir=True):
|
||||
"""
|
||||
Examples:
|
||||
@ -200,6 +211,13 @@ class WorkPaths(object):
|
||||
path = os.path.join(path, file_name)
|
||||
return path
|
||||
|
||||
def lookaside_package_list(self, arch, variant, create_dir=True):
|
||||
"""
|
||||
Examples:
|
||||
work/x86_64/package_list/Server.x86_64.lookaside.conf
|
||||
"""
|
||||
return self.package_list(arch, variant, pkg_type='lookaside', create_dir=create_dir)
|
||||
|
||||
def pungi_download_dir(self, arch, create_dir=True):
|
||||
"""
|
||||
Examples:
|
||||
|
||||
@ -19,16 +19,20 @@ import os
|
||||
import shutil
|
||||
|
||||
from kobo.rpmlib import parse_nvra
|
||||
from kobo.shortcuts import run
|
||||
from productmd.rpms import Rpms
|
||||
|
||||
from pungi.wrappers.scm import get_file_from_scm
|
||||
from .link import link_files
|
||||
from ...wrappers.createrepo import CreaterepoWrapper
|
||||
import pungi.wrappers.kojiwrapper
|
||||
|
||||
from pungi import Modulemd
|
||||
from pungi.arch import get_compatible_arches, split_name_arch
|
||||
from pungi.graph import SimpleAcyclicOrientedGraph
|
||||
from pungi.phases.base import PhaseBase
|
||||
from pungi.util import get_arch_data, get_arch_variant_data, get_variant_data
|
||||
from pungi.util import (get_arch_data, get_arch_variant_data, get_variant_data,
|
||||
makedirs)
|
||||
|
||||
|
||||
def get_gather_source(name):
|
||||
@ -308,6 +312,77 @@ def _prepare_variant_as_lookaside(compose):
|
||||
return list(variant_processing_order)
|
||||
|
||||
|
||||
def _make_lookaside_repo(compose, variant, arch, pkg_map):
|
||||
"""
|
||||
Create variant lookaside repo for given variant and architecture with
|
||||
packages from the map. If the repo repo already exists, then nothing will
|
||||
happen. This could happen if multiple variants depend on this one.
|
||||
"""
|
||||
repo = compose.paths.work.lookaside_repo(arch, variant, create_dir=False)
|
||||
if os.path.exists(repo):
|
||||
# We have already generated this, nothing to do.
|
||||
return repo
|
||||
|
||||
makedirs(repo)
|
||||
msg = 'Generating lookaside repo from %s.%s' % (variant.uid, arch)
|
||||
compose.log_info('[BEGIN] %s', msg)
|
||||
|
||||
prefixes = {
|
||||
'repos': lambda: os.path.join(compose.paths.work.topdir(
|
||||
arch="global"), "download") + "/",
|
||||
'koji': lambda: pungi.wrappers.kojiwrapper.KojiWrapper(
|
||||
compose.conf['koji_profile']).koji_module.config.topdir.rstrip("/") + "/"
|
||||
}
|
||||
path_prefix = prefixes[compose.conf['pkgset_source']]()
|
||||
pkglist = compose.paths.work.lookaside_package_list(arch=arch, variant=variant)
|
||||
with open(pkglist, 'w') as f:
|
||||
for packages in pkg_map[arch][variant.uid].values():
|
||||
for pkg in packages:
|
||||
pkg = pkg['path']
|
||||
if path_prefix and pkg.startswith(path_prefix):
|
||||
pkg = pkg[len(path_prefix):]
|
||||
f.write('%s\n' % pkg)
|
||||
|
||||
cr = CreaterepoWrapper(compose.conf['createrepo_c'])
|
||||
cmd = cr.get_createrepo_cmd(path_prefix, update=True, database=True, skip_stat=True,
|
||||
pkglist=pkglist,
|
||||
outputdir=repo,
|
||||
baseurl="file://%s" % path_prefix,
|
||||
workers=compose.conf["createrepo_num_workers"],
|
||||
update_md_path=compose.paths.work.arch_repo(arch))
|
||||
run(cmd,
|
||||
logfile=compose.paths.log.log_file(arch, "lookaside_repo_%s" % (variant.uid)),
|
||||
show_cmd=True)
|
||||
compose.log_info('[DONE ] %s', msg)
|
||||
|
||||
return repo
|
||||
|
||||
|
||||
def _update_config(compose, variant_uid, arch, repo):
|
||||
"""
|
||||
Add the variant lookaside repository into the configuration.
|
||||
"""
|
||||
lookasides = compose.conf.setdefault('gather_lookaside_repos', [])
|
||||
lookasides.append(('^%s$' % variant_uid, {arch: repo}))
|
||||
|
||||
|
||||
def _update_lookaside_config(compose, variant, arch, pkg_map):
|
||||
"""
|
||||
Make sure lookaside repo for all variants that the given one depends on
|
||||
exist, and that configuration is updated to use those repos.
|
||||
"""
|
||||
for dest, lookaside_variant_uid in compose.conf.get('variant_as_lookaside', []):
|
||||
lookaside_variant = compose.all_variants[lookaside_variant_uid]
|
||||
if dest != variant.uid:
|
||||
continue
|
||||
if arch not in lookaside_variant.arches:
|
||||
compose.log_warning('[SKIP] Skipping lookaside from %s for %s.%s due to arch mismatch',
|
||||
lookaside_variant.uid, variant.uid, arch)
|
||||
continue
|
||||
repo = _make_lookaside_repo(compose, lookaside_variant, arch, pkg_map)
|
||||
_update_config(compose, variant.uid, arch, repo)
|
||||
|
||||
|
||||
def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltree=False):
|
||||
"""Run gathering on all arches of all variants of given type.
|
||||
|
||||
@ -330,6 +405,12 @@ def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltr
|
||||
if exclude_fulltree:
|
||||
for pkg_name, pkg_arch in get_parent_pkgs(arch, variant, result)["srpm"]:
|
||||
fulltree_excludes.add(pkg_name)
|
||||
|
||||
# Get lookaside repos for this variant from other variants. Based
|
||||
# on the ordering we already know that we have the packages from
|
||||
# there.
|
||||
_update_lookaside_config(compose, variant, arch, result)
|
||||
|
||||
pkg_map = gather_packages(compose, arch, variant, package_sets, fulltree_excludes=fulltree_excludes)
|
||||
result.setdefault(arch, {})[variant.uid] = pkg_map
|
||||
|
||||
|
||||
@ -901,3 +901,176 @@ class TestGetPackagesToGather(helpers.PungiTestCase):
|
||||
|
||||
self.assertItemsEqual(packages, ["foo", "pkg", "foo2.x86_64"])
|
||||
self.assertItemsEqual(groups, ["core"])
|
||||
|
||||
|
||||
class TestUpdateConfig(unittest.TestCase):
|
||||
|
||||
def test_add_to_empty(self):
|
||||
compose = mock.Mock(conf={})
|
||||
gather._update_config(compose, 'Server', 'x86_64', '/tmp/foo')
|
||||
self.assertEqual(compose.conf,
|
||||
{'gather_lookaside_repos': [
|
||||
('^Server$', {'x86_64': '/tmp/foo'})
|
||||
]})
|
||||
|
||||
def test_add_to_existing(self):
|
||||
compose = mock.Mock(conf={'gather_lookaside_repos': [
|
||||
('^Server$', {'x86_64': '/tmp/bar'}),
|
||||
]})
|
||||
gather._update_config(compose, 'Server', 'x86_64', '/tmp/foo')
|
||||
self.assertEqual(compose.conf,
|
||||
{'gather_lookaside_repos': [
|
||||
('^Server$', {'x86_64': '/tmp/bar'}),
|
||||
('^Server$', {'x86_64': '/tmp/foo'})
|
||||
]})
|
||||
|
||||
|
||||
class TestUpdateLookasideConfig(helpers.PungiTestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestUpdateLookasideConfig, self).setUp()
|
||||
self.compose = helpers.DummyCompose(self.topdir, {})
|
||||
self.pkg_map = mock.Mock()
|
||||
|
||||
@mock.patch('pungi.phases.gather._update_config')
|
||||
@mock.patch('pungi.phases.gather._make_lookaside_repo')
|
||||
def test_no_config(self, mock_make_repo, mock_update_config):
|
||||
gather._update_lookaside_config(self.compose, self.compose.variants['Server'],
|
||||
'x86_64', self.pkg_map)
|
||||
self.assertEqual(mock_make_repo.call_args_list, [])
|
||||
self.assertEqual(mock_update_config.call_args_list, [])
|
||||
|
||||
@mock.patch('pungi.phases.gather._update_config')
|
||||
@mock.patch('pungi.phases.gather._make_lookaside_repo')
|
||||
def test_no_matching_config(self, mock_make_repo, mock_update_config):
|
||||
self.compose.conf['variant_as_lookaside'] = [('Everything', 'Client')]
|
||||
gather._update_lookaside_config(self.compose, self.compose.variants['Server'],
|
||||
'x86_64', self.pkg_map)
|
||||
self.assertEqual(mock_make_repo.call_args_list, [])
|
||||
self.assertEqual(mock_update_config.call_args_list, [])
|
||||
|
||||
@mock.patch('pungi.phases.gather._update_config')
|
||||
@mock.patch('pungi.phases.gather._make_lookaside_repo')
|
||||
def test_missing_arch(self, mock_make_repo, mock_update_config):
|
||||
# Client only has amd64
|
||||
self.compose.conf['variant_as_lookaside'] = [('Server', 'Client')]
|
||||
gather._update_lookaside_config(self.compose, self.compose.variants['Server'],
|
||||
'x86_64', self.pkg_map)
|
||||
self.assertEqual(len(self.compose.log_warning.call_args_list), 1)
|
||||
self.assertEqual(mock_make_repo.call_args_list, [])
|
||||
self.assertEqual(mock_update_config.call_args_list, [])
|
||||
|
||||
@mock.patch('pungi.phases.gather._update_config')
|
||||
@mock.patch('pungi.phases.gather._make_lookaside_repo')
|
||||
def test_match(self, mock_make_repo, mock_update_config):
|
||||
self.compose.conf['variant_as_lookaside'] = [('Server', 'Everything')]
|
||||
gather._update_lookaside_config(self.compose, self.compose.variants['Server'],
|
||||
'x86_64', self.pkg_map)
|
||||
self.assertEqual(len(self.compose.log_warning.call_args_list), 0)
|
||||
self.assertEqual(mock_make_repo.call_args_list,
|
||||
[mock.call(self.compose,
|
||||
self.compose.variants['Everything'],
|
||||
'x86_64',
|
||||
self.pkg_map)])
|
||||
self.assertEqual(mock_update_config.call_args_list,
|
||||
[mock.call(self.compose, 'Server', 'x86_64',
|
||||
mock_make_repo.return_value)])
|
||||
|
||||
|
||||
class TestMakeLookasideRepo(helpers.PungiTestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestMakeLookasideRepo, self).setUp()
|
||||
self.compose = helpers.DummyCompose(self.topdir, {})
|
||||
self.variant = self.compose.variants['Server']
|
||||
self.arch = 'x86_64'
|
||||
self.repodir = self.compose.paths.work.lookaside_repo(self.arch, self.variant, create_dir=False)
|
||||
self.pkglist = self.compose.paths.work.lookaside_package_list(self.arch, self.variant)
|
||||
|
||||
@mock.patch('pungi.phases.gather.run')
|
||||
def test_existing_repo(self, mock_run):
|
||||
helpers.touch(os.path.join(self.repodir, 'repodata', 'primary.xml'))
|
||||
repopath = gather._make_lookaside_repo(self.compose, self.variant, self.arch, {})
|
||||
self.assertEqual(self.repodir, repopath)
|
||||
self.assertFalse(os.path.exists(self.pkglist))
|
||||
self.assertEqual(mock_run.call_args_list, [])
|
||||
|
||||
def assertCorrect(self, repopath, path_prefix, MockCR, mock_run):
|
||||
with open(self.pkglist) as f:
|
||||
packages = f.read().splitlines()
|
||||
self.assertItemsEqual(packages,
|
||||
['pkg/pkg-1.0-1.x86_64.rpm',
|
||||
'pkg/pkg-debuginfo-1.0-1.x86_64.rpm',
|
||||
'pkg/pkg-1.0-1.src.rpm'])
|
||||
|
||||
self.assertEqual(self.repodir, repopath)
|
||||
print(MockCR.return_value.get_createrepo_cmd.call_args_list)
|
||||
print([mock.call(path_prefix, update=True, database=True, skip_stat=True,
|
||||
pkglist=self.pkglist, outputdir=repopath,
|
||||
baseurl="file://%s" % path_prefix, workers=3,
|
||||
update_md_path=self.compose.paths.work.arch_repo(self.arch))])
|
||||
self.assertEqual(MockCR.return_value.get_createrepo_cmd.call_args_list,
|
||||
[mock.call(path_prefix, update=True, database=True, skip_stat=True,
|
||||
pkglist=self.pkglist, outputdir=repopath,
|
||||
baseurl="file://%s" % path_prefix, workers=3,
|
||||
update_md_path=self.compose.paths.work.arch_repo(self.arch))])
|
||||
self.assertEqual(mock_run.call_args_list,
|
||||
[mock.call(MockCR.return_value.get_createrepo_cmd.return_value,
|
||||
logfile=os.path.join(
|
||||
self.topdir, 'logs', self.arch,
|
||||
'lookaside_repo_Server.%s.log' % self.arch),
|
||||
show_cmd=True)])
|
||||
|
||||
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
|
||||
@mock.patch('pungi.phases.gather.CreaterepoWrapper')
|
||||
@mock.patch('pungi.phases.gather.run')
|
||||
def test_create_repo_koji_pkgset(self, mock_run, MockCR, MockKW):
|
||||
self.compose.conf.update({
|
||||
'pkgset_source': 'koji',
|
||||
'koji_profile': 'koji',
|
||||
})
|
||||
|
||||
pkg_map = {
|
||||
self.arch: {
|
||||
self.variant.uid: {
|
||||
'rpm': [{'path': '/tmp/packages/pkg/pkg-1.0-1.x86_64.rpm'}],
|
||||
'debuginfo': [{'path': '/tmp/packages/pkg/pkg-debuginfo-1.0-1.x86_64.rpm'}],
|
||||
'srpm': [{'path': '/tmp/packages/pkg/pkg-1.0-1.src.rpm'}],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
MockKW.return_value.koji_module.config.topdir = '/tmp/packages'
|
||||
|
||||
repopath = gather._make_lookaside_repo(self.compose, self.variant, self.arch, pkg_map)
|
||||
|
||||
self.assertCorrect(repopath, '/tmp/packages/', MockCR, mock_run)
|
||||
|
||||
@mock.patch('pungi.phases.gather.CreaterepoWrapper')
|
||||
@mock.patch('pungi.phases.gather.run')
|
||||
def test_create_repo_repos_pkgset(self, mock_run, MockCR):
|
||||
self.compose.conf.update({
|
||||
'pkgset_source': 'repos',
|
||||
})
|
||||
|
||||
dl_dir = self.compose.paths.work.topdir('global')
|
||||
|
||||
pkg_map = {
|
||||
self.arch: {
|
||||
self.variant.uid: {
|
||||
'rpm': [
|
||||
{'path': os.path.join(dl_dir, 'download/pkg/pkg-1.0-1.x86_64.rpm')}
|
||||
],
|
||||
'debuginfo': [
|
||||
{'path': os.path.join(dl_dir, 'download/pkg/pkg-debuginfo-1.0-1.x86_64.rpm')}
|
||||
],
|
||||
'srpm': [
|
||||
{'path': os.path.join(dl_dir, 'download/pkg/pkg-1.0-1.src.rpm')}
|
||||
],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
repopath = gather._make_lookaside_repo(self.compose, self.variant, self.arch, pkg_map)
|
||||
|
||||
self.assertCorrect(repopath, dl_dir + '/download/', MockCR, mock_run)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user