pkgset: Cherry-pick packages from Koji when we know already what packages will end up in compose

Merges: https://pagure.io/pungi/pull-request/763
Signed-off-by: Jan Kaluza <jkaluza@redhat.com>
This commit is contained in:
Jan Kaluza 2017-10-16 14:20:37 +02:00 committed by Lubomír Sedlář
parent 2819311d6e
commit 6208dae869
7 changed files with 216 additions and 24 deletions

View File

@ -335,12 +335,13 @@ def write_prepopulate_file(compose):
shutil.rmtree(tmp_dir)
def get_prepopulate_packages(compose, arch, variant):
def get_prepopulate_packages(compose, arch, variant, include_arch=True):
"""Read prepopulate file and return list of packages for given tree.
If ``variant`` is ``None``, all variants in the file are considered. The
result of this function is a set of strings of format
``package_name.arch``.
``package_name.arch``. If ``include_arch`` is False, the ".arch" suffix
is not included in packages in returned list.
"""
result = set()
@ -361,7 +362,10 @@ def get_prepopulate_packages(compose, arch, variant):
raise ValueError(
"Incompatible package arch '%s' for tree arch '%s' in prepopulate package '%s'"
% (pkg_arch, arch, pkg_name))
if include_arch:
result.add(i)
else:
result.add(pkg_name)
return result
@ -491,3 +495,51 @@ def get_system_release_packages(compose, arch, variant, package_sets):
filter_packages.add((pkg.name, None))
return packages, filter_packages
def get_packages_to_gather(compose, arch=None, variant=None, include_arch=True,
include_prepopulated=False):
"""
Returns the list of names of packages and list of names of groups which
would be included in a compose as GATHER phase result.
This works only for "comps" or "json" gather_source. For "module"
gather_source, this always return an empty list, because it is not clear
what packages will end up in a compose before the gather phase is run.
:param str arch: Arch to return packages for. If not set, returns packages
for all arches.
:param Variant variant: Variant to return packages for, If not set, returns
packages for all variants of a compose.
:param include_arch: When True, the arch of package will be included in
returned list as ["pkg_name.arch", ...]. Otherwise only
["pkg_name", ...] is returned.
:param include_prepopulated: When True, the prepopulated packages will
be included in a list of packages.
"""
if compose.conf["gather_source"] == "module":
return []
arches = [arch] if arch else compose.get_arches()
GatherSource = get_gather_source(compose.conf["gather_source"])
src = GatherSource(compose)
packages = set([])
groups = set([])
for arch in arches:
pkgs, grps = src(arch, variant)
groups = groups.union(set(grps))
additional_packages = get_additional_packages(compose, arch, None)
for pkg_name, pkg_arch in pkgs | additional_packages:
if not include_arch or pkg_arch is None:
packages.add(pkg_name)
else:
packages.add("%s.%s" % (pkg_name, pkg_arch))
if include_prepopulated:
prepopulated = get_prepopulate_packages(
compose, arch, variant, include_arch)
packages = packages.union(prepopulated)
return list(packages), list(groups)

View File

@ -208,10 +208,13 @@ class FilelistPackageSet(PackageSetBase):
class KojiPackageSet(PackageSetBase):
def __init__(self, koji_wrapper, sigkey_ordering, arches=None, logger=None):
def __init__(self, koji_wrapper, sigkey_ordering, arches=None, logger=None,
packages=None):
super(KojiPackageSet, self).__init__(sigkey_ordering=sigkey_ordering,
arches=arches, logger=logger)
self.koji_wrapper = koji_wrapper
# Names of packages to look for in the Koji tag.
self.packages = packages
def __getstate__(self):
result = self.__dict__.copy()
@ -281,6 +284,7 @@ class KojiPackageSet(PackageSetBase):
builds_by_id.setdefault(build_info["build_id"], build_info)
skipped_arches = []
skipped_packages_count = 0
for rpm_info in rpms:
if self.arches and rpm_info["arch"] not in self.arches:
if rpm_info["arch"] not in skipped_arches:
@ -288,11 +292,20 @@ class KojiPackageSet(PackageSetBase):
skipped_arches.append(rpm_info["arch"])
continue
if self.packages and rpm_info['name'] not in self.packages:
skipped_packages_count += 1
continue
build_info = builds_by_id[rpm_info["build_id"]]
if rpm_info["arch"] in ("src", "nosrc"):
result_srpms.append((rpm_info, build_info))
else:
result_rpms.append((rpm_info, build_info))
if skipped_packages_count:
self.log_debug("Skipped %d packages, not marked as to be "
"included in a compose." % skipped_packages_count)
result = self.read_packages(result_rpms, result_srpms)
# Create a log with package NEVRAs and the tag they are coming from

View File

@ -21,11 +21,13 @@ import re
from kobo.shortcuts import force_list
import pungi.wrappers.kojiwrapper
from pungi.wrappers.comps import CompsWrapper
import pungi.phases.pkgset.pkgsets
from pungi.arch import get_valid_arches
from pungi.util import is_arch_multilib, retry
from pungi.phases.pkgset.common import create_arch_repos, create_global_repo, populate_arch_pkgsets
from pungi.phases.gather import get_packages_to_gather
import pungi.phases.pkgset.source
@ -163,6 +165,20 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
# List of compose_tags per variant
variant_tags = {}
# In case we use "nodeps" gather_method, we might now the final list of
# packages which will end up in the compose even now, so instead of
# reading all the packages from Koji tag, we can just cherry-pick the ones
# which are really needed to do the compose and safe lot of time and
# resources here.
packages_to_gather = []
if compose.conf["gather_method"] == "nodeps":
packages_to_gather, groups = get_packages_to_gather(
compose, include_arch=False, include_prepopulated=True)
if groups:
comps = CompsWrapper(compose.paths.work.comps())
for group in groups:
packages_to_gather += comps.get_packages(group)
session = get_pdc_client_session(compose)
for variant in compose.all_variants.values():
variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
@ -219,7 +235,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
"'%s'" % compose_tag)
pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
arches=all_arches)
arches=all_arches, packages=packages_to_gather)
# Create a filename for log with package-to-tag mapping. The tag
# name is included in filename, so any slashes in it are replaced
# with underscores just to be safe.

View File

@ -25,7 +25,7 @@ from pungi.util import makedirs, is_arch_multilib
from pungi.wrappers.pungi import PungiWrapper
from pungi.phases.pkgset.common import create_global_repo, create_arch_repos, populate_arch_pkgsets
from pungi.phases.gather import get_prepopulate_packages, get_additional_packages
from pungi.phases.gather import get_prepopulate_packages, get_packages_to_gather
from pungi.linker import LinkerThread, LinkerPool
@ -160,24 +160,7 @@ def write_pungi_config(compose, arch, variant, repos=None, comps_repo=None, pack
return
compose.log_info(msg)
# TODO move to a function
gather_source = "GatherSource%s" % compose.conf["gather_source"]
from pungi.phases.gather.source import GatherSourceContainer
import pungi.phases.gather.sources
GatherSourceContainer.register_module(pungi.phases.gather.sources)
container = GatherSourceContainer()
SourceClass = container[gather_source]
src = SourceClass(compose)
packages = []
pkgs, grps = src(arch, variant)
additional_packages = get_additional_packages(compose, arch, None)
for pkg_name, pkg_arch in pkgs | additional_packages:
if pkg_arch is None:
packages.append(pkg_name)
else:
packages.append("%s.%s" % (pkg_name, pkg_arch))
packages, grps = get_packages_to_gather(compose, arch, variant)
# include *all* packages providing system-release
if "system-release" not in packages:

View File

@ -755,6 +755,17 @@ class TestGetPrepopulate(helpers.PungiTestCase):
"bar.x86_64"]
)
def test_for_all_variants_include_arch_set_to_false(self):
helpers.copy_fixture('prepopulate.json',
os.path.join(self.topdir, 'work', 'global', 'prepopulate.json'))
self.assertItemsEqual(
gather.get_prepopulate_packages(self.compose, 'x86_64', None,
include_arch=False),
["foo-common",
"foo",
"bar"]
)
class TestGatherPhase(helpers.PungiTestCase):
@mock.patch('pungi.phases.gather.link_files')
@ -797,3 +808,74 @@ class TestGatherPhase(helpers.PungiTestCase):
self.assertEqual(gather_wrapper.call_args_list, [])
self.assertTrue(os.path.isfile(os.path.join(self.topdir, 'compose', 'metadata', 'rpms.json')))
class TestGetPackagesToGather(helpers.PungiTestCase):
def setUp(self):
super(TestGetPackagesToGather, self).setUp()
self.compose = helpers.DummyCompose(self.topdir, {
'additional_packages': [
('.*', {'*': ['pkg', 'foo2.x86_64']}),
]
})
helpers.copy_fixture('prepopulate.json',
os.path.join(self.topdir, 'work', 'global', 'prepopulate.json'))
@mock.patch('pungi.phases.gather.get_gather_source')
def test_all_arches(self, get_gather_source):
get_gather_source.return_value = mock.Mock(
return_value=mock.Mock(return_value=(set([('foo', None)]), set(['core'])))
)
packages, groups = gather.get_packages_to_gather(self.compose)
self.assertItemsEqual(packages, ["foo", "foo2.x86_64", "pkg"])
self.assertItemsEqual(groups, ["core"])
@mock.patch('pungi.phases.gather.get_gather_source')
def test_all_include_arch_set_to_false(self, get_gather_source):
get_gather_source.return_value = mock.Mock(
return_value=mock.Mock(return_value=(set([('foo', None)]), set(['core'])))
)
packages, groups = gather.get_packages_to_gather(self.compose, include_arch=False)
self.assertItemsEqual(packages, ["foo", "foo2", "pkg"])
self.assertItemsEqual(groups, ["core"])
@mock.patch('pungi.phases.gather.get_gather_source')
def test_all_include_prepopulated(self, get_gather_source):
get_gather_source.return_value = mock.Mock(
return_value=mock.Mock(return_value=(set([('foo', None)]), set(['core'])))
)
packages, groups = gather.get_packages_to_gather(self.compose, include_prepopulated=True)
self.assertItemsEqual(packages, ["foo", "pkg", "foo-common.noarch",
"foo.x86_64", "foo.i686", "foo2.x86_64",
"bar.x86_64"])
self.assertItemsEqual(groups, ["core"])
@mock.patch('pungi.phases.gather.get_gather_source')
def test_all_include_prepopulated_no_include_arch(self, get_gather_source):
get_gather_source.return_value = mock.Mock(
return_value=mock.Mock(return_value=(set([('foo', None)]), set(['core'])))
)
packages, groups = gather.get_packages_to_gather(self.compose, include_prepopulated=True,
include_arch=False)
self.assertItemsEqual(packages, ["foo", "pkg", "foo-common",
"foo2", "bar"])
self.assertItemsEqual(groups, ["core"])
@mock.patch('pungi.phases.gather.get_gather_source')
def test_all_one_arch(self, get_gather_source):
get_gather_source.return_value = mock.Mock(
return_value=mock.Mock(return_value=(set([('foo', None)]), set(['core'])))
)
packages, groups = gather.get_packages_to_gather(self.compose, "x86_64")
self.assertItemsEqual(packages, ["foo", "pkg", "foo2.x86_64"])
self.assertItemsEqual(groups, ["core"])

View File

@ -275,6 +275,31 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
self.assertRegexpMatches(str(ctx.exception),
r'^RPM\(s\) not found for sigs: .+Check log for details.+')
def test_packages_attribute(self):
self._touch_files([
'rpms/pungi@4.1.3@3.fc25@noarch',
'rpms/pungi@4.1.3@3.fc25@src',
'rpms/bash@4.3.42@4.fc24@i686',
'rpms/bash@4.3.42@4.fc24@x86_64',
'rpms/bash@4.3.42@4.fc24@src',
'rpms/bash-debuginfo@4.3.42@4.fc24@i686',
'rpms/bash-debuginfo@4.3.42@4.fc24@x86_64',
])
pkgset = pkgsets.KojiPackageSet(self.koji_wrapper, [None],
packages=["bash"])
result = pkgset.populate('f25', logfile=self.topdir + '/pkgset.log')
self.assertEqual(
self.koji_wrapper.koji_proxy.mock_calls,
[mock.call.listTaggedRPMS('f25', event=None, inherit=True, latest=True)])
self.assertPkgsetEqual(result,
{'src': ['rpms/bash@4.3.42@4.fc24@src'],
'i686': ['rpms/bash@4.3.42@4.fc24@i686'],
'x86_64': ['rpms/bash@4.3.42@4.fc24@x86_64']})
@mock.patch('kobo.pkgset.FileCache', new=MockFileCache)
class TestMergePackageSets(PkgsetCompareMixin, unittest.TestCase):

View File

@ -167,6 +167,27 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
[mock.call.save_file_list(self.topdir + '/work/global/package_list/global.conf',
remove_path_prefix='/prefix')])
@mock.patch('cPickle.dumps')
@mock.patch('pungi.phases.pkgset.pkgsets.KojiPackageSet.populate')
@mock.patch('pungi.phases.pkgset.pkgsets.KojiPackageSet.save_file_list')
def test_populate_packages_to_gather(self, save_file_list, popuplate,
pickle_dumps):
self.compose = helpers.DummyCompose(self.topdir, {
'gather_method': 'nodeps',
'gather_source': 'none',
'pkgset_koji_tag': 'f25',
'sigkeys': mock.Mock(),
'additional_packages': [
('.*', {'*': ['pkg', 'foo.x86_64']}),
]
})
self.compose.DEBUG = False
pickle_dumps.return_value = 'DATA'
pkgset = source_koji.populate_global_pkgset(
self.compose, self.koji_wrapper, '/prefix', 123456)
self.assertItemsEqual(pkgset.packages, ["pkg", "foo"])
class TestGetPackageSetFromKoji(helpers.PungiTestCase):
def setUp(self):