pkgset: Cherry-pick packages from Koji when we know already what packages will end up in compose
Merges: https://pagure.io/pungi/pull-request/763 Signed-off-by: Jan Kaluza <jkaluza@redhat.com>
This commit is contained in:
parent
2819311d6e
commit
6208dae869
@ -335,12 +335,13 @@ def write_prepopulate_file(compose):
|
|||||||
shutil.rmtree(tmp_dir)
|
shutil.rmtree(tmp_dir)
|
||||||
|
|
||||||
|
|
||||||
def get_prepopulate_packages(compose, arch, variant):
|
def get_prepopulate_packages(compose, arch, variant, include_arch=True):
|
||||||
"""Read prepopulate file and return list of packages for given tree.
|
"""Read prepopulate file and return list of packages for given tree.
|
||||||
|
|
||||||
If ``variant`` is ``None``, all variants in the file are considered. The
|
If ``variant`` is ``None``, all variants in the file are considered. The
|
||||||
result of this function is a set of strings of format
|
result of this function is a set of strings of format
|
||||||
``package_name.arch``.
|
``package_name.arch``. If ``include_arch`` is False, the ".arch" suffix
|
||||||
|
is not included in packages in returned list.
|
||||||
"""
|
"""
|
||||||
result = set()
|
result = set()
|
||||||
|
|
||||||
@ -361,7 +362,10 @@ def get_prepopulate_packages(compose, arch, variant):
|
|||||||
raise ValueError(
|
raise ValueError(
|
||||||
"Incompatible package arch '%s' for tree arch '%s' in prepopulate package '%s'"
|
"Incompatible package arch '%s' for tree arch '%s' in prepopulate package '%s'"
|
||||||
% (pkg_arch, arch, pkg_name))
|
% (pkg_arch, arch, pkg_name))
|
||||||
|
if include_arch:
|
||||||
result.add(i)
|
result.add(i)
|
||||||
|
else:
|
||||||
|
result.add(pkg_name)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
@ -491,3 +495,51 @@ def get_system_release_packages(compose, arch, variant, package_sets):
|
|||||||
filter_packages.add((pkg.name, None))
|
filter_packages.add((pkg.name, None))
|
||||||
|
|
||||||
return packages, filter_packages
|
return packages, filter_packages
|
||||||
|
|
||||||
|
|
||||||
|
def get_packages_to_gather(compose, arch=None, variant=None, include_arch=True,
|
||||||
|
include_prepopulated=False):
|
||||||
|
"""
|
||||||
|
Returns the list of names of packages and list of names of groups which
|
||||||
|
would be included in a compose as GATHER phase result.
|
||||||
|
This works only for "comps" or "json" gather_source. For "module"
|
||||||
|
gather_source, this always return an empty list, because it is not clear
|
||||||
|
what packages will end up in a compose before the gather phase is run.
|
||||||
|
|
||||||
|
:param str arch: Arch to return packages for. If not set, returns packages
|
||||||
|
for all arches.
|
||||||
|
:param Variant variant: Variant to return packages for, If not set, returns
|
||||||
|
packages for all variants of a compose.
|
||||||
|
:param include_arch: When True, the arch of package will be included in
|
||||||
|
returned list as ["pkg_name.arch", ...]. Otherwise only
|
||||||
|
["pkg_name", ...] is returned.
|
||||||
|
:param include_prepopulated: When True, the prepopulated packages will
|
||||||
|
be included in a list of packages.
|
||||||
|
"""
|
||||||
|
if compose.conf["gather_source"] == "module":
|
||||||
|
return []
|
||||||
|
|
||||||
|
arches = [arch] if arch else compose.get_arches()
|
||||||
|
|
||||||
|
GatherSource = get_gather_source(compose.conf["gather_source"])
|
||||||
|
src = GatherSource(compose)
|
||||||
|
|
||||||
|
packages = set([])
|
||||||
|
groups = set([])
|
||||||
|
for arch in arches:
|
||||||
|
pkgs, grps = src(arch, variant)
|
||||||
|
groups = groups.union(set(grps))
|
||||||
|
|
||||||
|
additional_packages = get_additional_packages(compose, arch, None)
|
||||||
|
for pkg_name, pkg_arch in pkgs | additional_packages:
|
||||||
|
if not include_arch or pkg_arch is None:
|
||||||
|
packages.add(pkg_name)
|
||||||
|
else:
|
||||||
|
packages.add("%s.%s" % (pkg_name, pkg_arch))
|
||||||
|
|
||||||
|
if include_prepopulated:
|
||||||
|
prepopulated = get_prepopulate_packages(
|
||||||
|
compose, arch, variant, include_arch)
|
||||||
|
packages = packages.union(prepopulated)
|
||||||
|
|
||||||
|
return list(packages), list(groups)
|
||||||
|
@ -208,10 +208,13 @@ class FilelistPackageSet(PackageSetBase):
|
|||||||
|
|
||||||
|
|
||||||
class KojiPackageSet(PackageSetBase):
|
class KojiPackageSet(PackageSetBase):
|
||||||
def __init__(self, koji_wrapper, sigkey_ordering, arches=None, logger=None):
|
def __init__(self, koji_wrapper, sigkey_ordering, arches=None, logger=None,
|
||||||
|
packages=None):
|
||||||
super(KojiPackageSet, self).__init__(sigkey_ordering=sigkey_ordering,
|
super(KojiPackageSet, self).__init__(sigkey_ordering=sigkey_ordering,
|
||||||
arches=arches, logger=logger)
|
arches=arches, logger=logger)
|
||||||
self.koji_wrapper = koji_wrapper
|
self.koji_wrapper = koji_wrapper
|
||||||
|
# Names of packages to look for in the Koji tag.
|
||||||
|
self.packages = packages
|
||||||
|
|
||||||
def __getstate__(self):
|
def __getstate__(self):
|
||||||
result = self.__dict__.copy()
|
result = self.__dict__.copy()
|
||||||
@ -281,6 +284,7 @@ class KojiPackageSet(PackageSetBase):
|
|||||||
builds_by_id.setdefault(build_info["build_id"], build_info)
|
builds_by_id.setdefault(build_info["build_id"], build_info)
|
||||||
|
|
||||||
skipped_arches = []
|
skipped_arches = []
|
||||||
|
skipped_packages_count = 0
|
||||||
for rpm_info in rpms:
|
for rpm_info in rpms:
|
||||||
if self.arches and rpm_info["arch"] not in self.arches:
|
if self.arches and rpm_info["arch"] not in self.arches:
|
||||||
if rpm_info["arch"] not in skipped_arches:
|
if rpm_info["arch"] not in skipped_arches:
|
||||||
@ -288,11 +292,20 @@ class KojiPackageSet(PackageSetBase):
|
|||||||
skipped_arches.append(rpm_info["arch"])
|
skipped_arches.append(rpm_info["arch"])
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
if self.packages and rpm_info['name'] not in self.packages:
|
||||||
|
skipped_packages_count += 1
|
||||||
|
continue
|
||||||
|
|
||||||
build_info = builds_by_id[rpm_info["build_id"]]
|
build_info = builds_by_id[rpm_info["build_id"]]
|
||||||
if rpm_info["arch"] in ("src", "nosrc"):
|
if rpm_info["arch"] in ("src", "nosrc"):
|
||||||
result_srpms.append((rpm_info, build_info))
|
result_srpms.append((rpm_info, build_info))
|
||||||
else:
|
else:
|
||||||
result_rpms.append((rpm_info, build_info))
|
result_rpms.append((rpm_info, build_info))
|
||||||
|
|
||||||
|
if skipped_packages_count:
|
||||||
|
self.log_debug("Skipped %d packages, not marked as to be "
|
||||||
|
"included in a compose." % skipped_packages_count)
|
||||||
|
|
||||||
result = self.read_packages(result_rpms, result_srpms)
|
result = self.read_packages(result_rpms, result_srpms)
|
||||||
|
|
||||||
# Create a log with package NEVRAs and the tag they are coming from
|
# Create a log with package NEVRAs and the tag they are coming from
|
||||||
|
@ -21,11 +21,13 @@ import re
|
|||||||
from kobo.shortcuts import force_list
|
from kobo.shortcuts import force_list
|
||||||
|
|
||||||
import pungi.wrappers.kojiwrapper
|
import pungi.wrappers.kojiwrapper
|
||||||
|
from pungi.wrappers.comps import CompsWrapper
|
||||||
import pungi.phases.pkgset.pkgsets
|
import pungi.phases.pkgset.pkgsets
|
||||||
from pungi.arch import get_valid_arches
|
from pungi.arch import get_valid_arches
|
||||||
from pungi.util import is_arch_multilib, retry
|
from pungi.util import is_arch_multilib, retry
|
||||||
|
|
||||||
from pungi.phases.pkgset.common import create_arch_repos, create_global_repo, populate_arch_pkgsets
|
from pungi.phases.pkgset.common import create_arch_repos, create_global_repo, populate_arch_pkgsets
|
||||||
|
from pungi.phases.gather import get_packages_to_gather
|
||||||
|
|
||||||
|
|
||||||
import pungi.phases.pkgset.source
|
import pungi.phases.pkgset.source
|
||||||
@ -163,6 +165,20 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
|
|||||||
# List of compose_tags per variant
|
# List of compose_tags per variant
|
||||||
variant_tags = {}
|
variant_tags = {}
|
||||||
|
|
||||||
|
# In case we use "nodeps" gather_method, we might now the final list of
|
||||||
|
# packages which will end up in the compose even now, so instead of
|
||||||
|
# reading all the packages from Koji tag, we can just cherry-pick the ones
|
||||||
|
# which are really needed to do the compose and safe lot of time and
|
||||||
|
# resources here.
|
||||||
|
packages_to_gather = []
|
||||||
|
if compose.conf["gather_method"] == "nodeps":
|
||||||
|
packages_to_gather, groups = get_packages_to_gather(
|
||||||
|
compose, include_arch=False, include_prepopulated=True)
|
||||||
|
if groups:
|
||||||
|
comps = CompsWrapper(compose.paths.work.comps())
|
||||||
|
for group in groups:
|
||||||
|
packages_to_gather += comps.get_packages(group)
|
||||||
|
|
||||||
session = get_pdc_client_session(compose)
|
session = get_pdc_client_session(compose)
|
||||||
for variant in compose.all_variants.values():
|
for variant in compose.all_variants.values():
|
||||||
variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
|
variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
|
||||||
@ -219,7 +235,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
|
|||||||
"'%s'" % compose_tag)
|
"'%s'" % compose_tag)
|
||||||
pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
|
pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
|
||||||
koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
|
koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
|
||||||
arches=all_arches)
|
arches=all_arches, packages=packages_to_gather)
|
||||||
# Create a filename for log with package-to-tag mapping. The tag
|
# Create a filename for log with package-to-tag mapping. The tag
|
||||||
# name is included in filename, so any slashes in it are replaced
|
# name is included in filename, so any slashes in it are replaced
|
||||||
# with underscores just to be safe.
|
# with underscores just to be safe.
|
||||||
|
@ -25,7 +25,7 @@ from pungi.util import makedirs, is_arch_multilib
|
|||||||
from pungi.wrappers.pungi import PungiWrapper
|
from pungi.wrappers.pungi import PungiWrapper
|
||||||
|
|
||||||
from pungi.phases.pkgset.common import create_global_repo, create_arch_repos, populate_arch_pkgsets
|
from pungi.phases.pkgset.common import create_global_repo, create_arch_repos, populate_arch_pkgsets
|
||||||
from pungi.phases.gather import get_prepopulate_packages, get_additional_packages
|
from pungi.phases.gather import get_prepopulate_packages, get_packages_to_gather
|
||||||
from pungi.linker import LinkerThread, LinkerPool
|
from pungi.linker import LinkerThread, LinkerPool
|
||||||
|
|
||||||
|
|
||||||
@ -160,24 +160,7 @@ def write_pungi_config(compose, arch, variant, repos=None, comps_repo=None, pack
|
|||||||
return
|
return
|
||||||
|
|
||||||
compose.log_info(msg)
|
compose.log_info(msg)
|
||||||
|
packages, grps = get_packages_to_gather(compose, arch, variant)
|
||||||
# TODO move to a function
|
|
||||||
gather_source = "GatherSource%s" % compose.conf["gather_source"]
|
|
||||||
from pungi.phases.gather.source import GatherSourceContainer
|
|
||||||
import pungi.phases.gather.sources
|
|
||||||
GatherSourceContainer.register_module(pungi.phases.gather.sources)
|
|
||||||
container = GatherSourceContainer()
|
|
||||||
SourceClass = container[gather_source]
|
|
||||||
src = SourceClass(compose)
|
|
||||||
|
|
||||||
packages = []
|
|
||||||
pkgs, grps = src(arch, variant)
|
|
||||||
additional_packages = get_additional_packages(compose, arch, None)
|
|
||||||
for pkg_name, pkg_arch in pkgs | additional_packages:
|
|
||||||
if pkg_arch is None:
|
|
||||||
packages.append(pkg_name)
|
|
||||||
else:
|
|
||||||
packages.append("%s.%s" % (pkg_name, pkg_arch))
|
|
||||||
|
|
||||||
# include *all* packages providing system-release
|
# include *all* packages providing system-release
|
||||||
if "system-release" not in packages:
|
if "system-release" not in packages:
|
||||||
|
@ -755,6 +755,17 @@ class TestGetPrepopulate(helpers.PungiTestCase):
|
|||||||
"bar.x86_64"]
|
"bar.x86_64"]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_for_all_variants_include_arch_set_to_false(self):
|
||||||
|
helpers.copy_fixture('prepopulate.json',
|
||||||
|
os.path.join(self.topdir, 'work', 'global', 'prepopulate.json'))
|
||||||
|
self.assertItemsEqual(
|
||||||
|
gather.get_prepopulate_packages(self.compose, 'x86_64', None,
|
||||||
|
include_arch=False),
|
||||||
|
["foo-common",
|
||||||
|
"foo",
|
||||||
|
"bar"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestGatherPhase(helpers.PungiTestCase):
|
class TestGatherPhase(helpers.PungiTestCase):
|
||||||
@mock.patch('pungi.phases.gather.link_files')
|
@mock.patch('pungi.phases.gather.link_files')
|
||||||
@ -797,3 +808,74 @@ class TestGatherPhase(helpers.PungiTestCase):
|
|||||||
|
|
||||||
self.assertEqual(gather_wrapper.call_args_list, [])
|
self.assertEqual(gather_wrapper.call_args_list, [])
|
||||||
self.assertTrue(os.path.isfile(os.path.join(self.topdir, 'compose', 'metadata', 'rpms.json')))
|
self.assertTrue(os.path.isfile(os.path.join(self.topdir, 'compose', 'metadata', 'rpms.json')))
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetPackagesToGather(helpers.PungiTestCase):
|
||||||
|
def setUp(self):
|
||||||
|
super(TestGetPackagesToGather, self).setUp()
|
||||||
|
self.compose = helpers.DummyCompose(self.topdir, {
|
||||||
|
'additional_packages': [
|
||||||
|
('.*', {'*': ['pkg', 'foo2.x86_64']}),
|
||||||
|
]
|
||||||
|
})
|
||||||
|
helpers.copy_fixture('prepopulate.json',
|
||||||
|
os.path.join(self.topdir, 'work', 'global', 'prepopulate.json'))
|
||||||
|
|
||||||
|
@mock.patch('pungi.phases.gather.get_gather_source')
|
||||||
|
def test_all_arches(self, get_gather_source):
|
||||||
|
get_gather_source.return_value = mock.Mock(
|
||||||
|
return_value=mock.Mock(return_value=(set([('foo', None)]), set(['core'])))
|
||||||
|
)
|
||||||
|
|
||||||
|
packages, groups = gather.get_packages_to_gather(self.compose)
|
||||||
|
|
||||||
|
self.assertItemsEqual(packages, ["foo", "foo2.x86_64", "pkg"])
|
||||||
|
self.assertItemsEqual(groups, ["core"])
|
||||||
|
|
||||||
|
@mock.patch('pungi.phases.gather.get_gather_source')
|
||||||
|
def test_all_include_arch_set_to_false(self, get_gather_source):
|
||||||
|
get_gather_source.return_value = mock.Mock(
|
||||||
|
return_value=mock.Mock(return_value=(set([('foo', None)]), set(['core'])))
|
||||||
|
)
|
||||||
|
|
||||||
|
packages, groups = gather.get_packages_to_gather(self.compose, include_arch=False)
|
||||||
|
|
||||||
|
self.assertItemsEqual(packages, ["foo", "foo2", "pkg"])
|
||||||
|
self.assertItemsEqual(groups, ["core"])
|
||||||
|
|
||||||
|
@mock.patch('pungi.phases.gather.get_gather_source')
|
||||||
|
def test_all_include_prepopulated(self, get_gather_source):
|
||||||
|
get_gather_source.return_value = mock.Mock(
|
||||||
|
return_value=mock.Mock(return_value=(set([('foo', None)]), set(['core'])))
|
||||||
|
)
|
||||||
|
|
||||||
|
packages, groups = gather.get_packages_to_gather(self.compose, include_prepopulated=True)
|
||||||
|
|
||||||
|
self.assertItemsEqual(packages, ["foo", "pkg", "foo-common.noarch",
|
||||||
|
"foo.x86_64", "foo.i686", "foo2.x86_64",
|
||||||
|
"bar.x86_64"])
|
||||||
|
self.assertItemsEqual(groups, ["core"])
|
||||||
|
|
||||||
|
@mock.patch('pungi.phases.gather.get_gather_source')
|
||||||
|
def test_all_include_prepopulated_no_include_arch(self, get_gather_source):
|
||||||
|
get_gather_source.return_value = mock.Mock(
|
||||||
|
return_value=mock.Mock(return_value=(set([('foo', None)]), set(['core'])))
|
||||||
|
)
|
||||||
|
|
||||||
|
packages, groups = gather.get_packages_to_gather(self.compose, include_prepopulated=True,
|
||||||
|
include_arch=False)
|
||||||
|
|
||||||
|
self.assertItemsEqual(packages, ["foo", "pkg", "foo-common",
|
||||||
|
"foo2", "bar"])
|
||||||
|
self.assertItemsEqual(groups, ["core"])
|
||||||
|
|
||||||
|
@mock.patch('pungi.phases.gather.get_gather_source')
|
||||||
|
def test_all_one_arch(self, get_gather_source):
|
||||||
|
get_gather_source.return_value = mock.Mock(
|
||||||
|
return_value=mock.Mock(return_value=(set([('foo', None)]), set(['core'])))
|
||||||
|
)
|
||||||
|
|
||||||
|
packages, groups = gather.get_packages_to_gather(self.compose, "x86_64")
|
||||||
|
|
||||||
|
self.assertItemsEqual(packages, ["foo", "pkg", "foo2.x86_64"])
|
||||||
|
self.assertItemsEqual(groups, ["core"])
|
||||||
|
@ -275,6 +275,31 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
|
|||||||
self.assertRegexpMatches(str(ctx.exception),
|
self.assertRegexpMatches(str(ctx.exception),
|
||||||
r'^RPM\(s\) not found for sigs: .+Check log for details.+')
|
r'^RPM\(s\) not found for sigs: .+Check log for details.+')
|
||||||
|
|
||||||
|
def test_packages_attribute(self):
|
||||||
|
self._touch_files([
|
||||||
|
'rpms/pungi@4.1.3@3.fc25@noarch',
|
||||||
|
'rpms/pungi@4.1.3@3.fc25@src',
|
||||||
|
'rpms/bash@4.3.42@4.fc24@i686',
|
||||||
|
'rpms/bash@4.3.42@4.fc24@x86_64',
|
||||||
|
'rpms/bash@4.3.42@4.fc24@src',
|
||||||
|
'rpms/bash-debuginfo@4.3.42@4.fc24@i686',
|
||||||
|
'rpms/bash-debuginfo@4.3.42@4.fc24@x86_64',
|
||||||
|
])
|
||||||
|
|
||||||
|
pkgset = pkgsets.KojiPackageSet(self.koji_wrapper, [None],
|
||||||
|
packages=["bash"])
|
||||||
|
|
||||||
|
result = pkgset.populate('f25', logfile=self.topdir + '/pkgset.log')
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
self.koji_wrapper.koji_proxy.mock_calls,
|
||||||
|
[mock.call.listTaggedRPMS('f25', event=None, inherit=True, latest=True)])
|
||||||
|
|
||||||
|
self.assertPkgsetEqual(result,
|
||||||
|
{'src': ['rpms/bash@4.3.42@4.fc24@src'],
|
||||||
|
'i686': ['rpms/bash@4.3.42@4.fc24@i686'],
|
||||||
|
'x86_64': ['rpms/bash@4.3.42@4.fc24@x86_64']})
|
||||||
|
|
||||||
|
|
||||||
@mock.patch('kobo.pkgset.FileCache', new=MockFileCache)
|
@mock.patch('kobo.pkgset.FileCache', new=MockFileCache)
|
||||||
class TestMergePackageSets(PkgsetCompareMixin, unittest.TestCase):
|
class TestMergePackageSets(PkgsetCompareMixin, unittest.TestCase):
|
||||||
|
@ -167,6 +167,27 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
|
|||||||
[mock.call.save_file_list(self.topdir + '/work/global/package_list/global.conf',
|
[mock.call.save_file_list(self.topdir + '/work/global/package_list/global.conf',
|
||||||
remove_path_prefix='/prefix')])
|
remove_path_prefix='/prefix')])
|
||||||
|
|
||||||
|
@mock.patch('cPickle.dumps')
|
||||||
|
@mock.patch('pungi.phases.pkgset.pkgsets.KojiPackageSet.populate')
|
||||||
|
@mock.patch('pungi.phases.pkgset.pkgsets.KojiPackageSet.save_file_list')
|
||||||
|
def test_populate_packages_to_gather(self, save_file_list, popuplate,
|
||||||
|
pickle_dumps):
|
||||||
|
self.compose = helpers.DummyCompose(self.topdir, {
|
||||||
|
'gather_method': 'nodeps',
|
||||||
|
'gather_source': 'none',
|
||||||
|
'pkgset_koji_tag': 'f25',
|
||||||
|
'sigkeys': mock.Mock(),
|
||||||
|
'additional_packages': [
|
||||||
|
('.*', {'*': ['pkg', 'foo.x86_64']}),
|
||||||
|
]
|
||||||
|
})
|
||||||
|
self.compose.DEBUG = False
|
||||||
|
pickle_dumps.return_value = 'DATA'
|
||||||
|
|
||||||
|
pkgset = source_koji.populate_global_pkgset(
|
||||||
|
self.compose, self.koji_wrapper, '/prefix', 123456)
|
||||||
|
self.assertItemsEqual(pkgset.packages, ["pkg", "foo"])
|
||||||
|
|
||||||
|
|
||||||
class TestGetPackageSetFromKoji(helpers.PungiTestCase):
|
class TestGetPackageSetFromKoji(helpers.PungiTestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
|
Loading…
Reference in New Issue
Block a user