Support multiple sources in one variant

With this patch the gather_source option is no longer used. Instead, all
sources are always used. If they return at least some input packages,
then a configured method is used and the returned lists of packages from
all sources are merged.

The method used for gathering can be configured for each variant and
gather source separately.

Additional packages are only added to the comps source.

Each gathering step is logged separately. All the logs are preserved for
later inspection.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2017-11-07 14:16:37 +01:00
parent 0074fe3f2c
commit 364d7f5229
12 changed files with 199 additions and 136 deletions

View File

@ -35,7 +35,6 @@ Minimal Config Example
createrepo_checksum = "sha256" createrepo_checksum = "sha256"
# GATHER # GATHER
gather_source = "comps"
gather_method = "deps" gather_method = "deps"
greedy_method = "build" greedy_method = "build"
check_deps = False check_deps = False
@ -546,16 +545,13 @@ Gather Settings
Options Options
------- -------
**gather_source** [mandatory]
(*str*) -- from where to read initial package list; expected values:
``comps``, ``none``, ``module``
When ``comps`` is selected, you have to specify ``comps_file`` option. When
``module`` is selected, you have to :ref:`configure PDC API url <pdc-settings>`.
**gather_method** [mandatory] **gather_method** [mandatory]
(*str*) -- Options are ``deps`` and ``nodeps``. Specifies whether package (*str*|*dict*) -- Options are ``deps`` and ``nodeps``. Specifies whether
dependencies should be pulled in as well. package dependencies should be pulled in as well. Either a single value or
a dictionary mapping variant UID and source type to a value. Make sure only
one regex matches each variant, as there is no guarantee which value will
be used if there are multiple matching ones. All used sources must have a
configured method.
**gather_fulltree** = False **gather_fulltree** = False
(*bool*) -- When set to ``True`` all RPMs built from an SRPM will always be (*bool*) -- When set to ``True`` all RPMs built from an SRPM will always be
@ -671,9 +667,9 @@ Options
such cases are still reported as warnings in the log. such cases are still reported as warnings in the log.
**gather_source_mapping** **gather_source_mapping**
(*str*) -- Only use when ``gather_source = "json"``. The value should be a (*str*) -- JSON mapping with initial packages for the compose. The value
path to JSON file with following mapping: ``{variant: {arch: {rpm_name: should be a path to JSON file with following mapping: ``{variant: {arch:
[rpm_arch|None]}}}``. {rpm_name: [rpm_arch|None]}}}``.
**gather_profiler** = False **gather_profiler** = False
(*bool*) -- When set to ``True`` the gather tool will produce additional (*bool*) -- When set to ``True`` the gather tool will produce additional
@ -685,12 +681,24 @@ Example
------- -------
:: ::
gather_source = "comps"
gather_method = "deps" gather_method = "deps"
greedy_method = "build" greedy_method = "build"
check_deps = False check_deps = False
hashed_directories = True hashed_directories = True
gather_method = {
"^Everything$": {
"comps": "deps" # traditional content defined by comps groups
},
"^Modular$": {
"module": "nodeps" # Modules do not need dependencies
},
"^Mixed$": { # Mixed content in one variant
"comps": "deps",
"module": "nodeps"
}
}
additional_packages = [ additional_packages = [
# bz#123456 # bz#123456
('^(Workstation|Server)$', { ('^(Workstation|Server)$', {

View File

@ -8,9 +8,9 @@ a subset of the content targeted at a particular use case.
There are different types of variants. The type affects how packages are There are different types of variants. The type affects how packages are
gathered into the variant. gathered into the variant.
The inputs for gathering are defined by the ``gather_source`` option. It The inputs for gathering are defined by various gather sources. Packages from
provides a list of package names, comps groups names and a list of packages all sources are collected to create a big list of package names, comps groups
that should be filtered out. names and a list of packages that should be filtered out.
.. note:: .. note::
The inputs for both explicit package list and comps file are interpreted as The inputs for both explicit package list and comps file are interpreted as

View File

@ -554,12 +554,30 @@ def make_schema():
}, },
"gather_method": { "gather_method": {
"type": "string", "oneOf": [
"enum": ["deps", "nodeps"], {
"type": "object",
"patternProperties": {
".+": {
"type": "object",
"patternProperties": {
"^module|comps|json$": {
"type": "string",
"enum": ["deps", "nodeps"],
}
}
}
},
"additionalProperties": False,
},
{
"type": "string",
"enum": ["deps", "nodeps"],
}
],
}, },
"gather_source": { "gather_source": {
"type": "string", "deprecated": "remove it",
"enum": ["module", "json", "comps", "none"],
}, },
"gather_fulltree": { "gather_fulltree": {
"type": "boolean", "type": "boolean",
@ -706,7 +724,7 @@ def make_schema():
"type": "string", "type": "string",
"enum": ["lorax", "buildinstall"], "enum": ["lorax", "buildinstall"],
}, },
"buildinstall_topdir": {"type": "string"}, "buildinstall_topdir": {"type": "string"},
"buildinstall_kickstart": {"$ref": "#/definitions/str_or_scm_dict"}, "buildinstall_kickstart": {"$ref": "#/definitions/str_or_scm_dict"},
"buildinstall_use_guestmount": {"type": "boolean", "default": True}, "buildinstall_use_guestmount": {"type": "boolean", "default": True},
@ -1067,7 +1085,7 @@ def make_schema():
"release_is_layered", "release_is_layered",
"variants_file", "sigkeys", "variants_file", "sigkeys",
"runroot", "pkgset_source", "runroot", "pkgset_source",
"gather_source", "gather_method"], "gather_method"],
"additionalProperties": False, "additionalProperties": False,
} }
@ -1118,15 +1136,6 @@ def get_num_cpus():
# encountered and its value satisfies the lambda, an error is reported for each # encountered and its value satisfies the lambda, an error is reported for each
# missing (for requires) option in the list. # missing (for requires) option in the list.
CONFIG_DEPS = { CONFIG_DEPS = {
"gather_source": {
"conflicts": [
(lambda val: val != 'json', ['gather_source_mapping']),
],
"requires": [
(lambda val: val == 'json', ['gather_source_mapping']),
(lambda val: val == 'comps', ['comps_file']),
]
},
"productimg": { "productimg": {
"requires": ( "requires": (
(lambda x: bool(x), ["productimg_install_class"]), (lambda x: bool(x), ["productimg_install_class"]),

View File

@ -107,32 +107,37 @@ class WorkPaths(object):
path = os.path.join(path, file_name) path = os.path.join(path, file_name)
return path return path
def pungi_conf(self, arch=None, variant=None, create_dir=True): def pungi_conf(self, arch=None, variant=None, create_dir=True, source_name=None):
""" """
Examples: Examples:
work/x86_64/pungi/x86_64.conf work/x86_64/pungi/x86_64.conf
work/x86_64/pungi/Server.x86_64.conf work/x86_64/pungi/Server.x86_64.conf
""" """
arch = arch or "global" arch = arch or "global"
if variant is None: file_name = ''
file_name = "%s.conf" % arch if variant:
else: file_name += variant.uid + '.'
file_name = "%s.%s.conf" % (variant.uid, arch) file_name += arch + '.'
if source_name:
file_name += source_name + '.'
file_name += 'conf'
path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi") path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi")
if create_dir: if create_dir:
makedirs(path) makedirs(path)
path = os.path.join(path, file_name) path = os.path.join(path, file_name)
return path return path
def pungi_log(self, arch=None, variant=None, create_dir=True): def pungi_log(self, arch=None, variant=None, create_dir=True, source_name=None):
""" """
Examples: Examples:
work/x86_64/pungi/x86_64.log work/x86_64/pungi/x86_64.log
work/x86_64/pungi/Server.x86_64.log work/x86_64/pungi/Server.x86_64.log
""" """
path = self.pungi_conf(arch, variant, create_dir=create_dir) path = self.pungi_conf(arch, variant, create_dir=create_dir)
path = path[:-5] + ".log" path = path[:-5]
return path if source_name:
path += '.' + source_name
return path + ".log"
def pungi_cache_dir(self, arch, variant=None, create_dir=True): def pungi_cache_dir(self, arch, variant=None, create_dir=True):
""" """

View File

@ -24,7 +24,7 @@ from productmd.rpms import Rpms
from pungi.wrappers.scm import get_file_from_scm from pungi.wrappers.scm import get_file_from_scm
from .link import link_files from .link import link_files
from pungi.util import get_arch_variant_data, get_arch_data from pungi.util import get_arch_variant_data, get_arch_data, get_variant_data
from pungi.phases.base import PhaseBase from pungi.phases.base import PhaseBase
from pungi.arch import split_name_arch, get_compatible_arches from pungi.arch import split_name_arch, get_compatible_arches
@ -68,10 +68,13 @@ class GatherPhase(PhaseBase):
except ValueError as exc: except ValueError as exc:
errors = exc.message.split('\n') errors = exc.message.split('\n')
if self.compose.conf['gather_source'] == 'module': # This must be imported here to avoid circular deps problems.
from pungi.phases.pkgset.sources import source_koji from pungi.phases.pkgset.sources import source_koji
if not source_koji.WITH_MODULES: if not source_koji.WITH_MODULES:
errors.append('Modular compose requires pdc_client and modulemd packages.') # Modules are not supported, check if we need them
for variant in self.compose.variants.values():
if variant.modules:
errors.append('Modular compose requires pdc_client and modulemd packages.')
if errors: if errors:
raise ValueError('\n'.join(errors)) raise ValueError('\n'.join(errors))
@ -126,7 +129,14 @@ def gather_packages(compose, arch, variant, package_sets, fulltree_excludes=None
# multilib white/black-list is per-arch, common for all variants # multilib white/black-list is per-arch, common for all variants
multilib_whitelist = get_multilib_whitelist(compose, arch) multilib_whitelist = get_multilib_whitelist(compose, arch)
multilib_blacklist = get_multilib_blacklist(compose, arch) multilib_blacklist = get_multilib_blacklist(compose, arch)
GatherMethod = get_gather_method(compose.conf["gather_method"]) methods = compose.conf["gather_method"]
global_method_name = methods
if isinstance(methods, dict):
try:
methods = get_variant_data(compose.conf, 'gather_method', variant)[-1]
global_method_name = None
except IndexError:
raise RuntimeError("Variant %s has no configured gather_method" % variant.uid)
msg = "Gathering packages (arch: %s, variant: %s)" % (arch, variant) msg = "Gathering packages (arch: %s, variant: %s)" % (arch, variant)
@ -136,17 +146,43 @@ def gather_packages(compose, arch, variant, package_sets, fulltree_excludes=None
compose.log_info("[BEGIN] %s" % msg) compose.log_info("[BEGIN] %s" % msg)
packages, groups, filter_packages = get_variant_packages(compose, arch, variant, package_sets) result = {
"rpm": [],
"srpm": [],
"debuginfo": [],
}
prepopulate = get_prepopulate_packages(compose, arch, variant) prepopulate = get_prepopulate_packages(compose, arch, variant)
fulltree_excludes = fulltree_excludes or set() fulltree_excludes = fulltree_excludes or set()
method = GatherMethod(compose) for source_name in ('module', 'comps', 'json'):
pkg_map = method(arch, variant, packages, groups, filter_packages,
multilib_whitelist, multilib_blacklist, package_sets, packages, groups, filter_packages = get_variant_packages(compose, arch, variant,
fulltree_excludes=fulltree_excludes, prepopulate=prepopulate) source_name, package_sets)
if not packages and not groups:
# No inputs, nothing to do really.
continue
try:
method_name = global_method_name or methods[source_name]
except KeyError:
raise RuntimeError("Variant %s has no configured gather_method for source %s"
% (variant.uid, source_name))
GatherMethod = get_gather_method(method_name)
method = GatherMethod(compose)
method.source_name = source_name
compose.log_debug("Gathering source %s, method %s" % (source_name, method_name))
pkg_map = method(arch, variant, packages, groups, filter_packages,
multilib_whitelist, multilib_blacklist, package_sets,
fulltree_excludes=fulltree_excludes,
prepopulate=prepopulate if source_name == 'comps' else set())
for t in ('rpm', 'srpm', 'debuginfo'):
result[t].extend(pkg_map.get(t, []))
compose.log_info("[DONE ] %s" % msg) compose.log_info("[DONE ] %s" % msg)
return pkg_map return result
def write_packages(compose, arch, variant, pkg_map, path_prefix): def write_packages(compose, arch, variant, pkg_map, path_prefix):
@ -415,7 +451,7 @@ def get_lookaside_repos(compose, arch, variant):
return get_arch_variant_data(compose.conf, "gather_lookaside_repos", arch, variant) return get_arch_variant_data(compose.conf, "gather_lookaside_repos", arch, variant)
def get_variant_packages(compose, arch, variant, package_sets=None): def get_variant_packages(compose, arch, variant, source_name, package_sets=None):
"""Find inputs for depsolving of variant.arch combination. """Find inputs for depsolving of variant.arch combination.
Returns a triple: a list of input packages, a list of input comps groups Returns a triple: a list of input packages, a list of input comps groups
@ -429,17 +465,27 @@ def get_variant_packages(compose, arch, variant, package_sets=None):
When system-release packages should be filtered, the ``package_sets`` When system-release packages should be filtered, the ``package_sets``
argument is required. argument is required.
""" """
GatherSource = get_gather_source(compose.conf["gather_source"]) packages, groups, filter_packages = set(), set(), set()
GatherSource = get_gather_source(source_name)
source = GatherSource(compose) source = GatherSource(compose)
packages, groups = source(arch, variant) p, g = source(arch, variant)
filter_packages = set()
if source_name != "comps" and not p and not g:
# For modules and json source, if the source did not return anything,
# we should skip all further work. Additional packages and possibly
# system-release will be added to comps source.
return packages, groups, filter_packages
packages |= p
groups |= g
if variant is None: if variant is None:
# no variant -> no parent -> we have everything we need # no variant -> no parent -> we have everything we need
# doesn't make sense to do any package filtering # doesn't make sense to do any package filtering
return packages, groups, filter_packages return packages, groups, filter_packages
packages |= get_additional_packages(compose, arch, variant) if source_name == 'comps':
packages |= get_additional_packages(compose, arch, variant)
filter_packages |= get_filter_packages(compose, arch, variant) filter_packages |= get_filter_packages(compose, arch, variant)
if compose.conf['filter_system_release_packages']: if compose.conf['filter_system_release_packages']:
@ -452,13 +498,13 @@ def get_variant_packages(compose, arch, variant, package_sets=None):
for var in variant.parent.get_variants( for var in variant.parent.get_variants(
arch=arch, types=["self", "variant", "addon", "layered-product"]): arch=arch, types=["self", "variant", "addon", "layered-product"]):
var_packages, var_groups, _ = get_variant_packages( var_packages, var_groups, _ = get_variant_packages(
compose, arch, var, package_sets=package_sets) compose, arch, var, source_name, package_sets=package_sets)
packages |= var_packages packages |= var_packages
groups |= var_groups groups |= var_groups
if variant.type in ["addon", "layered-product"]: if variant.type in ["addon", "layered-product"]:
var_packages, var_groups, _ = get_variant_packages( var_packages, var_groups, _ = get_variant_packages(
compose, arch, variant.parent, package_sets=package_sets) compose, arch, variant.parent, source_name, package_sets=package_sets)
packages |= var_packages packages |= var_packages
groups |= var_groups groups |= var_groups
@ -517,9 +563,6 @@ def get_packages_to_gather(compose, arch=None, variant=None, include_arch=True,
""" """
Returns the list of names of packages and list of names of groups which Returns the list of names of packages and list of names of groups which
would be included in a compose as GATHER phase result. would be included in a compose as GATHER phase result.
This works only for "comps" or "json" gather_source. For "module"
gather_source, this always return an empty list, because it is not clear
what packages will end up in a compose before the gather phase is run.
:param str arch: Arch to return packages for. If not set, returns packages :param str arch: Arch to return packages for. If not set, returns packages
for all arches. for all arches.
@ -531,30 +574,28 @@ def get_packages_to_gather(compose, arch=None, variant=None, include_arch=True,
:param include_prepopulated: When True, the prepopulated packages will :param include_prepopulated: When True, the prepopulated packages will
be included in a list of packages. be included in a list of packages.
""" """
if compose.conf["gather_source"] == "module":
return ([], [])
arches = [arch] if arch else compose.get_arches()
GatherSource = get_gather_source(compose.conf["gather_source"])
src = GatherSource(compose)
packages = set([]) packages = set([])
groups = set([]) groups = set([])
for arch in arches: for source_name in ('module', 'comps', 'json'):
pkgs, grps = src(arch, variant) GatherSource = get_gather_source(source_name)
groups = groups.union(set(grps)) src = GatherSource(compose)
additional_packages = get_additional_packages(compose, arch, None) arches = [arch] if arch else compose.get_arches()
for pkg_name, pkg_arch in pkgs | additional_packages:
if not include_arch or pkg_arch is None:
packages.add(pkg_name)
else:
packages.add("%s.%s" % (pkg_name, pkg_arch))
if include_prepopulated: for arch in arches:
prepopulated = get_prepopulate_packages( pkgs, grps = src(arch, variant)
compose, arch, variant, include_arch) groups = groups.union(set(grps))
packages = packages.union(prepopulated)
additional_packages = get_additional_packages(compose, arch, None)
for pkg_name, pkg_arch in pkgs | additional_packages:
if not include_arch or pkg_arch is None:
packages.add(pkg_name)
else:
packages.add("%s.%s" % (pkg_name, pkg_arch))
if include_prepopulated:
prepopulated = get_prepopulate_packages(
compose, arch, variant, include_arch)
packages = packages.union(prepopulated)
return list(packages), list(groups) return list(packages), list(groups)

View File

@ -40,8 +40,9 @@ class GatherMethodDeps(pungi.phases.gather.method.GatherMethodBase):
write_pungi_config(self.compose, arch, variant, packages, groups, filter_packages, write_pungi_config(self.compose, arch, variant, packages, groups, filter_packages,
multilib_whitelist, multilib_blacklist, multilib_whitelist, multilib_blacklist,
fulltree_excludes=fulltree_excludes, prepopulate=prepopulate) fulltree_excludes=fulltree_excludes, prepopulate=prepopulate,
result, missing_deps = resolve_deps(self.compose, arch, variant) source_name=self.source_name)
result, missing_deps = resolve_deps(self.compose, arch, variant, source_name=self.source_name)
check_deps(self.compose, arch, variant, missing_deps) check_deps(self.compose, arch, variant, missing_deps)
return result return result
@ -61,10 +62,10 @@ def _format_packages(pkgs):
def write_pungi_config(compose, arch, variant, packages, groups, filter_packages, def write_pungi_config(compose, arch, variant, packages, groups, filter_packages,
multilib_whitelist, multilib_blacklist, fulltree_excludes=None, multilib_whitelist, multilib_blacklist, fulltree_excludes=None,
prepopulate=None): prepopulate=None, source_name=None):
"""write pungi config (kickstart) for arch/variant""" """write pungi config (kickstart) for arch/variant"""
pungi_wrapper = PungiWrapper() pungi_wrapper = PungiWrapper()
pungi_cfg = compose.paths.work.pungi_conf(variant=variant, arch=arch) pungi_cfg = compose.paths.work.pungi_conf(variant=variant, arch=arch, source_name=source_name)
msg = "Writing pungi config (arch: %s, variant: %s): %s" % (arch, variant, pungi_cfg) msg = "Writing pungi config (arch: %s, variant: %s): %s" % (arch, variant, pungi_cfg)
if compose.DEBUG and os.path.isfile(pungi_cfg): if compose.DEBUG and os.path.isfile(pungi_cfg):
@ -95,9 +96,9 @@ def write_pungi_config(compose, arch, variant, packages, groups, filter_packages
prepopulate=prepopulate) prepopulate=prepopulate)
def resolve_deps(compose, arch, variant): def resolve_deps(compose, arch, variant, source_name=None):
pungi_wrapper = PungiWrapper() pungi_wrapper = PungiWrapper()
pungi_log = compose.paths.work.pungi_log(arch, variant) pungi_log = compose.paths.work.pungi_log(arch, variant, source_name=source_name)
msg = "Running pungi (arch: %s, variant: %s)" % (arch, variant) msg = "Running pungi (arch: %s, variant: %s)" % (arch, variant)
if compose.DEBUG and os.path.exists(pungi_log): if compose.DEBUG and os.path.exists(pungi_log):
@ -107,7 +108,7 @@ def resolve_deps(compose, arch, variant):
return res, broken_deps return res, broken_deps
compose.log_info("[BEGIN] %s" % msg) compose.log_info("[BEGIN] %s" % msg)
pungi_conf = compose.paths.work.pungi_conf(arch, variant) pungi_conf = compose.paths.work.pungi_conf(arch, variant, source_name=source_name)
multilib_methods = get_arch_variant_data(compose.conf, 'multilib', arch, variant) multilib_methods = get_arch_variant_data(compose.conf, 'multilib', arch, variant)

View File

@ -28,7 +28,10 @@ class GatherMethodNodeps(pungi.phases.gather.method.GatherMethodBase):
enabled = True enabled = True
def __call__(self, arch, variant, *args, **kwargs): def __call__(self, arch, variant, *args, **kwargs):
log_file = self.compose.paths.log.log_file(arch, 'gather-nodeps-%s' % variant.uid) fname = 'gather-nodeps-%s' % variant.uid
if self.source_name:
fname += '-' + self.source_name
log_file = self.compose.paths.log.log_file(arch, fname)
with open(log_file, 'w') as log: with open(log_file, 'w') as log:
return self.worker(log, arch, variant, *args, **kwargs) return self.worker(log, arch, variant, *args, **kwargs)

View File

@ -36,8 +36,12 @@ createrepo_checksum = "sha256"
# GATHER # GATHER
gather_source = "comps" gather_method = {
gather_method = "deps" "^.*$": {
"module": "nodeps",
"comps": "deps",
}
}
greedy_method = "build" greedy_method = "build"
check_deps = False check_deps = False
hashed_directories = True hashed_directories = True

View File

@ -186,7 +186,6 @@ BASE_CONFIG = dict(
runroot=False, runroot=False,
createrepo_checksum='sha256', createrepo_checksum='sha256',
gather_method='deps', gather_method='deps',
gather_source='none',
sigkeys=[], sigkeys=[],
) )

View File

@ -213,31 +213,6 @@ class CreaterepoConfigTestCase(ConfigTestCase):
class GatherConfigTestCase(ConfigTestCase): class GatherConfigTestCase(ConfigTestCase):
def test_source_comps_requires_comps(self):
cfg = load_config(
pkgset_source='koji',
pkgset_koji_tag="f25",
gather_source='comps',
gather_source_mapping='foo'
)
self.assertValidation(
cfg,
[checks.REQUIRES.format('gather_source', 'comps', 'comps_file'),
checks.CONFLICTS.format('gather_source', 'comps', 'gather_source_mapping')])
def test_source_json_requires_mapping(self):
cfg = load_config(
pkgset_source='koji',
pkgset_koji_tag="f25",
gather_source='json',
comps_file='comps',
)
self.assertValidation(
cfg,
[checks.REQUIRES.format('gather_source', 'json', 'gather_source_mapping')])
def test_dnf_backend_is_default_on_py3(self): def test_dnf_backend_is_default_on_py3(self):
cfg = load_config( cfg = load_config(
pkgset_source='koji', pkgset_source='koji',

View File

@ -460,7 +460,7 @@ class TestGetVariantPackages(helpers.PungiTestCase):
def test_no_variant(self): def test_no_variant(self):
compose = helpers.DummyCompose(self.topdir, {}) compose = helpers.DummyCompose(self.topdir, {})
packages, groups, filter_packages = gather.get_variant_packages( packages, groups, filter_packages = gather.get_variant_packages(
compose, 'x86_64', None) compose, 'x86_64', None, 'comps')
self.assertItemsEqual(packages, []) self.assertItemsEqual(packages, [])
self.assertItemsEqual(groups, []) self.assertItemsEqual(groups, [])
self.assertItemsEqual(filter_packages, []) self.assertItemsEqual(filter_packages, [])
@ -473,7 +473,7 @@ class TestGetVariantPackages(helpers.PungiTestCase):
compose = helpers.DummyCompose(self.topdir, {}) compose = helpers.DummyCompose(self.topdir, {})
packages, groups, filter_packages = gather.get_variant_packages( packages, groups, filter_packages = gather.get_variant_packages(
compose, 'x86_64', compose.variants['Server']) compose, 'x86_64', compose.variants['Server'], 'comps')
self.assertItemsEqual(packages, ['foo']) self.assertItemsEqual(packages, ['foo'])
self.assertItemsEqual(groups, ['core']) self.assertItemsEqual(groups, ['core'])
self.assertItemsEqual(filter_packages, []) self.assertItemsEqual(filter_packages, [])
@ -490,7 +490,7 @@ class TestGetVariantPackages(helpers.PungiTestCase):
) )
packages, groups, filter_packages = gather.get_variant_packages( packages, groups, filter_packages = gather.get_variant_packages(
compose, 'x86_64', compose.variants['Server'], package_sets={'x86_64': pkgset}) compose, 'x86_64', compose.variants['Server'], 'comps', package_sets={'x86_64': pkgset})
self.assertItemsEqual(packages, [('system-release-server', None)]) self.assertItemsEqual(packages, [('system-release-server', None)])
self.assertItemsEqual(groups, []) self.assertItemsEqual(groups, [])
self.assertItemsEqual(filter_packages, [('system-release', None)]) self.assertItemsEqual(filter_packages, [('system-release', None)])
@ -509,7 +509,7 @@ class TestGetVariantPackages(helpers.PungiTestCase):
) )
packages, groups, filter_packages = gather.get_variant_packages( packages, groups, filter_packages = gather.get_variant_packages(
compose, 'x86_64', compose.variants['Server'], package_sets={'x86_64': pkgset}) compose, 'x86_64', compose.variants['Server'], 'comps', package_sets={'x86_64': pkgset})
self.assertItemsEqual(packages, []) self.assertItemsEqual(packages, [])
self.assertItemsEqual(groups, []) self.assertItemsEqual(groups, [])
self.assertItemsEqual(filter_packages, []) self.assertItemsEqual(filter_packages, [])
@ -534,7 +534,7 @@ class TestGetVariantPackages(helpers.PungiTestCase):
) )
packages, groups, filter_packages = gather.get_variant_packages( packages, groups, filter_packages = gather.get_variant_packages(
compose, 'x86_64', compose.all_variants['Server-optional']) compose, 'x86_64', compose.all_variants['Server-optional'], 'comps')
self.assertItemsEqual(packages, ['server-pkg', 'addon-pkg', 'opt-pkg']) self.assertItemsEqual(packages, ['server-pkg', 'addon-pkg', 'opt-pkg'])
self.assertItemsEqual(groups, ['server-group', 'addon-group', 'opt-group']) self.assertItemsEqual(groups, ['server-group', 'addon-group', 'opt-group'])
self.assertItemsEqual(filter_packages, []) self.assertItemsEqual(filter_packages, [])
@ -554,7 +554,7 @@ class TestGetVariantPackages(helpers.PungiTestCase):
) )
packages, groups, filter_packages = gather.get_variant_packages( packages, groups, filter_packages = gather.get_variant_packages(
compose, 'x86_64', compose.all_variants['Server-optional']) compose, 'x86_64', compose.all_variants['Server-optional'], 'comps')
self.assertItemsEqual(packages, []) self.assertItemsEqual(packages, [])
self.assertItemsEqual(groups, []) self.assertItemsEqual(groups, [])
self.assertItemsEqual(filter_packages, []) self.assertItemsEqual(filter_packages, [])
@ -572,7 +572,7 @@ class TestGetVariantPackages(helpers.PungiTestCase):
) )
packages, groups, filter_packages = gather.get_variant_packages( packages, groups, filter_packages = gather.get_variant_packages(
compose, 'x86_64', compose.all_variants['Server']) compose, 'x86_64', compose.all_variants['Server'], 'comps')
self.assertItemsEqual(packages, [('pkg', None), ('foo', 'x86_64')]) self.assertItemsEqual(packages, [('pkg', None), ('foo', 'x86_64')])
self.assertItemsEqual(groups, []) self.assertItemsEqual(groups, [])
self.assertItemsEqual(filter_packages, []) self.assertItemsEqual(filter_packages, [])
@ -591,7 +591,7 @@ class TestGetVariantPackages(helpers.PungiTestCase):
with self.assertRaises(ValueError) as ctx: with self.assertRaises(ValueError) as ctx:
packages, groups, filter_packages = gather.get_variant_packages( packages, groups, filter_packages = gather.get_variant_packages(
compose, 'x86_64', compose.all_variants['Server']) compose, 'x86_64', compose.all_variants['Server'], 'comps')
self.assertIn('Incompatible package arch', str(ctx.exception)) self.assertIn('Incompatible package arch', str(ctx.exception))
@ -641,17 +641,19 @@ class TestGatherPackages(helpers.PungiTestCase):
pkg_set = mock.Mock() pkg_set = mock.Mock()
self.assertEqual( self.assertEqual(
gather.gather_packages(compose, 'x86_64', compose.variants['Server'], pkg_set), gather.gather_packages(compose, 'x86_64', compose.variants['Server'], pkg_set),
get_gather_method.return_value.return_value.return_value {'rpm': [], 'srpm': [], 'debuginfo': []}
) )
self.assertEqual(get_gather_method.call_args_list, self.assertEqual(get_gather_method.call_args_list,
[mock.call(compose.conf['gather_method'])]) [mock.call(compose.conf['gather_method'])] * 3)
self.assertEqual(get_variant_packages.call_args_list, self.assertEqual(get_variant_packages.call_args_list,
[mock.call(compose, 'x86_64', compose.variants['Server'], pkg_set)]) [mock.call(compose, 'x86_64', compose.variants['Server'], 'module', pkg_set),
mock.call(compose, 'x86_64', compose.variants['Server'], 'comps', pkg_set),
mock.call(compose, 'x86_64', compose.variants['Server'], 'json', pkg_set)])
self.assertEqual( self.assertEqual(
get_gather_method.return_value.return_value.call_args_list, get_gather_method.return_value.return_value.call_args_list,
[mock.call('x86_64', compose.variants['Server'], packages, groups, [mock.call('x86_64', compose.variants['Server'], packages, groups,
filters, set(), set(), pkg_set, fulltree_excludes=set(), filters, set(), set(), pkg_set, fulltree_excludes=set(),
prepopulate=set())] prepopulate=set())] * 3
) )
@mock.patch('pungi.phases.gather.get_variant_packages') @mock.patch('pungi.phases.gather.get_variant_packages')
@ -679,19 +681,36 @@ class TestGatherPackages(helpers.PungiTestCase):
pkg_set = mock.Mock() pkg_set = mock.Mock()
self.assertEqual( self.assertEqual(
gather.gather_packages(compose, 'x86_64', compose.variants['Server'], pkg_set), gather.gather_packages(compose, 'x86_64', compose.variants['Server'], pkg_set),
get_gather_method.return_value.return_value.return_value {'rpm': [], 'srpm': [], 'debuginfo': []}
) )
self.assertEqual(get_gather_method.call_args_list, self.assertEqual(get_gather_method.call_args_list,
[mock.call(compose.conf['gather_method'])]) [mock.call(compose.conf['gather_method'])] * 3)
self.assertEqual(get_variant_packages.call_args_list, self.assertEqual(get_variant_packages.call_args_list,
[mock.call(compose, 'x86_64', compose.variants['Server'], pkg_set)]) [mock.call(compose, 'x86_64', compose.variants['Server'], 'module', pkg_set),
mock.call(compose, 'x86_64', compose.variants['Server'], 'comps', pkg_set),
mock.call(compose, 'x86_64', compose.variants['Server'], 'json', pkg_set)])
self.assertEqual( self.assertEqual(
get_gather_method.return_value.return_value.call_args_list, get_gather_method.return_value.return_value.call_args_list,
[mock.call('x86_64', compose.variants['Server'], packages, groups, [mock.call('x86_64', compose.variants['Server'], packages, groups,
filters, set(['white']), set(['black']), pkg_set, filters, set(['white']), set(['black']), pkg_set,
fulltree_excludes=set(), prepopulate=set())] fulltree_excludes=set(), prepopulate=set())] * 3
) )
@mock.patch('pungi.phases.gather.get_variant_packages')
@mock.patch('pungi.phases.gather.get_gather_method')
def test_per_source_method(self, get_gather_method, get_variant_packages):
packages, groups, filters = mock.Mock(), mock.Mock(), mock.Mock()
get_variant_packages.return_value = (packages, groups, filters)
compose = helpers.DummyCompose(self.topdir, {
'multilib_whitelist': {'*': ['white']},
'multilib_blacklist': {'*': ['black']},
'gather_method': {'^Server$': {'comps': 'deps', 'module': 'nodeps', 'json': 'deps'}},
})
pkg_set = mock.Mock()
gather.gather_packages(compose, 'x86_64', compose.variants['Server'], pkg_set),
self.assertEqual(get_gather_method.call_args_list,
[mock.call('nodeps'), mock.call('deps'), mock.call('deps')])
class TestWritePrepopulate(helpers.PungiTestCase): class TestWritePrepopulate(helpers.PungiTestCase):
def test_without_config(self): def test_without_config(self):

View File

@ -204,7 +204,6 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
pickle_dumps): pickle_dumps):
self.compose = helpers.DummyCompose(self.topdir, { self.compose = helpers.DummyCompose(self.topdir, {
'gather_method': 'nodeps', 'gather_method': 'nodeps',
'gather_source': 'none',
'pkgset_koji_tag': 'f25', 'pkgset_koji_tag': 'f25',
'sigkeys': mock.Mock(), 'sigkeys': mock.Mock(),
'additional_packages': [ 'additional_packages': [