almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

gather: Add a hybrid depsolver backend

Browse Source 
This patch adds a new gather method called `hybrid`, which uses a `fus`
binary, which must exist somewhere on the `$PATH`. It will call it
multiple times to add multilib packages.

The solver can handle packages, modules and comps groups as input.
However comps groups are never passed in. Pungi will expand it to a list
of packages to avoid issues with comps handling in fus. It ignores
optional packages, and if the group mentions a package that does not
exist, nothing else from the group is included.

Multilib is also handled outside of fus. Pungi will run it, parse the
packages from output, determines multilib packages and adds them as
input. Then it runs the solver again. This is done until nothing new is
added. Usually two passes should be enough.

Source packages and debuginfo are added as a final step. All debuginfo
packages from any included source are added. If the source or debuginfo
package is included in any lookaside repo, it will be skipped.

The tool expects to get a platform stream that should be provided for
modules to depend on. Pungi looks into the modules and gets the platform
from there. If there are more requests, an error is raised.

There is some missing functionality and options that are ignored.
Particularly these are:

 * gather_fulltree
 * gather_selfhosting
 * greedy_method

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2018-06-22 08:58:54 +02:00
parent 2c6b784f70
commit ba0193ca28
10 changed files with 1264 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
doc/configuration.rst
View File

@ -588,12 +588,14 @@ Options
-------
**gather_method** [mandatory]
(*str*|*dict*) -- Options are ``deps`` and ``nodeps``. Specifies whether
package dependencies should be pulled in as well. Either a single value or
a dictionary mapping variant UID and source type to a value. Make sure only
one regex matches each variant, as there is no guarantee which value will
be used if there are multiple matching ones. All used sources must have a
configured method.
(*str*|*dict*) -- Options are ``deps``, ``nodeps`` and ``hybrid``.
Specifies whether and how package dependencies should be pulled in.
Possible configuration can be one value for all variants, or if configured
per-variant it can be a simple string ``hybrid`` or a a dictionary mapping
source type to a value of ``deps`` or ``nodeps``. Make sure only one regex
matches each variant, as there is no guarantee which value will be used if
there are multiple matching ones. All used sources must have a configured
method unless hybrid solving is used.
**gather_fulltree** = False
(*bool*) -- When set to ``True`` all RPMs built from an SRPM will always be
@ -745,6 +747,7 @@ Example
"comps": "deps",
"module": "nodeps"
}
"^OtherMixed$": "hybrid", # Using hybrid depsolver
}
additional_packages = [

20
pungi/checks.py
View File

@ -561,20 +561,28 @@ def make_schema():
"type": "object",
"patternProperties": {
".+": {
"type": "object",
"patternProperties": {
"^module|comps|json$": {
"oneOf": [
{
"type": "string",
"enum": ["deps", "nodeps"],
"enum": ["hybrid"],
},
{
"type": "object",
"patternProperties": {
"^module|comps|json$": {
"type": "string",
"enum": ["deps", "nodeps"],
}
},
}
}
]
}
},
"additionalProperties": False,
},
{
"type": "string",
"enum": ["deps", "nodeps"],
"enum": ["deps", "nodeps", "hybrid"],
}
],
},

27
pungi/paths.py
View File

@ -153,20 +153,31 @@ class WorkPaths(object):
makedirs(path)
return path
def comps_repo(self, arch=None, variant=None, create_dir=True):
"""
Examples:
work/x86_64/comps-repo
work/global/comps-repo
"""
def _repo(self, type, arch=None, variant=None, create_dir=True):
arch = arch or "global"
path = os.path.join(self.topdir(arch, create_dir=create_dir), "comps_repo")
path = os.path.join(self.topdir(arch, create_dir=create_dir), "%s_repo" % type)
if variant:
path += '_' + variant.uid
path += "_" + variant.uid
if create_dir:
makedirs(path)
return path
def comps_repo(self, arch=None, variant=None, create_dir=True):
"""
Examples:
work/x86_64/comps_repo_Server
work/global/comps_repo
"""
return self._repo("comps", arch, variant, create_dir=create_dir)
def module_repo(self, arch=None, variant=None, create_dir=True):
"""
Examples:
work/x86_64/module_repo_Server
work/global/module_repo
"""
return self._repo("module", arch, variant, create_dir=create_dir)
def arch_repo(self, arch=None, create_dir=True):
"""
Examples:

75
pungi/phases/gather/__init__.py
View File

@ -162,31 +162,64 @@ def gather_packages(compose, arch, variant, package_sets, fulltree_excludes=None
prepopulate = get_prepopulate_packages(compose, arch, variant)
fulltree_excludes = fulltree_excludes or set()
for source_name in ('module', 'comps', 'json'):
if methods == "hybrid":
# This variant is using a hybrid solver. Gather all inputs and run the
# method once.
packages, groups, filter_packages = get_variant_packages(compose, arch, variant,
source_name, package_sets)
if not packages and not groups:
# No inputs, nothing to do really.
continue
packages = []
groups = []
filter_packages = []
try:
method_name = global_method_name or methods[source_name]
except KeyError:
raise RuntimeError("Variant %s has no configured gather_method for source %s"
% (variant.uid, source_name))
# Run the module source. This is needed to set up module metadata for
# the variant, but we don't really care about the returned packages.
# They will be pulled in based on the actual module.
get_variant_packages(compose, arch, variant, "module", package_sets)
GatherMethod = get_gather_method(method_name)
method = GatherMethod(compose)
method.source_name = source_name
compose.log_debug("Gathering source %s, method %s" % (source_name, method_name))
pkg_map = method(arch, variant, packages, groups, filter_packages,
multilib_whitelist, multilib_blacklist, package_sets,
fulltree_excludes=fulltree_excludes,
prepopulate=prepopulate if source_name == 'comps' else set())
# Here we do want to get list of comps groups and additional packages.
packages, groups, filter_packages = get_variant_packages(
compose, arch, variant, "comps", package_sets
)
for t in ('rpm', 'srpm', 'debuginfo'):
result[t].extend(pkg_map.get(t, []))
result = get_gather_method("hybrid")(compose)(
arch,
variant,
packages=packages,
groups=groups,
filter_packages=filter_packages,
multilib_whitelist=multilib_whitelist,
multilib_blacklist=multilib_blacklist,
package_sets=package_sets,
fulltree_excludes=fulltree_excludes,
prepopulate=prepopulate,
)
else:
for source_name in ('module', 'comps', 'json'):
packages, groups, filter_packages = get_variant_packages(compose, arch, variant,
source_name, package_sets)
if not packages and not groups:
# No inputs, nothing to do really.
continue
try:
method_name = global_method_name or methods[source_name]
except KeyError:
raise RuntimeError("Variant %s has no configured gather_method for source %s"
% (variant.uid, source_name))
GatherMethod = get_gather_method(method_name)
method = GatherMethod(compose)
method.source_name = source_name
compose.log_debug("Gathering source %s, method %s" % (source_name, method_name))
pkg_map = method(arch, variant, packages, groups, filter_packages,
multilib_whitelist, multilib_blacklist, package_sets,
fulltree_excludes=fulltree_excludes,
prepopulate=prepopulate if source_name == 'comps' else set())
for t in ('rpm', 'srpm', 'debuginfo'):
result[t].extend(pkg_map.get(t, []))
compose.log_info("[DONE ] %s" % msg)
return result

408
pungi/phases/gather/methods/method_hybrid.py Normal file
View File

@ -0,0 +1,408 @@
# -*- coding: utf-8 -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <https://gnu.org/licenses/>.
from collections import defaultdict
import os
from kobo.shortcuts import run
import kobo.rpmlib
import pungi.phases.gather.method
from pungi import Modulemd, multilib_dnf
from pungi.arch import get_valid_arches, tree_arch_to_yum_arch
from pungi.phases.gather import _mk_pkg_map
from pungi.util import (
get_arch_variant_data,
iter_module_defaults,
pkg_is_debug,
temp_dir,
)
from pungi.wrappers import fus
from pungi.wrappers.createrepo import CreaterepoWrapper
from .method_nodeps import expand_groups
import createrepo_c as cr
class FakePackage(object):
"""This imitates a DNF package object and can be passed to python-multilib
library.
"""
def __init__(self, pkg):
self.pkg = pkg
def __getattr__(self, attr):
return getattr(self.pkg, attr)
@property
def files(self):
return [
os.path.join(dirname, basename) for (_, dirname, basename) in self.pkg.files
]
@property
def provides(self):
# This is supposed to match what yum package object returns. It's a
# nested tuple (name, flag, (epoch, version, release)). This code only
# fills in the name, because that's all that python-multilib is using..
return [(p[0].split()[0], None, (None, None, None)) for p in self.pkg.provides]
class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
enabled = True
def __init__(self, *args, **kwargs):
super(GatherMethodHybrid, self).__init__(*args, **kwargs)
self.package_maps = {}
self.packages = {}
def _get_pkg_map(self, arch):
"""Create a mapping from NEVRA to actual package object. This will be
done once for each architecture, since the package set is the same for
all variants.
The keys are in NEVRA format and only include the epoch if it's not
zero. This makes it easier to query by results for the depsolver.
"""
if arch not in self.package_maps:
pkg_map = {}
for pkg_arch in self.package_sets[arch].rpms_by_arch:
for pkg in self.package_sets[arch].rpms_by_arch[pkg_arch]:
pkg_map[_fmt_nevra(pkg, pkg_arch)] = pkg
self.package_maps[arch] = pkg_map
return self.package_maps[arch]
def _prepare_packages(self):
repo_path = self.compose.paths.work.arch_repo(arch=self.arch)
md = cr.Metadata()
md.locate_and_load_xml(repo_path)
for key in md.keys():
pkg = md.get(key)
if pkg.arch in self.valid_arches:
self.packages[_fmt_nevra(pkg, arch=pkg.arch)] = FakePackage(pkg)
def _get_package(self, nevra):
if not self.packages:
self._prepare_packages()
return self.packages[nevra]
def __call__(
self,
arch,
variant,
package_sets,
packages=[],
groups=[],
multilib_whitelist=[],
multilib_blacklist=[],
**kwargs
):
self.arch = arch
self.valid_arches = get_valid_arches(arch, multilib=True)
self.package_sets = package_sets
self.multilib_methods = get_arch_variant_data(
self.compose.conf, "multilib", arch, variant
)
self.multilib = multilib_dnf.Multilib(
self.multilib_methods, multilib_blacklist, multilib_whitelist
)
platform, modular_rpms = create_module_repo(self.compose, variant, arch)
packages.update(
expand_groups(self.compose, arch, variant, groups, set_pkg_arch=False)
)
nvrs, modules = self.run_solver(variant, arch, packages, platform, modular_rpms)
return expand_packages(
self._get_pkg_map(arch),
variant.arch_mmds.get(arch, {}),
pungi.phases.gather.get_lookaside_repos(self.compose, arch, variant),
nvrs,
modules,
)
# maybe check invalid sigkeys
def run_solver(self, variant, arch, packages, platform, modular_rpms):
repos = [self.compose.paths.work.arch_repo(arch=arch)]
modules = []
if variant.arch_mmds.get(arch):
repos.append(self.compose.paths.work.module_repo(arch, variant))
for mmd in variant.arch_mmds[arch].values():
modules.append("%s:%s" % (mmd.peek_name(), mmd.peek_stream()))
input_packages = [
_fmt_pkg(pkg_name, pkg_arch) for pkg_name, pkg_arch in packages
]
step = 0
while True:
step += 1
cmd = fus.get_cmd(
tree_arch_to_yum_arch(arch),
repos,
pungi.phases.gather.get_lookaside_repos(self.compose, arch, variant),
input_packages,
modules,
platform=platform,
)
logfile = self.compose.paths.log.log_file(
arch, "hybrid-depsolver-%s-iter-%d" % (variant, step)
)
run(cmd, logfile=logfile, show_cmd=True)
output, output_modules = fus.parse_output(logfile)
new_multilib = self.add_multilib(variant, arch, output, modular_rpms)
if not new_multilib:
# No new multilib packages were added, we're done.
break
input_packages.extend(
_fmt_pkg(pkg_name, pkg_arch) for pkg_name, pkg_arch in new_multilib
)
return output, output_modules
def add_multilib(self, variant, arch, nvrs, modular_rpms):
added = set()
if not self.multilib_methods:
return []
for nvr, pkg_arch in nvrs:
if pkg_arch != arch:
# Not a native package, not checking to add multilib
continue
nevr = kobo.rpmlib.parse_nvr(nvr)
nevr_copy = nevr.copy()
nevr_copy["arch"] = pkg_arch
if kobo.rpmlib.make_nvra(nevr_copy, force_epoch=True) in modular_rpms:
# Skip modular package
continue
if self.multilib.is_multilib(self._get_package("%s.%s" % (nvr, pkg_arch))):
for add_arch in self.valid_arches:
if add_arch == arch:
continue
if _nevra(arch=add_arch, **nevr) in self._get_pkg_map(arch):
added.add((nevr["name"], add_arch))
# Remove packages that are already present
for nvr, pkg_arch in nvrs:
existing = (nvr.rsplit("-", 2)[0], pkg_arch)
if existing in added:
added.remove(existing)
return sorted(added)
def create_module_repo(compose, variant, arch):
"""Create repository with module metadata. There are no packages otherwise."""
createrepo_c = compose.conf["createrepo_c"]
createrepo_checksum = compose.conf["createrepo_checksum"]
msg = "Creating repo with modular metadata for %s.%s" % (variant, arch)
if not variant.arch_mmds.get(arch):
compose.log_debug("[SKIP ] %s: no modules found" % msg)
return None, []
compose.log_debug("[BEGIN] %s" % msg)
platforms = set()
modular_rpms = set()
repo_path = compose.paths.work.module_repo(arch, variant)
# Add modular metadata to it
modules = []
for mmd in variant.arch_mmds[arch].values():
# Set the arch field, but no other changes are needed.
repo_mmd = mmd.copy()
repo_mmd.set_arch(tree_arch_to_yum_arch(arch))
for dep in repo_mmd.peek_dependencies():
streams = dep.peek_requires().get("platform")
if streams:
platforms.update(streams.dup())
# Collect all modular NEVRAs
artifacts = repo_mmd.get_rpm_artifacts()
if artifacts:
modular_rpms.update(artifacts.dup())
modules.append(repo_mmd)
if len(platforms) > 1:
raise RuntimeError("There are conflicting requests for platform.")
module_names = set([x.get_name() for x in modules])
defaults_dir = compose.paths.work.module_defaults_dir()
for mmddef in iter_module_defaults(defaults_dir):
if mmddef.peek_module_name() in module_names:
modules.append(mmddef)
# Initialize empty repo
repo = CreaterepoWrapper(createrepo_c=createrepo_c)
cmd = repo.get_createrepo_cmd(
repo_path, database=False, outputdir=repo_path, checksum=createrepo_checksum
)
logfile = "module_repo-%s" % variant
run(cmd, logfile=compose.paths.log.log_file(arch, logfile), show_cmd=True)
with temp_dir() as tmp_dir:
modules_path = os.path.join(tmp_dir, "modules.yaml")
Modulemd.dump(modules, modules_path)
cmd = repo.get_modifyrepo_cmd(
os.path.join(repo_path, "repodata"),
modules_path,
mdtype="modules",
compress_type="gz",
)
log_file = compose.paths.log.log_file(
arch, "gather-modifyrepo-modules-%s" % variant
)
run(cmd, logfile=log_file, show_cmd=True)
compose.log_debug("[DONE ] %s" % msg)
return list(platforms)[0] if platforms else None, modular_rpms
def _fmt_pkg(pkg_name, arch):
if arch:
pkg_name += ".%s" % arch
return pkg_name
def _nevra(**kwargs):
if kwargs.get("epoch") not in (None, "", 0, "0"):
return "%(name)s-%(epoch)s:%(version)s-%(release)s.%(arch)s" % kwargs
return "%(name)s-%(version)s-%(release)s.%(arch)s" % kwargs
def _fmt_nevra(pkg, arch):
return _nevra(
name=pkg.name,
epoch=pkg.epoch,
version=pkg.version,
release=pkg.release,
arch=arch,
)
def _get_srpm_nevra(pkg):
nevra = kobo.rpmlib.parse_nvra(pkg.sourcerpm)
nevra["epoch"] = nevra["epoch"] or pkg.epoch
return _nevra(**nevra)
def _make_result(paths):
return [{"path": path, "flags": []} for path in sorted(paths)]
def expand_packages(nevra_to_pkg, variant_modules, lookasides, nvrs, modules):
"""For each package add source RPM and possibly also debuginfo."""
# This will server as the final result. We collect sets of paths to the
# packages.
rpms = set()
srpms = set()
debuginfo = set()
# Collect list of all packages in lookaside. These will not be added to the
# result. Fus handles this in part: if a package is explicitly mentioned as
# input (which can happen with comps group expansion), it will be in the
# output even if it's in lookaside.
lookaside_packages = set()
for repo in lookasides:
md = cr.Metadata()
md.locate_and_load_xml(repo)
for key in md.keys():
pkg = md.get(key)
url = os.path.join(pkg.location_base, pkg.location_href)
# Strip file:// prefix
lookaside_packages.add(url[7:])
# Get all packages in modules and include them in rpms or debuginfo.
variant_mmd = {}
for mmd in variant_modules.values():
nsvc = "%s:%s:%s:%s" % (
mmd.peek_name(),
mmd.peek_stream(),
mmd.peek_version(),
mmd.peek_context(),
)
variant_mmd[nsvc] = mmd
for module in modules:
mmd = variant_mmd.get(module)
if not mmd:
continue
artifacts = mmd.get_rpm_artifacts()
if not artifacts:
continue
for rpm in artifacts.dup():
pkg = nevra_to_pkg[_nevra(**kobo.rpmlib.parse_nvra(rpm))]
if pkg_is_debug(pkg):
debuginfo.add(pkg.file_path)
else:
rpms.add(pkg.file_path)
# Add source package. We don't need modular packages, those are
# listed in modulemd.
try:
srpm_nevra = _get_srpm_nevra(pkg)
srpm = nevra_to_pkg[srpm_nevra]
if srpm.file_path not in lookaside_packages:
srpms.add(srpm.file_path)
except KeyError:
# Didn't find source RPM.. this should be logged
pass
# This is used to figure out which debuginfo packages to include. We keep
# track of package architectures from each SRPM.
srpm_arches = defaultdict(set)
for nvr, arch in nvrs:
pkg = nevra_to_pkg["%s.%s" % (nvr, arch)]
if pkg.file_path in lookaside_packages:
# Package is in lookaside, don't add it and ignore sources and
# debuginfo too.
continue
rpms.add(pkg.file_path)
try:
srpm_nevra = _get_srpm_nevra(pkg)
srpm = nevra_to_pkg[srpm_nevra]
srpm_arches[srpm_nevra].add(arch)
if srpm.file_path not in lookaside_packages:
srpms.add(srpm.file_path)
except KeyError:
# Didn't find source RPM.. this should be logged
pass
# Get all debuginfo packages from all included sources. We iterate over all
# available packages and if we see a debug package from included SRPM built
# for architecture that has at least one binary package, we include it too.
for pkg in nevra_to_pkg.values():
if pkg_is_debug(pkg) and pkg.arch in srpm_arches[_get_srpm_nevra(pkg)]:
if pkg.file_path not in lookaside_packages:
debuginfo.add(pkg.file_path)
return _mk_pkg_map(_make_result(rpms), _make_result(srpms), _make_result(debuginfo))

6
pungi/phases/gather/methods/method_nodeps.py
View File

@ -117,7 +117,7 @@ class GatherMethodNodeps(pungi.phases.gather.method.GatherMethodBase):
return result
def expand_groups(compose, arch, variant, groups):
def expand_groups(compose, arch, variant, groups, set_pkg_arch=True):
"""Read comps file filtered for given architecture and variant and return
all packages in given groups.
@ -131,7 +131,9 @@ def expand_groups(compose, arch, variant, groups):
comps = CompsWrapper(comps_file)
packages = set()
pkg_arch = arch if set_pkg_arch else None
for group in groups:
packages.update([(pkg, arch) for pkg in comps.get_packages(group)])
packages.update([(pkg, pkg_arch) for pkg in comps.get_packages(group)])
return packages

70
pungi/wrappers/fus.py Normal file
View File

@ -0,0 +1,70 @@
# -*- coding: utf-8 -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <https://gnu.org/licenses/>.
"""
This is a wrapper for a hybrid depsolver that understands how module
dependencies work. It's Funny Solver, because it does funny things.
https://github.com/fedora-modularity/fus
The executable basically provides one iteration of the traditional DNF based
depsolver. It has to be run multiple times to explicitly add multilib packages,
or source packages to include build dependencies (which is not yet supported in
Pungi).
"""
def get_cmd(
arch,
repos,
lookasides,
packages,
modules,
platform=None,
filter_packages=None, # TODO not supported yet
):
cmd = ["fus", "--verbose", "--arch", arch]
for idx, repo in enumerate(repos):
cmd.append("--repo=repo-%s,repo,%s" % (idx, repo))
for idx, repo in enumerate(lookasides):
cmd.append("--repo=lookaside-%s,lookaside,%s" % (idx, repo))
if platform:
cmd.append("--platform=%s" % platform)
for module in modules:
cmd.append("module(%s)" % module)
cmd.extend(packages)
return cmd
def parse_output(output):
"""Read output of fus from the given filepath, and return a set of tuples
(NVR, arch) and a set of module NSVCs.
"""
packages = set()
modules = set()
with open(output) as f:
for line in f:
if " " in line or "@" not in line:
continue
nevra, _ = line.strip().rsplit("@", 1)
if nevra.startswith("module:"):
modules.add(nevra[7:].rsplit(".", 1)[0])
else:
packages.add(tuple(nevra.rsplit(".", 1)))
return packages, modules

77
tests/test_fus_wrapper.py Normal file
View File

@ -0,0 +1,77 @@
# -*- coding: utf-8 -*-
try:
import unittest2 as unittest
except ImportError:
import unittest
import tempfile
import os
import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from pungi.wrappers import fus
from .helpers import touch
class TestGetCmd(unittest.TestCase):
def test_minimum_command(self):
cmd = fus.get_cmd("x86_64", [], [], [], [], [])
self.assertEqual(cmd, ["fus", "--verbose", "--arch", "x86_64"])
def test_full_command(self):
cmd = fus.get_cmd(
"x86_64",
["/tmp/first", "/tmp/second"],
["/tmp/fst", "/tmp/snd"],
["pkg"],
["mod:1.0"],
platform="f29",
)
self.assertEqual(
cmd,
[
"fus",
"--verbose",
"--arch",
"x86_64",
"--repo=repo-0,repo,/tmp/first",
"--repo=repo-1,repo,/tmp/second",
"--repo=lookaside-0,lookaside,/tmp/fst",
"--repo=lookaside-1,lookaside,/tmp/snd",
"--platform=f29",
"module(mod:1.0)",
"pkg",
],
)
class TestParseOutput(unittest.TestCase):
def setUp(self):
_, self.file = tempfile.mkstemp(prefix="test-parse-fus-out-")
def tearDown(self):
os.remove(self.file)
def test_skips_debug_line(self):
touch(self.file, "debug line\n")
packages, modules = fus.parse_output(self.file)
self.assertItemsEqual(packages, [])
self.assertItemsEqual(modules, [])
def test_separates_arch(self):
touch(self.file, "pkg-1.0-1.x86_64@repo-0\npkg-1.0-1.i686@repo-0\n")
packages, modules = fus.parse_output(self.file)
self.assertItemsEqual(
packages,
[("pkg-1.0-1", "x86_64"), ("pkg-1.0-1", "i686")],
)
self.assertItemsEqual(modules, [])
def test_returns_modules(self):
touch(self.file, "module:foo:1:201807131350:deadcafe.x86_64@repo-0\n")
packages, modules = fus.parse_output(self.file)
self.assertItemsEqual(packages, [])
self.assertItemsEqual(modules, ["foo:1:201807131350:deadcafe"])

584
tests/test_gather_method_hybrid.py Normal file
View File

@ -0,0 +1,584 @@
# -*- coding: utf-8 -*-
from collections import namedtuple
import copy
import mock
import os
import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from pungi.phases.gather.methods import method_hybrid as hybrid
from tests import helpers
MockPkg = namedtuple(
"MockPkg", ["name", "version", "release", "epoch", "sourcerpm", "file_path", "arch"]
)
class NamedMock(mock.Mock):
def __init__(self, name=None, **kwargs):
super(NamedMock, self).__init__(**kwargs)
self.name = name
class TestMethodHybrid(helpers.PungiTestCase):
@mock.patch("pungi.phases.gather.get_lookaside_repos")
@mock.patch("pungi.phases.gather.methods.method_hybrid.expand_groups")
@mock.patch("pungi.phases.gather.methods.method_hybrid.expand_packages")
@mock.patch("pungi.phases.gather.methods.method_hybrid.create_module_repo")
def test_call_method(self, cmr, ep, eg, glr):
compose = helpers.DummyCompose(self.topdir, {})
cmr.return_value = (mock.Mock(), mock.Mock())
m = hybrid.GatherMethodHybrid(compose)
m.run_solver = mock.Mock(return_value=(mock.Mock(), mock.Mock()))
pkg = MockPkg(
name="pkg",
version="1",
release="2",
arch="x86_64",
epoch=3,
sourcerpm=None,
file_path=None,
)
eg.return_value = ["foo", "bar"]
package_sets = {"x86_64": mock.Mock(rpms_by_arch={"x86_64": [pkg]})}
arch = "x86_64"
variant = compose.variants["Server"]
res = m(arch, variant, package_sets, set(["pkg"]), ["standard"])
self.assertEqual(res, ep.return_value)
self.assertEqual(cmr.call_args_list, [mock.call(compose, variant, arch)])
self.assertEqual(
m.run_solver.call_args_list,
[mock.call(variant, arch, set(["pkg", "foo", "bar"]), *cmr.return_value)],
)
self.assertEqual(
ep.call_args_list,
[
mock.call(
{"pkg-3:1-2.x86_64": pkg},
{},
glr.return_value,
m.run_solver.return_value[0],
m.run_solver.return_value[1],
)
],
)
self.assertEqual(
eg.call_args_list,
[mock.call(compose, arch, variant, ["standard"], set_pkg_arch=False)],
)
class MockModule(object):
def __init__(
self, name, platform=None, stream=None, version=None, context=None, rpms=None
):
self.name = name
self.platform = platform
self.stream = stream
self.version = version
self.context = context
self.rpms = rpms or ["pkg-1.0-1.x86_64"]
def get_name(self):
return self.name
def peek_name(self):
return self.name
def peek_stream(self):
return self.stream
def peek_version(self):
return self.version
def peek_context(self):
return self.context
def peek_dependencies(self):
return [
mock.Mock(
peek_requires=mock.Mock(
return_value={
"platform": mock.Mock(
dup=mock.Mock(return_value=[self.platform])
)
}
)
)
]
def copy(self):
return self
def set_arch(self, arch):
pass
def get_rpm_artifacts(self):
return mock.Mock(dup=mock.Mock(return_value=self.rpms))
class HelperMixin(object):
def _repo(self, name):
return os.path.join(self.compose.topdir, "work/x86_64/%s" % name)
@mock.patch("pungi.phases.gather.methods.method_hybrid.Modulemd")
@mock.patch("pungi.phases.gather.methods.method_hybrid.run")
class TestCreateModuleRepo(HelperMixin, helpers.PungiTestCase):
def setUp(self):
super(TestCreateModuleRepo, self).setUp()
self.compose = helpers.DummyCompose(self.topdir, {})
self.variant = self.compose.variants["Server"]
def test_no_modules(self, run, Modulemd):
plat, pkgs = hybrid.create_module_repo(self.compose, self.variant, "x86_64")
self.assertIsNone(plat)
self.assertItemsEqual(pkgs, [])
self.assertEqual(run.call_args_list, [])
self.assertEqual(Modulemd.mock_calls, [])
def test_more_than_one_platform(self, run, Modulemd):
self.variant.arch_mmds["x86_64"] = {
"mod:1": MockModule("mod", platform="f29"),
"mod:2": MockModule("mod", platform="f30"),
}
with self.assertRaises(RuntimeError) as ctx:
hybrid.create_module_repo(self.compose, self.variant, "x86_64")
self.assertIn("conflicting requests for platform", str(ctx.exception))
self.assertEqual(run.call_args_list, [])
self.assertEqual(Modulemd.mock_calls, [])
@mock.patch("pungi.phases.gather.methods.method_hybrid.iter_module_defaults")
def test_creating_repo_with_module_and_default(self, imd, run, Modulemd):
mod = MockModule("mod", platform="f29")
self.variant.arch_mmds["x86_64"] = {"mod:1": mod}
default = mock.Mock(peek_module_name=mock.Mock(return_value="mod"))
imd.return_value = [default]
plat, pkgs = hybrid.create_module_repo(self.compose, self.variant, "x86_64")
self.assertEqual(plat, "f29")
self.assertItemsEqual(pkgs, ["pkg-1.0-1.x86_64"])
self.assertEqual(
Modulemd.mock_calls, [mock.call.dump([mod, default], mock.ANY)]
)
create, modify = run.call_args_list
self.assertEqual(
create[0][0][:2], ["createrepo_c", self._repo("module_repo_Server")]
)
self.assertEqual(
modify[0][0][:4],
[
"modifyrepo_c",
Modulemd.mock_calls[0][1][1],
self._repo("module_repo_Server/repodata"),
"--mdtype=modules",
],
)
class ModifiedMagicMock(mock.MagicMock):
"""Like MagicMock, but remembers original values or mutable arguments."""
def _mock_call(_mock_self, *args, **kwargs):
return super(ModifiedMagicMock, _mock_self)._mock_call(
*copy.deepcopy(args), **copy.deepcopy(kwargs)
)
@mock.patch("pungi.wrappers.fus.parse_output")
@mock.patch("pungi.wrappers.fus.get_cmd", new_callable=ModifiedMagicMock)
@mock.patch("pungi.phases.gather.methods.method_hybrid.run")
class TestRunSolver(HelperMixin, helpers.PungiTestCase):
def setUp(self):
super(TestRunSolver, self).setUp()
self.compose = helpers.DummyCompose(self.topdir, {})
self.phase = hybrid.GatherMethodHybrid(self.compose)
self.phase.multilib_methods = []
self.logfile1 = os.path.join(
self.compose.topdir, "logs/x86_64/hybrid-depsolver-Server-iter-1.x86_64.log"
)
self.logfile2 = os.path.join(
self.compose.topdir, "logs/x86_64/hybrid-depsolver-Server-iter-2.x86_64.log"
)
def test_with_modules(self, run, gc, po):
self.compose.has_comps = None
self.compose.variants["Server"].arch_mmds["x86_64"] = {
"mod:master": mock.Mock(
peek_name=mock.Mock(return_value="mod"),
peek_stream=mock.Mock(return_value="master"),
)
}
po.return_value = (mock.Mock(), mock.Mock())
res = self.phase.run_solver(
self.compose.variants["Server"],
"x86_64",
[],
platform="pl",
modular_rpms=[],
)
self.assertEqual(res, po.return_value)
self.assertEqual(po.call_args_list, [mock.call(self.logfile1)])
self.assertEqual(
run.call_args_list,
[mock.call(gc.return_value, logfile=self.logfile1, show_cmd=True)],
)
self.assertEqual(
gc.call_args_list,
[
mock.call(
"x86_64",
[self._repo("repo"), self._repo("module_repo_Server")],
[],
[],
["mod:master"],
platform="pl",
)
],
)
def test_with_comps(self, run, gc, po):
po.return_value = (mock.Mock(), mock.Mock())
res = self.phase.run_solver(
self.compose.variants["Server"],
"x86_64",
[("pkg", None)],
platform=None,
modular_rpms=[],
)
self.assertEqual(res, po.return_value)
self.assertEqual(po.call_args_list, [mock.call(self.logfile1)])
self.assertEqual(
run.call_args_list,
[mock.call(gc.return_value, logfile=self.logfile1, show_cmd=True)],
)
self.assertEqual(
gc.call_args_list,
[mock.call("x86_64", [self._repo("repo")], [], ["pkg"], [], platform=None)],
)
@mock.patch("pungi.phases.gather.methods.method_hybrid.cr")
def test_multilib_devel(self, cr, run, gc, po):
self.phase.arch = "x86_64"
self.phase.multilib_methods = ["devel"]
self.phase.multilib = mock.Mock()
self.phase.multilib.is_multilib.side_effect = (
lambda pkg: pkg.name == "pkg-devel"
)
self.phase.valid_arches = ["x86_64", "i686", "noarch"]
cr.Metadata.return_value.keys.return_value = []
self.phase.package_maps = {
"x86_64": {
"pkg-devel-1.0-1.x86_64": NamedMock(name="pkg-devel"),
"pkg-devel-1.0-1.i686": NamedMock(name="pkg-devel"),
"foo-1.0-1.x86_64": NamedMock(name="foo"),
}
}
self.phase.packages = self.phase.package_maps["x86_64"]
final = [
("pkg-devel-1.0-1", "x86_64"),
("foo-1.0-1", "x86_64"),
("pkg-devel-1.0-1", "i686"),
]
po.side_effect = [
[[("pkg-devel-1.0-1", "x86_64"), ("foo-1.0-1", "x86_64")], set()],
[final, set()],
]
res = self.phase.run_solver(
self.compose.variants["Server"],
"x86_64",
[("pkg-devel", None), ("foo", None)],
platform=None,
modular_rpms=[],
)
self.assertEqual(res, (final, set()))
self.assertEqual(
po.call_args_list, [mock.call(self.logfile1), mock.call(self.logfile2)]
)
self.assertEqual(
run.call_args_list,
[
mock.call(gc.return_value, logfile=self.logfile1, show_cmd=True),
mock.call(gc.return_value, logfile=self.logfile2, show_cmd=True),
],
)
self.assertEqual(
gc.call_args_list,
[
mock.call(
"x86_64",
[self._repo("repo")],
[],
["pkg-devel", "foo"],
[],
platform=None,
),
mock.call(
"x86_64",
[self._repo("repo")],
[],
["pkg-devel", "foo", "pkg-devel.i686"],
[],
platform=None,
),
],
)
@mock.patch("pungi.phases.gather.methods.method_hybrid.cr")
def test_multilib_runtime(self, cr, run, gc, po):
packages = {
"abc": NamedMock(
name="foo",
epoch=None,
version="1.0",
release="1",
arch="x86_64",
provides=[("/usr/lib/libfoo.1.so.1", None, None)],
),
"def": NamedMock(
name="foo",
epoch=None,
version="1.0",
release="1",
arch="i686",
provides=[("/usr/lib/libfoo.1.so.1", None, None)],
),
"ghi": NamedMock(
name="pkg-devel",
epoch=None,
version="1.0",
release="1",
arch="x86_64",
provides=[],
),
}
cr.Metadata.return_value.keys.return_value = packages.keys()
cr.Metadata.return_value.get.side_effect = lambda key: packages[key]
self.phase.multilib_methods = ["runtime"]
self.phase.multilib = mock.Mock()
self.phase.multilib.is_multilib.side_effect = lambda pkg: pkg.name == "foo"
self.phase.valid_arches = ["x86_64", "i686", "noarch"]
self.phase.arch = "x86_64"
self.phase.package_maps = {
"x86_64": {
"pkg-devel-1.0-1.x86_64": mock.Mock(),
"pkg-devel-1.0-1.i686": mock.Mock(),
"foo-1.0-1.x86_64": mock.Mock(),
"foo-1.0-1.i686": mock.Mock(),
}
}
final = [
("pkg-devel-1.0-1", "x86_64"),
("foo-1.0-1", "x86_64"),
("foo-1.0-1", "i686"),
]
po.side_effect = [
([("pkg-devel-1.0-1", "x86_64"), ("foo-1.0-1", "x86_64")], set()),
(final, set()),
]
res = self.phase.run_solver(
self.compose.variants["Server"],
"x86_64",
[("pkg-devel", None), ("foo", None)],
platform=None,
modular_rpms=[],
)
self.assertEqual(res, (final, set()))
self.assertEqual(
po.call_args_list, [mock.call(self.logfile1), mock.call(self.logfile2)]
)
self.assertEqual(
run.call_args_list,
[
mock.call(gc.return_value, logfile=self.logfile1, show_cmd=True),
mock.call(gc.return_value, logfile=self.logfile2, show_cmd=True),
],
)
self.assertEqual(
gc.call_args_list,
[
mock.call(
"x86_64",
[self._repo("repo")],
[],
["pkg-devel", "foo"],
[],
platform=None,
),
mock.call(
"x86_64",
[self._repo("repo")],
[],
["pkg-devel", "foo", "foo.i686"],
[],
platform=None,
),
],
)
class TestExpandPackages(helpers.PungiTestCase):
def _mk_packages(self, src=None, debug_arch=None):
pkg = MockPkg(
name="pkg",
version="1",
release="2",
arch="x86_64",
epoch=3,
sourcerpm="pkg-1-2.src",
file_path="/tmp/pkg.rpm",
)
nevra_to_pkg = {"pkg-3:1-2.x86_64": pkg}
if src or debug_arch:
nevra_to_pkg["pkg-3:1-2.src"] = pkg._replace(
arch="src", file_path="/tmp/spkg.rpm"
)
if debug_arch:
nevra_to_pkg["pkg-debuginfo-3:1-2.%s" % debug_arch] = pkg._replace(
name="pkg-debuginfo", arch=debug_arch, file_path="/tmp/d1.rpm"
)
return nevra_to_pkg
def test_single_package(self):
nevra_to_pkg = self._mk_packages()
res = hybrid.expand_packages(
nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64")], []
)
self.assertEqual(
res,
{
"rpm": [{"path": "/tmp/pkg.rpm", "flags": []}],
"srpm": [],
"debuginfo": [],
},
)
def test_include_src_and_debuginfo(self):
nevra_to_pkg = self._mk_packages(debug_arch="x86_64")
res = hybrid.expand_packages(
nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64")], []
)
self.assertEqual(
res,
{
"rpm": [{"path": "/tmp/pkg.rpm", "flags": []}],
"srpm": [{"path": "/tmp/spkg.rpm", "flags": []}],
"debuginfo": [{"path": "/tmp/d1.rpm", "flags": []}],
},
)
def test_skip_debuginfo_for_different_arch(self):
nevra_to_pkg = self._mk_packages(debug_arch="i686")
res = hybrid.expand_packages(
nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64")], []
)
self.assertEqual(
res,
{
"rpm": [{"path": "/tmp/pkg.rpm", "flags": []}],
"srpm": [{"path": "/tmp/spkg.rpm", "flags": []}],
"debuginfo": [],
},
)
@mock.patch("pungi.phases.gather.methods.method_hybrid.cr")
def test_skip_lookaside_source_and_debuginfo(self, cr):
nevra_to_pkg = self._mk_packages(debug_arch="x86_64")
lookasides = [mock.Mock()]
repo = {
"abc": NamedMock(
name="pkg",
arch="src",
location_base="file:///tmp/",
location_href="spkg.rpm",
),
"def": NamedMock(
name="pkg-debuginfo",
arch="x86_64",
location_base="file:///tmp/",
location_href="d1.rpm",
),
}
cr.Metadata.return_value.keys.return_value = repo.keys()
cr.Metadata.return_value.get.side_effect = lambda key: repo[key]
res = hybrid.expand_packages(
nevra_to_pkg, {}, lookasides, [("pkg-3:1-2", "x86_64")], []
)
self.assertEqual(
res,
{
"rpm": [{"path": "/tmp/pkg.rpm", "flags": []}],
"srpm": [],
"debuginfo": [],
},
)
@mock.patch("pungi.phases.gather.methods.method_hybrid.cr")
def test_skip_lookaside_packages(self, cr):
nevra_to_pkg = self._mk_packages(debug_arch="x86_64")
lookasides = [mock.Mock()]
repo = {
"abc": NamedMock(
name="pkg",
arch="x86_64",
location_base="file:///tmp/",
location_href="pkg.rpm",
)
}
cr.Metadata.return_value.keys.return_value = repo.keys()
cr.Metadata.return_value.get.side_effect = lambda key: repo[key]
res = hybrid.expand_packages(
nevra_to_pkg, {}, lookasides, [("pkg-3:1-2", "x86_64")], []
)
self.assertEqual(res, {"rpm": [], "srpm": [], "debuginfo": []})
def test_expand_module_packages(self):
nevra_to_pkg = self._mk_packages(src=True)
mod = MockModule(
"foo",
stream="1.0",
version="201807131350",
context="deadcafe",
rpms=["pkg-3:1-2.x86_64"],
)
res = hybrid.expand_packages(
nevra_to_pkg, {"foo-1.0": mod}, [], [], ["foo:1.0:201807131350:deadcafe"]
)
self.assertEqual(
res,
{
"rpm": [{"flags": [], "path": "/tmp/pkg.rpm"}],
"srpm": [{"flags": [], "path": "/tmp/spkg.rpm"}],
"debuginfo": [],
},
)

25
tests/test_gather_phase.py
View File

@ -711,6 +711,31 @@ class TestGatherPackages(helpers.PungiTestCase):
self.assertEqual(get_gather_method.call_args_list,
[mock.call('nodeps'), mock.call('deps'), mock.call('deps')])
@mock.patch("pungi.phases.gather.get_variant_packages")
@mock.patch("pungi.phases.gather.get_gather_method")
def test_hybrid_method(self, get_gather_method, get_variant_packages):
packages, groups, filters = mock.Mock(), mock.Mock(), mock.Mock()
get_variant_packages.side_effect = (
lambda c, v, a, s, p: (packages, groups, filters)
if s == "comps"
else (None, None, None)
)
compose = helpers.DummyCompose(self.topdir, {"gather_method": "hybrid"})
variant = compose.variants["Server"]
pkg_set = mock.Mock()
gather.gather_packages(compose, "x86_64", variant, pkg_set),
self.assertItemsEqual(
get_variant_packages.call_args_list,
[
mock.call(compose, "x86_64", variant, "module", pkg_set),
mock.call(compose, "x86_64", variant, "comps", pkg_set)
],
)
self.assertEqual(get_gather_method.call_args_list, [mock.call("hybrid")])
method_kwargs = get_gather_method.return_value.return_value.call_args_list[0][1]
self.assertEqual(method_kwargs["packages"], packages)
self.assertEqual(method_kwargs["groups"], groups)
class TestWritePrepopulate(helpers.PungiTestCase):
def test_without_config(self):