gather: Link files in order for dependant variants

If there are variants that depend on another, they should be processed
in order to make sure packages from the base variant are linked first.
That way the srpm cache is populated and any package in layered variant
but with source in base will have access to correct epoch information.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2019-01-31 15:52:34 +01:00
parent 8065239e04
commit 940a581bd9
2 changed files with 45 additions and 30 deletions

View File

@ -32,6 +32,7 @@ from productmd.images import Images
from dogpile.cache import make_region from dogpile.cache import make_region
from pungi.graph import SimpleAcyclicOrientedGraph
from pungi.wrappers.variants import VariantsXmlParser from pungi.wrappers.variants import VariantsXmlParser
from pungi.paths import Paths from pungi.paths import Paths
from pungi.wrappers.scm import get_file_from_scm from pungi.wrappers.scm import get_file_from_scm
@ -428,3 +429,40 @@ class Compose(kobo.log.LoggingBase):
""" """
path = os.path.join(self.paths.work.tmp_dir(arch=arch, variant=variant)) path = os.path.join(self.paths.work.tmp_dir(arch=arch, variant=variant))
return tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=path) return tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=path)
def get_ordered_variant_uids(compose):
if not hasattr(compose, "_ordered_variant_uids"):
ordered_variant_uids = _prepare_variant_as_lookaside(compose)
# Some variants were not mentioned in configuration value
# 'variant_as_lookaside' and its run order is not crucial (that
# means there are no dependencies inside this group). They will be
# processed first. A-Z sorting is for reproducibility.
unordered_variant_uids = sorted(
set(compose.all_variants.keys()) - set(ordered_variant_uids)
)
setattr(
compose,
"_ordered_variant_uids",
unordered_variant_uids + ordered_variant_uids
)
return getattr(compose, "_ordered_variant_uids")
def _prepare_variant_as_lookaside(compose):
"""
Configuration value 'variant_as_lookaside' contains variant pairs <variant,
its lookaside>. In that pair lookaside variant have to be processed first.
Structure can be represented as a oriented graph. Its spanning line shows
order how to process this set of variants.
"""
variant_as_lookaside = compose.conf.get("variant_as_lookaside", [])
graph = SimpleAcyclicOrientedGraph()
for variant, lookaside_variant in variant_as_lookaside:
try:
graph.add_edge(variant, lookaside_variant)
except ValueError as e:
raise ValueError("There is a bad configuration in 'variant_as_lookaside': %s" % e)
variant_processing_order = reversed(graph.prune_graph())
return list(variant_processing_order)

View File

@ -28,8 +28,8 @@ from ...wrappers.createrepo import CreaterepoWrapper
import pungi.wrappers.kojiwrapper import pungi.wrappers.kojiwrapper
from pungi import Modulemd from pungi import Modulemd
from pungi.compose import get_ordered_variant_uids
from pungi.arch import get_compatible_arches, split_name_arch, tree_arch_to_yum_arch from pungi.arch import get_compatible_arches, split_name_arch, tree_arch_to_yum_arch
from pungi.graph import SimpleAcyclicOrientedGraph
from pungi.phases.base import PhaseBase from pungi.phases.base import PhaseBase
from pungi.util import (get_arch_data, get_arch_variant_data, get_variant_data, from pungi.util import (get_arch_data, get_arch_variant_data, get_variant_data,
makedirs, iter_module_defaults) makedirs, iter_module_defaults)
@ -100,10 +100,11 @@ class GatherPhase(PhaseBase):
pkg_map = gather_wrapper(self.compose, self.pkgset_phase.package_sets, pkg_map = gather_wrapper(self.compose, self.pkgset_phase.package_sets,
self.pkgset_phase.path_prefix) self.pkgset_phase.path_prefix)
for arch in self.compose.get_arches(): for variant_uid in get_ordered_variant_uids(self.compose):
for variant in self.compose.get_variants(arch=arch): variant = self.compose.all_variants[variant_uid]
if variant.is_empty: if variant.is_empty:
continue continue
for arch in variant.arches:
link_files(self.compose, arch, variant, link_files(self.compose, arch, variant,
pkg_map[arch][variant.uid], pkg_map[arch][variant.uid],
self.pkgset_phase.package_sets, self.pkgset_phase.package_sets,
@ -343,24 +344,6 @@ def trim_packages(compose, arch, variant, pkg_map, parent_pkgs=None, remove_pkgs
return addon_pkgs, move_to_parent_pkgs, removed_pkgs return addon_pkgs, move_to_parent_pkgs, removed_pkgs
def _prepare_variant_as_lookaside(compose):
"""
Configuration value 'variant_as_lookaside' contains variant pairs: <variant - its lookaside>
In that pair lookaside variant have to be processed first. Structure can be represented
as a oriented graph. Its spanning line shows order how to process this set of variants.
"""
variant_as_lookaside = compose.conf.get("variant_as_lookaside", [])
graph = SimpleAcyclicOrientedGraph()
for variant, lookaside_variant in variant_as_lookaside:
try:
graph.add_edge(variant, lookaside_variant)
except ValueError as e:
raise ValueError("There is a bad configuration in 'variant_as_lookaside': %s" % e)
variant_processing_order = reversed(graph.prune_graph())
return list(variant_processing_order)
def _make_lookaside_repo(compose, variant, arch, pkg_map): def _make_lookaside_repo(compose, variant, arch, pkg_map):
""" """
Create variant lookaside repo for given variant and architecture with Create variant lookaside repo for given variant and architecture with
@ -460,13 +443,7 @@ def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltr
will be added to fulltree excludes for the processed variants. will be added to fulltree excludes for the processed variants.
""" """
ordered_variants_uids = _prepare_variant_as_lookaside(compose) for variant_uid in get_ordered_variant_uids(compose):
# Some variants were not mentioned in configuration value 'variant_as_lookaside'
# and its run order is not crucial (that means there are no dependencies inside this group).
# They will be processed first. A-Z sorting is for reproducibility.
unordered_variants_uids = sorted(set(compose.all_variants.keys()) - set(ordered_variants_uids))
for variant_uid in unordered_variants_uids + ordered_variants_uids:
variant = compose.all_variants[variant_uid] variant = compose.all_variants[variant_uid]
if variant.type != variant_type: if variant.type != variant_type:
continue continue