diff --git a/doc/configuration.rst b/doc/configuration.rst
index 2ed89342..47eca587 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -703,6 +703,12 @@ Options
performance profiling information at the end of its logs. Only takes
effect when ``gather_backend = "dnf"``.
+**variant_as_lookaside**
+ (*list*) -- a variant/variant mapping that tells one or more variants in compose
+ has other variant(s) in compose as a lookaside. Only top level variants are
+ supported (not addons/layered products). Format:
+ ``[(variant_uid, variant_uid)]``
+
Example
-------
diff --git a/pungi/checks.py b/pungi/checks.py
index 9f684b46..2c2b7443 100644
--- a/pungi/checks.py
+++ b/pungi/checks.py
@@ -1086,6 +1086,19 @@ def make_schema():
"gather_lookaside_repos": _variant_arch_mapping({
"$ref": "#/definitions/strings",
}),
+
+ "variant_as_lookaside": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": [
+ {"type": "string"},
+ {"type": "string"},
+ ],
+ "minItems": 2,
+ "maxItems": 2,
+ },
+ },
},
"required": ["release_name", "release_short", "release_version",
diff --git a/pungi/graph.py b/pungi/graph.py
new file mode 100755
index 00000000..bec4833d
--- /dev/null
+++ b/pungi/graph.py
@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+
+
+class SimpleAcyclicOrientedGraph(object):
+ """
+ Stores a graph data structure and allows operation with it.
+ Example data: {'P1': ['P2'], 'P3': ['P4', 'P5'], 'P2': 'P3'}
+ Graph is constructed by adding oriented edges one by one. It can not contain cycles.
+ Main result is spanning line, it determines ordering of the nodes.
+ """
+ def __init__(self):
+ self._graph = {}
+ self._all_nodes = set()
+
+ def add_edge(self, start, end):
+ """
+ Add one edge from node 'start' to node 'end'.
+ This operation must not create a cycle in the graph.
+ """
+ if start == end:
+ raise ValueError("Can not add this kind of edge into graph: %s-%s" % (start, end))
+ self._graph.setdefault(start, [])
+ if end not in self._graph[start]:
+ self._graph[start].append(end)
+ self._all_nodes.add(start)
+ self._all_nodes.add(end)
+ # try to find opposite direction path (from end to start) to detect newly created cycle
+ path = SimpleAcyclicOrientedGraph.find_path(self._graph, end, start)
+ if path:
+ raise ValueError("There is a cycle in the graph: %s" % path)
+
+ def get_active_nodes(self):
+ """
+ nodes connected to any edge
+ """
+ active_nodes = set()
+ for start, ends in self._graph.items():
+ active_nodes.add(start)
+ active_nodes.update(ends)
+ return active_nodes
+
+ def is_final_endpoint(self, node):
+ """
+ edge(s) ends in this node; no other edge starts in this node
+ """
+ if node not in self._all_nodes:
+ return ValueError("This node is not found in the graph: %s" % node)
+ if node not in self.get_active_nodes():
+ return False
+ return False if node in self._graph else True
+
+ def remove_final_endpoint(self, node):
+ """
+ """
+ remove_start_points = []
+ for start, ends in self._graph.items():
+ if node in ends:
+ ends.remove(node)
+ if not ends:
+ remove_start_points.append(start)
+ for start in remove_start_points:
+ del self._graph[start]
+
+ @staticmethod
+ def find_path(graph, start, end, path=[]):
+ """
+ find path among nodes 'start' and 'end' recursively
+ """
+ path = path + [start]
+ if start == end:
+ return path
+ if start not in graph:
+ return None
+ for node in graph[start]:
+ if node not in path:
+ newpath = SimpleAcyclicOrientedGraph.find_path(graph, node, end, path)
+ if newpath:
+ return newpath
+ return None
+
+ def prune_graph(self):
+ """
+ Construct spanning_line by pruning the graph.
+ Looking for endpoints and remove them one by one until graph is empty.
+ """
+ spanning_line = []
+ while self._graph:
+ for node in sorted(self._all_nodes):
+ if self.is_final_endpoint(node):
+ self.remove_final_endpoint(node)
+ spanning_line.insert(0, node)
+ # orphan node = no edge is connected with this node
+ orphans = self._all_nodes - self.get_active_nodes()
+ if orphans:
+ break # restart iteration not to set size self._all_nodes during iteration
+ for orphan in orphans:
+ if orphan not in spanning_line:
+ spanning_line.insert(0, orphan)
+ self._all_nodes.remove(orphan)
+ return spanning_line
diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py
index 58e69c2b..a842a774 100644
--- a/pungi/phases/gather/__init__.py
+++ b/pungi/phases/gather/__init__.py
@@ -14,9 +14,9 @@
# along with this program; if not, see .
+import json
import os
import shutil
-import json
from kobo.rpmlib import parse_nvra
from productmd.rpms import Rpms
@@ -24,10 +24,11 @@ from productmd.rpms import Rpms
from pungi.wrappers.scm import get_file_from_scm
from .link import link_files
-from pungi.util import get_arch_variant_data, get_arch_data, get_variant_data
-from pungi.phases.base import PhaseBase
-from pungi.arch import split_name_arch, get_compatible_arches
from pungi import Modulemd
+from pungi.arch import get_compatible_arches, split_name_arch
+from pungi.graph import SimpleAcyclicOrientedGraph
+from pungi.phases.base import PhaseBase
+from pungi.util import get_arch_data, get_arch_variant_data, get_variant_data
def get_gather_source(name):
@@ -75,6 +76,13 @@ class GatherPhase(PhaseBase):
if variant.modules:
errors.append('Modular compose requires pdc_client and libmodulemd packages.')
+ # check whether variants from configuration value 'variant_as_lookaside' are correct
+ variant_as_lookaside = self.compose.conf.get("variant_as_lookaside", [])
+ for variant_pair in variant_as_lookaside:
+ for variant_uid in variant_pair:
+ if variant_uid not in self.compose.all_variants:
+ errors.append("Variant uid '%s' does't exists in 'variant_as_lookaside'" % variant_uid)
+
if errors:
raise ValueError('\n'.join(errors))
@@ -282,14 +290,42 @@ def trim_packages(compose, arch, variant, pkg_map, parent_pkgs=None, remove_pkgs
return addon_pkgs, move_to_parent_pkgs, removed_pkgs
+def _prepare_variant_as_lookaside(compose):
+ """
+ Configuration value 'variant_as_lookaside' contains variant pairs:
+ In that pair lookaside variant have to be processed first. Structure can be represented
+ as a oriented graph. Its spanning line shows order how to process this set of variants.
+ """
+ variant_as_lookaside = compose.conf.get("variant_as_lookaside", [])
+ graph = SimpleAcyclicOrientedGraph()
+ for variant, lookaside_variant in variant_as_lookaside:
+ try:
+ graph.add_edge(variant, lookaside_variant)
+ except ValueError as e:
+ raise ValueError("There is a bad configuration in 'variant_as_lookaside': %s" % e.message)
+
+ variant_processing_order = reversed(graph.prune_graph())
+ return list(variant_processing_order)
+
+
def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltree=False):
"""Run gathering on all arches of all variants of given type.
If ``exclude_fulltree`` is set, all source packages from parent variants
will be added to fulltree excludes for the processed variants.
"""
- for arch in compose.get_arches():
- for variant in compose.get_variants(arch=arch, types=[variant_type]):
+
+ ordered_variants_uids = _prepare_variant_as_lookaside(compose)
+ # Some variants were not mentioned in configuration value 'variant_as_lookaside'
+ # and its run order is not crucial (that means there are no dependencies inside this group).
+ # They will be processed first. A-Z sorting is for reproducibility.
+ unordered_variants_uids = sorted(set(compose.all_variants.keys()) - set(ordered_variants_uids))
+
+ for variant_uid in unordered_variants_uids + ordered_variants_uids:
+ variant = compose.all_variants[variant_uid]
+ if variant.type != variant_type:
+ continue
+ for arch in variant.arches:
fulltree_excludes = set()
if exclude_fulltree:
for pkg_name, pkg_arch in get_parent_pkgs(arch, variant, result)["srpm"]:
diff --git a/tests/test_config.py b/tests/test_config.py
index b2c1c8fb..9423eb6a 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -406,5 +406,27 @@ class RepoclosureTestCase(ConfigTestCase):
["Failed validation in repoclosure_backend: 'fnd' is not one of %s" % options])
+class VariantAsLookasideTestCase(ConfigTestCase):
+ def test_empty(self):
+ variant_as_lookaside = []
+ cfg = load_config(
+ PKGSET_REPOS,
+ variant_as_lookaside=variant_as_lookaside,
+ )
+ self.assertValidation(cfg)
+
+ def test_basic(self):
+ variant_as_lookaside = [
+ ("Client", "Base"),
+ ("Server", "Client"),
+ ("Everything", "Spin"),
+ ]
+ cfg = load_config(
+ PKGSET_REPOS,
+ variant_as_lookaside=variant_as_lookaside,
+ )
+ self.assertValidation(cfg)
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/tests/test_graph.py b/tests/test_graph.py
new file mode 100644
index 00000000..977ce2a2
--- /dev/null
+++ b/tests/test_graph.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+try:
+ import unittest2 as unittest
+except ImportError:
+ import unittest
+
+import os
+import sys
+
+from pungi.graph import SimpleAcyclicOrientedGraph
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+
+class SimpleAcyclicOrientedGraphTestCase(unittest.TestCase):
+
+ def setUp(self):
+ self.g = SimpleAcyclicOrientedGraph()
+
+ def test_simple_graph(self):
+ graph_data = (
+ ('Client', 'Base'),
+ ('Server', 'Base'),
+ ('Workstation', 'Base'),
+ )
+
+ for start, end in graph_data:
+ self.g.add_edge(start, end)
+ spanning_line = self.g.prune_graph()
+
+ self.assertEqual(4, len(spanning_line))
+ # 'Base' as a lookaside should be at the end of the spanning line, order of others is not crucial
+ self.assertEqual("Base", spanning_line[-1])
+
+ def test_complex_graph(self):
+ graph_data = (
+ ('1', '3'), # 1 --> 3 --> 4 --> 5 ...
+ ('3', '4'),
+ ('4', '5'),
+ ('4', '6'),
+ ('2', '4'),
+ ('7', '6'),
+ ('6', '5'),
+ )
+
+ for start, end in graph_data:
+ self.g.add_edge(start, end)
+ spanning_line = self.g.prune_graph()
+
+ # spanning line have to match completely to given graph
+ self.assertEqual(['1', '3', '2', '4', '7', '6', '5'], spanning_line)
+
+ def test_cyclic_graph(self):
+ graph_data = (
+ ('1', '2'),
+ ('2', '3'),
+ ('3', '1'),
+ )
+
+ with self.assertRaises(ValueError):
+ for start, end in graph_data:
+ self.g.add_edge(start, end)
+
+ def test_two_separate_graph_lines(self):
+ graph_data = (
+ ('1', '3'), # 1st graph
+ ('3', '2'), # 1st graph
+ ('6', '5'), # 2nd graph
+ )
+
+ for start, end in graph_data:
+ self.g.add_edge(start, end)
+ spanning_line = self.g.prune_graph()
+ spanning_line_str = ''.join(spanning_line)
+
+ self.assertEqual(5, len(spanning_line))
+ # Particular parts should match. Order of these parts is not crucial.
+ self.assertTrue(
+ "132" in spanning_line_str and "65" in spanning_line_str,
+ "Spanning line '%s' does not match to graphs" % spanning_line_str
+ )
+
+ def alternative_route_in_graph(self):
+ graph_data = (
+ ('1', '3'),
+ ('3', '2'),
+ ('1', '2'),
+ )
+
+ for start, end in graph_data:
+ self.g.add_edge(start, end)
+ spanning_line = self.g.prune_graph()
+
+ # spanning line have to match completely to given graph
+ self.assertEqual(['1', '3', '2'], spanning_line)