Variant as a lookaside - configuration

Relates: COMPOSE-2425

Signed-off-by: Ondrej Nosek <onosek@redhat.com>

doc/configuration.rst

@@ -703,6 +703,12 @@ Options
     performance profiling information at the end of its logs.  Only takes
     effect when ``gather_backend = "dnf"``.
 
+**variant_as_lookaside**
+    (*list*) -- a variant/variant mapping that tells one or more variants in compose
+    has other variant(s) in compose as a lookaside. Only top level variants are
+    supported (not addons/layered products). Format:
+    ``[(variant_uid, variant_uid)]``
+
 
 Example
 -------

pungi/checks.py

@@ -1086,6 +1086,19 @@ def make_schema():
             "gather_lookaside_repos": _variant_arch_mapping({
                 "$ref": "#/definitions/strings",
             }),
+
+            "variant_as_lookaside": {
+                "type": "array",
+                "items": {
+                    "type": "array",
+                    "items": [
+                        {"type": "string"},
+                        {"type": "string"},
+                    ],
+                    "minItems": 2,
+                    "maxItems": 2,
+                },
+            },
         },
 
         "required": ["release_name", "release_short", "release_version",

pungi/graph.py

@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+
+
+class SimpleAcyclicOrientedGraph(object):
+    """
+    Stores a graph data structure and allows operation with it.
+    Example data: {'P1': ['P2'], 'P3': ['P4', 'P5'], 'P2': 'P3'}
+    Graph is constructed by adding oriented edges one by one. It can not contain cycles.
+    Main result is spanning line, it determines ordering of the nodes.
+    """
+    def __init__(self):
+        self._graph = {}
+        self._all_nodes = set()
+
+    def add_edge(self, start, end):
+        """
+        Add one edge from node 'start' to node 'end'.
+        This operation must not create a cycle in the graph.
+        """
+        if start == end:
+            raise ValueError("Can not add this kind of edge into graph: %s-%s" % (start, end))
+        self._graph.setdefault(start, [])
+        if end not in self._graph[start]:
+            self._graph[start].append(end)
+        self._all_nodes.add(start)
+        self._all_nodes.add(end)
+        # try to find opposite direction path (from end to start) to detect newly created cycle
+        path = SimpleAcyclicOrientedGraph.find_path(self._graph, end, start)
+        if path:
+            raise ValueError("There is a cycle in the graph: %s" % path)
+
+    def get_active_nodes(self):
+        """
+        nodes connected to any edge
+        """
+        active_nodes = set()
+        for start, ends in self._graph.items():
+            active_nodes.add(start)
+            active_nodes.update(ends)
+        return active_nodes
+
+    def is_final_endpoint(self, node):
+        """
+        edge(s) ends in this node; no other edge starts in this node
+        """
+        if node not in self._all_nodes:
+            return ValueError("This node is not found in the graph: %s" % node)
+        if node not in self.get_active_nodes():
+            return False
+        return False if node in self._graph else True
+
+    def remove_final_endpoint(self, node):
+        """
+        """
+        remove_start_points = []
+        for start, ends in self._graph.items():
+            if node in ends:
+                ends.remove(node)
+                if not ends:
+                    remove_start_points.append(start)
+        for start in remove_start_points:
+            del self._graph[start]
+
+    @staticmethod
+    def find_path(graph, start, end, path=[]):
+        """
+        find path among nodes 'start' and 'end' recursively
+        """
+        path = path + [start]
+        if start == end:
+            return path
+        if start not in graph:
+            return None
+        for node in graph[start]:
+            if node not in path:
+                newpath = SimpleAcyclicOrientedGraph.find_path(graph, node, end, path)
+                if newpath:
+                    return newpath
+        return None
+
+    def prune_graph(self):
+        """
+        Construct spanning_line by pruning the graph.
+        Looking for endpoints and remove them one by one until graph is empty.
+        """
+        spanning_line = []
+        while self._graph:
+            for node in sorted(self._all_nodes):
+                if self.is_final_endpoint(node):
+                    self.remove_final_endpoint(node)
+                    spanning_line.insert(0, node)
+                    # orphan node = no edge is connected with this node
+                    orphans = self._all_nodes - self.get_active_nodes()
+                    if orphans:
+                        break  # restart iteration not to set size self._all_nodes during iteration
+            for orphan in orphans:
+                if orphan not in spanning_line:
+                    spanning_line.insert(0, orphan)
+                self._all_nodes.remove(orphan)
+        return spanning_line

pungi/phases/gather/__init__.py

@@ -14,9 +14,9 @@
 # along with this program; if not, see <https://gnu.org/licenses/>.
 
 
+import json
 import os
 import shutil
-import json
 
 from kobo.rpmlib import parse_nvra
 from productmd.rpms import Rpms
@@ -24,10 +24,11 @@ from productmd.rpms import Rpms
 from pungi.wrappers.scm import get_file_from_scm
 from .link import link_files
 
-from pungi.util import get_arch_variant_data, get_arch_data, get_variant_data
-from pungi.phases.base import PhaseBase
-from pungi.arch import split_name_arch, get_compatible_arches
 from pungi import Modulemd
+from pungi.arch import get_compatible_arches, split_name_arch
+from pungi.graph import SimpleAcyclicOrientedGraph
+from pungi.phases.base import PhaseBase
+from pungi.util import get_arch_data, get_arch_variant_data, get_variant_data
 
 
 def get_gather_source(name):
@@ -75,6 +76,13 @@ class GatherPhase(PhaseBase):
                 if variant.modules:
                     errors.append('Modular compose requires pdc_client and libmodulemd packages.')
 
+        # check whether variants from configuration value 'variant_as_lookaside' are correct
+        variant_as_lookaside = self.compose.conf.get("variant_as_lookaside", [])
+        for variant_pair in variant_as_lookaside:
+            for variant_uid in variant_pair:
+                if variant_uid not in self.compose.all_variants:
+                    errors.append("Variant uid '%s' does't exists in 'variant_as_lookaside'" % variant_uid)
+
         if errors:
             raise ValueError('\n'.join(errors))
 
@@ -282,14 +290,42 @@ def trim_packages(compose, arch, variant, pkg_map, parent_pkgs=None, remove_pkgs
     return addon_pkgs, move_to_parent_pkgs, removed_pkgs
 
 
+def _prepare_variant_as_lookaside(compose):
+    """
+    Configuration value 'variant_as_lookaside' contains variant pairs: <variant - its lookaside>
+    In that pair lookaside variant have to be processed first. Structure can be represented
+    as a oriented graph. Its spanning line shows order how to process this set of variants.
+    """
+    variant_as_lookaside = compose.conf.get("variant_as_lookaside", [])
+    graph = SimpleAcyclicOrientedGraph()
+    for variant, lookaside_variant in variant_as_lookaside:
+        try:
+            graph.add_edge(variant, lookaside_variant)
+        except ValueError as e:
+            raise ValueError("There is a bad configuration in 'variant_as_lookaside': %s" % e.message)
+
+    variant_processing_order = reversed(graph.prune_graph())
+    return list(variant_processing_order)
+
+
 def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltree=False):
     """Run gathering on all arches of all variants of given type.
 
     If ``exclude_fulltree`` is set, all source packages from parent variants
     will be added to fulltree excludes for the processed variants.
     """
-    for arch in compose.get_arches():
+
-        for variant in compose.get_variants(arch=arch, types=[variant_type]):
+    ordered_variants_uids = _prepare_variant_as_lookaside(compose)
+    # Some variants were not mentioned in configuration value 'variant_as_lookaside'
+    # and its run order is not crucial (that means there are no dependencies inside this group).
+    # They will be processed first. A-Z sorting is for reproducibility.
+    unordered_variants_uids = sorted(set(compose.all_variants.keys()) - set(ordered_variants_uids))
+
+    for variant_uid in unordered_variants_uids + ordered_variants_uids:
+        variant = compose.all_variants[variant_uid]
+        if variant.type != variant_type:
+            continue
+        for arch in variant.arches:
             fulltree_excludes = set()
             if exclude_fulltree:
                 for pkg_name, pkg_arch in get_parent_pkgs(arch, variant, result)["srpm"]:

tests/test_config.py

@@ -406,5 +406,27 @@ class RepoclosureTestCase(ConfigTestCase):
             ["Failed validation in repoclosure_backend: 'fnd' is not one of %s" % options])
 
 
+class VariantAsLookasideTestCase(ConfigTestCase):
+    def test_empty(self):
+        variant_as_lookaside = []
+        cfg = load_config(
+            PKGSET_REPOS,
+            variant_as_lookaside=variant_as_lookaside,
+        )
+        self.assertValidation(cfg)
+
+    def test_basic(self):
+        variant_as_lookaside = [
+            ("Client", "Base"),
+            ("Server", "Client"),
+            ("Everything", "Spin"),
+        ]
+        cfg = load_config(
+            PKGSET_REPOS,
+            variant_as_lookaside=variant_as_lookaside,
+        )
+        self.assertValidation(cfg)
+
+
 if __name__ == '__main__':
     unittest.main()

tests/test_graph.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
+
+import os
+import sys
+
+from pungi.graph import SimpleAcyclicOrientedGraph
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+
+class SimpleAcyclicOrientedGraphTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.g = SimpleAcyclicOrientedGraph()
+
+    def test_simple_graph(self):
+        graph_data = (
+            ('Client', 'Base'),
+            ('Server', 'Base'),
+            ('Workstation', 'Base'),
+        )
+
+        for start, end in graph_data:
+            self.g.add_edge(start, end)
+        spanning_line = self.g.prune_graph()
+
+        self.assertEqual(4, len(spanning_line))
+        # 'Base' as a lookaside should be at the end of the spanning line, order of others is not crucial
+        self.assertEqual("Base", spanning_line[-1])
+
+    def test_complex_graph(self):
+        graph_data = (
+            ('1', '3'),  # 1 --> 3 --> 4 --> 5 ...
+            ('3', '4'),
+            ('4', '5'),
+            ('4', '6'),
+            ('2', '4'),
+            ('7', '6'),
+            ('6', '5'),
+        )
+
+        for start, end in graph_data:
+            self.g.add_edge(start, end)
+        spanning_line = self.g.prune_graph()
+
+        # spanning line have to match completely to given graph
+        self.assertEqual(['1', '3', '2', '4', '7', '6', '5'], spanning_line)
+
+    def test_cyclic_graph(self):
+        graph_data = (
+            ('1', '2'),
+            ('2', '3'),
+            ('3', '1'),
+        )
+
+        with self.assertRaises(ValueError):
+            for start, end in graph_data:
+                self.g.add_edge(start, end)
+
+    def test_two_separate_graph_lines(self):
+        graph_data = (
+            ('1', '3'),  # 1st graph
+            ('3', '2'),  # 1st graph
+            ('6', '5'),  # 2nd graph
+        )
+
+        for start, end in graph_data:
+            self.g.add_edge(start, end)
+        spanning_line = self.g.prune_graph()
+        spanning_line_str = ''.join(spanning_line)
+
+        self.assertEqual(5, len(spanning_line))
+        # Particular parts should match. Order of these parts is not crucial.
+        self.assertTrue(
+            "132" in spanning_line_str and "65" in spanning_line_str,
+            "Spanning line '%s' does not match to graphs" % spanning_line_str
+        )
+
+    def alternative_route_in_graph(self):
+        graph_data = (
+            ('1', '3'),
+            ('3', '2'),
+            ('1', '2'),
+        )
+
+        for start, end in graph_data:
+            self.g.add_edge(start, end)
+        spanning_line = self.g.prune_graph()
+
+        # spanning line have to match completely to given graph
+        self.assertEqual(['1', '3', '2'], spanning_line)