From cb3d36be5d51c01363fef3e97efd53d0dd60c7cc Mon Sep 17 00:00:00 2001 From: Ondrej Nosek Date: Tue, 10 Apr 2018 14:27:55 +0200 Subject: [PATCH] Variant as a lookaside - configuration Relates: COMPOSE-2425 Signed-off-by: Ondrej Nosek --- doc/configuration.rst | 6 ++ pungi/checks.py | 13 +++++ pungi/graph.py | 100 ++++++++++++++++++++++++++++++++ pungi/phases/gather/__init__.py | 48 +++++++++++++-- tests/test_config.py | 22 +++++++ tests/test_graph.py | 97 +++++++++++++++++++++++++++++++ 6 files changed, 280 insertions(+), 6 deletions(-) create mode 100755 pungi/graph.py create mode 100644 tests/test_graph.py diff --git a/doc/configuration.rst b/doc/configuration.rst index 2ed89342..47eca587 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -703,6 +703,12 @@ Options performance profiling information at the end of its logs. Only takes effect when ``gather_backend = "dnf"``. +**variant_as_lookaside** + (*list*) -- a variant/variant mapping that tells one or more variants in compose + has other variant(s) in compose as a lookaside. Only top level variants are + supported (not addons/layered products). Format: + ``[(variant_uid, variant_uid)]`` + Example ------- diff --git a/pungi/checks.py b/pungi/checks.py index 9f684b46..2c2b7443 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -1086,6 +1086,19 @@ def make_schema(): "gather_lookaside_repos": _variant_arch_mapping({ "$ref": "#/definitions/strings", }), + + "variant_as_lookaside": { + "type": "array", + "items": { + "type": "array", + "items": [ + {"type": "string"}, + {"type": "string"}, + ], + "minItems": 2, + "maxItems": 2, + }, + }, }, "required": ["release_name", "release_short", "release_version", diff --git a/pungi/graph.py b/pungi/graph.py new file mode 100755 index 00000000..bec4833d --- /dev/null +++ b/pungi/graph.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- + + +class SimpleAcyclicOrientedGraph(object): + """ + Stores a graph data structure and allows operation with it. + Example data: {'P1': ['P2'], 'P3': ['P4', 'P5'], 'P2': 'P3'} + Graph is constructed by adding oriented edges one by one. It can not contain cycles. + Main result is spanning line, it determines ordering of the nodes. + """ + def __init__(self): + self._graph = {} + self._all_nodes = set() + + def add_edge(self, start, end): + """ + Add one edge from node 'start' to node 'end'. + This operation must not create a cycle in the graph. + """ + if start == end: + raise ValueError("Can not add this kind of edge into graph: %s-%s" % (start, end)) + self._graph.setdefault(start, []) + if end not in self._graph[start]: + self._graph[start].append(end) + self._all_nodes.add(start) + self._all_nodes.add(end) + # try to find opposite direction path (from end to start) to detect newly created cycle + path = SimpleAcyclicOrientedGraph.find_path(self._graph, end, start) + if path: + raise ValueError("There is a cycle in the graph: %s" % path) + + def get_active_nodes(self): + """ + nodes connected to any edge + """ + active_nodes = set() + for start, ends in self._graph.items(): + active_nodes.add(start) + active_nodes.update(ends) + return active_nodes + + def is_final_endpoint(self, node): + """ + edge(s) ends in this node; no other edge starts in this node + """ + if node not in self._all_nodes: + return ValueError("This node is not found in the graph: %s" % node) + if node not in self.get_active_nodes(): + return False + return False if node in self._graph else True + + def remove_final_endpoint(self, node): + """ + """ + remove_start_points = [] + for start, ends in self._graph.items(): + if node in ends: + ends.remove(node) + if not ends: + remove_start_points.append(start) + for start in remove_start_points: + del self._graph[start] + + @staticmethod + def find_path(graph, start, end, path=[]): + """ + find path among nodes 'start' and 'end' recursively + """ + path = path + [start] + if start == end: + return path + if start not in graph: + return None + for node in graph[start]: + if node not in path: + newpath = SimpleAcyclicOrientedGraph.find_path(graph, node, end, path) + if newpath: + return newpath + return None + + def prune_graph(self): + """ + Construct spanning_line by pruning the graph. + Looking for endpoints and remove them one by one until graph is empty. + """ + spanning_line = [] + while self._graph: + for node in sorted(self._all_nodes): + if self.is_final_endpoint(node): + self.remove_final_endpoint(node) + spanning_line.insert(0, node) + # orphan node = no edge is connected with this node + orphans = self._all_nodes - self.get_active_nodes() + if orphans: + break # restart iteration not to set size self._all_nodes during iteration + for orphan in orphans: + if orphan not in spanning_line: + spanning_line.insert(0, orphan) + self._all_nodes.remove(orphan) + return spanning_line diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 58e69c2b..a842a774 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -14,9 +14,9 @@ # along with this program; if not, see . +import json import os import shutil -import json from kobo.rpmlib import parse_nvra from productmd.rpms import Rpms @@ -24,10 +24,11 @@ from productmd.rpms import Rpms from pungi.wrappers.scm import get_file_from_scm from .link import link_files -from pungi.util import get_arch_variant_data, get_arch_data, get_variant_data -from pungi.phases.base import PhaseBase -from pungi.arch import split_name_arch, get_compatible_arches from pungi import Modulemd +from pungi.arch import get_compatible_arches, split_name_arch +from pungi.graph import SimpleAcyclicOrientedGraph +from pungi.phases.base import PhaseBase +from pungi.util import get_arch_data, get_arch_variant_data, get_variant_data def get_gather_source(name): @@ -75,6 +76,13 @@ class GatherPhase(PhaseBase): if variant.modules: errors.append('Modular compose requires pdc_client and libmodulemd packages.') + # check whether variants from configuration value 'variant_as_lookaside' are correct + variant_as_lookaside = self.compose.conf.get("variant_as_lookaside", []) + for variant_pair in variant_as_lookaside: + for variant_uid in variant_pair: + if variant_uid not in self.compose.all_variants: + errors.append("Variant uid '%s' does't exists in 'variant_as_lookaside'" % variant_uid) + if errors: raise ValueError('\n'.join(errors)) @@ -282,14 +290,42 @@ def trim_packages(compose, arch, variant, pkg_map, parent_pkgs=None, remove_pkgs return addon_pkgs, move_to_parent_pkgs, removed_pkgs +def _prepare_variant_as_lookaside(compose): + """ + Configuration value 'variant_as_lookaside' contains variant pairs: + In that pair lookaside variant have to be processed first. Structure can be represented + as a oriented graph. Its spanning line shows order how to process this set of variants. + """ + variant_as_lookaside = compose.conf.get("variant_as_lookaside", []) + graph = SimpleAcyclicOrientedGraph() + for variant, lookaside_variant in variant_as_lookaside: + try: + graph.add_edge(variant, lookaside_variant) + except ValueError as e: + raise ValueError("There is a bad configuration in 'variant_as_lookaside': %s" % e.message) + + variant_processing_order = reversed(graph.prune_graph()) + return list(variant_processing_order) + + def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltree=False): """Run gathering on all arches of all variants of given type. If ``exclude_fulltree`` is set, all source packages from parent variants will be added to fulltree excludes for the processed variants. """ - for arch in compose.get_arches(): - for variant in compose.get_variants(arch=arch, types=[variant_type]): + + ordered_variants_uids = _prepare_variant_as_lookaside(compose) + # Some variants were not mentioned in configuration value 'variant_as_lookaside' + # and its run order is not crucial (that means there are no dependencies inside this group). + # They will be processed first. A-Z sorting is for reproducibility. + unordered_variants_uids = sorted(set(compose.all_variants.keys()) - set(ordered_variants_uids)) + + for variant_uid in unordered_variants_uids + ordered_variants_uids: + variant = compose.all_variants[variant_uid] + if variant.type != variant_type: + continue + for arch in variant.arches: fulltree_excludes = set() if exclude_fulltree: for pkg_name, pkg_arch in get_parent_pkgs(arch, variant, result)["srpm"]: diff --git a/tests/test_config.py b/tests/test_config.py index b2c1c8fb..9423eb6a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -406,5 +406,27 @@ class RepoclosureTestCase(ConfigTestCase): ["Failed validation in repoclosure_backend: 'fnd' is not one of %s" % options]) +class VariantAsLookasideTestCase(ConfigTestCase): + def test_empty(self): + variant_as_lookaside = [] + cfg = load_config( + PKGSET_REPOS, + variant_as_lookaside=variant_as_lookaside, + ) + self.assertValidation(cfg) + + def test_basic(self): + variant_as_lookaside = [ + ("Client", "Base"), + ("Server", "Client"), + ("Everything", "Spin"), + ] + cfg = load_config( + PKGSET_REPOS, + variant_as_lookaside=variant_as_lookaside, + ) + self.assertValidation(cfg) + + if __name__ == '__main__': unittest.main() diff --git a/tests/test_graph.py b/tests/test_graph.py new file mode 100644 index 00000000..977ce2a2 --- /dev/null +++ b/tests/test_graph.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +try: + import unittest2 as unittest +except ImportError: + import unittest + +import os +import sys + +from pungi.graph import SimpleAcyclicOrientedGraph + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + + +class SimpleAcyclicOrientedGraphTestCase(unittest.TestCase): + + def setUp(self): + self.g = SimpleAcyclicOrientedGraph() + + def test_simple_graph(self): + graph_data = ( + ('Client', 'Base'), + ('Server', 'Base'), + ('Workstation', 'Base'), + ) + + for start, end in graph_data: + self.g.add_edge(start, end) + spanning_line = self.g.prune_graph() + + self.assertEqual(4, len(spanning_line)) + # 'Base' as a lookaside should be at the end of the spanning line, order of others is not crucial + self.assertEqual("Base", spanning_line[-1]) + + def test_complex_graph(self): + graph_data = ( + ('1', '3'), # 1 --> 3 --> 4 --> 5 ... + ('3', '4'), + ('4', '5'), + ('4', '6'), + ('2', '4'), + ('7', '6'), + ('6', '5'), + ) + + for start, end in graph_data: + self.g.add_edge(start, end) + spanning_line = self.g.prune_graph() + + # spanning line have to match completely to given graph + self.assertEqual(['1', '3', '2', '4', '7', '6', '5'], spanning_line) + + def test_cyclic_graph(self): + graph_data = ( + ('1', '2'), + ('2', '3'), + ('3', '1'), + ) + + with self.assertRaises(ValueError): + for start, end in graph_data: + self.g.add_edge(start, end) + + def test_two_separate_graph_lines(self): + graph_data = ( + ('1', '3'), # 1st graph + ('3', '2'), # 1st graph + ('6', '5'), # 2nd graph + ) + + for start, end in graph_data: + self.g.add_edge(start, end) + spanning_line = self.g.prune_graph() + spanning_line_str = ''.join(spanning_line) + + self.assertEqual(5, len(spanning_line)) + # Particular parts should match. Order of these parts is not crucial. + self.assertTrue( + "132" in spanning_line_str and "65" in spanning_line_str, + "Spanning line '%s' does not match to graphs" % spanning_line_str + ) + + def alternative_route_in_graph(self): + graph_data = ( + ('1', '3'), + ('3', '2'), + ('1', '2'), + ) + + for start, end in graph_data: + self.g.add_edge(start, end) + spanning_line = self.g.prune_graph() + + # spanning line have to match completely to given graph + self.assertEqual(['1', '3', '2'], spanning_line)