almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Variant as a lookaside - configuration

Browse Source 
Relates: COMPOSE-2425

Signed-off-by: Ondrej Nosek <onosek@redhat.com>
This commit is contained in:
Ondrej Nosek 2018-04-10 14:27:55 +02:00
parent 15ccd309fa
commit cb3d36be5d
6 changed files with 280 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
doc/configuration.rst
View File

@ -703,6 +703,12 @@ Options
performance profiling information at the end of its logs. Only takes
effect when ``gather_backend = "dnf"``.
**variant_as_lookaside**
(*list*) -- a variant/variant mapping that tells one or more variants in compose
has other variant(s) in compose as a lookaside. Only top level variants are
supported (not addons/layered products). Format:
``[(variant_uid, variant_uid)]``
Example
-------

13
pungi/checks.py
View File

@ -1086,6 +1086,19 @@ def make_schema():
"gather_lookaside_repos": _variant_arch_mapping({
"$ref": "#/definitions/strings",
}),
"variant_as_lookaside": {
"type": "array",
"items": {
"type": "array",
"items": [
{"type": "string"},
{"type": "string"},
],
"minItems": 2,
"maxItems": 2,
},
},
},
"required": ["release_name", "release_short", "release_version",

100
pungi/graph.py Executable file
View File

@ -0,0 +1,100 @@
# -*- coding: utf-8 -*-
class SimpleAcyclicOrientedGraph(object):
"""
Stores a graph data structure and allows operation with it.
Example data: {'P1': ['P2'], 'P3': ['P4', 'P5'], 'P2': 'P3'}
Graph is constructed by adding oriented edges one by one. It can not contain cycles.
Main result is spanning line, it determines ordering of the nodes.
"""
def __init__(self):
self._graph = {}
self._all_nodes = set()
def add_edge(self, start, end):
"""
Add one edge from node 'start' to node 'end'.
This operation must not create a cycle in the graph.
"""
if start == end:
raise ValueError("Can not add this kind of edge into graph: %s-%s" % (start, end))
self._graph.setdefault(start, [])
if end not in self._graph[start]:
self._graph[start].append(end)
self._all_nodes.add(start)
self._all_nodes.add(end)
# try to find opposite direction path (from end to start) to detect newly created cycle
path = SimpleAcyclicOrientedGraph.find_path(self._graph, end, start)
if path:
raise ValueError("There is a cycle in the graph: %s" % path)
def get_active_nodes(self):
"""
nodes connected to any edge
"""
active_nodes = set()
for start, ends in self._graph.items():
active_nodes.add(start)
active_nodes.update(ends)
return active_nodes
def is_final_endpoint(self, node):
"""
edge(s) ends in this node; no other edge starts in this node
"""
if node not in self._all_nodes:
return ValueError("This node is not found in the graph: %s" % node)
if node not in self.get_active_nodes():
return False
return False if node in self._graph else True
def remove_final_endpoint(self, node):
"""
"""
remove_start_points = []
for start, ends in self._graph.items():
if node in ends:
ends.remove(node)
if not ends:
remove_start_points.append(start)
for start in remove_start_points:
del self._graph[start]
@staticmethod
def find_path(graph, start, end, path=[]):
"""
find path among nodes 'start' and 'end' recursively
"""
path = path + [start]
if start == end:
return path
if start not in graph:
return None
for node in graph[start]:
if node not in path:
newpath = SimpleAcyclicOrientedGraph.find_path(graph, node, end, path)
if newpath:
return newpath
return None
def prune_graph(self):
"""
Construct spanning_line by pruning the graph.
Looking for endpoints and remove them one by one until graph is empty.
"""
spanning_line = []
while self._graph:
for node in sorted(self._all_nodes):
if self.is_final_endpoint(node):
self.remove_final_endpoint(node)
spanning_line.insert(0, node)
# orphan node = no edge is connected with this node
orphans = self._all_nodes - self.get_active_nodes()
if orphans:
break # restart iteration not to set size self._all_nodes during iteration
for orphan in orphans:
if orphan not in spanning_line:
spanning_line.insert(0, orphan)
self._all_nodes.remove(orphan)
return spanning_line

48
pungi/phases/gather/__init__.py
View File

@ -14,9 +14,9 @@
# along with this program; if not, see <https://gnu.org/licenses/>.
import json
import os
import shutil
import json
from kobo.rpmlib import parse_nvra
from productmd.rpms import Rpms
@ -24,10 +24,11 @@ from productmd.rpms import Rpms
from pungi.wrappers.scm import get_file_from_scm
from .link import link_files
from pungi.util import get_arch_variant_data, get_arch_data, get_variant_data
from pungi.phases.base import PhaseBase
from pungi.arch import split_name_arch, get_compatible_arches
from pungi import Modulemd
from pungi.arch import get_compatible_arches, split_name_arch
from pungi.graph import SimpleAcyclicOrientedGraph
from pungi.phases.base import PhaseBase
from pungi.util import get_arch_data, get_arch_variant_data, get_variant_data
def get_gather_source(name):
@ -75,6 +76,13 @@ class GatherPhase(PhaseBase):
if variant.modules:
errors.append('Modular compose requires pdc_client and libmodulemd packages.')
# check whether variants from configuration value 'variant_as_lookaside' are correct
variant_as_lookaside = self.compose.conf.get("variant_as_lookaside", [])
for variant_pair in variant_as_lookaside:
for variant_uid in variant_pair:
if variant_uid not in self.compose.all_variants:
errors.append("Variant uid '%s' does't exists in 'variant_as_lookaside'" % variant_uid)
if errors:
raise ValueError('\n'.join(errors))
@ -282,14 +290,42 @@ def trim_packages(compose, arch, variant, pkg_map, parent_pkgs=None, remove_pkgs
return addon_pkgs, move_to_parent_pkgs, removed_pkgs
def _prepare_variant_as_lookaside(compose):
"""
Configuration value 'variant_as_lookaside' contains variant pairs: <variant - its lookaside>
In that pair lookaside variant have to be processed first. Structure can be represented
as a oriented graph. Its spanning line shows order how to process this set of variants.
"""
variant_as_lookaside = compose.conf.get("variant_as_lookaside", [])
graph = SimpleAcyclicOrientedGraph()
for variant, lookaside_variant in variant_as_lookaside:
try:
graph.add_edge(variant, lookaside_variant)
except ValueError as e:
raise ValueError("There is a bad configuration in 'variant_as_lookaside': %s" % e.message)
variant_processing_order = reversed(graph.prune_graph())
return list(variant_processing_order)
def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltree=False):
"""Run gathering on all arches of all variants of given type.
If ``exclude_fulltree`` is set, all source packages from parent variants
will be added to fulltree excludes for the processed variants.
"""
for arch in compose.get_arches():
for variant in compose.get_variants(arch=arch, types=[variant_type]):
ordered_variants_uids = _prepare_variant_as_lookaside(compose)
# Some variants were not mentioned in configuration value 'variant_as_lookaside'
# and its run order is not crucial (that means there are no dependencies inside this group).
# They will be processed first. A-Z sorting is for reproducibility.
unordered_variants_uids = sorted(set(compose.all_variants.keys()) - set(ordered_variants_uids))
for variant_uid in unordered_variants_uids + ordered_variants_uids:
variant = compose.all_variants[variant_uid]
if variant.type != variant_type:
continue
for arch in variant.arches:
fulltree_excludes = set()
if exclude_fulltree:
for pkg_name, pkg_arch in get_parent_pkgs(arch, variant, result)["srpm"]:

22
tests/test_config.py
View File

@ -406,5 +406,27 @@ class RepoclosureTestCase(ConfigTestCase):
["Failed validation in repoclosure_backend: 'fnd' is not one of %s" % options])
class VariantAsLookasideTestCase(ConfigTestCase):
def test_empty(self):
variant_as_lookaside = []
cfg = load_config(
PKGSET_REPOS,
variant_as_lookaside=variant_as_lookaside,
)
self.assertValidation(cfg)
def test_basic(self):
variant_as_lookaside = [
("Client", "Base"),
("Server", "Client"),
("Everything", "Spin"),
]
cfg = load_config(
PKGSET_REPOS,
variant_as_lookaside=variant_as_lookaside,
)
self.assertValidation(cfg)
if __name__ == '__main__':
unittest.main()

97
tests/test_graph.py Normal file
View File

@ -0,0 +1,97 @@
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
try:
import unittest2 as unittest
except ImportError:
import unittest
import os
import sys
from pungi.graph import SimpleAcyclicOrientedGraph
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
class SimpleAcyclicOrientedGraphTestCase(unittest.TestCase):
def setUp(self):
self.g = SimpleAcyclicOrientedGraph()
def test_simple_graph(self):
graph_data = (
('Client', 'Base'),
('Server', 'Base'),
('Workstation', 'Base'),
)
for start, end in graph_data:
self.g.add_edge(start, end)
spanning_line = self.g.prune_graph()
self.assertEqual(4, len(spanning_line))
# 'Base' as a lookaside should be at the end of the spanning line, order of others is not crucial
self.assertEqual("Base", spanning_line[-1])
def test_complex_graph(self):
graph_data = (
('1', '3'), # 1 --> 3 --> 4 --> 5 ...
('3', '4'),
('4', '5'),
('4', '6'),
('2', '4'),
('7', '6'),
('6', '5'),
)
for start, end in graph_data:
self.g.add_edge(start, end)
spanning_line = self.g.prune_graph()
# spanning line have to match completely to given graph
self.assertEqual(['1', '3', '2', '4', '7', '6', '5'], spanning_line)
def test_cyclic_graph(self):
graph_data = (
('1', '2'),
('2', '3'),
('3', '1'),
)
with self.assertRaises(ValueError):
for start, end in graph_data:
self.g.add_edge(start, end)
def test_two_separate_graph_lines(self):
graph_data = (
('1', '3'), # 1st graph
('3', '2'), # 1st graph
('6', '5'), # 2nd graph
)
for start, end in graph_data:
self.g.add_edge(start, end)
spanning_line = self.g.prune_graph()
spanning_line_str = ''.join(spanning_line)
self.assertEqual(5, len(spanning_line))
# Particular parts should match. Order of these parts is not crucial.
self.assertTrue(
"132" in spanning_line_str and "65" in spanning_line_str,
"Spanning line '%s' does not match to graphs" % spanning_line_str
)
def alternative_route_in_graph(self):
graph_data = (
('1', '3'),
('3', '2'),
('1', '2'),
)
for start, end in graph_data:
self.g.add_edge(start, end)
spanning_line = self.g.prune_graph()
# spanning line have to match completely to given graph
self.assertEqual(['1', '3', '2'], spanning_line)