diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 2ca22db0..23cc09e1 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -23,6 +23,13 @@ from itertools import groupby from kobo.rpmlib import parse_nvra from kobo.shortcuts import force_list +from typing import ( + Dict, + AnyStr, + List, + Tuple, + Set, +) import pungi.wrappers.kojiwrapper from pungi.wrappers.comps import CompsWrapper @@ -507,6 +514,45 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules) return modules_to_keep +def _filter_expected_modules( + variant_name: AnyStr, + variant_arches: List[AnyStr], + expected_modules: Set[AnyStr], + filtered_modules: List[Tuple[AnyStr, Dict[AnyStr, List[AnyStr]]]], +) -> set: + """ + Function filters out all modules which are listed in Pungi config. + Those modules can be absent in koji env so we must remove it from + the expected modules list otherwise Pungi will fail + """ + for variant_regexp, filters_dict in filtered_modules: + for arch, modules in filters_dict.items(): + arch = '.*' if arch == '*' else arch + variant_regexp = '.*' if variant_regexp == '*' else variant_regexp + modules = ['.*' if module == '*' else module for module in modules] + cond1 = re.findall( + variant_regexp, + variant_name, + ) + cond2 = any( + re.findall( + arch, + variant_arch, + ) for variant_arch in variant_arches + ) + if cond1 and cond2: + expected_modules = { + expected_module for expected_module in expected_modules if + not all( + re.findall( + filtered_module, + expected_module, + ) for filtered_module in modules + ) + } + return expected_modules + + def _get_modules_from_koji_tags( compose, koji_wrapper, event_id, variant, variant_tags, tag_to_mmd ): @@ -622,7 +668,12 @@ def _get_modules_from_koji_tags( # needed in createrepo phase where metadata is exposed by # productmd variant.module_uid_to_koji_tag[nsvc] = module_tag - + expected_modules = _filter_expected_modules( + variant_name=variant.name, + variant_arches=variant.arches, + expected_modules=expected_modules, + filtered_modules=compose.conf['filter_modules'], + ) if expected_modules: # There are some module names that were listed in configuration and not # found in any tag... diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index 8d72a608..9d844bc6 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -1,15 +1,19 @@ # -*- coding: utf-8 -*- import json -import mock + import os import re import six +from ddt import ddt, data, unpack +from typing import AnyStr, List, Set, Dict, Tuple try: import unittest2 as unittest + from unittest2 import mock except ImportError: import unittest + from unittest import mock from pungi.phases.pkgset.sources import source_koji from tests import helpers @@ -968,3 +972,124 @@ class TestAddScratchModuleToVariant(helpers.PungiTestCase): self.compose.log_warning.assert_called_once_with( "Only test composes could include scratch module builds" ) + + +@ddt +class TestSourceKoji(unittest.TestCase): + + @unpack + @data( + ( + 'AppStream', + [ + 'x86_64', + 'i386' + ], + { + 'python39-devel:3.9', + 'python39:3.9', + }, + [ + ( + '^(BaseOS|AppStream|PowerTools)$', + { + 'x86_64': [ + 'python39:3.9', + ], + 'aarch64': [ + 'python39-devel:3.9', + ] + } + ) + ], + { + 'python39-devel:3.9', + } + ), + ( + 'AppStream', + [ + 'x86_64', + 'i386' + ], + { + 'python39-devel:3.9', + 'python39:3.9', + 'python38-devel:3.8', + 'python38:3.8', + }, + [ + ( + '^(BaseOS|AppStream|PowerTools)$', + { + 'x86_64': [ + 'python39:3.9', + ], + '*': [ + 'python38-devel:3.8', + ] + } + ) + ], + { + 'python39-devel:3.9', + 'python38:3.8', + } + ), + ( + 'AppStream', + [ + 'x86_64', + 'i386' + ], + { + 'python39-devel:3.9', + 'python39:3.9', + 'python38-devel:3.8', + 'python38:3.8', + }, + [ + ( + '^(BaseOS|AppStream|PowerTools)$', + { + 'x86_64': [ + 'python39:3.9', + ], + 'aarch64': [ + 'python38-devel:3.8', + ] + } + ), + ( + '*', + { + '*': [ + 'python38-devel:3.8', + ] + } + ), + ], + { + 'python39-devel:3.9', + 'python38:3.8', + } + ), + ) + def test__filter_expected_modules( + self, + variant_name: AnyStr, + variant_arches: List[AnyStr], + expected_modules: Set[AnyStr], + filtered_modules: List[Tuple[AnyStr, Dict[AnyStr, List[AnyStr]]]], + expected_result: Set[AnyStr], + ) -> None: + real_result = source_koji._filter_expected_modules( + variant_name=variant_name, + variant_arches=variant_arches, + expected_modules=expected_modules, + filtered_modules=filtered_modules, + ) + self.assertSetEqual( + real_result, + expected_result, + ) \ No newline at end of file