From 8b11bb81afd97cce7ed092611a07878d2344a0aa Mon Sep 17 00:00:00 2001 From: Stepan Oksanichenko Date: Mon, 24 Jul 2023 18:26:51 +0300 Subject: [PATCH] AL-5220: Investigate why CL9 can't built on the new nebula - Exclude the packages for using in a build --- pungi/scripts/gather_rpms.py | 40 ++++++++++++++++++++++++++++++------ tests/test_gather_rpms.py | 38 +++++++++++++++++++--------------- 2 files changed, 56 insertions(+), 22 deletions(-) diff --git a/pungi/scripts/gather_rpms.py b/pungi/scripts/gather_rpms.py index 1ce770c2..4d35929a 100644 --- a/pungi/scripts/gather_rpms.py +++ b/pungi/scripts/gather_rpms.py @@ -1,3 +1,4 @@ +import re from argparse import ArgumentParser import os @@ -11,7 +12,7 @@ from productmd.common import parse_nvra @dataclass class Package: - nvra: str + nvra: dict path: Path @@ -23,12 +24,30 @@ def search_rpms(top_dir: Path) -> List[Package]: list: list of paths """ return [Package( - nvra=Path(path).stem, + nvra=parse_nvra(Path(path).stem), path=Path(path), ) for path in iglob(str(top_dir.joinpath('**/*.rpm')), recursive=True)] -def copy_rpms(packages: List[Package], target_top_dir: Path): +def is_excluded_package( + package: Package, + excluded_packages: List[str], +) -> bool: + package_key = f'{package.nvra["name"]}.{package.nvra["arch"]}' + return any( + re.search( + f'^{excluded_pkg}$', + package_key, + ) or excluded_pkg in (package.nvra['name'], package_key) + for excluded_pkg in excluded_packages + ) + + +def copy_rpms( + packages: List[Package], + target_top_dir: Path, + excluded_packages: List[str], +): """ Search synced repos for rpms and prepare koji-like structure for pungi @@ -40,8 +59,9 @@ def copy_rpms(packages: List[Package], target_top_dir: Path): Nothing: """ for package in packages: - info = parse_nvra(package.nvra) - target_arch_dir = target_top_dir.joinpath(info['arch']) + if is_excluded_package(package, excluded_packages): + continue + target_arch_dir = target_top_dir.joinpath(package.nvra['arch']) target_file = target_arch_dir.joinpath(package.path.name) os.makedirs(target_arch_dir, exist_ok=True) @@ -57,11 +77,19 @@ def cli_main(): parser = ArgumentParser() parser.add_argument('-p', '--path', required=True, type=Path) parser.add_argument('-t', '--target', required=True, type=Path) + parser.add_argument( + '-e', + '--excluded-packages', + required=False, + nargs='+', + type=str, + default=[], + ) namespace = parser.parse_args() rpms = search_rpms(namespace.path) - copy_rpms(rpms, namespace.target) + copy_rpms(rpms, namespace.target, namespace.excluded_packages) if __name__ == '__main__': diff --git a/tests/test_gather_rpms.py b/tests/test_gather_rpms.py index 460349f3..d1b0fa1e 100644 --- a/tests/test_gather_rpms.py +++ b/tests/test_gather_rpms.py @@ -6,6 +6,7 @@ from pathlib import Path from pyfakefs.fake_filesystem_unittest import TestCase from pungi.scripts.gather_rpms import search_rpms, copy_rpms, Package +from productmd.common import parse_nvra PATH_TO_REPOS = '/path/to/repos' MODULES_YAML_GZ = 'modules.yaml.gz' @@ -41,37 +42,39 @@ class TestGatherRpms(TestCase): def test_gather_rpms(self): self.assertEqual( - [Package(nvra='libvirt-6.0.0-28.module_el8.3.0+555+a55c8938.i686', + [Package(nvra=parse_nvra('libvirt-6.0.0-28.module_' + 'el8.3.0+555+a55c8938.i686'), path=Path( f'{PATH_TO_REPOS}/powertools/Packages/' f'libvirt-6.0.0-28.module_el' f'8.3.0+555+a55c8938.i686.rpm' )), - Package(nvra='libgit2-devel-0.26.8-2.el8.x86_64', + Package(nvra=parse_nvra('libgit2-devel-0.26.8-2.el8.x86_64'), path=Path( f'{PATH_TO_REPOS}/powertools/Packages/' f'libgit2-devel-0.26.8-2.el8.x86_64.rpm' )), - Package(nvra='xalan-j2-2.7.1-38.module_el' - '8.0.0+30+832da3a1.noarch', + Package(nvra=parse_nvra('xalan-j2-2.7.1-38.module_el' + '8.0.0+30+832da3a1.noarch'), path=Path( f'{PATH_TO_REPOS}/powertools/Packages/' f'xalan-j2-2.7.1-38.module_el' f'8.0.0+30+832da3a1.noarch.rpm' )), - Package(nvra='bnd-maven-plugin-3.5.0-4.module_el' - '8.0.0+30+832da3a1.noarch', + Package(nvra=parse_nvra('bnd-maven-plugin-3.5.0-4.module_el' + '8.0.0+30+832da3a1.noarch'), path=Path( '/path/to/repos/appstream/Packages/' 'bnd-maven-plugin-3.5.0-4.module_el' '8.0.0+30+832da3a1.noarch.rpm' )), - Package(nvra='OpenEXR-devel-2.2.0-11.el8.i686', + Package(nvra=parse_nvra('OpenEXR-devel-2.2.0-11.el8.i686'), path=Path( f'{PATH_TO_REPOS}/appstream/Packages/' f'OpenEXR-devel-2.2.0-11.el8.i686.rpm' )), - Package(nvra='mingw-binutils-generic-2.30-1.el8.x86_64', + Package(nvra=parse_nvra('mingw-binutils-generic-' + '2.30-1.el8.x86_64'), path=Path( f'{PATH_TO_REPOS}/appstream/Packages/' f'mingw-binutils-generic-2.30-1.el8.x86_64.rpm' @@ -84,42 +87,45 @@ class TestGatherRpms(TestCase): target_path = Path('/mnt/koji') packages = [ - Package(nvra='libvirt-6.0.0-28.module_el8.3.0+555+a55c8938.i686', + Package(nvra=parse_nvra('libvirt-6.0.0-28.module_' + 'el8.3.0+555+a55c8938.i686'), path=Path( f'{PATH_TO_REPOS}/powertools/Packages/' f'libvirt-6.0.0-28.module_el' f'8.3.0+555+a55c8938.i686.rpm' )), - Package(nvra='libgit2-devel-0.26.8-2.el8.x86_64', + Package(nvra=parse_nvra('libgit2-devel-0.26.8-2.el8.x86_64'), path=Path( f'{PATH_TO_REPOS}/powertools/Packages/' f'libgit2-devel-0.26.8-2.el8.x86_64.rpm' )), - Package(nvra='xalan-j2-2.7.1-38.module_el8.0.0+30+832da3a1.noarch', + Package(nvra=parse_nvra('xalan-j2-2.7.1-38.module_' + 'el8.0.0+30+832da3a1.noarch'), path=Path( f'{PATH_TO_REPOS}/powertools/Packages/' f'xalan-j2-2.7.1-38.module_el' f'8.0.0+30+832da3a1.noarch.rpm' )), - Package(nvra='bnd-maven-plugin-3.5.0-4.module_el' - '8.0.0+30+832da3a1.noarch', + Package(nvra=parse_nvra('bnd-maven-plugin-3.5.0-4.module_el' + '8.0.0+30+832da3a1.noarch'), path=Path( '/path/to/repos/appstream/Packages/' 'bnd-maven-plugin-3.5.0-4.module_el' '8.0.0+30+832da3a1.noarch.rpm' )), - Package(nvra='OpenEXR-devel-2.2.0-11.el8.i686', + Package(nvra=parse_nvra('OpenEXR-devel-2.2.0-11.el8.i686'), path=Path( f'{PATH_TO_REPOS}/appstream/Packages/' f'OpenEXR-devel-2.2.0-11.el8.i686.rpm' )), - Package(nvra='mingw-binutils-generic-2.30-1.el8.x86_64', + Package(nvra=parse_nvra('mingw-binutils-generic-' + '2.30-1.el8.x86_64'), path=Path( f'{PATH_TO_REPOS}/appstream/Packages/' f'mingw-binutils-generic-2.30-1.el8.x86_64.rpm' )) ] - copy_rpms(packages, target_path) + copy_rpms(packages, target_path, []) self.assertCountEqual([ 'xalan-j2-2.7.1-38.module_el8.0.0+30+832da3a1.noarch.rpm',