diff --git a/pungi/scripts/gather_rpms.py b/pungi/scripts/gather_rpms.py new file mode 100644 index 00000000..9a9f9a50 --- /dev/null +++ b/pungi/scripts/gather_rpms.py @@ -0,0 +1,75 @@ +from argparse import ArgumentParser + +import os +from typing import List + +from attr import dataclass +from productmd.common import parse_nvra + + +@dataclass +class Package: + nvra: str + path: str + + +def search_rpms(top_dir) -> List[Package]: + """ + Search for all *.rpm files recursively + in given top directory + Returns: + list: list of paths + """ + rpms = [] + for root, dirs, files in os.walk(top_dir): + path = root.split(os.sep) + for file in files: + if not file.endswith('.rpm'): + continue + nvra, _ = os.path.splitext(file) + rpms.append( + Package(nvra=nvra, path=os.path.join('/', *path, file)) + ) + return rpms + + +def copy_rpms(packages: List[Package], target_top_dir: str): + """ + Search synced repos for rpms and prepare + koji-like structure for pungi + + Instead of repos, use following structure: + # ls /mnt/koji/ + i686/ noarch/ x86_64/ + Returns: + Nothing: + """ + for package in packages: + info = parse_nvra(package.nvra) + + target_arch_dir = os.path.join(target_top_dir, info['arch']) + os.makedirs(target_arch_dir, exist_ok=True) + + target_file = os.path.join(target_arch_dir, os.path.basename(package.path)) + + if not os.path.exists(target_file): + try: + os.link(package.path, target_file) + except OSError: + # hardlink failed, try symlinking + os.symlink(package.path, target_file) + + +def cli_main(): + parser = ArgumentParser() + parser.add_argument('-p', '--path', required=True) + parser.add_argument('-t', '--target', required=True) + + namespace = parser.parse_args() + + rpms = search_rpms(namespace.path) + copy_rpms(rpms, namespace.target) + + +if __name__ == '__main__': + cli_main() diff --git a/setup.py b/setup.py index 404a359e..d740c1dd 100755 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ setup( "pungi-config-dump = pungi.scripts.config_dump:cli_main", "pungi-config-validate = pungi.scripts.config_validate:cli_main", "pungi-gather-modules = pungi.scripts.gather_modules:cli_main", + "pungi-gather-rpms = pungi.scripts.gather_rpms:cli_main", ] }, scripts=["contrib/yum-dnf-compare/pungi-compare-depsolving"], diff --git a/tests/test_gather_rpms.py b/tests/test_gather_rpms.py new file mode 100644 index 00000000..9a6f7f2d --- /dev/null +++ b/tests/test_gather_rpms.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +import os +import unittest +from pathlib import Path + +from pyfakefs.fake_filesystem_unittest import TestCase + +from pungi.scripts.gather_rpms import search_rpms, copy_rpms, Package + +PATH_TO_REPOS = '/path/to/repos' +MODULES_YAML_GZ = 'modules.yaml.gz' + + +class TestGatherRpms(TestCase): + maxDiff = None + + FILES_TO_CREATE = [ + 'powertools/Packages/libvirt-6.0.0-28.module_el8.3.0+555+a55c8938.i686.rpm', + 'powertools/Packages/libgit2-devel-0.26.8-2.el8.x86_64.rpm', + 'powertools/Packages/xalan-j2-2.7.1-38.module_el8.0.0+30+832da3a1.noarch.rpm', + 'appstream/Packages/bnd-maven-plugin-3.5.0-4.module_el8.0.0+30+832da3a1.noarch.rpm', + 'appstream/Packages/OpenEXR-devel-2.2.0-11.el8.i686.rpm', + 'appstream/Packages/mingw-binutils-generic-2.30-1.el8.x86_64.rpm', + 'appstream/Packages/somenonrpm', + ] + + def setUp(self): + self.setUpPyfakefs() + + os.makedirs(PATH_TO_REPOS) + + for filepath in self.FILES_TO_CREATE: + os.makedirs(os.path.join(PATH_TO_REPOS, os.path.dirname(filepath)), exist_ok=True) + open(os.path.join(PATH_TO_REPOS, filepath), 'w').close() + + def test_gather_rpms(self): + self.assertEqual( + [Package(nvra='libvirt-6.0.0-28.module_el8.3.0+555+a55c8938.i686', + path=f'{PATH_TO_REPOS}/powertools/Packages/' + f'libvirt-6.0.0-28.module_el8.3.0+555+a55c8938.i686.rpm'), + Package(nvra='libgit2-devel-0.26.8-2.el8.x86_64', + path=f'{PATH_TO_REPOS}/powertools/Packages/' + f'libgit2-devel-0.26.8-2.el8.x86_64.rpm'), + Package(nvra='xalan-j2-2.7.1-38.module_el8.0.0+30+832da3a1.noarch', + path=f'{PATH_TO_REPOS}/powertools/Packages/' + f'xalan-j2-2.7.1-38.module_el8.0.0+30+832da3a1.noarch.rpm'), + Package(nvra='bnd-maven-plugin-3.5.0-4.module_el8.0.0+30+832da3a1.noarch', + path='/path/to/repos/appstream/Packages/' + 'bnd-maven-plugin-3.5.0-4.module_el8.0.0+30+832da3a1.noarch.rpm'), + Package(nvra='OpenEXR-devel-2.2.0-11.el8.i686', + path=f'{PATH_TO_REPOS}/appstream/Packages/' + f'OpenEXR-devel-2.2.0-11.el8.i686.rpm'), + Package(nvra='mingw-binutils-generic-2.30-1.el8.x86_64', + path=f'{PATH_TO_REPOS}/appstream/Packages/' + f'mingw-binutils-generic-2.30-1.el8.x86_64.rpm')], + search_rpms(PATH_TO_REPOS) + ) + + def test_copy_rpms(self): + target_path = Path('/mnt/koji') + packages = [ + + Package(nvra='libvirt-6.0.0-28.module_el8.3.0+555+a55c8938.i686', + path=f'{PATH_TO_REPOS}/powertools/Packages/' + f'libvirt-6.0.0-28.module_el8.3.0+555+a55c8938.i686.rpm'), + Package(nvra='libgit2-devel-0.26.8-2.el8.x86_64', + path=f'{PATH_TO_REPOS}/powertools/Packages/' + f'libgit2-devel-0.26.8-2.el8.x86_64.rpm'), + Package(nvra='xalan-j2-2.7.1-38.module_el8.0.0+30+832da3a1.noarch', + path=f'{PATH_TO_REPOS}/powertools/Packages/' + f'xalan-j2-2.7.1-38.module_el8.0.0+30+832da3a1.noarch.rpm'), + Package(nvra='bnd-maven-plugin-3.5.0-4.module_el8.0.0+30+832da3a1.noarch', + path='/path/to/repos/appstream/Packages/' + 'bnd-maven-plugin-3.5.0-4.module_el8.0.0+30+832da3a1.noarch.rpm'), + Package(nvra='OpenEXR-devel-2.2.0-11.el8.i686', + path=f'{PATH_TO_REPOS}/appstream/Packages/' + f'OpenEXR-devel-2.2.0-11.el8.i686.rpm'), + Package(nvra='mingw-binutils-generic-2.30-1.el8.x86_64', + path=f'{PATH_TO_REPOS}/appstream/Packages/' + f'mingw-binutils-generic-2.30-1.el8.x86_64.rpm') + ] + copy_rpms(packages, target_path) + + self.assertCountEqual([ + 'xalan-j2-2.7.1-38.module_el8.0.0+30+832da3a1.noarch.rpm', + 'bnd-maven-plugin-3.5.0-4.module_el8.0.0+30+832da3a1.noarch.rpm' + ], os.listdir(target_path / 'noarch')) + + self.assertCountEqual([ + 'libgit2-devel-0.26.8-2.el8.x86_64.rpm', + 'mingw-binutils-generic-2.30-1.el8.x86_64.rpm' + ], os.listdir(target_path / 'x86_64')) + + self.assertCountEqual([ + 'libvirt-6.0.0-28.module_el8.3.0+555+a55c8938.i686.rpm', + 'OpenEXR-devel-2.2.0-11.el8.i686.rpm' + ], os.listdir(target_path / 'i686')) + + self.assertCountEqual([ + 'i686', 'x86_64', 'noarch' + ], os.listdir(target_path)) + + +if __name__ == '__main__': + unittest.main()