LNX-102: Add tool that gathers directory for all rpms

Tool that finds all available rpm files in directory
and creates special tree for pungi:
 # ls /mnt/koji/
   i686/  noarch/  x86_64/

Change-Id: Ibcf2d23c46411ad89477058f4d56e07ca117f0d1
This commit is contained in:
oshyshatskyi 2020-12-29 11:51:37 +02:00 committed by Oleksandr Shyshatskyi
parent 4404246db4
commit 478018f6f2
3 changed files with 181 additions and 0 deletions

View File

@ -0,0 +1,75 @@
from argparse import ArgumentParser
import os
from typing import List
from attr import dataclass
from productmd.common import parse_nvra
@dataclass
class Package:
nvra: str
path: str
def search_rpms(top_dir) -> List[Package]:
"""
Search for all *.rpm files recursively
in given top directory
Returns:
list: list of paths
"""
rpms = []
for root, dirs, files in os.walk(top_dir):
path = root.split(os.sep)
for file in files:
if not file.endswith('.rpm'):
continue
nvra, _ = os.path.splitext(file)
rpms.append(
Package(nvra=nvra, path=os.path.join('/', *path, file))
)
return rpms
def copy_rpms(packages: List[Package], target_top_dir: str):
"""
Search synced repos for rpms and prepare
koji-like structure for pungi
Instead of repos, use following structure:
# ls /mnt/koji/
i686/ noarch/ x86_64/
Returns:
Nothing:
"""
for package in packages:
info = parse_nvra(package.nvra)
target_arch_dir = os.path.join(target_top_dir, info['arch'])
os.makedirs(target_arch_dir, exist_ok=True)
target_file = os.path.join(target_arch_dir, os.path.basename(package.path))
if not os.path.exists(target_file):
try:
os.link(package.path, target_file)
except OSError:
# hardlink failed, try symlinking
os.symlink(package.path, target_file)
def cli_main():
parser = ArgumentParser()
parser.add_argument('-p', '--path', required=True)
parser.add_argument('-t', '--target', required=True)
namespace = parser.parse_args()
rpms = search_rpms(namespace.path)
copy_rpms(rpms, namespace.target)
if __name__ == '__main__':
cli_main()

View File

@ -48,6 +48,7 @@ setup(
"pungi-config-dump = pungi.scripts.config_dump:cli_main",
"pungi-config-validate = pungi.scripts.config_validate:cli_main",
"pungi-gather-modules = pungi.scripts.gather_modules:cli_main",
"pungi-gather-rpms = pungi.scripts.gather_rpms:cli_main",
]
},
scripts=["contrib/yum-dnf-compare/pungi-compare-depsolving"],

105
tests/test_gather_rpms.py Normal file
View File

@ -0,0 +1,105 @@
# -*- coding: utf-8 -*-
import os
import unittest
from pathlib import Path
from pyfakefs.fake_filesystem_unittest import TestCase
from pungi.scripts.gather_rpms import search_rpms, copy_rpms, Package
PATH_TO_REPOS = '/path/to/repos'
MODULES_YAML_GZ = 'modules.yaml.gz'
class TestGatherRpms(TestCase):
maxDiff = None
FILES_TO_CREATE = [
'powertools/Packages/libvirt-6.0.0-28.module_el8.3.0+555+a55c8938.i686.rpm',
'powertools/Packages/libgit2-devel-0.26.8-2.el8.x86_64.rpm',
'powertools/Packages/xalan-j2-2.7.1-38.module_el8.0.0+30+832da3a1.noarch.rpm',
'appstream/Packages/bnd-maven-plugin-3.5.0-4.module_el8.0.0+30+832da3a1.noarch.rpm',
'appstream/Packages/OpenEXR-devel-2.2.0-11.el8.i686.rpm',
'appstream/Packages/mingw-binutils-generic-2.30-1.el8.x86_64.rpm',
'appstream/Packages/somenonrpm',
]
def setUp(self):
self.setUpPyfakefs()
os.makedirs(PATH_TO_REPOS)
for filepath in self.FILES_TO_CREATE:
os.makedirs(os.path.join(PATH_TO_REPOS, os.path.dirname(filepath)), exist_ok=True)
open(os.path.join(PATH_TO_REPOS, filepath), 'w').close()
def test_gather_rpms(self):
self.assertEqual(
[Package(nvra='libvirt-6.0.0-28.module_el8.3.0+555+a55c8938.i686',
path=f'{PATH_TO_REPOS}/powertools/Packages/'
f'libvirt-6.0.0-28.module_el8.3.0+555+a55c8938.i686.rpm'),
Package(nvra='libgit2-devel-0.26.8-2.el8.x86_64',
path=f'{PATH_TO_REPOS}/powertools/Packages/'
f'libgit2-devel-0.26.8-2.el8.x86_64.rpm'),
Package(nvra='xalan-j2-2.7.1-38.module_el8.0.0+30+832da3a1.noarch',
path=f'{PATH_TO_REPOS}/powertools/Packages/'
f'xalan-j2-2.7.1-38.module_el8.0.0+30+832da3a1.noarch.rpm'),
Package(nvra='bnd-maven-plugin-3.5.0-4.module_el8.0.0+30+832da3a1.noarch',
path='/path/to/repos/appstream/Packages/'
'bnd-maven-plugin-3.5.0-4.module_el8.0.0+30+832da3a1.noarch.rpm'),
Package(nvra='OpenEXR-devel-2.2.0-11.el8.i686',
path=f'{PATH_TO_REPOS}/appstream/Packages/'
f'OpenEXR-devel-2.2.0-11.el8.i686.rpm'),
Package(nvra='mingw-binutils-generic-2.30-1.el8.x86_64',
path=f'{PATH_TO_REPOS}/appstream/Packages/'
f'mingw-binutils-generic-2.30-1.el8.x86_64.rpm')],
search_rpms(PATH_TO_REPOS)
)
def test_copy_rpms(self):
target_path = Path('/mnt/koji')
packages = [
Package(nvra='libvirt-6.0.0-28.module_el8.3.0+555+a55c8938.i686',
path=f'{PATH_TO_REPOS}/powertools/Packages/'
f'libvirt-6.0.0-28.module_el8.3.0+555+a55c8938.i686.rpm'),
Package(nvra='libgit2-devel-0.26.8-2.el8.x86_64',
path=f'{PATH_TO_REPOS}/powertools/Packages/'
f'libgit2-devel-0.26.8-2.el8.x86_64.rpm'),
Package(nvra='xalan-j2-2.7.1-38.module_el8.0.0+30+832da3a1.noarch',
path=f'{PATH_TO_REPOS}/powertools/Packages/'
f'xalan-j2-2.7.1-38.module_el8.0.0+30+832da3a1.noarch.rpm'),
Package(nvra='bnd-maven-plugin-3.5.0-4.module_el8.0.0+30+832da3a1.noarch',
path='/path/to/repos/appstream/Packages/'
'bnd-maven-plugin-3.5.0-4.module_el8.0.0+30+832da3a1.noarch.rpm'),
Package(nvra='OpenEXR-devel-2.2.0-11.el8.i686',
path=f'{PATH_TO_REPOS}/appstream/Packages/'
f'OpenEXR-devel-2.2.0-11.el8.i686.rpm'),
Package(nvra='mingw-binutils-generic-2.30-1.el8.x86_64',
path=f'{PATH_TO_REPOS}/appstream/Packages/'
f'mingw-binutils-generic-2.30-1.el8.x86_64.rpm')
]
copy_rpms(packages, target_path)
self.assertCountEqual([
'xalan-j2-2.7.1-38.module_el8.0.0+30+832da3a1.noarch.rpm',
'bnd-maven-plugin-3.5.0-4.module_el8.0.0+30+832da3a1.noarch.rpm'
], os.listdir(target_path / 'noarch'))
self.assertCountEqual([
'libgit2-devel-0.26.8-2.el8.x86_64.rpm',
'mingw-binutils-generic-2.30-1.el8.x86_64.rpm'
], os.listdir(target_path / 'x86_64'))
self.assertCountEqual([
'libvirt-6.0.0-28.module_el8.3.0+555+a55c8938.i686.rpm',
'OpenEXR-devel-2.2.0-11.el8.i686.rpm'
], os.listdir(target_path / 'i686'))
self.assertCountEqual([
'i686', 'x86_64', 'noarch'
], os.listdir(target_path))
if __name__ == '__main__':
unittest.main()