From 4b64d20826337b2f737c4bdb5aa78161df8aff01 Mon Sep 17 00:00:00 2001 From: soksanichenko Date: Fri, 24 Mar 2023 12:45:28 +0200 Subject: [PATCH] ALBS-987: Generate i686 and dev repositories with pungi on building new distr. version automatically - Path.rglob/glob doesn't work with symlinks (it's the bug and reported) - Refactoring --- pungi/scripts/create_packages_json.py | 128 ++++++++++++++++---------- pungi/scripts/gather_rpms.py | 7 +- 2 files changed, 84 insertions(+), 51 deletions(-) diff --git a/pungi/scripts/create_packages_json.py b/pungi/scripts/create_packages_json.py index acf1cb83..2f2bdcb1 100644 --- a/pungi/scripts/create_packages_json.py +++ b/pungi/scripts/create_packages_json.py @@ -15,6 +15,7 @@ import os import re import tempfile from collections import defaultdict +from itertools import tee from pathlib import Path from typing import ( AnyStr, @@ -27,6 +28,8 @@ from typing import ( ) import binascii +from urllib.parse import urljoin + import createrepo_c as cr import dnf.subject import hawkey @@ -98,6 +101,7 @@ class PackagesGenerator: included_packages: List[AnyStr], ): self.repos = repos + self.pkgs_iterators = dict() self.excluded_packages = excluded_packages self.included_packages = included_packages self.tmp_files = [] @@ -110,6 +114,19 @@ class PackagesGenerator: if os.path.exists(tmp_file): os.remove(tmp_file) + @staticmethod + def _get_full_repo_path(repo_info: RepoInfo): + if repo_info.is_remote: + return urljoin( + repo_info.path + '/', + repo_info.folder, + ) + else: + return os.path.join( + repo_info.path, + repo_info.folder + ) + @staticmethod def _warning_callback(warning_type, message): """ @@ -151,7 +168,7 @@ class PackagesGenerator: """ Parse modules.yaml.gz and returns parsed data :param modules_file_path: path to local modules.yaml.gz - :return: List of dict for each modules in a repo + :return: List of dict for each module in a repo """ with open(modules_file_path, 'rb') as modules_file: @@ -174,16 +191,22 @@ class PackagesGenerator: :param repo_info: structure which contains info about a current repo :return: list with repomd records """ - repomd_file_path = os.path.join( - repo_info.path, - repo_info.folder, - 'repodata', - 'repomd.xml', - ) if repo_info.is_remote: + repomd_file_path = urljoin( + urljoin( + repo_info.path + '/', + repo_info.folder + ) + '/', + 'repodata/repomd.xml' + ) repomd_file_path = self.get_remote_file_content(repomd_file_path) else: - repomd_file_path = repomd_file_path + repomd_file_path = os.path.join( + repo_info.path, + repo_info.folder, + 'repodata', + 'repomd.xml', + ) repomd_object = self._parse_repomd(repomd_file_path) if repo_info.is_remote: os.remove(repomd_file_path) @@ -273,33 +296,43 @@ class PackagesGenerator: ) ) ) - all_packages = defaultdict(lambda: {'variants': list()}) + all_packages = defaultdict(lambda: { + 'variants': list(), + 'package_info': dict(), + }) for repo_info in sorted( self.repos, key=lambda i: i.repo_type, reverse=True, ): - repomd_records = self._get_repomd_records( - repo_info=repo_info, - ) - repomd_records_dict = {} # type: Dict[str, str] - self._download_repomd_records( - repo_info=repo_info, - repomd_records=repomd_records, - repomd_records_dict=repomd_records_dict, - ) - packages_iterator = PackageIterator( - primary_path=repomd_records_dict['primary'], - filelists_path=repomd_records_dict['filelists'], - other_path=repomd_records_dict['other'], - warningcb=self._warning_callback, - ) - for package in packages_iterator: + full_repo_path = self._get_full_repo_path(repo_info) + if full_repo_path in self.pkgs_iterators: + pkgs_iterator = tee(self.pkgs_iterators[full_repo_path]) + else: + repomd_records = self._get_repomd_records( + repo_info=repo_info, + ) + repomd_records_dict = {} # type: Dict[str, str] + self._download_repomd_records( + repo_info=repo_info, + repomd_records=repomd_records, + repomd_records_dict=repomd_records_dict, + ) + pkgs_iterator = PackageIterator( + primary_path=repomd_records_dict['primary'], + filelists_path=repomd_records_dict['filelists'], + other_path=repomd_records_dict['other'], + warningcb=self._warning_callback, + ) + self.pkgs_iterators[full_repo_path] = tee(pkgs_iterator) + for package in pkgs_iterator: if package.arch not in self.repo_arches[repo_info.arch]: package_arch = repo_info.arch else: package_arch = package.arch package_key = f'{package.name}.{package_arch}' + package_variants = all_packages[package_key]['variants'] + package_info = all_packages[package_key]['package_info'] if 'module' in package.release and not any( re.search(included_package, package.name) for included_package in self.included_packages @@ -307,41 +340,41 @@ class PackagesGenerator: # Even a module package will be added to packages.json if # it presents in the list of included packages continue - if package_key not in all_packages: - all_packages[package_key]['variants'].append( - (repo_info.name, repo_info.arch) - ) - all_packages[package_key]['arch'] = package_arch - all_packages[package_key]['package'] = package - all_packages[package_key]['type'] = repo_info.is_reference - elif repo_info.repo_type == 'absent' and (repo_info.name, repo_info.arch) in all_packages[package_key]['variants']: - all_packages[package_key]['variants'].remove((repo_info.name, repo_info.arch)) + if repo_info.repo_type == 'present' and not package_info: + package_variants.append((repo_info.name, repo_info.arch)) + package_info['arch'] = package_arch + package_info['package'] = package + package_info['type'] = repo_info.is_reference + elif repo_info.repo_type == 'absent' and \ + (repo_info.name, repo_info.arch) in package_variants: + package_variants.remove((repo_info.name, repo_info.arch)) # replace an older package if it's not reference or # a newer package is from reference repo - elif (not all_packages[package_key]['type'] or - all_packages[package_key]['type'] == + elif (not package_info['type'] or + package_info['type'] == repo_info.is_reference) and \ self.compare_pkgs_version( package, - all_packages[package_key]['package'] - ) > 0: + package_info['package'] + ) > 0 and repo_info.repo_type == 'present': all_packages[package_key]['variants'] = [ (repo_info.name, repo_info.arch) ] - all_packages[package_key]['arch'] = package_arch - all_packages[package_key]['package'] = package + package_info['arch'] = package_arch + package_info['package'] = package elif self.compare_pkgs_version( package, - all_packages[package_key]['package'] - ) == 0: - all_packages[package_key]['variants'].append( + package_info['package'] + ) == 0 and repo_info.repo_type == 'present': + package_variants.append( (repo_info.name, repo_info.arch) ) for package_dict in all_packages.values(): for variant_name, variant_arch in package_dict['variants']: - package_arch = package_dict['arch'] - package = package_dict['package'] + package_info = package_dict['package_info'] + package_arch = package_info['arch'] + package = package_info['package'] package_name = f'{package.name}.{package_arch}' if any(re.search(excluded_package, package_name) for excluded_package in self.excluded_packages): @@ -373,9 +406,8 @@ class PackagesGenerator: # pkgs_list.append(added_pkg) pkgs_list = packages_json[variant_name][ variant_arch][src_package_name] - added_pkg = f'{package_name}.{package_arch}' - if added_pkg not in pkgs_list: - pkgs_list.append(added_pkg) + if package_name not in pkgs_list: + pkgs_list.append(package_name) return packages_json diff --git a/pungi/scripts/gather_rpms.py b/pungi/scripts/gather_rpms.py index 369e2e79..1ce770c2 100644 --- a/pungi/scripts/gather_rpms.py +++ b/pungi/scripts/gather_rpms.py @@ -1,6 +1,7 @@ from argparse import ArgumentParser import os +from glob import iglob from typing import List from pathlib import Path @@ -22,9 +23,9 @@ def search_rpms(top_dir: Path) -> List[Package]: list: list of paths """ return [Package( - nvra=path.stem, - path=path, - ) for path in top_dir.rglob('*.rpm')] + nvra=Path(path).stem, + path=Path(path), + ) for path in iglob(str(top_dir.joinpath('**/*.rpm')), recursive=True)] def copy_rpms(packages: List[Package], target_top_dir: Path):