almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Replace list of cr.packages by cr.PackageIterator

Browse Source
This commit is contained in:
soksanichenko 2022-10-18 22:53:50 +03:00
parent 885b3376d6
commit 941a02722c
2 changed files with 62 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
pungi/scripts/create_extra_repo.py
View File

@ -241,9 +241,9 @@ class CreateExtraRepo(PackagesGenerator):
"""
Download all defined modularity packages and their data from
a remote repo
:param modules_data: information about all of modules in a remote repo
:param modules_data: information about all modules in a remote repo
:param repo_info: information about a remote repo
:param packages: information about all of packages (including
:param packages: information about all packages (including
modularity) in a remote repo
"""
for module in modules_data:
@ -264,7 +264,7 @@ class CreateExtraRepo(PackagesGenerator):
continue
for rpm in module['data']['artifacts']['rpms']:
# Empty repo_info.packages means that we will download
# all of packages from repo including
# all packages from repo including
# the modularity packages
if not repo_info.packages:
break
@ -290,10 +290,9 @@ class CreateExtraRepo(PackagesGenerator):
repo_info=repo_info,
)
# parse the repodata (including modules.yaml.gz)
modules_data = self._parse_repomd_records(
modules_data = self._parse_module_repomd_record(
repo_info=repo_info,
repomd_records=repomd_records,
packages=packages,
)
# convert the packages dict to more usable form
# for future checking that a rpm from the module's artifacts

92
pungi/scripts/create_packages_json.py
View File

@ -14,7 +14,7 @@ import os
import re
import tempfile
from collections import defaultdict
from typing import AnyStr, Dict, List, Optional
from typing import AnyStr, Dict, List, Optional, Any, Iterator
import binascii
import createrepo_c as cr
@ -23,7 +23,7 @@ import hawkey
import requests
import rpm
import yaml
from createrepo_c import Package
from createrepo_c import Package, PackageIterator
from dataclasses import dataclass
@ -60,10 +60,10 @@ class RepoInfo:
arch: AnyStr
# Is a repo remote or local
is_remote: bool
# Is an reference repository (usually it's a RHEL repo)
# Is a reference repository (usually it's a RHEL repo)
# Layout of packages from such repository will be taken as example
# Only layout of specific package (which don't exist
# in an reference repository) will be taken as example
# in a reference repository) will be taken as example
is_reference: bool = False
@ -77,6 +77,12 @@ class PackagesGenerator:
self.repos = repos
self.excluded_packages = excluded_packages
self.included_packages = included_packages
self.tmp_files = []
def __del__(self):
for tmp_file in self.tmp_files:
if os.path.exists(tmp_file):
os.remove(tmp_file)
@staticmethod
def _warning_callback(warning_type, message):
@ -119,7 +125,7 @@ class PackagesGenerator:
Parse primary.xml.gz, take from it info about packages and put it to
dict packages
:param primary_file_path: path to local primary.xml.gz
:param packages: dictionary which will be contain info about packages
:param packages: dictionary which will contain info about packages
from repository
"""
cr.xml_parse_primary(
@ -140,7 +146,7 @@ class PackagesGenerator:
Parse filelists.xml.gz, take from it info about packages and put it to
dict packages
:param filelists_file_path: path to local filelists.xml.gz
:param packages: dictionary which will be contain info about packages
:param packages: dictionary which will contain info about packages
from repository
"""
cr.xml_parse_filelists(
@ -161,7 +167,7 @@ class PackagesGenerator:
Parse other.xml.gz, take from it info about packages and put it to
dict packages
:param other_file_path: path to local other.xml.gz
:param packages: dictionary which will be contain info about packages
:param packages: dictionary which will contain info about packages
from repository
"""
cr.xml_parse_other(
@ -178,11 +184,11 @@ class PackagesGenerator:
cls,
modules_file_path: AnyStr,
) -> List[Dict]:
) -> Iterator[Any]:
"""
Parse modules.yaml.gz and returns parsed data
:param modules_file_path: path to local modules.yaml.gz
:return: List of dict for an each modules in a repo
:return: List of dict for each modules in a repo
"""
with open(modules_file_path, 'rb') as modules_file:
@ -220,28 +226,23 @@ class PackagesGenerator:
os.remove(repomd_file_path)
return repomd_object.records
def _parse_repomd_records(
def _download_repomd_records(
self,
repo_info: RepoInfo,
repomd_records: List[cr.RepomdRecord],
packages: Dict[AnyStr, cr.Package],
) -> Optional[List[Dict]]:
repomd_records_dict: Dict[str, str],
):
"""
Parse repomd records and extract from repodata file info about packages
Download repomd records
:param repo_info: structure which contains info about a current repo
:param repomd_records: list with repomd records
:param packages: dictionary which will be contain info about packages
from repository
:return: List of dict for an each modules in a repo if it contains
modules info otherwise returns None
:param repomd_records_dict: dict with paths to repodata files
"""
modules_data = []
for repomd_record in repomd_records:
if repomd_record.type not in (
'primary',
'filelists',
'other',
'modules',
):
continue
repomd_record_file_path = os.path.join(
@ -252,22 +253,35 @@ class PackagesGenerator:
if repo_info.is_remote:
repomd_record_file_path = self.get_remote_file_content(
repomd_record_file_path)
if repomd_record.type == 'modules':
modules_data = self._parse_modules_file(
repomd_record_file_path,
)
else:
parse_file_method = getattr(
self.tmp_files.append(repomd_record_file_path)
repomd_records_dict[repomd_record.type] = repomd_record_file_path
def _parse_module_repomd_record(
self,
f'_parse_{repomd_record.type}_file'
)
parse_file_method(
repomd_record_file_path,
packages,
repo_info: RepoInfo,
repomd_records: List[cr.RepomdRecord],
) -> List[Dict]:
"""
Download repomd records
:param repo_info: structure which contains info about a current repo
:param repomd_records: list with repomd records
:param repomd_records_dict: dict with paths to repodata files
"""
for repomd_record in repomd_records:
if repomd_record.type != 'modules':
continue
repomd_record_file_path = os.path.join(
repo_info.path,
repo_info.folder,
repomd_record.location_href,
)
if repo_info.is_remote:
os.remove(repomd_record_file_path)
return list(modules_data)
repomd_record_file_path = self.get_remote_file_content(
repomd_record_file_path)
self.tmp_files.append(repomd_record_file_path)
return list(self._parse_modules_file(
repomd_record_file_path,
))
@staticmethod
def compare_pkgs_version(package_1: Package, package_2: Package) -> int:
@ -307,16 +321,20 @@ class PackagesGenerator:
'i686',
'i386',
])
packages = {} # type: Dict[AnyStr, cr.Package]
repomd_records = self._get_repomd_records(
repo_info=repo_info,
)
self._parse_repomd_records(
repo_info=repo_info,
repomd_records_dict = {} # type: Dict[str, str]
self._download_repomd_records(repo_info=repo_info,
repomd_records=repomd_records,
packages=packages,
repomd_records_dict=repomd_records_dict)
packages_iterator = PackageIterator(
primary_path=repomd_records_dict['primary'],
filelists_path=repomd_records_dict['filelists'],
other_path=repomd_records_dict['other'],
warningcb=self._warning_callback,
)
for package in packages.values():
for package in packages_iterator:
if package.arch not in repo_arches:
package_arch = repo_info.arch
else: