almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Replace list of cr.packages by cr.PackageIterator #7

Merged
soksanichenko merged 2 commits from package_iterator_aln9 into aln9 2022-10-19 01:38:37 +00:00
Conversation 0 Commits 2 Files Changed 3 +62 -45
2 changed files with 62 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 941a02722c - Show all commits

9
pungi/scripts/create_extra_repo.py
View File

@ -241,9 +241,9 @@ class CreateExtraRepo(PackagesGenerator):
""" """
Download all defined modularity packages and their data from Download all defined modularity packages and their data from
a remote repo a remote repo
:param modules_data: information about all of modules in a remote repo :param modules_data: information about all modules in a remote repo
:param repo_info: information about a remote repo :param repo_info: information about a remote repo
:param packages: information about all of packages (including :param packages: information about all packages (including
modularity) in a remote repo modularity) in a remote repo
""" """
for module in modules_data: for module in modules_data:
@ -264,7 +264,7 @@ class CreateExtraRepo(PackagesGenerator):
continue continue
for rpm in module['data']['artifacts']['rpms']: for rpm in module['data']['artifacts']['rpms']:
# Empty repo_info.packages means that we will download # Empty repo_info.packages means that we will download
# all of packages from repo including # all packages from repo including
# the modularity packages # the modularity packages
if not repo_info.packages: if not repo_info.packages:
break break
@ -290,10 +290,9 @@ class CreateExtraRepo(PackagesGenerator):
repo_info=repo_info, repo_info=repo_info,
) )
# parse the repodata (including modules.yaml.gz) # parse the repodata (including modules.yaml.gz)
modules_data = self._parse_repomd_records( modules_data = self._parse_module_repomd_record(
repo_info=repo_info, repo_info=repo_info,
repomd_records=repomd_records, repomd_records=repomd_records,
packages=packages,
) )
# convert the packages dict to more usable form # convert the packages dict to more usable form
# for future checking that a rpm from the module's artifacts # for future checking that a rpm from the module's artifacts

98
pungi/scripts/create_packages_json.py
View File

@ -14,7 +14,7 @@ import os
import re import re
import tempfile import tempfile
from collections import defaultdict from collections import defaultdict
from typing import AnyStr, Dict, List, Optional from typing import AnyStr, Dict, List, Optional, Any, Iterator
import binascii import binascii
import createrepo_c as cr import createrepo_c as cr
@ -23,7 +23,7 @@ import hawkey
import requests import requests
import rpm import rpm
import yaml import yaml
from createrepo_c import Package from createrepo_c import Package, PackageIterator
from dataclasses import dataclass from dataclasses import dataclass
@ -60,10 +60,10 @@ class RepoInfo:
arch: AnyStr arch: AnyStr
# Is a repo remote or local # Is a repo remote or local
is_remote: bool is_remote: bool
# Is an reference repository (usually it's a RHEL repo) # Is a reference repository (usually it's a RHEL repo)
# Layout of packages from such repository will be taken as example # Layout of packages from such repository will be taken as example
# Only layout of specific package (which don't exist # Only layout of specific package (which don't exist
# in an reference repository) will be taken as example # in a reference repository) will be taken as example
is_reference: bool = False is_reference: bool = False
@ -77,6 +77,12 @@ class PackagesGenerator:
self.repos = repos self.repos = repos
self.excluded_packages = excluded_packages self.excluded_packages = excluded_packages
self.included_packages = included_packages self.included_packages = included_packages
self.tmp_files = []
def __del__(self):
for tmp_file in self.tmp_files:
if os.path.exists(tmp_file):
os.remove(tmp_file)
@staticmethod @staticmethod
def _warning_callback(warning_type, message): def _warning_callback(warning_type, message):
@ -119,7 +125,7 @@ class PackagesGenerator:
Parse primary.xml.gz, take from it info about packages and put it to Parse primary.xml.gz, take from it info about packages and put it to
dict packages dict packages
:param primary_file_path: path to local primary.xml.gz :param primary_file_path: path to local primary.xml.gz
:param packages: dictionary which will be contain info about packages :param packages: dictionary which will contain info about packages
from repository from repository
""" """
cr.xml_parse_primary( cr.xml_parse_primary(
@ -140,7 +146,7 @@ class PackagesGenerator:
Parse filelists.xml.gz, take from it info about packages and put it to Parse filelists.xml.gz, take from it info about packages and put it to
dict packages dict packages
:param filelists_file_path: path to local filelists.xml.gz :param filelists_file_path: path to local filelists.xml.gz
:param packages: dictionary which will be contain info about packages :param packages: dictionary which will contain info about packages
from repository from repository
""" """
cr.xml_parse_filelists( cr.xml_parse_filelists(
@ -161,7 +167,7 @@ class PackagesGenerator:
Parse other.xml.gz, take from it info about packages and put it to Parse other.xml.gz, take from it info about packages and put it to
dict packages dict packages
:param other_file_path: path to local other.xml.gz :param other_file_path: path to local other.xml.gz
:param packages: dictionary which will be contain info about packages :param packages: dictionary which will contain info about packages
from repository from repository
""" """
cr.xml_parse_other( cr.xml_parse_other(
@ -178,11 +184,11 @@ class PackagesGenerator:
cls, cls,
modules_file_path: AnyStr, modules_file_path: AnyStr,
) -> List[Dict]: ) -> Iterator[Any]:
""" """
Parse modules.yaml.gz and returns parsed data Parse modules.yaml.gz and returns parsed data
:param modules_file_path: path to local modules.yaml.gz :param modules_file_path: path to local modules.yaml.gz
:return: List of dict for an each modules in a repo :return: List of dict for each modules in a repo
""" """
with open(modules_file_path, 'rb') as modules_file: with open(modules_file_path, 'rb') as modules_file:
@ -220,28 +226,23 @@ class PackagesGenerator:
os.remove(repomd_file_path) os.remove(repomd_file_path)
return repomd_object.records return repomd_object.records
def _parse_repomd_records( def _download_repomd_records(
self, self,
repo_info: RepoInfo, repo_info: RepoInfo,
repomd_records: List[cr.RepomdRecord], repomd_records: List[cr.RepomdRecord],
packages: Dict[AnyStr, cr.Package], repomd_records_dict: Dict[str, str],
) -> Optional[List[Dict]]: ):
""" """
Parse repomd records and extract from repodata file info about packages Download repomd records
:param repo_info: structure which contains info about a current repo :param repo_info: structure which contains info about a current repo
:param repomd_records: list with repomd records :param repomd_records: list with repomd records
:param packages: dictionary which will be contain info about packages :param repomd_records_dict: dict with paths to repodata files
from repository
:return: List of dict for an each modules in a repo if it contains
modules info otherwise returns None
""" """
modules_data = []
for repomd_record in repomd_records: for repomd_record in repomd_records:
if repomd_record.type not in ( if repomd_record.type not in (
'primary', 'primary',
'filelists', 'filelists',
'other', 'other',
'modules',
): ):
continue continue
repomd_record_file_path = os.path.join( repomd_record_file_path = os.path.join(
@ -252,22 +253,35 @@ class PackagesGenerator:
if repo_info.is_remote: if repo_info.is_remote:
repomd_record_file_path = self.get_remote_file_content( repomd_record_file_path = self.get_remote_file_content(
repomd_record_file_path) repomd_record_file_path)
if repomd_record.type == 'modules': self.tmp_files.append(repomd_record_file_path)
modules_data = self._parse_modules_file( repomd_records_dict[repomd_record.type] = repomd_record_file_path
repomd_record_file_path,
) def _parse_module_repomd_record(
else: self,
parse_file_method = getattr( repo_info: RepoInfo,
self, repomd_records: List[cr.RepomdRecord],
f'_parse_{repomd_record.type}_file' ) -> List[Dict]:
) """
parse_file_method( Download repomd records
repomd_record_file_path, :param repo_info: structure which contains info about a current repo
packages, :param repomd_records: list with repomd records
) :param repomd_records_dict: dict with paths to repodata files
"""
for repomd_record in repomd_records:
if repomd_record.type != 'modules':
continue
repomd_record_file_path = os.path.join(
repo_info.path,
repo_info.folder,
repomd_record.location_href,
)
if repo_info.is_remote: if repo_info.is_remote:
os.remove(repomd_record_file_path) repomd_record_file_path = self.get_remote_file_content(
return list(modules_data) repomd_record_file_path)
self.tmp_files.append(repomd_record_file_path)
return list(self._parse_modules_file(
repomd_record_file_path,
))
@staticmethod @staticmethod
def compare_pkgs_version(package_1: Package, package_2: Package) -> int: def compare_pkgs_version(package_1: Package, package_2: Package) -> int:
@ -307,16 +321,20 @@ class PackagesGenerator:
'i686', 'i686',
'i386', 'i386',
]) ])
packages = {} # type: Dict[AnyStr, cr.Package]
repomd_records = self._get_repomd_records( repomd_records = self._get_repomd_records(
repo_info=repo_info, repo_info=repo_info,
) )
self._parse_repomd_records( repomd_records_dict = {} # type: Dict[str, str]
repo_info=repo_info, self._download_repomd_records(repo_info=repo_info,
repomd_records=repomd_records, repomd_records=repomd_records,
packages=packages, repomd_records_dict=repomd_records_dict)
packages_iterator = PackageIterator(
primary_path=repomd_records_dict['primary'],
filelists_path=repomd_records_dict['filelists'],
other_path=repomd_records_dict['other'],
warningcb=self._warning_callback,
) )
for package in packages.values(): for package in packages_iterator:
if package.arch not in repo_arches: if package.arch not in repo_arches:
package_arch = repo_info.arch package_arch = repo_info.arch
else: else: