almalinux
/
pungi
15
1
Fork 0
Code Issues Pull Requests Releases Activity

Merge pull request 'Replace list of cr.packages by cr.PackageIterator' (#6) from package_iterator into aln8 

Reviewed-on: #6
This commit is contained in:
Stepan Oksanichenko 2022-10-19 01:38:44 +00:00
parent f21ed6f607 b157a1825a
commit dd22d94a9e
3 changed files with 83 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
pungi/scripts/create_extra_repo.py
View File

@ -241,9 +241,9 @@ class CreateExtraRepo(PackagesGenerator):
""" """
Download all defined modularity packages and their data from Download all defined modularity packages and their data from
a remote repo a remote repo
:param modules_data: information about all of modules in a remote repo :param modules_data: information about all modules in a remote repo
:param repo_info: information about a remote repo :param repo_info: information about a remote repo
:param packages: information about all of packages (including :param packages: information about all packages (including
modularity) in a remote repo modularity) in a remote repo
""" """
for module in modules_data: for module in modules_data:
@ -264,7 +264,7 @@ class CreateExtraRepo(PackagesGenerator):
continue continue
for rpm in module['data']['artifacts']['rpms']: for rpm in module['data']['artifacts']['rpms']:
# Empty repo_info.packages means that we will download # Empty repo_info.packages means that we will download
# all of packages from repo including # all packages from repo including
# the modularity packages # the modularity packages
if not repo_info.packages: if not repo_info.packages:
break break
@ -290,10 +290,9 @@ class CreateExtraRepo(PackagesGenerator):
repo_info=repo_info, repo_info=repo_info,
) )
# parse the repodata (including modules.yaml.gz) # parse the repodata (including modules.yaml.gz)
modules_data = self._parse_repomd_records( modules_data = self._parse_module_repomd_record(
repo_info=repo_info, repo_info=repo_info,
repomd_records=repomd_records, repomd_records=repomd_records,
packages=packages,
) )
# convert the packages dict to more usable form # convert the packages dict to more usable form
# for future checking that a rpm from the module's artifacts # for future checking that a rpm from the module's artifacts

92
pungi/scripts/create_packages_json.py
View File

@ -14,7 +14,7 @@ import os
import re import re
import tempfile import tempfile
from collections import defaultdict from collections import defaultdict
from typing import AnyStr, Dict, List, Optional from typing import AnyStr, Dict, List, Optional, Any, Iterator
import binascii import binascii
import createrepo_c as cr import createrepo_c as cr
@ -23,7 +23,7 @@ import hawkey
import requests import requests
import rpm import rpm
import yaml import yaml
from createrepo_c import Package from createrepo_c import Package, PackageIterator
from dataclasses import dataclass from dataclasses import dataclass
@ -60,10 +60,10 @@ class RepoInfo:
arch: AnyStr arch: AnyStr
# Is a repo remote or local # Is a repo remote or local
is_remote: bool is_remote: bool
# Is an reference repository (usually it's a RHEL repo) # Is a reference repository (usually it's a RHEL repo)
# Layout of packages from such repository will be taken as example # Layout of packages from such repository will be taken as example
# Only layout of specific package (which don't exist # Only layout of specific package (which don't exist
# in an reference repository) will be taken as example # in a reference repository) will be taken as example
is_reference: bool = False is_reference: bool = False
@ -77,6 +77,12 @@ class PackagesGenerator:
self.repos = repos self.repos = repos
self.excluded_packages = excluded_packages self.excluded_packages = excluded_packages
self.included_packages = included_packages self.included_packages = included_packages
self.tmp_files = []
def __del__(self):
for tmp_file in self.tmp_files:
if os.path.exists(tmp_file):
os.remove(tmp_file)
@staticmethod @staticmethod
def _warning_callback(warning_type, message): def _warning_callback(warning_type, message):
@ -119,7 +125,7 @@ class PackagesGenerator:
Parse primary.xml.gz, take from it info about packages and put it to Parse primary.xml.gz, take from it info about packages and put it to
dict packages dict packages
:param primary_file_path: path to local primary.xml.gz :param primary_file_path: path to local primary.xml.gz
:param packages: dictionary which will be contain info about packages :param packages: dictionary which will contain info about packages
from repository from repository
""" """
cr.xml_parse_primary( cr.xml_parse_primary(
@ -140,7 +146,7 @@ class PackagesGenerator:
Parse filelists.xml.gz, take from it info about packages and put it to Parse filelists.xml.gz, take from it info about packages and put it to
dict packages dict packages
:param filelists_file_path: path to local filelists.xml.gz :param filelists_file_path: path to local filelists.xml.gz
:param packages: dictionary which will be contain info about packages :param packages: dictionary which will contain info about packages
from repository from repository
""" """
cr.xml_parse_filelists( cr.xml_parse_filelists(
@ -161,7 +167,7 @@ class PackagesGenerator:
Parse other.xml.gz, take from it info about packages and put it to Parse other.xml.gz, take from it info about packages and put it to
dict packages dict packages
:param other_file_path: path to local other.xml.gz :param other_file_path: path to local other.xml.gz
:param packages: dictionary which will be contain info about packages :param packages: dictionary which will contain info about packages
from repository from repository
""" """
cr.xml_parse_other( cr.xml_parse_other(
@ -178,11 +184,11 @@ class PackagesGenerator:
cls, cls,
modules_file_path: AnyStr, modules_file_path: AnyStr,
) -> List[Dict]: ) -> Iterator[Any]:
""" """
Parse modules.yaml.gz and returns parsed data Parse modules.yaml.gz and returns parsed data
:param modules_file_path: path to local modules.yaml.gz :param modules_file_path: path to local modules.yaml.gz
:return: List of dict for an each modules in a repo :return: List of dict for each modules in a repo
""" """
with open(modules_file_path, 'rb') as modules_file: with open(modules_file_path, 'rb') as modules_file:
@ -220,28 +226,23 @@ class PackagesGenerator:
os.remove(repomd_file_path) os.remove(repomd_file_path)
return repomd_object.records return repomd_object.records
def _parse_repomd_records( def _download_repomd_records(
self, self,
repo_info: RepoInfo, repo_info: RepoInfo,
repomd_records: List[cr.RepomdRecord], repomd_records: List[cr.RepomdRecord],
packages: Dict[AnyStr, cr.Package], repomd_records_dict: Dict[str, str],
) -> Optional[List[Dict]]: ):
""" """
Parse repomd records and extract from repodata file info about packages Download repomd records
:param repo_info: structure which contains info about a current repo :param repo_info: structure which contains info about a current repo
:param repomd_records: list with repomd records :param repomd_records: list with repomd records
:param packages: dictionary which will be contain info about packages :param repomd_records_dict: dict with paths to repodata files
from repository
:return: List of dict for an each modules in a repo if it contains
modules info otherwise returns None
""" """
modules_data = []
for repomd_record in repomd_records: for repomd_record in repomd_records:
if repomd_record.type not in ( if repomd_record.type not in (
'primary', 'primary',
'filelists', 'filelists',
'other', 'other',
'modules',
): ):
continue continue
repomd_record_file_path = os.path.join( repomd_record_file_path = os.path.join(
@ -252,22 +253,35 @@ class PackagesGenerator:
if repo_info.is_remote: if repo_info.is_remote:
repomd_record_file_path = self.get_remote_file_content( repomd_record_file_path = self.get_remote_file_content(
repomd_record_file_path) repomd_record_file_path)
if repomd_record.type == 'modules': self.tmp_files.append(repomd_record_file_path)
modules_data = self._parse_modules_file( repomd_records_dict[repomd_record.type] = repomd_record_file_path
repomd_record_file_path,
) def _parse_module_repomd_record(
else:
parse_file_method = getattr(
self, self,
f'_parse_{repomd_record.type}_file' repo_info: RepoInfo,
) repomd_records: List[cr.RepomdRecord],
parse_file_method( ) -> List[Dict]:
repomd_record_file_path, """
packages, Download repomd records
:param repo_info: structure which contains info about a current repo
:param repomd_records: list with repomd records
:param repomd_records_dict: dict with paths to repodata files
"""
for repomd_record in repomd_records:
if repomd_record.type != 'modules':
continue
repomd_record_file_path = os.path.join(
repo_info.path,
repo_info.folder,
repomd_record.location_href,
) )
if repo_info.is_remote: if repo_info.is_remote:
os.remove(repomd_record_file_path) repomd_record_file_path = self.get_remote_file_content(
return list(modules_data) repomd_record_file_path)
self.tmp_files.append(repomd_record_file_path)
return list(self._parse_modules_file(
repomd_record_file_path,
))
@staticmethod @staticmethod
def compare_pkgs_version(package_1: Package, package_2: Package) -> int: def compare_pkgs_version(package_1: Package, package_2: Package) -> int:
@ -307,16 +321,20 @@ class PackagesGenerator:
'i686', 'i686',
'i386', 'i386',
]) ])
packages = {} # type: Dict[AnyStr, cr.Package]
repomd_records = self._get_repomd_records( repomd_records = self._get_repomd_records(
repo_info=repo_info, repo_info=repo_info,
) )
self._parse_repomd_records( repomd_records_dict = {} # type: Dict[str, str]
repo_info=repo_info, self._download_repomd_records(repo_info=repo_info,
repomd_records=repomd_records, repomd_records=repomd_records,
packages=packages, repomd_records_dict=repomd_records_dict)
packages_iterator = PackageIterator(
primary_path=repomd_records_dict['primary'],
filelists_path=repomd_records_dict['filelists'],
other_path=repomd_records_dict['other'],
warningcb=self._warning_callback,
) )
for package in packages.values(): for package in packages_iterator:
if package.arch not in repo_arches: if package.arch not in repo_arches:
package_arch = repo_info.arch package_arch = repo_info.arch
else: else:

22
pungi/wrappers/kojimock.py
View File

@ -1,6 +1,6 @@
import os import os
import subprocess
import time import time
from pathlib import Path
from attr import dataclass from attr import dataclass
from kobo.rpmlib import parse_nvra from kobo.rpmlib import parse_nvra
@ -48,14 +48,14 @@ class KojiMock:
self._modules_dir = modules_dir self._modules_dir = modules_dir
self._packages_dir = packages_dir self._packages_dir = packages_dir
def _gather_modules(self, modules_dir): @staticmethod
def _gather_modules(modules_dir):
modules = {} modules = {}
for arch in os.listdir(modules_dir): for index, (f, arch) in enumerate(
arch_dir = os.path.join( (sub_path.name, sub_path.parent.name)
modules_dir, for path in Path(modules_dir).glob('*')
arch, for sub_path in path.iterdir()
) ):
for index, f in enumerate(os.listdir(arch_dir)):
parsed = parse_nvra(f) parsed = parse_nvra(f)
modules[index] = Module( modules[index] = Module(
name=parsed['name'], name=parsed['name'],
@ -68,7 +68,8 @@ class KojiMock:
) )
return modules return modules
def getLastEvent(self, *args, **kwargs): @staticmethod
def getLastEvent(*args, **kwargs):
return {'id': LAST_EVENT_ID, 'ts': LAST_EVENT_TIME} return {'id': LAST_EVENT_ID, 'ts': LAST_EVENT_TIME}
def listTagged(self, tag_name, *args, **kwargs): def listTagged(self, tag_name, *args, **kwargs):
@ -111,7 +112,8 @@ class KojiMock:
return builds return builds
def getFullInheritance(self, *args, **kwargs): @staticmethod
def getFullInheritance(*args, **kwargs):
""" """
Unneeded because we use local storage. Unneeded because we use local storage.
""" """