ALBS-334: Make the ability of Pungi to give module_defaults from remote sources

This commit is contained in:
soksanichenko 2022-05-03 18:18:17 +03:00
parent 75dcabbfae
commit 885b3376d6

View File

@ -4,7 +4,7 @@ import os
from argparse import ArgumentParser, FileType from argparse import ArgumentParser, FileType
from io import BytesIO from io import BytesIO
from pathlib import Path from pathlib import Path
from typing import List, AnyStr, Iterable, Union from typing import List, AnyStr, Iterable, Union, Optional
import logging import logging
from urllib.parse import urljoin from urllib.parse import urljoin
@ -30,16 +30,19 @@ def grep_list_of_modules_yaml(repos_path: AnyStr) -> Iterable[BytesIO]:
""" """
return ( return (
read_modules_yaml(modules_yaml_path=path.parent) for path in read_modules_yaml_from_specific_repo(repo_path=path.parent)
Path(repos_path).rglob('repodata') for path in Path(repos_path).rglob('repodata')
) )
def _is_remote(path: str): def _is_remote(path: str):
return any(path.startswith(protocol) for protocol in ('http', 'https')) return any(str(path).startswith(protocol)
for protocol in ('http', 'https'))
def read_modules_yaml_from_specific_repo(repo_path: AnyStr) -> List[BytesIO]: def read_modules_yaml_from_specific_repo(
repo_path: Union[str, Path]
) -> Optional[BytesIO]:
""" """
Read modules_yaml from a specific repo (remote or local) Read modules_yaml from a specific repo (remote or local)
:param repo_path: path/url to a specific repo :param repo_path: path/url to a specific repo
@ -78,9 +81,32 @@ def read_modules_yaml_from_specific_repo(repo_path: AnyStr) -> List[BytesIO]:
repo_path, repo_path,
record.location_href, record.location_href,
) )
return [read_modules_yaml(modules_yaml_path=modules_yaml_path)] return read_modules_yaml(modules_yaml_path=modules_yaml_path)
else: else:
return [] return None
def _should_grep_defaults(
document_type: str,
grep_only_modules_data: bool = False,
grep_only_modules_defaults_data: bool = False,
) -> bool:
xor_flag = grep_only_modules_data == grep_only_modules_defaults_data
if document_type == 'modulemd' and (xor_flag or grep_only_modules_data):
return True
return False
def _should_grep_modules(
document_type: str,
grep_only_modules_data: bool = False,
grep_only_modules_defaults_data: bool = False,
) -> bool:
xor_flag = grep_only_modules_data == grep_only_modules_defaults_data
if document_type == 'modulemd-defaults' and \
(xor_flag or grep_only_modules_defaults_data):
return True
return False
def collect_modules( def collect_modules(
@ -115,12 +141,19 @@ def collect_modules(
documents = yaml.load_all(data, Loader=yaml.BaseLoader) documents = yaml.load_all(data, Loader=yaml.BaseLoader)
for doc in documents: for doc in documents:
path = None path = None
if doc['document'] == 'modulemd-defaults' and \ if _should_grep_modules(
(grep_only_modules_defaults_data or xor_flag): doc['document'],
grep_only_modules_data,
grep_only_modules_defaults_data,
):
name = f"{doc['data']['module']}.yaml" name = f"{doc['data']['module']}.yaml"
path = os.path.join(module_defaults_path, name) path = os.path.join(module_defaults_path, name)
logging.info('Found %s module defaults', name) logging.info('Found %s module defaults', name)
elif grep_only_modules_data or xor_flag: elif _should_grep_defaults(
doc['document'],
grep_only_modules_data,
grep_only_modules_defaults_data,
):
# pungi.phases.pkgset.sources.source_koji.get_koji_modules # pungi.phases.pkgset.sources.source_koji.get_koji_modules
stream = doc['data']['stream'].replace('-', '_') stream = doc['data']['stream'].replace('-', '_')
doc_data = doc['data'] doc_data = doc['data']
@ -188,13 +221,14 @@ def cli_main():
if namespace.repodata_paths: if namespace.repodata_paths:
modules = [] modules = []
for repodata_path in namespace.repodata_paths: for repodata_path in namespace.repodata_paths:
modules.extend(read_modules_yaml_from_specific_repo( modules.append(read_modules_yaml_from_specific_repo(
repodata_path, repodata_path,
)) ))
elif namespace.path is not None: elif namespace.path is not None:
modules = namespace.path modules = namespace.path
else: else:
modules = grep_list_of_modules_yaml(namespace.repo_path) modules = grep_list_of_modules_yaml(namespace.repo_path)
modules = list(filter(lambda i: i is not None, modules))
collect_modules( collect_modules(
modules, modules,
namespace.target, namespace.target,