almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

ALBS-334: Make the ability of Pungi to give module_defaults from remote sources

Browse Source
This commit is contained in:
soksanichenko 2022-05-03 18:18:17 +03:00
parent 75dcabbfae
commit 885b3376d6
1 changed files with 45 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
pungi/scripts/gather_modules.py
View File

@ -4,7 +4,7 @@ import os
from argparse import ArgumentParser, FileType
from io import BytesIO
from pathlib import Path
from typing import List, AnyStr, Iterable, Union
from typing import List, AnyStr, Iterable, Union, Optional
import logging
from urllib.parse import urljoin
@ -30,16 +30,19 @@ def grep_list_of_modules_yaml(repos_path: AnyStr) -> Iterable[BytesIO]:
"""
return (
read_modules_yaml(modules_yaml_path=path.parent) for path in
Path(repos_path).rglob('repodata')
read_modules_yaml_from_specific_repo(repo_path=path.parent)
for path in Path(repos_path).rglob('repodata')
)
def _is_remote(path: str):
return any(path.startswith(protocol) for protocol in ('http', 'https'))
return any(str(path).startswith(protocol)
for protocol in ('http', 'https'))
def read_modules_yaml_from_specific_repo(repo_path: AnyStr) -> List[BytesIO]:
def read_modules_yaml_from_specific_repo(
repo_path: Union[str, Path]
) -> Optional[BytesIO]:
"""
Read modules_yaml from a specific repo (remote or local)
:param repo_path: path/url to a specific repo
@ -78,9 +81,32 @@ def read_modules_yaml_from_specific_repo(repo_path: AnyStr) -> List[BytesIO]:
repo_path,
record.location_href,
)
return [read_modules_yaml(modules_yaml_path=modules_yaml_path)]
return read_modules_yaml(modules_yaml_path=modules_yaml_path)
else:
return []
return None
def _should_grep_defaults(
document_type: str,
grep_only_modules_data: bool = False,
grep_only_modules_defaults_data: bool = False,
) -> bool:
xor_flag = grep_only_modules_data == grep_only_modules_defaults_data
if document_type == 'modulemd' and (xor_flag or grep_only_modules_data):
return True
return False
def _should_grep_modules(
document_type: str,
grep_only_modules_data: bool = False,
grep_only_modules_defaults_data: bool = False,
) -> bool:
xor_flag = grep_only_modules_data == grep_only_modules_defaults_data
if document_type == 'modulemd-defaults' and \
(xor_flag or grep_only_modules_defaults_data):
return True
return False
def collect_modules(
@ -115,12 +141,19 @@ def collect_modules(
documents = yaml.load_all(data, Loader=yaml.BaseLoader)
for doc in documents:
path = None
if doc['document'] == 'modulemd-defaults' and \
(grep_only_modules_defaults_data or xor_flag):
if _should_grep_modules(
doc['document'],
grep_only_modules_data,
grep_only_modules_defaults_data,
):
name = f"{doc['data']['module']}.yaml"
path = os.path.join(module_defaults_path, name)
logging.info('Found %s module defaults', name)
elif grep_only_modules_data or xor_flag:
elif _should_grep_defaults(
doc['document'],
grep_only_modules_data,
grep_only_modules_defaults_data,
):
# pungi.phases.pkgset.sources.source_koji.get_koji_modules
stream = doc['data']['stream'].replace('-', '_')
doc_data = doc['data']
@ -188,13 +221,14 @@ def cli_main():
if namespace.repodata_paths:
modules = []
for repodata_path in namespace.repodata_paths:
modules.extend(read_modules_yaml_from_specific_repo(
modules.append(read_modules_yaml_from_specific_repo(
repodata_path,
))
elif namespace.path is not None:
modules = namespace.path
else:
modules = grep_list_of_modules_yaml(namespace.repo_path)
modules = list(filter(lambda i: i is not None, modules))
collect_modules(
modules,
namespace.target,