ALBS-334: Make the ability of Pungi to give module_defaults from remote sources

pungi/scripts/gather_modules.py

@@ -4,7 +4,7 @@ import os
 from argparse import ArgumentParser, FileType
 from io import BytesIO
 from pathlib import Path
-from typing import List, AnyStr, Iterable, Union
+from typing import List, AnyStr, Iterable, Union, Optional
 import logging
 from urllib.parse import urljoin
 
@@ -30,16 +30,19 @@ def grep_list_of_modules_yaml(repos_path: AnyStr) -> Iterable[BytesIO]:
     """
 
     return (
-        read_modules_yaml(modules_yaml_path=path.parent) for path in
+        read_modules_yaml_from_specific_repo(repo_path=path.parent)
-        Path(repos_path).rglob('repodata')
+        for path in Path(repos_path).rglob('repodata')
     )
 
 
 def _is_remote(path: str):
-    return any(path.startswith(protocol) for protocol in ('http', 'https'))
+    return any(str(path).startswith(protocol)
+               for protocol in ('http', 'https'))
 
 
-def read_modules_yaml_from_specific_repo(repo_path: AnyStr) -> List[BytesIO]:
+def read_modules_yaml_from_specific_repo(
+        repo_path: Union[str, Path]
+) -> Optional[BytesIO]:
     """
     Read modules_yaml from a specific repo (remote or local)
     :param repo_path: path/url to a specific repo
@@ -78,9 +81,32 @@ def read_modules_yaml_from_specific_repo(repo_path: AnyStr) -> List[BytesIO]:
                     repo_path,
                     record.location_href,
                 )
-            return [read_modules_yaml(modules_yaml_path=modules_yaml_path)]
+            return read_modules_yaml(modules_yaml_path=modules_yaml_path)
     else:
-        return []
+        return None
+
+
+def _should_grep_defaults(
+        document_type: str,
+        grep_only_modules_data: bool = False,
+        grep_only_modules_defaults_data: bool = False,
+) -> bool:
+    xor_flag = grep_only_modules_data == grep_only_modules_defaults_data
+    if document_type == 'modulemd' and (xor_flag or grep_only_modules_data):
+        return True
+    return False
+
+
+def _should_grep_modules(
+        document_type: str,
+        grep_only_modules_data: bool = False,
+        grep_only_modules_defaults_data: bool = False,
+) -> bool:
+    xor_flag = grep_only_modules_data == grep_only_modules_defaults_data
+    if document_type == 'modulemd-defaults' and \
+            (xor_flag or grep_only_modules_defaults_data):
+        return True
+    return False
 
 
 def collect_modules(
@@ -115,12 +141,19 @@ def collect_modules(
         documents = yaml.load_all(data, Loader=yaml.BaseLoader)
         for doc in documents:
             path = None
-            if doc['document'] == 'modulemd-defaults' and \
+            if _should_grep_modules(
-                    (grep_only_modules_defaults_data or xor_flag):
+                doc['document'],
+                grep_only_modules_data,
+                grep_only_modules_defaults_data,
+            ):
                 name = f"{doc['data']['module']}.yaml"
                 path = os.path.join(module_defaults_path, name)
                 logging.info('Found %s module defaults', name)
-            elif grep_only_modules_data or xor_flag:
+            elif _should_grep_defaults(
+                    doc['document'],
+                    grep_only_modules_data,
+                    grep_only_modules_defaults_data,
+            ):
                 # pungi.phases.pkgset.sources.source_koji.get_koji_modules
                 stream = doc['data']['stream'].replace('-', '_')
                 doc_data = doc['data']
@@ -188,13 +221,14 @@ def cli_main():
     if namespace.repodata_paths:
         modules = []
         for repodata_path in namespace.repodata_paths:
-            modules.extend(read_modules_yaml_from_specific_repo(
+            modules.append(read_modules_yaml_from_specific_repo(
                 repodata_path,
             ))
     elif namespace.path is not None:
         modules = namespace.path
     else:
         modules = grep_list_of_modules_yaml(namespace.repo_path)
+    modules = list(filter(lambda i: i is not None, modules))
     collect_modules(
         modules,
         namespace.target,