From 103c3dc608bcd1152e4be127806f8638c6d3551c Mon Sep 17 00:00:00 2001
From: Stepan Oksanichenko <soksanichenko@cloudlinux.com>
Date: Thu, 21 Jan 2021 19:27:29 +0200
Subject: [PATCH] LNX-133: Create a server for building nightly builds of
 AlmaLinux

- Script `pungi-gather-modules` can find valid *modules.yaml.gz in the repo dirs by itself

@BS-LINKED-5ffda6156f44affc6c5ea239  # pungi & dependencies
@BS-TARGET-CL8

Change-Id: I3cddc0cf41ea1087183e23de39126a52c69bc9ac
---
 pungi/scripts/gather_modules.py | 65 +++++++++++++++++++++++++++------
 1 file changed, 54 insertions(+), 11 deletions(-)

diff --git a/pungi/scripts/gather_modules.py b/pungi/scripts/gather_modules.py
index 1a21e51c..1da8097a 100644
--- a/pungi/scripts/gather_modules.py
+++ b/pungi/scripts/gather_modules.py
@@ -1,14 +1,40 @@
-import argparse
 import gzip
 import os
-import typing
-from argparse import ArgumentParser
-from typing import List
+from argparse import ArgumentParser, FileType
+from io import BytesIO
+from pathlib import Path
+from typing import List, AnyStr
 import logging
 import yaml
+import createrepo_c as cr
+from typing.io import BinaryIO
 
 
-def collect_modules(modules_paths: List[typing.BinaryIO], target_dir: str):
+def grep_list_of_modules_yaml_gz(repo_path: AnyStr) -> List[BytesIO]:
+    """
+    Find all of valid *modules.yaml.gz in repos
+    :param repo_path: path to a directory which contains repodirs
+    :return: list of content from *modules.yaml.gz
+    """
+
+    result = []
+    for path in Path(repo_path).rglob('repomd.xml'):
+        repo_dir_path = Path(path.parent).parent
+        repomd_obj = cr.Repomd(str(path))
+        for record in repomd_obj.records:
+            if record.type != 'modules':
+                continue
+            with open(os.path.join(
+                    repo_dir_path,
+                    record.location_href,
+            ), 'rb') as fp:
+                result.append(
+                    BytesIO(fp.read())
+                )
+    return result
+
+
+def collect_modules(modules_paths: List[BinaryIO], target_dir: str):
     """
     Read given modules.yaml.gz files and export modules
     and modulemd files from it.
@@ -39,7 +65,10 @@ def collect_modules(modules_paths: List[typing.BinaryIO], target_dir: str):
                 logging.info('Found module %s', name)
 
                 if 'artifacts' not in doc['data']:
-                    logging.warning('RPM %s does not have explicit list of artifacts', name)
+                    logging.warning(
+                        'RPM %s does not have explicit list of artifacts',
+                        name
+                    )
 
             with open(path, 'w') as f:
                 yaml.dump(doc, f, default_flow_style=False)
@@ -47,17 +76,31 @@ def collect_modules(modules_paths: List[typing.BinaryIO], target_dir: str):
 
 def cli_main():
     parser = ArgumentParser()
-    parser.add_argument(
-        '-p', '--path', required=True,
-        type=argparse.FileType('rb'), nargs='+',
+    path_group = parser.add_mutually_exclusive_group(required=True)
+    path_group.add_argument(
+        '-p', '--path',
+        type=FileType('rb'), nargs='+',
         help='Path to modules.yaml.gz file. '
              'You may pass multiple files by passing -p path1 path2'
     )
+    path_group.add_argument(
+        '-rp', '--repo-path',
+        required=False,
+        type=str,
+        default=None,
+        help='Path to a directory which contains repodirs. E.g. /var/repos'
+    )
     parser.add_argument('-t', '--target', required=True)
 
     namespace = parser.parse_args()
-
-    collect_modules(namespace.path, namespace.target)
+    if namespace.repo_path is None:
+        modules = namespace.path
+    else:
+        modules = grep_list_of_modules_yaml_gz(namespace.repo_path)
+    collect_modules(
+        modules,
+        namespace.target,
+    )
 
 
 if __name__ == '__main__':