From 757a6ed653dfa1799834585e430d768e13bf505e Mon Sep 17 00:00:00 2001
From: oshyshatskyi <oshyshatskyi@cloudlinux.com>
Date: Wed, 30 Dec 2020 13:58:06 +0200
Subject: [PATCH] Revert unneeded commit to match upstream sources

This reverts commit b2e439e5

Change-Id: Ia6706415039681a6fe7b5ec6a735c3bda66d6bb1
---
 pungi/collect_modules.py                   |  60 -----
 pungi/collect_rpms.py                      |  66 -----
 pungi/kojimock.py                          | 293 ---------------------
 pungi/phases/pkgset/pkgsets.py             |  42 ++-
 pungi/phases/pkgset/sources/source_koji.py |  13 +-
 5 files changed, 35 insertions(+), 439 deletions(-)
 delete mode 100644 pungi/collect_modules.py
 delete mode 100644 pungi/collect_rpms.py
 delete mode 100644 pungi/kojimock.py

diff --git a/pungi/collect_modules.py b/pungi/collect_modules.py
deleted file mode 100644
index aad214bb..00000000
--- a/pungi/collect_modules.py
+++ /dev/null
@@ -1,60 +0,0 @@
-import argparse
-import gzip
-import os
-import typing
-from argparse import ArgumentParser
-from typing import List
-
-import yaml
-
-
-def collect_modules(modules_paths: List[typing.BinaryIO], target_dir: str):
-    """
-    Read given modules.yaml.gz files and export modules
-    and modulemd files from it.
-    Returns:
-        object:
-    """
-    modules_path = os.path.join(target_dir, 'modules')
-    module_defaults_path = os.path.join(target_dir, 'module_defaults')
-    os.makedirs(modules_path, exist_ok=True)
-    os.makedirs(module_defaults_path, exist_ok=True)
-
-    for module_file in modules_paths:
-        data = gzip.decompress(module_file.read())
-        documents = yaml.load_all(data, Loader=yaml.BaseLoader)
-        for doc in documents:
-            if doc['document'] == 'modulemd-defaults':
-                name = doc['data']['module']
-                path = os.path.join(module_defaults_path, name)
-                print('INFO', 'Found', name, 'module defaults')
-            else:
-                name = '%s-%s-%s.%s' % (
-                    doc['data']['name'],
-                    doc['data']['stream'],
-                    doc['data']['version'],
-                    doc['data']['context']
-                )
-                path = os.path.join(modules_path, name)
-                print('INFO', 'Found', name, 'module')
-
-                if 'artifacts' not in doc['data']:
-                    print('WARN', 'RPM', name, 'does not have explicit list of artifacts')
-
-            with open(path, 'w') as f:
-                yaml.dump(doc, f, default_flow_style=False)
-
-
-if __name__ == '__main__':
-    parser = ArgumentParser()
-    parser.add_argument(
-        '-p', '--path', required=True,
-        type=argparse.FileType('rb'), nargs='+',
-        help='Path to modules.yaml.gz file. '
-             'You may pass multiple files by passing -p path1 path2'
-    )
-    parser.add_argument('-t', '--target', required=True)
-
-    namespace = parser.parse_args()
-
-    collect_modules(namespace.path, namespace.target)
\ No newline at end of file
diff --git a/pungi/collect_rpms.py b/pungi/collect_rpms.py
deleted file mode 100644
index b7ab2ff5..00000000
--- a/pungi/collect_rpms.py
+++ /dev/null
@@ -1,66 +0,0 @@
-from argparse import ArgumentParser
-
-import os
-from typing import List
-
-from attr import dataclass
-from productmd.common import parse_nvra
-
-
-@dataclass
-class Package:
-    nvra: str
-    path: str
-
-
-def search_rpms(top_dir) -> List[Package]:
-    """
-    Search for all *.rpm files recursively
-    in given top directory
-    Returns:
-        list: list of paths
-    """
-    rpms = []
-    for root, dirs, files in os.walk(top_dir):
-        path = root.split(os.sep)
-        for file in files:
-            if not file.endswith('.rpm'):
-                continue
-            rpms.append(
-                Package(nvra=file[:-4], path=os.path.join('/', *path, file))
-            )
-    return rpms
-
-
-def copy_rpms(packages: List[Package], target_top_dir: str):
-    """
-    Search synced repos for rpms and prepare
-    koji-like structure for pungi
-
-    Instead of repos, use following structure:
-        # ls /mnt/koji/
-            i686/  noarch/  x86_64/
-    Returns:
-        Nothing:
-    """
-    for package in packages:
-        info = parse_nvra(package.nvra)
-
-        target_arch_dir = os.path.join(target_top_dir, info['arch'])
-        os.makedirs(target_arch_dir, exist_ok=True)
-
-        target_file = os.path.join(target_arch_dir, os.path.basename(package.path))
-
-        if not os.path.exists(target_file):
-            os.link(package.path, target_file)
-
-
-if __name__ == '__main__':
-    parser = ArgumentParser()
-    parser.add_argument('-p', '--path', required=True)
-    parser.add_argument('-t', '--target', required=True)
-
-    namespace = parser.parse_args()
-
-    rpms = search_rpms(namespace.path)
-    copy_rpms(rpms, namespace.target)
diff --git a/pungi/kojimock.py b/pungi/kojimock.py
deleted file mode 100644
index 5816381c..00000000
--- a/pungi/kojimock.py
+++ /dev/null
@@ -1,293 +0,0 @@
-import time
-
-import json
-
-import subprocess
-
-import os
-
-from attr import dataclass
-from flask import Flask
-from flask_xmlrpcre import xmlrpcre
-import xmlrpc.client
-import xmlrpc.server
-
-from pungi.module_util import Modulemd
-from kobo.rpmlib import parse_nvra
-
-# hacky hack for int64 (xmlrpc sucks)
-xmlrpc.client.Marshaller.dispatch[type(0)] = lambda _, v, w: w("<value><i8>%d</i8></value>" % v)
-
-from werkzeug.routing import BaseConverter
-
-MODULES_DIR = '/mnt/koji/modules/'
-MODULES = {}
-
-
-@dataclass
-class Module:
-    build_id: int
-    name: str
-    nvr: str
-    stream: str
-    version: str
-    context: str
-
-
-for index, f in enumerate(os.listdir(MODULES_DIR)):
-    parsed = parse_nvra(f)
-    MODULES[index] = Module(
-        name=parsed['name'],
-        nvr=f,
-        version=parsed['release'],
-        context=parsed['arch'],
-        stream=parsed['version'],
-        build_id=index
-    )
-
-app = Flask(__name__)
-
-handler = xmlrpcre.XMLRPCHandler('api')
-handler.connect(app, '/kojihub')
-
-@handler.register
-def getLastEvent(*args, **kwargs):
-    return {'id': 999999, 'ts': time.time()}
-
-
-@handler.register
-def listTagged(*args, **kwargs):
-    builds = []
-    for module in MODULES.values():
-        builds.append({
-            'build_id': module.build_id,
-            'owner_name': 'centos',
-            'package_name': module.name,
-            # 'package_name': 'perl-libwww-perl-devel',
-            'task_id': None,
-            'state': 1,
-            'nvr': f,
-            # 'nvr': 'perl-libwww-perl-devel-6.34-8030020201223164340.5839bc99',
-            'start_time': '2020-12-23 16:43:59',
-            'creation_event_id': 309485,
-            'creation_time': '2020-12-23 17:05:33.553748',
-            'epoch': None, 'tag_id': 533,
-            'completion_time': '2020-12-23 17:05:23',
-            'tag_name': 'dist-c8-module-compose',
-            'version': module.stream,
-            # 'version': '6.34',
-            'volume_id': 0,
-            # 'release': '8030020201223164340.5839bc99',
-            'release': '%s.%s' % (module.version, module.context),
-            'package_id': 3221,
-            'owner_id': 11,
-            'id': module.build_id,
-            'volume_name': 'DEFAULT',
-            # 'name': 'perl-libwww-perl-devel'
-            'name': module.name
-        })
-
-    return builds
-
-
-@handler.register
-def getFullInheritance(*args, **kwargs):
-    return []
-
-
-@handler.register
-def getBuild(*args, **kwargs):
-
-    build_id = args[0]
-    module = MODULES[build_id]
-
-    result = {
-        'id': build_id,
-        'name': module.name,
-        'version': module.stream,
-        'release': '%s.%s' % (module.version, module.context),
-        'completion_ts': 0,
-        'state': 'COMPLETE',
-        'extra': {
-            'typeinfo': {
-                'module': {
-                    'stream': module.stream,
-                    'version': module.version,
-                    'name': module.name,
-                    'context': module.context,
-                    'content_koji_tag': '-'.join([
-                        module.name,
-                        module.stream,
-                        module.version
-                    ]) + '.' + module.context
-                }
-            }
-        }
-    }
-    return result
-
-
-@handler.register
-def listArchives(*args, **kwargs):
-    module = MODULES[args[0]]
-
-    return [
-        {
-            'build_id': module.build_id,
-            'filename': 'modulemd.x86_64.txt',
-            'btype': 'module'
-        },
-        # noone ever uses this file
-        # but it should be because pungi ignores builds
-        # with len(files) <= 1
-        {
-            'build_id': module.build_id,
-            'filename': 'modulemd.txt',
-            'btype': 'module'
-        }
-    ]
-
-
-@handler.register
-def listTaggedRPMS(*args, **kwargs):
-    print('listTaggedRPMS', args, kwargs)
-    if args[0] == 'dist-c8-compose':
-        packages = []
-        rpms = subprocess.check_output('find /var/repos/ | grep .rpm', shell=True, universal_newlines=True).strip()
-        all_rpms = rpms.split('\n')
-
-        nvras = set()
-        for module in MODULES.values():
-            path = os.path.join(MODULES_DIR, module.nvr)
-            info = Modulemd.ModuleStream.read_file(path, strict=True)
-
-            for package in info.get_rpm_artifacts():
-                data = parse_nvra(package)
-                nvras.add((data['name'], data['version'], data['release'], data['arch']))
-
-        for rpm in all_rpms[:]:
-            data = parse_nvra(os.path.basename(rpm[:-4]))
-            if (data['name'], data['version'], data['release'], data['arch']) in nvras:
-                all_rpms.remove(rpm)
-
-        for rpm in all_rpms:
-            # print(os.path.basename(rpm))
-            info = parse_nvra(os.path.basename(rpm))
-            packages.append({
-                "build_id": 15270,
-                "name": info['name'],
-                "extra": None,
-                "arch": info['arch'],
-                "epoch": info['epoch'] or None,
-                "version": info['version'],
-                "metadata_only": False,
-                "release": info['release'],
-                "id": 262555,
-                "size": 0
-            })
-        builds = [
-            {
-              "build_id": 15270,
-              "owner_name": "mbox-mbs-backend",
-              "package_name": "389-ds-base",
-              "task_id": 195937,
-              "state": 1,
-              "nvr": "389-ds-base-1.4.3.8-6.module_el8.3.0+604+ab7bf9cc",
-              "start_time": "2020-12-22 19:20:12.504578",
-              "creation_event_id": 306731,
-              "creation_time": "2020-12-22 19:20:12.504578",
-              "epoch": None,
-              "tag_id": 1192,
-              "completion_time": "2020-12-22 19:34:34.716615",
-              "tag_name": "module-389-ds-1.4-8030020201222185615-618f7055",
-              "version": "1.4.3.8",
-              "volume_id": 0,
-              "release": "6.module_el8.3.0+604+ab7bf9cc",
-              "package_id": 104,
-              "owner_id": 6,
-              "id": 15270,
-              "volume_name": "DEFAULT",
-              "name": "389-ds-base"
-            }
-          ]
-    else:
-        module = args[0]
-        path = os.path.join('/root/pungi-centos/modules', module)
-        print('moduleinfo', path, os.path.exists(path))
-
-        for module in MODULES.values():
-            if module.nvr == args[0]:
-                builds = [
-                    {
-                        "build_id": module.build_id,
-                        "owner_name": "mbox-mbs-backend",
-                        "package_name": module.name,
-                        "task_id": 195937,
-                        "state": 1,
-                        "nvr": module.nvr,
-                        "start_time": "2020-12-22 19:20:12.504578",
-                        "creation_event_id": 306731,
-                        "creation_time": "2020-12-22 19:20:12.504578",
-                        "epoch": None,
-                        "tag_id": 1192,
-                        "completion_time": "2020-12-22 19:34:34.716615",
-                        "tag_name": module.nvr,
-                        "version": module.stream,
-                        "volume_id": 0,
-                        "release": module.version,
-                        "package_id": 104,
-                        "owner_id": 6,
-                        "id": module.build_id,
-                        "volume_name": "DEFAULT",
-                        "name": module.name
-                    }
-                ]
-                break
-
-        packages = []
-        if os.path.exists(path):
-            info = Modulemd.ModuleStream.read_file(path, strict=True)
-            print(info.get_rpm_artifacts())
-            for art in info.get_rpm_artifacts():
-                data = parse_nvra(art)
-                packages.append({
-                    "build_id": module.build_id,
-                    "name": data['name'],
-                    "extra": None,
-                    "arch": data['arch'],
-                    "epoch": data['epoch'] or None,
-                    "version": data['version'],
-                    "metadata_only": False,
-                    "release": data['release'],
-                    "id": 262555,
-                    "size": 0
-                })
-        else:
-            raise Exception
-
-
-    return [
-      packages,
-      builds
-    ]
-
-
-class RegexConverter(BaseConverter):
-    def __init__(self, url_map, *items):
-        super(RegexConverter, self).__init__(url_map)
-        self.regex = items[0]
-
-
-app.url_map.converters['regex'] = RegexConverter
-
-@app.route('/pkgs/packages/<regex(".*"):name>/<regex(".*"):rhel>/<regex(".*"):stream>/files/module/<regex(".*.txt"):filename>')
-def example(*args, name, rhel, stream, **kwargs):
-    print(args, name, rhel, kwargs)
-    path = os.path.join('/root/pungi-centos/modules', '%s-%s-%s' % (name, rhel, stream))
-    print('moduleinfo', path, os.path.exists(path))
-    if os.path.exists(path):
-        return open(path).read()
-    raise NotImplementedError
-
-if __name__ == "__main__":
-    app.run(port=8080, host='0.0.0.0')
\ No newline at end of file
diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py
index daaf0257..e82a47cf 100644
--- a/pungi/phases/pkgset/pkgsets.py
+++ b/pungi/phases/pkgset/pkgsets.py
@@ -22,8 +22,6 @@ It automatically finds a signed copies according to *sigkey_ordering*.
 import itertools
 import json
 import os
-
-import subprocess
 from six.moves import cPickle as pickle
 
 import kobo.log
@@ -504,12 +502,40 @@ class KojiPackageSet(PackageSetBase):
             return rpm_info["path_from_task"]
 
         pathinfo = self.koji_wrapper.koji_module.pathinfo
-        rpm_path = os.path.join(pathinfo.topdir, pathinfo.rpm(rpm_info))
-        if os.path.isfile(rpm_path):
-            return rpm_path
-        else:
-            self.log_warning("RPM %s not found" % rpm_path)
-            return None
+        paths = []
+        for sigkey in self.sigkey_ordering:
+            if not sigkey:
+                # we're looking for *signed* copies here
+                continue
+            sigkey = sigkey.lower()
+            rpm_path = os.path.join(
+                pathinfo.build(build_info), pathinfo.signed(rpm_info, sigkey)
+            )
+            paths.append(rpm_path)
+            if os.path.isfile(rpm_path):
+                return rpm_path
+
+        if None in self.sigkey_ordering or "" in self.sigkey_ordering:
+            # use an unsigned copy (if allowed)
+            rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info))
+            paths.append(rpm_path)
+            if os.path.isfile(rpm_path):
+                return rpm_path
+
+        if self._allow_invalid_sigkeys and rpm_info["name"] not in self.packages:
+            # use an unsigned copy (if allowed)
+            rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info))
+            paths.append(rpm_path)
+            if os.path.isfile(rpm_path):
+                self._invalid_sigkey_rpms.append(rpm_info)
+                return rpm_path
+
+        self._invalid_sigkey_rpms.append(rpm_info)
+        self.log_error(
+            "RPM %s not found for sigs: %s. Paths checked: %s"
+            % (rpm_info, self.sigkey_ordering, paths)
+        )
+        return None
 
     def populate(self, tag, event=None, inherit=True, include_packages=None):
         """Populate the package set with packages from given tag.
diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py
index 5da3aeb1..a557f604 100644
--- a/pungi/phases/pkgset/sources/source_koji.py
+++ b/pungi/phases/pkgset/sources/source_koji.py
@@ -21,7 +21,6 @@ import functools
 from fnmatch import fnmatch
 from itertools import groupby
 
-import requests
 from kobo.rpmlib import parse_nvra
 from kobo.shortcuts import force_list
 
@@ -30,7 +29,7 @@ from pungi.wrappers.comps import CompsWrapper
 from pungi.wrappers.mbs import MBSWrapper
 import pungi.phases.pkgset.pkgsets
 from pungi.arch import getBaseArch
-from pungi.util import retry, get_arch_variant_data, get_variant_data, translate_path
+from pungi.util import retry, get_arch_variant_data, get_variant_data
 from pungi.module_util import Modulemd
 
 from pungi.phases.pkgset.common import MaterializedPackageSet, get_all_arches
@@ -253,16 +252,6 @@ def _add_module_to_variant(
             continue
 
         try:
-            print(mmds["modulemd.%s.txt" % arch])
-
-            url = translate_path(compose, mmds["modulemd.%s.txt" % arch])
-            print(url)
-            r = requests.get(url)
-            r.raise_for_status()
-            os.makedirs(os.path.dirname(mmds["modulemd.%s.txt" % arch]), exist_ok=True)
-            with open(mmds["modulemd.%s.txt" % arch], 'wb') as f:
-                f.write(r.content)
-
             mmd = Modulemd.ModuleStream.read_file(
                 mmds["modulemd.%s.txt" % arch], strict=True
             )