current

pungi/collect_modules.py

@@ -0,0 +1,60 @@
+import argparse
+import gzip
+import os
+import typing
+from argparse import ArgumentParser
+from typing import List
+
+import yaml
+
+
+def collect_modules(modules_paths: List[typing.BinaryIO], target_dir: str):
+    """
+    Read given modules.yaml.gz files and export modules
+    and modulemd files from it.
+    Returns:
+        object:
+    """
+    modules_path = os.path.join(target_dir, 'modules')
+    module_defaults_path = os.path.join(target_dir, 'module_defaults')
+    os.makedirs(modules_path, exist_ok=True)
+    os.makedirs(module_defaults_path, exist_ok=True)
+
+    for module_file in modules_paths:
+        data = gzip.decompress(module_file.read())
+        documents = yaml.load_all(data, Loader=yaml.BaseLoader)
+        for doc in documents:
+            if doc['document'] == 'modulemd-defaults':
+                name = doc['data']['module']
+                path = os.path.join(module_defaults_path, name)
+                print('INFO', 'Found', name, 'module defaults')
+            else:
+                name = '%s-%s-%s.%s' % (
+                    doc['data']['name'],
+                    doc['data']['stream'],
+                    doc['data']['version'],
+                    doc['data']['context']
+                )
+                path = os.path.join(modules_path, name)
+                print('INFO', 'Found', name, 'module')
+
+                if 'artifacts' not in doc['data']:
+                    print('WARN', 'RPM', name, 'does not have explicit list of artifacts')
+
+            with open(path, 'w') as f:
+                yaml.dump(doc, f, default_flow_style=False)
+
+
+if __name__ == '__main__':
+    parser = ArgumentParser()
+    parser.add_argument(
+        '-p', '--path', required=True,
+        type=argparse.FileType('rb'), nargs='+',
+        help='Path to modules.yaml.gz file. '
+             'You may pass multiple files by passing -p path1 path2'
+    )
+    parser.add_argument('-t', '--target', required=True)
+
+    namespace = parser.parse_args()
+
+    collect_modules(namespace.path, namespace.target)

pungi/collect_rpms.py

@@ -0,0 +1,66 @@
+from argparse import ArgumentParser
+
+import os
+from typing import List
+
+from attr import dataclass
+from productmd.common import parse_nvra
+
+
+@dataclass
+class Package:
+    nvra: str
+    path: str
+
+
+def search_rpms(top_dir) -> List[Package]:
+    """
+    Search for all *.rpm files recursively
+    in given top directory
+    Returns:
+        list: list of paths
+    """
+    rpms = []
+    for root, dirs, files in os.walk(top_dir):
+        path = root.split(os.sep)
+        for file in files:
+            if not file.endswith('.rpm'):
+                continue
+            rpms.append(
+                Package(nvra=file[:-4], path=os.path.join('/', *path, file))
+            )
+    return rpms
+
+
+def copy_rpms(packages: List[Package], target_top_dir: str):
+    """
+    Search synced repos for rpms and prepare
+    koji-like structure for pungi
+
+    Instead of repos, use following structure:
+        # ls /mnt/koji/
+            i686/  noarch/  x86_64/
+    Returns:
+        Nothing:
+    """
+    for package in packages:
+        info = parse_nvra(package.nvra)
+
+        target_arch_dir = os.path.join(target_top_dir, info['arch'])
+        os.makedirs(target_arch_dir, exist_ok=True)
+
+        target_file = os.path.join(target_arch_dir, os.path.basename(package.path))
+
+        if not os.path.exists(target_file):
+            os.link(package.path, target_file)
+
+
+if __name__ == '__main__':
+    parser = ArgumentParser()
+    parser.add_argument('-p', '--path', required=True)
+    parser.add_argument('-t', '--target', required=True)
+
+    namespace = parser.parse_args()
+
+    rpms = search_rpms(namespace.path)
+    copy_rpms(rpms, namespace.target)

pungi/kojimock.py

@@ -0,0 +1,293 @@
+import time
+
+import json
+
+import subprocess
+
+import os
+
+from attr import dataclass
+from flask import Flask
+from flask_xmlrpcre import xmlrpcre
+import xmlrpc.client
+import xmlrpc.server
+
+from pungi.module_util import Modulemd
+from kobo.rpmlib import parse_nvra
+
+# hacky hack for int64 (xmlrpc sucks)
+xmlrpc.client.Marshaller.dispatch[type(0)] = lambda _, v, w: w("<value><i8>%d</i8></value>" % v)
+
+from werkzeug.routing import BaseConverter
+
+MODULES_DIR = '/mnt/koji/modules/'
+MODULES = {}
+
+
+@dataclass
+class Module:
+    build_id: int
+    name: str
+    nvr: str
+    stream: str
+    version: str
+    context: str
+
+
+for index, f in enumerate(os.listdir(MODULES_DIR)):
+    parsed = parse_nvra(f)
+    MODULES[index] = Module(
+        name=parsed['name'],
+        nvr=f,
+        version=parsed['release'],
+        context=parsed['arch'],
+        stream=parsed['version'],
+        build_id=index
+    )
+
+app = Flask(__name__)
+
+handler = xmlrpcre.XMLRPCHandler('api')
+handler.connect(app, '/kojihub')
+
+@handler.register
+def getLastEvent(*args, **kwargs):
+    return {'id': 999999, 'ts': time.time()}
+
+
+@handler.register
+def listTagged(*args, **kwargs):
+    builds = []
+    for module in MODULES.values():
+        builds.append({
+            'build_id': module.build_id,
+            'owner_name': 'centos',
+            'package_name': module.name,
+            # 'package_name': 'perl-libwww-perl-devel',
+            'task_id': None,
+            'state': 1,
+            'nvr': f,
+            # 'nvr': 'perl-libwww-perl-devel-6.34-8030020201223164340.5839bc99',
+            'start_time': '2020-12-23 16:43:59',
+            'creation_event_id': 309485,
+            'creation_time': '2020-12-23 17:05:33.553748',
+            'epoch': None, 'tag_id': 533,
+            'completion_time': '2020-12-23 17:05:23',
+            'tag_name': 'dist-c8-module-compose',
+            'version': module.stream,
+            # 'version': '6.34',
+            'volume_id': 0,
+            # 'release': '8030020201223164340.5839bc99',
+            'release': '%s.%s' % (module.version, module.context),
+            'package_id': 3221,
+            'owner_id': 11,
+            'id': module.build_id,
+            'volume_name': 'DEFAULT',
+            # 'name': 'perl-libwww-perl-devel'
+            'name': module.name
+        })
+
+    return builds
+
+
+@handler.register
+def getFullInheritance(*args, **kwargs):
+    return []
+
+
+@handler.register
+def getBuild(*args, **kwargs):
+
+    build_id = args[0]
+    module = MODULES[build_id]
+
+    result = {
+        'id': build_id,
+        'name': module.name,
+        'version': module.stream,
+        'release': '%s.%s' % (module.version, module.context),
+        'completion_ts': 0,
+        'state': 'COMPLETE',
+        'extra': {
+            'typeinfo': {
+                'module': {
+                    'stream': module.stream,
+                    'version': module.version,
+                    'name': module.name,
+                    'context': module.context,
+                    'content_koji_tag': '-'.join([
+                        module.name,
+                        module.stream,
+                        module.version
+                    ]) + '.' + module.context
+                }
+            }
+        }
+    }
+    return result
+
+
+@handler.register
+def listArchives(*args, **kwargs):
+    module = MODULES[args[0]]
+
+    return [
+        {
+            'build_id': module.build_id,
+            'filename': 'modulemd.x86_64.txt',
+            'btype': 'module'
+        },
+        # noone ever uses this file
+        # but it should be because pungi ignores builds
+        # with len(files) <= 1
+        {
+            'build_id': module.build_id,
+            'filename': 'modulemd.txt',
+            'btype': 'module'
+        }
+    ]
+
+
+@handler.register
+def listTaggedRPMS(*args, **kwargs):
+    print('listTaggedRPMS', args, kwargs)
+    if args[0] == 'dist-c8-compose':
+        packages = []
+        rpms = subprocess.check_output('find /var/repos/ | grep .rpm', shell=True, universal_newlines=True).strip()
+        all_rpms = rpms.split('\n')
+
+        nvras = set()
+        for module in MODULES.values():
+            path = os.path.join(MODULES_DIR, module.nvr)
+            info = Modulemd.ModuleStream.read_file(path, strict=True)
+
+            for package in info.get_rpm_artifacts():
+                data = parse_nvra(package)
+                nvras.add((data['name'], data['version'], data['release'], data['arch']))
+
+        for rpm in all_rpms[:]:
+            data = parse_nvra(os.path.basename(rpm[:-4]))
+            if (data['name'], data['version'], data['release'], data['arch']) in nvras:
+                all_rpms.remove(rpm)
+
+        for rpm in all_rpms:
+            # print(os.path.basename(rpm))
+            info = parse_nvra(os.path.basename(rpm))
+            packages.append({
+                "build_id": 15270,
+                "name": info['name'],
+                "extra": None,
+                "arch": info['arch'],
+                "epoch": info['epoch'] or None,
+                "version": info['version'],
+                "metadata_only": False,
+                "release": info['release'],
+                "id": 262555,
+                "size": 0
+            })
+        builds = [
+            {
+              "build_id": 15270,
+              "owner_name": "mbox-mbs-backend",
+              "package_name": "389-ds-base",
+              "task_id": 195937,
+              "state": 1,
+              "nvr": "389-ds-base-1.4.3.8-6.module_el8.3.0+604+ab7bf9cc",
+              "start_time": "2020-12-22 19:20:12.504578",
+              "creation_event_id": 306731,
+              "creation_time": "2020-12-22 19:20:12.504578",
+              "epoch": None,
+              "tag_id": 1192,
+              "completion_time": "2020-12-22 19:34:34.716615",
+              "tag_name": "module-389-ds-1.4-8030020201222185615-618f7055",
+              "version": "1.4.3.8",
+              "volume_id": 0,
+              "release": "6.module_el8.3.0+604+ab7bf9cc",
+              "package_id": 104,
+              "owner_id": 6,
+              "id": 15270,
+              "volume_name": "DEFAULT",
+              "name": "389-ds-base"
+            }
+          ]
+    else:
+        module = args[0]
+        path = os.path.join('/root/pungi-centos/modules', module)
+        print('moduleinfo', path, os.path.exists(path))
+
+        for module in MODULES.values():
+            if module.nvr == args[0]:
+                builds = [
+                    {
+                        "build_id": module.build_id,
+                        "owner_name": "mbox-mbs-backend",
+                        "package_name": module.name,
+                        "task_id": 195937,
+                        "state": 1,
+                        "nvr": module.nvr,
+                        "start_time": "2020-12-22 19:20:12.504578",
+                        "creation_event_id": 306731,
+                        "creation_time": "2020-12-22 19:20:12.504578",
+                        "epoch": None,
+                        "tag_id": 1192,
+                        "completion_time": "2020-12-22 19:34:34.716615",
+                        "tag_name": module.nvr,
+                        "version": module.stream,
+                        "volume_id": 0,
+                        "release": module.version,
+                        "package_id": 104,
+                        "owner_id": 6,
+                        "id": module.build_id,
+                        "volume_name": "DEFAULT",
+                        "name": module.name
+                    }
+                ]
+                break
+
+        packages = []
+        if os.path.exists(path):
+            info = Modulemd.ModuleStream.read_file(path, strict=True)
+            print(info.get_rpm_artifacts())
+            for art in info.get_rpm_artifacts():
+                data = parse_nvra(art)
+                packages.append({
+                    "build_id": module.build_id,
+                    "name": data['name'],
+                    "extra": None,
+                    "arch": data['arch'],
+                    "epoch": data['epoch'] or None,
+                    "version": data['version'],
+                    "metadata_only": False,
+                    "release": data['release'],
+                    "id": 262555,
+                    "size": 0
+                })
+        else:
+            raise Exception
+
+
+    return [
+      packages,
+      builds
+    ]
+
+
+class RegexConverter(BaseConverter):
+    def __init__(self, url_map, *items):
+        super(RegexConverter, self).__init__(url_map)
+        self.regex = items[0]
+
+
+app.url_map.converters['regex'] = RegexConverter
+
+@app.route('/pkgs/packages/<regex(".*"):name>/<regex(".*"):rhel>/<regex(".*"):stream>/files/module/<regex(".*.txt"):filename>')
+def example(*args, name, rhel, stream, **kwargs):
+    print(args, name, rhel, kwargs)
+    path = os.path.join('/root/pungi-centos/modules', '%s-%s-%s' % (name, rhel, stream))
+    print('moduleinfo', path, os.path.exists(path))
+    if os.path.exists(path):
+        return open(path).read()
+    raise NotImplementedError
+
+if __name__ == "__main__":
+    app.run(port=8080, host='0.0.0.0')

pungi/phases/pkgset/pkgsets.py

@@ -22,6 +22,8 @@ It automatically finds a signed copies according to *sigkey_ordering*.
 import itertools
 import json
 import os
+
+import subprocess
 from six.moves import cPickle as pickle
 
 import kobo.log
@@ -502,40 +504,12 @@ class KojiPackageSet(PackageSetBase):
             return rpm_info["path_from_task"]
 
         pathinfo = self.koji_wrapper.koji_module.pathinfo
-        paths = []
+        rpm_path = os.path.join(pathinfo.topdir, pathinfo.rpm(rpm_info))
-        for sigkey in self.sigkey_ordering:
+        if os.path.isfile(rpm_path):
-            if not sigkey:
+            return rpm_path
-                # we're looking for *signed* copies here
+        else:
-                continue
+            self.log_warning("RPM %s not found" % rpm_path)
-            sigkey = sigkey.lower()
+            return None
-            rpm_path = os.path.join(
-                pathinfo.build(build_info), pathinfo.signed(rpm_info, sigkey)
-            )
-            paths.append(rpm_path)
-            if os.path.isfile(rpm_path):
-                return rpm_path
-
-        if None in self.sigkey_ordering or "" in self.sigkey_ordering:
-            # use an unsigned copy (if allowed)
-            rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info))
-            paths.append(rpm_path)
-            if os.path.isfile(rpm_path):
-                return rpm_path
-
-        if self._allow_invalid_sigkeys and rpm_info["name"] not in self.packages:
-            # use an unsigned copy (if allowed)
-            rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info))
-            paths.append(rpm_path)
-            if os.path.isfile(rpm_path):
-                self._invalid_sigkey_rpms.append(rpm_info)
-                return rpm_path
-
-        self._invalid_sigkey_rpms.append(rpm_info)
-        self.log_error(
-            "RPM %s not found for sigs: %s. Paths checked: %s"
-            % (rpm_info, self.sigkey_ordering, paths)
-        )
-        return None
 
     def populate(self, tag, event=None, inherit=True, include_packages=None):
         """Populate the package set with packages from given tag.

pungi/phases/pkgset/sources/source_koji.py

@@ -21,6 +21,7 @@ import functools
 from fnmatch import fnmatch
 from itertools import groupby
 
+import requests
 from kobo.rpmlib import parse_nvra
 from kobo.shortcuts import force_list
 
@@ -29,7 +30,7 @@ from pungi.wrappers.comps import CompsWrapper
 from pungi.wrappers.mbs import MBSWrapper
 import pungi.phases.pkgset.pkgsets
 from pungi.arch import getBaseArch
-from pungi.util import retry, get_arch_variant_data, get_variant_data
+from pungi.util import retry, get_arch_variant_data, get_variant_data, translate_path
 from pungi.module_util import Modulemd
 
 from pungi.phases.pkgset.common import MaterializedPackageSet, get_all_arches
@@ -252,6 +253,16 @@ def _add_module_to_variant(
             continue
 
         try:
+            print(mmds["modulemd.%s.txt" % arch])
+
+            url = translate_path(compose, mmds["modulemd.%s.txt" % arch])
+            print(url)
+            r = requests.get(url)
+            r.raise_for_status()
+            os.makedirs(os.path.dirname(mmds["modulemd.%s.txt" % arch]), exist_ok=True)
+            with open(mmds["modulemd.%s.txt" % arch], 'wb') as f:
+                f.write(r.content)
+
             mmd = Modulemd.ModuleStream.read_file(
                 mmds["modulemd.%s.txt" % arch], strict=True
             )