diff --git a/pungi/collect_modules.py b/pungi/collect_modules.py
new file mode 100644
index 00000000..aad214bb
--- /dev/null
+++ b/pungi/collect_modules.py
@@ -0,0 +1,60 @@
+import argparse
+import gzip
+import os
+import typing
+from argparse import ArgumentParser
+from typing import List
+
+import yaml
+
+
+def collect_modules(modules_paths: List[typing.BinaryIO], target_dir: str):
+ """
+ Read given modules.yaml.gz files and export modules
+ and modulemd files from it.
+ Returns:
+ object:
+ """
+ modules_path = os.path.join(target_dir, 'modules')
+ module_defaults_path = os.path.join(target_dir, 'module_defaults')
+ os.makedirs(modules_path, exist_ok=True)
+ os.makedirs(module_defaults_path, exist_ok=True)
+
+ for module_file in modules_paths:
+ data = gzip.decompress(module_file.read())
+ documents = yaml.load_all(data, Loader=yaml.BaseLoader)
+ for doc in documents:
+ if doc['document'] == 'modulemd-defaults':
+ name = doc['data']['module']
+ path = os.path.join(module_defaults_path, name)
+ print('INFO', 'Found', name, 'module defaults')
+ else:
+ name = '%s-%s-%s.%s' % (
+ doc['data']['name'],
+ doc['data']['stream'],
+ doc['data']['version'],
+ doc['data']['context']
+ )
+ path = os.path.join(modules_path, name)
+ print('INFO', 'Found', name, 'module')
+
+ if 'artifacts' not in doc['data']:
+ print('WARN', 'RPM', name, 'does not have explicit list of artifacts')
+
+ with open(path, 'w') as f:
+ yaml.dump(doc, f, default_flow_style=False)
+
+
+if __name__ == '__main__':
+ parser = ArgumentParser()
+ parser.add_argument(
+ '-p', '--path', required=True,
+ type=argparse.FileType('rb'), nargs='+',
+ help='Path to modules.yaml.gz file. '
+ 'You may pass multiple files by passing -p path1 path2'
+ )
+ parser.add_argument('-t', '--target', required=True)
+
+ namespace = parser.parse_args()
+
+ collect_modules(namespace.path, namespace.target)
\ No newline at end of file
diff --git a/pungi/collect_rpms.py b/pungi/collect_rpms.py
new file mode 100644
index 00000000..b7ab2ff5
--- /dev/null
+++ b/pungi/collect_rpms.py
@@ -0,0 +1,66 @@
+from argparse import ArgumentParser
+
+import os
+from typing import List
+
+from attr import dataclass
+from productmd.common import parse_nvra
+
+
+@dataclass
+class Package:
+ nvra: str
+ path: str
+
+
+def search_rpms(top_dir) -> List[Package]:
+ """
+ Search for all *.rpm files recursively
+ in given top directory
+ Returns:
+ list: list of paths
+ """
+ rpms = []
+ for root, dirs, files in os.walk(top_dir):
+ path = root.split(os.sep)
+ for file in files:
+ if not file.endswith('.rpm'):
+ continue
+ rpms.append(
+ Package(nvra=file[:-4], path=os.path.join('/', *path, file))
+ )
+ return rpms
+
+
+def copy_rpms(packages: List[Package], target_top_dir: str):
+ """
+ Search synced repos for rpms and prepare
+ koji-like structure for pungi
+
+ Instead of repos, use following structure:
+ # ls /mnt/koji/
+ i686/ noarch/ x86_64/
+ Returns:
+ Nothing:
+ """
+ for package in packages:
+ info = parse_nvra(package.nvra)
+
+ target_arch_dir = os.path.join(target_top_dir, info['arch'])
+ os.makedirs(target_arch_dir, exist_ok=True)
+
+ target_file = os.path.join(target_arch_dir, os.path.basename(package.path))
+
+ if not os.path.exists(target_file):
+ os.link(package.path, target_file)
+
+
+if __name__ == '__main__':
+ parser = ArgumentParser()
+ parser.add_argument('-p', '--path', required=True)
+ parser.add_argument('-t', '--target', required=True)
+
+ namespace = parser.parse_args()
+
+ rpms = search_rpms(namespace.path)
+ copy_rpms(rpms, namespace.target)
diff --git a/pungi/kojimock.py b/pungi/kojimock.py
new file mode 100644
index 00000000..5816381c
--- /dev/null
+++ b/pungi/kojimock.py
@@ -0,0 +1,293 @@
+import time
+
+import json
+
+import subprocess
+
+import os
+
+from attr import dataclass
+from flask import Flask
+from flask_xmlrpcre import xmlrpcre
+import xmlrpc.client
+import xmlrpc.server
+
+from pungi.module_util import Modulemd
+from kobo.rpmlib import parse_nvra
+
+# hacky hack for int64 (xmlrpc sucks)
+xmlrpc.client.Marshaller.dispatch[type(0)] = lambda _, v, w: w("%d" % v)
+
+from werkzeug.routing import BaseConverter
+
+MODULES_DIR = '/mnt/koji/modules/'
+MODULES = {}
+
+
+@dataclass
+class Module:
+ build_id: int
+ name: str
+ nvr: str
+ stream: str
+ version: str
+ context: str
+
+
+for index, f in enumerate(os.listdir(MODULES_DIR)):
+ parsed = parse_nvra(f)
+ MODULES[index] = Module(
+ name=parsed['name'],
+ nvr=f,
+ version=parsed['release'],
+ context=parsed['arch'],
+ stream=parsed['version'],
+ build_id=index
+ )
+
+app = Flask(__name__)
+
+handler = xmlrpcre.XMLRPCHandler('api')
+handler.connect(app, '/kojihub')
+
+@handler.register
+def getLastEvent(*args, **kwargs):
+ return {'id': 999999, 'ts': time.time()}
+
+
+@handler.register
+def listTagged(*args, **kwargs):
+ builds = []
+ for module in MODULES.values():
+ builds.append({
+ 'build_id': module.build_id,
+ 'owner_name': 'centos',
+ 'package_name': module.name,
+ # 'package_name': 'perl-libwww-perl-devel',
+ 'task_id': None,
+ 'state': 1,
+ 'nvr': f,
+ # 'nvr': 'perl-libwww-perl-devel-6.34-8030020201223164340.5839bc99',
+ 'start_time': '2020-12-23 16:43:59',
+ 'creation_event_id': 309485,
+ 'creation_time': '2020-12-23 17:05:33.553748',
+ 'epoch': None, 'tag_id': 533,
+ 'completion_time': '2020-12-23 17:05:23',
+ 'tag_name': 'dist-c8-module-compose',
+ 'version': module.stream,
+ # 'version': '6.34',
+ 'volume_id': 0,
+ # 'release': '8030020201223164340.5839bc99',
+ 'release': '%s.%s' % (module.version, module.context),
+ 'package_id': 3221,
+ 'owner_id': 11,
+ 'id': module.build_id,
+ 'volume_name': 'DEFAULT',
+ # 'name': 'perl-libwww-perl-devel'
+ 'name': module.name
+ })
+
+ return builds
+
+
+@handler.register
+def getFullInheritance(*args, **kwargs):
+ return []
+
+
+@handler.register
+def getBuild(*args, **kwargs):
+
+ build_id = args[0]
+ module = MODULES[build_id]
+
+ result = {
+ 'id': build_id,
+ 'name': module.name,
+ 'version': module.stream,
+ 'release': '%s.%s' % (module.version, module.context),
+ 'completion_ts': 0,
+ 'state': 'COMPLETE',
+ 'extra': {
+ 'typeinfo': {
+ 'module': {
+ 'stream': module.stream,
+ 'version': module.version,
+ 'name': module.name,
+ 'context': module.context,
+ 'content_koji_tag': '-'.join([
+ module.name,
+ module.stream,
+ module.version
+ ]) + '.' + module.context
+ }
+ }
+ }
+ }
+ return result
+
+
+@handler.register
+def listArchives(*args, **kwargs):
+ module = MODULES[args[0]]
+
+ return [
+ {
+ 'build_id': module.build_id,
+ 'filename': 'modulemd.x86_64.txt',
+ 'btype': 'module'
+ },
+ # noone ever uses this file
+ # but it should be because pungi ignores builds
+ # with len(files) <= 1
+ {
+ 'build_id': module.build_id,
+ 'filename': 'modulemd.txt',
+ 'btype': 'module'
+ }
+ ]
+
+
+@handler.register
+def listTaggedRPMS(*args, **kwargs):
+ print('listTaggedRPMS', args, kwargs)
+ if args[0] == 'dist-c8-compose':
+ packages = []
+ rpms = subprocess.check_output('find /var/repos/ | grep .rpm', shell=True, universal_newlines=True).strip()
+ all_rpms = rpms.split('\n')
+
+ nvras = set()
+ for module in MODULES.values():
+ path = os.path.join(MODULES_DIR, module.nvr)
+ info = Modulemd.ModuleStream.read_file(path, strict=True)
+
+ for package in info.get_rpm_artifacts():
+ data = parse_nvra(package)
+ nvras.add((data['name'], data['version'], data['release'], data['arch']))
+
+ for rpm in all_rpms[:]:
+ data = parse_nvra(os.path.basename(rpm[:-4]))
+ if (data['name'], data['version'], data['release'], data['arch']) in nvras:
+ all_rpms.remove(rpm)
+
+ for rpm in all_rpms:
+ # print(os.path.basename(rpm))
+ info = parse_nvra(os.path.basename(rpm))
+ packages.append({
+ "build_id": 15270,
+ "name": info['name'],
+ "extra": None,
+ "arch": info['arch'],
+ "epoch": info['epoch'] or None,
+ "version": info['version'],
+ "metadata_only": False,
+ "release": info['release'],
+ "id": 262555,
+ "size": 0
+ })
+ builds = [
+ {
+ "build_id": 15270,
+ "owner_name": "mbox-mbs-backend",
+ "package_name": "389-ds-base",
+ "task_id": 195937,
+ "state": 1,
+ "nvr": "389-ds-base-1.4.3.8-6.module_el8.3.0+604+ab7bf9cc",
+ "start_time": "2020-12-22 19:20:12.504578",
+ "creation_event_id": 306731,
+ "creation_time": "2020-12-22 19:20:12.504578",
+ "epoch": None,
+ "tag_id": 1192,
+ "completion_time": "2020-12-22 19:34:34.716615",
+ "tag_name": "module-389-ds-1.4-8030020201222185615-618f7055",
+ "version": "1.4.3.8",
+ "volume_id": 0,
+ "release": "6.module_el8.3.0+604+ab7bf9cc",
+ "package_id": 104,
+ "owner_id": 6,
+ "id": 15270,
+ "volume_name": "DEFAULT",
+ "name": "389-ds-base"
+ }
+ ]
+ else:
+ module = args[0]
+ path = os.path.join('/root/pungi-centos/modules', module)
+ print('moduleinfo', path, os.path.exists(path))
+
+ for module in MODULES.values():
+ if module.nvr == args[0]:
+ builds = [
+ {
+ "build_id": module.build_id,
+ "owner_name": "mbox-mbs-backend",
+ "package_name": module.name,
+ "task_id": 195937,
+ "state": 1,
+ "nvr": module.nvr,
+ "start_time": "2020-12-22 19:20:12.504578",
+ "creation_event_id": 306731,
+ "creation_time": "2020-12-22 19:20:12.504578",
+ "epoch": None,
+ "tag_id": 1192,
+ "completion_time": "2020-12-22 19:34:34.716615",
+ "tag_name": module.nvr,
+ "version": module.stream,
+ "volume_id": 0,
+ "release": module.version,
+ "package_id": 104,
+ "owner_id": 6,
+ "id": module.build_id,
+ "volume_name": "DEFAULT",
+ "name": module.name
+ }
+ ]
+ break
+
+ packages = []
+ if os.path.exists(path):
+ info = Modulemd.ModuleStream.read_file(path, strict=True)
+ print(info.get_rpm_artifacts())
+ for art in info.get_rpm_artifacts():
+ data = parse_nvra(art)
+ packages.append({
+ "build_id": module.build_id,
+ "name": data['name'],
+ "extra": None,
+ "arch": data['arch'],
+ "epoch": data['epoch'] or None,
+ "version": data['version'],
+ "metadata_only": False,
+ "release": data['release'],
+ "id": 262555,
+ "size": 0
+ })
+ else:
+ raise Exception
+
+
+ return [
+ packages,
+ builds
+ ]
+
+
+class RegexConverter(BaseConverter):
+ def __init__(self, url_map, *items):
+ super(RegexConverter, self).__init__(url_map)
+ self.regex = items[0]
+
+
+app.url_map.converters['regex'] = RegexConverter
+
+@app.route('/pkgs/packages////files/module/')
+def example(*args, name, rhel, stream, **kwargs):
+ print(args, name, rhel, kwargs)
+ path = os.path.join('/root/pungi-centos/modules', '%s-%s-%s' % (name, rhel, stream))
+ print('moduleinfo', path, os.path.exists(path))
+ if os.path.exists(path):
+ return open(path).read()
+ raise NotImplementedError
+
+if __name__ == "__main__":
+ app.run(port=8080, host='0.0.0.0')
\ No newline at end of file
diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py
index e82a47cf..daaf0257 100644
--- a/pungi/phases/pkgset/pkgsets.py
+++ b/pungi/phases/pkgset/pkgsets.py
@@ -22,6 +22,8 @@ It automatically finds a signed copies according to *sigkey_ordering*.
import itertools
import json
import os
+
+import subprocess
from six.moves import cPickle as pickle
import kobo.log
@@ -502,40 +504,12 @@ class KojiPackageSet(PackageSetBase):
return rpm_info["path_from_task"]
pathinfo = self.koji_wrapper.koji_module.pathinfo
- paths = []
- for sigkey in self.sigkey_ordering:
- if not sigkey:
- # we're looking for *signed* copies here
- continue
- sigkey = sigkey.lower()
- rpm_path = os.path.join(
- pathinfo.build(build_info), pathinfo.signed(rpm_info, sigkey)
- )
- paths.append(rpm_path)
- if os.path.isfile(rpm_path):
- return rpm_path
-
- if None in self.sigkey_ordering or "" in self.sigkey_ordering:
- # use an unsigned copy (if allowed)
- rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info))
- paths.append(rpm_path)
- if os.path.isfile(rpm_path):
- return rpm_path
-
- if self._allow_invalid_sigkeys and rpm_info["name"] not in self.packages:
- # use an unsigned copy (if allowed)
- rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info))
- paths.append(rpm_path)
- if os.path.isfile(rpm_path):
- self._invalid_sigkey_rpms.append(rpm_info)
- return rpm_path
-
- self._invalid_sigkey_rpms.append(rpm_info)
- self.log_error(
- "RPM %s not found for sigs: %s. Paths checked: %s"
- % (rpm_info, self.sigkey_ordering, paths)
- )
- return None
+ rpm_path = os.path.join(pathinfo.topdir, pathinfo.rpm(rpm_info))
+ if os.path.isfile(rpm_path):
+ return rpm_path
+ else:
+ self.log_warning("RPM %s not found" % rpm_path)
+ return None
def populate(self, tag, event=None, inherit=True, include_packages=None):
"""Populate the package set with packages from given tag.
diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py
index a557f604..5da3aeb1 100644
--- a/pungi/phases/pkgset/sources/source_koji.py
+++ b/pungi/phases/pkgset/sources/source_koji.py
@@ -21,6 +21,7 @@ import functools
from fnmatch import fnmatch
from itertools import groupby
+import requests
from kobo.rpmlib import parse_nvra
from kobo.shortcuts import force_list
@@ -29,7 +30,7 @@ from pungi.wrappers.comps import CompsWrapper
from pungi.wrappers.mbs import MBSWrapper
import pungi.phases.pkgset.pkgsets
from pungi.arch import getBaseArch
-from pungi.util import retry, get_arch_variant_data, get_variant_data
+from pungi.util import retry, get_arch_variant_data, get_variant_data, translate_path
from pungi.module_util import Modulemd
from pungi.phases.pkgset.common import MaterializedPackageSet, get_all_arches
@@ -252,6 +253,16 @@ def _add_module_to_variant(
continue
try:
+ print(mmds["modulemd.%s.txt" % arch])
+
+ url = translate_path(compose, mmds["modulemd.%s.txt" % arch])
+ print(url)
+ r = requests.get(url)
+ r.raise_for_status()
+ os.makedirs(os.path.dirname(mmds["modulemd.%s.txt" % arch]), exist_ok=True)
+ with open(mmds["modulemd.%s.txt" % arch], 'wb') as f:
+ f.write(r.content)
+
mmd = Modulemd.ModuleStream.read_file(
mmds["modulemd.%s.txt" % arch], strict=True
)