From b2e439e561a508f4eeddd38faaddcfa75b3610ba Mon Sep 17 00:00:00 2001 From: Oleksandr Shyshatskyi Date: Sat, 26 Dec 2020 15:04:41 +0200 Subject: [PATCH] current --- pungi/collect_modules.py | 60 +++++ pungi/collect_rpms.py | 66 +++++ pungi/kojimock.py | 293 +++++++++++++++++++++ pungi/phases/pkgset/pkgsets.py | 42 +-- pungi/phases/pkgset/sources/source_koji.py | 13 +- 5 files changed, 439 insertions(+), 35 deletions(-) create mode 100644 pungi/collect_modules.py create mode 100644 pungi/collect_rpms.py create mode 100644 pungi/kojimock.py diff --git a/pungi/collect_modules.py b/pungi/collect_modules.py new file mode 100644 index 00000000..aad214bb --- /dev/null +++ b/pungi/collect_modules.py @@ -0,0 +1,60 @@ +import argparse +import gzip +import os +import typing +from argparse import ArgumentParser +from typing import List + +import yaml + + +def collect_modules(modules_paths: List[typing.BinaryIO], target_dir: str): + """ + Read given modules.yaml.gz files and export modules + and modulemd files from it. + Returns: + object: + """ + modules_path = os.path.join(target_dir, 'modules') + module_defaults_path = os.path.join(target_dir, 'module_defaults') + os.makedirs(modules_path, exist_ok=True) + os.makedirs(module_defaults_path, exist_ok=True) + + for module_file in modules_paths: + data = gzip.decompress(module_file.read()) + documents = yaml.load_all(data, Loader=yaml.BaseLoader) + for doc in documents: + if doc['document'] == 'modulemd-defaults': + name = doc['data']['module'] + path = os.path.join(module_defaults_path, name) + print('INFO', 'Found', name, 'module defaults') + else: + name = '%s-%s-%s.%s' % ( + doc['data']['name'], + doc['data']['stream'], + doc['data']['version'], + doc['data']['context'] + ) + path = os.path.join(modules_path, name) + print('INFO', 'Found', name, 'module') + + if 'artifacts' not in doc['data']: + print('WARN', 'RPM', name, 'does not have explicit list of artifacts') + + with open(path, 'w') as f: + yaml.dump(doc, f, default_flow_style=False) + + +if __name__ == '__main__': + parser = ArgumentParser() + parser.add_argument( + '-p', '--path', required=True, + type=argparse.FileType('rb'), nargs='+', + help='Path to modules.yaml.gz file. ' + 'You may pass multiple files by passing -p path1 path2' + ) + parser.add_argument('-t', '--target', required=True) + + namespace = parser.parse_args() + + collect_modules(namespace.path, namespace.target) \ No newline at end of file diff --git a/pungi/collect_rpms.py b/pungi/collect_rpms.py new file mode 100644 index 00000000..b7ab2ff5 --- /dev/null +++ b/pungi/collect_rpms.py @@ -0,0 +1,66 @@ +from argparse import ArgumentParser + +import os +from typing import List + +from attr import dataclass +from productmd.common import parse_nvra + + +@dataclass +class Package: + nvra: str + path: str + + +def search_rpms(top_dir) -> List[Package]: + """ + Search for all *.rpm files recursively + in given top directory + Returns: + list: list of paths + """ + rpms = [] + for root, dirs, files in os.walk(top_dir): + path = root.split(os.sep) + for file in files: + if not file.endswith('.rpm'): + continue + rpms.append( + Package(nvra=file[:-4], path=os.path.join('/', *path, file)) + ) + return rpms + + +def copy_rpms(packages: List[Package], target_top_dir: str): + """ + Search synced repos for rpms and prepare + koji-like structure for pungi + + Instead of repos, use following structure: + # ls /mnt/koji/ + i686/ noarch/ x86_64/ + Returns: + Nothing: + """ + for package in packages: + info = parse_nvra(package.nvra) + + target_arch_dir = os.path.join(target_top_dir, info['arch']) + os.makedirs(target_arch_dir, exist_ok=True) + + target_file = os.path.join(target_arch_dir, os.path.basename(package.path)) + + if not os.path.exists(target_file): + os.link(package.path, target_file) + + +if __name__ == '__main__': + parser = ArgumentParser() + parser.add_argument('-p', '--path', required=True) + parser.add_argument('-t', '--target', required=True) + + namespace = parser.parse_args() + + rpms = search_rpms(namespace.path) + copy_rpms(rpms, namespace.target) diff --git a/pungi/kojimock.py b/pungi/kojimock.py new file mode 100644 index 00000000..5816381c --- /dev/null +++ b/pungi/kojimock.py @@ -0,0 +1,293 @@ +import time + +import json + +import subprocess + +import os + +from attr import dataclass +from flask import Flask +from flask_xmlrpcre import xmlrpcre +import xmlrpc.client +import xmlrpc.server + +from pungi.module_util import Modulemd +from kobo.rpmlib import parse_nvra + +# hacky hack for int64 (xmlrpc sucks) +xmlrpc.client.Marshaller.dispatch[type(0)] = lambda _, v, w: w("%d" % v) + +from werkzeug.routing import BaseConverter + +MODULES_DIR = '/mnt/koji/modules/' +MODULES = {} + + +@dataclass +class Module: + build_id: int + name: str + nvr: str + stream: str + version: str + context: str + + +for index, f in enumerate(os.listdir(MODULES_DIR)): + parsed = parse_nvra(f) + MODULES[index] = Module( + name=parsed['name'], + nvr=f, + version=parsed['release'], + context=parsed['arch'], + stream=parsed['version'], + build_id=index + ) + +app = Flask(__name__) + +handler = xmlrpcre.XMLRPCHandler('api') +handler.connect(app, '/kojihub') + +@handler.register +def getLastEvent(*args, **kwargs): + return {'id': 999999, 'ts': time.time()} + + +@handler.register +def listTagged(*args, **kwargs): + builds = [] + for module in MODULES.values(): + builds.append({ + 'build_id': module.build_id, + 'owner_name': 'centos', + 'package_name': module.name, + # 'package_name': 'perl-libwww-perl-devel', + 'task_id': None, + 'state': 1, + 'nvr': f, + # 'nvr': 'perl-libwww-perl-devel-6.34-8030020201223164340.5839bc99', + 'start_time': '2020-12-23 16:43:59', + 'creation_event_id': 309485, + 'creation_time': '2020-12-23 17:05:33.553748', + 'epoch': None, 'tag_id': 533, + 'completion_time': '2020-12-23 17:05:23', + 'tag_name': 'dist-c8-module-compose', + 'version': module.stream, + # 'version': '6.34', + 'volume_id': 0, + # 'release': '8030020201223164340.5839bc99', + 'release': '%s.%s' % (module.version, module.context), + 'package_id': 3221, + 'owner_id': 11, + 'id': module.build_id, + 'volume_name': 'DEFAULT', + # 'name': 'perl-libwww-perl-devel' + 'name': module.name + }) + + return builds + + +@handler.register +def getFullInheritance(*args, **kwargs): + return [] + + +@handler.register +def getBuild(*args, **kwargs): + + build_id = args[0] + module = MODULES[build_id] + + result = { + 'id': build_id, + 'name': module.name, + 'version': module.stream, + 'release': '%s.%s' % (module.version, module.context), + 'completion_ts': 0, + 'state': 'COMPLETE', + 'extra': { + 'typeinfo': { + 'module': { + 'stream': module.stream, + 'version': module.version, + 'name': module.name, + 'context': module.context, + 'content_koji_tag': '-'.join([ + module.name, + module.stream, + module.version + ]) + '.' + module.context + } + } + } + } + return result + + +@handler.register +def listArchives(*args, **kwargs): + module = MODULES[args[0]] + + return [ + { + 'build_id': module.build_id, + 'filename': 'modulemd.x86_64.txt', + 'btype': 'module' + }, + # noone ever uses this file + # but it should be because pungi ignores builds + # with len(files) <= 1 + { + 'build_id': module.build_id, + 'filename': 'modulemd.txt', + 'btype': 'module' + } + ] + + +@handler.register +def listTaggedRPMS(*args, **kwargs): + print('listTaggedRPMS', args, kwargs) + if args[0] == 'dist-c8-compose': + packages = [] + rpms = subprocess.check_output('find /var/repos/ | grep .rpm', shell=True, universal_newlines=True).strip() + all_rpms = rpms.split('\n') + + nvras = set() + for module in MODULES.values(): + path = os.path.join(MODULES_DIR, module.nvr) + info = Modulemd.ModuleStream.read_file(path, strict=True) + + for package in info.get_rpm_artifacts(): + data = parse_nvra(package) + nvras.add((data['name'], data['version'], data['release'], data['arch'])) + + for rpm in all_rpms[:]: + data = parse_nvra(os.path.basename(rpm[:-4])) + if (data['name'], data['version'], data['release'], data['arch']) in nvras: + all_rpms.remove(rpm) + + for rpm in all_rpms: + # print(os.path.basename(rpm)) + info = parse_nvra(os.path.basename(rpm)) + packages.append({ + "build_id": 15270, + "name": info['name'], + "extra": None, + "arch": info['arch'], + "epoch": info['epoch'] or None, + "version": info['version'], + "metadata_only": False, + "release": info['release'], + "id": 262555, + "size": 0 + }) + builds = [ + { + "build_id": 15270, + "owner_name": "mbox-mbs-backend", + "package_name": "389-ds-base", + "task_id": 195937, + "state": 1, + "nvr": "389-ds-base-1.4.3.8-6.module_el8.3.0+604+ab7bf9cc", + "start_time": "2020-12-22 19:20:12.504578", + "creation_event_id": 306731, + "creation_time": "2020-12-22 19:20:12.504578", + "epoch": None, + "tag_id": 1192, + "completion_time": "2020-12-22 19:34:34.716615", + "tag_name": "module-389-ds-1.4-8030020201222185615-618f7055", + "version": "1.4.3.8", + "volume_id": 0, + "release": "6.module_el8.3.0+604+ab7bf9cc", + "package_id": 104, + "owner_id": 6, + "id": 15270, + "volume_name": "DEFAULT", + "name": "389-ds-base" + } + ] + else: + module = args[0] + path = os.path.join('/root/pungi-centos/modules', module) + print('moduleinfo', path, os.path.exists(path)) + + for module in MODULES.values(): + if module.nvr == args[0]: + builds = [ + { + "build_id": module.build_id, + "owner_name": "mbox-mbs-backend", + "package_name": module.name, + "task_id": 195937, + "state": 1, + "nvr": module.nvr, + "start_time": "2020-12-22 19:20:12.504578", + "creation_event_id": 306731, + "creation_time": "2020-12-22 19:20:12.504578", + "epoch": None, + "tag_id": 1192, + "completion_time": "2020-12-22 19:34:34.716615", + "tag_name": module.nvr, + "version": module.stream, + "volume_id": 0, + "release": module.version, + "package_id": 104, + "owner_id": 6, + "id": module.build_id, + "volume_name": "DEFAULT", + "name": module.name + } + ] + break + + packages = [] + if os.path.exists(path): + info = Modulemd.ModuleStream.read_file(path, strict=True) + print(info.get_rpm_artifacts()) + for art in info.get_rpm_artifacts(): + data = parse_nvra(art) + packages.append({ + "build_id": module.build_id, + "name": data['name'], + "extra": None, + "arch": data['arch'], + "epoch": data['epoch'] or None, + "version": data['version'], + "metadata_only": False, + "release": data['release'], + "id": 262555, + "size": 0 + }) + else: + raise Exception + + + return [ + packages, + builds + ] + + +class RegexConverter(BaseConverter): + def __init__(self, url_map, *items): + super(RegexConverter, self).__init__(url_map) + self.regex = items[0] + + +app.url_map.converters['regex'] = RegexConverter + +@app.route('/pkgs/packages////files/module/') +def example(*args, name, rhel, stream, **kwargs): + print(args, name, rhel, kwargs) + path = os.path.join('/root/pungi-centos/modules', '%s-%s-%s' % (name, rhel, stream)) + print('moduleinfo', path, os.path.exists(path)) + if os.path.exists(path): + return open(path).read() + raise NotImplementedError + +if __name__ == "__main__": + app.run(port=8080, host='0.0.0.0') \ No newline at end of file diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index e82a47cf..daaf0257 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -22,6 +22,8 @@ It automatically finds a signed copies according to *sigkey_ordering*. import itertools import json import os + +import subprocess from six.moves import cPickle as pickle import kobo.log @@ -502,40 +504,12 @@ class KojiPackageSet(PackageSetBase): return rpm_info["path_from_task"] pathinfo = self.koji_wrapper.koji_module.pathinfo - paths = [] - for sigkey in self.sigkey_ordering: - if not sigkey: - # we're looking for *signed* copies here - continue - sigkey = sigkey.lower() - rpm_path = os.path.join( - pathinfo.build(build_info), pathinfo.signed(rpm_info, sigkey) - ) - paths.append(rpm_path) - if os.path.isfile(rpm_path): - return rpm_path - - if None in self.sigkey_ordering or "" in self.sigkey_ordering: - # use an unsigned copy (if allowed) - rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info)) - paths.append(rpm_path) - if os.path.isfile(rpm_path): - return rpm_path - - if self._allow_invalid_sigkeys and rpm_info["name"] not in self.packages: - # use an unsigned copy (if allowed) - rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info)) - paths.append(rpm_path) - if os.path.isfile(rpm_path): - self._invalid_sigkey_rpms.append(rpm_info) - return rpm_path - - self._invalid_sigkey_rpms.append(rpm_info) - self.log_error( - "RPM %s not found for sigs: %s. Paths checked: %s" - % (rpm_info, self.sigkey_ordering, paths) - ) - return None + rpm_path = os.path.join(pathinfo.topdir, pathinfo.rpm(rpm_info)) + if os.path.isfile(rpm_path): + return rpm_path + else: + self.log_warning("RPM %s not found" % rpm_path) + return None def populate(self, tag, event=None, inherit=True, include_packages=None): """Populate the package set with packages from given tag. diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index a557f604..5da3aeb1 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -21,6 +21,7 @@ import functools from fnmatch import fnmatch from itertools import groupby +import requests from kobo.rpmlib import parse_nvra from kobo.shortcuts import force_list @@ -29,7 +30,7 @@ from pungi.wrappers.comps import CompsWrapper from pungi.wrappers.mbs import MBSWrapper import pungi.phases.pkgset.pkgsets from pungi.arch import getBaseArch -from pungi.util import retry, get_arch_variant_data, get_variant_data +from pungi.util import retry, get_arch_variant_data, get_variant_data, translate_path from pungi.module_util import Modulemd from pungi.phases.pkgset.common import MaterializedPackageSet, get_all_arches @@ -252,6 +253,16 @@ def _add_module_to_variant( continue try: + print(mmds["modulemd.%s.txt" % arch]) + + url = translate_path(compose, mmds["modulemd.%s.txt" % arch]) + print(url) + r = requests.get(url) + r.raise_for_status() + os.makedirs(os.path.dirname(mmds["modulemd.%s.txt" % arch]), exist_ok=True) + with open(mmds["modulemd.%s.txt" % arch], 'wb') as f: + f.write(r.content) + mmd = Modulemd.ModuleStream.read_file( mmds["modulemd.%s.txt" % arch], strict=True )