import os import time from pathlib import Path from attr import dataclass from kobo.rpmlib import parse_nvra from pungi.module_util import Modulemd # just a random value which we don't # use in mock currently # originally builds are filtered by this value # to get consistent snapshot of tags and packages from pungi.scripts.gather_rpms import search_rpms LAST_EVENT_ID = 999999 # last event time is not important but build # time should be less then it LAST_EVENT_TIME = time.time() BUILD_TIME = 0 # virtual build that collects all # packages built for some arch RELEASE_BUILD_ID = 15270 # tag that should have all packages available ALL_PACKAGES_TAG = 'dist-c8-compose' # tag that should have all modules available ALL_MODULES_TAG = 'dist-c8-module-compose' @dataclass class Module: build_id: int name: str nvr: str stream: str version: str context: str arch: str class KojiMock: """ Class that acts like real koji (for some needed methods) but uses local storage as data source """ def __init__(self, packages_dir, modules_dir, all_arches): self._modules = self._gather_modules(modules_dir) self._modules_dir = modules_dir self._packages_dir = packages_dir self._all_arches = all_arches @staticmethod def _gather_modules(modules_dir): modules = {} for index, (f, arch) in enumerate( (sub_path.name, sub_path.parent.name) for path in Path(modules_dir).glob('*') for sub_path in path.iterdir() ): parsed = parse_nvra(f) modules[index] = Module( name=parsed['name'], nvr=f, version=parsed['release'], context=parsed['arch'], stream=parsed['version'], build_id=index, arch=arch, ) return modules @staticmethod def getLastEvent(*args, **kwargs): return {'id': LAST_EVENT_ID, 'ts': LAST_EVENT_TIME} def listTagged(self, tag_name, *args, **kwargs): """ Returns list of virtual 'builds' that contain packages by given tag There are two kinds of tags: modular and distributive. For now, only one kind, distributive one, is needed. """ if tag_name != ALL_MODULES_TAG: raise ValueError("I don't know what tag is %s" % tag_name) builds = [] for module in self._modules.values(): builds.append({ 'build_id': module.build_id, 'owner_name': 'centos', 'package_name': module.name, 'nvr': module.nvr, 'version': module.stream, 'release': '%s.%s' % (module.version, module.context), 'name': module.name, 'id': module.build_id, 'tag_name': tag_name, 'arch': module.arch, # Following fields are currently not # used but returned by real koji # left them here just for reference # # 'task_id': None, # 'state': 1, # 'start_time': '2020-12-23 16:43:59', # 'creation_event_id': 309485, # 'creation_time': '2020-12-23 17:05:33.553748', # 'epoch': None, 'tag_id': 533, # 'completion_time': '2020-12-23 17:05:23', # 'volume_id': 0, # 'package_id': 3221, # 'owner_id': 11, # 'volume_name': 'DEFAULT', }) return builds @staticmethod def getFullInheritance(*args, **kwargs): """ Unneeded because we use local storage. """ return [] def getBuild(self, build_id, *args, **kwargs): """ Used to get information about build (used in pungi only for modules currently) """ module = self._modules[build_id] result = { 'id': build_id, 'name': module.name, 'version': module.stream, 'release': '%s.%s' % (module.version, module.context), 'completion_ts': BUILD_TIME, 'state': 'COMPLETE', 'arch': module.arch, 'extra': { 'typeinfo': { 'module': { 'stream': module.stream, 'version': module.version, 'name': module.name, 'context': module.context, 'content_koji_tag': '-'.join([ module.name, module.stream, module.version ]) + '.' + module.context } } } } return result def listArchives(self, build_id, *args, **kwargs): """ Originally lists artifacts for build, but in pungi used only to get list of modulemd files for some module """ module = self._modules[build_id] return [ { 'build_id': module.build_id, 'filename': f'modulemd.{module.arch}.txt', 'btype': 'module' }, # noone ever uses this file # but it should be because pungi ignores builds # with len(files) <= 1 { 'build_id': module.build_id, 'filename': 'modulemd.txt', 'btype': 'module' } ] def listTaggedRPMS(self, tag_name, *args, **kwargs): """ Get information about packages that are tagged by tag. There are two kings of tags: per-module and per-distr. """ if tag_name == ALL_PACKAGES_TAG: builds, packages = self._get_release_packages() else: builds, packages = self._get_module_packages(tag_name) return [ packages, builds ] def _get_release_packages(self): """ Search packages dir and keep only packages that are non-modular. This is quite the way how real koji works: - modular packages are tagged by module-* tag - all other packages are tagged with dist* tag """ packages = [] # get all rpms in folder rpms = search_rpms(self._packages_dir) all_rpms = [package.path for package in rpms] # get nvras for modular packages nvras = set() for module in self._modules.values(): path = os.path.join( self._modules_dir, module.arch, module.nvr, ) info = Modulemd.ModuleStream.read_string(open(path).read(), strict=True) for package in info.get_rpm_artifacts(): data = parse_nvra(package) nvras.add((data['name'], data['version'], data['release'], data['arch'])) # and remove modular packages from global list for rpm in all_rpms[:]: data = parse_nvra(os.path.basename(rpm[:-4])) if (data['name'], data['version'], data['release'], data['arch']) in nvras: all_rpms.remove(rpm) for rpm in all_rpms: info = parse_nvra(os.path.basename(rpm)) packages.append({ "build_id": RELEASE_BUILD_ID, "name": info['name'], "extra": None, "arch": info['arch'], "epoch": info['epoch'] or None, "version": info['version'], "metadata_only": False, "release": info['release'], # not used currently # "id": 262555, # "size": 0 }) builds = [] return builds, packages def _get_module_packages(self, tag_name): """ Get list of builds for module and given module tag name. """ builds = [] packages = [] modules = self._get_modules_by_name(tag_name) for module in modules: if module is None: raise ValueError('Module %s is not found' % tag_name) path = os.path.join( self._modules_dir, module.arch, tag_name, ) builds.append({ "build_id": module.build_id, "package_name": module.name, "nvr": module.nvr, "tag_name": module.nvr, "version": module.stream, "release": module.version, "id": module.build_id, "name": module.name, "volume_name": "DEFAULT", # Following fields are currently not # used but returned by real koji # left them here just for reference # # "owner_name": "mbox-mbs-backend", # "task_id": 195937, # "state": 1, # "start_time": "2020-12-22 19:20:12.504578", # "creation_event_id": 306731, # "creation_time": "2020-12-22 19:20:12.504578", # "epoch": None, # "tag_id": 1192, # "completion_time": "2020-12-22 19:34:34.716615", # "volume_id": 0, # "package_id": 104, # "owner_id": 6, }) if os.path.exists(path): info = Modulemd.ModuleStream.read_string(open(path).read(), strict=True) for art in info.get_rpm_artifacts(): data = parse_nvra(art) packages.append({ "build_id": module.build_id, "name": data['name'], "extra": None, "arch": data['arch'], "epoch": data['epoch'] or None, "version": data['version'], "metadata_only": False, "release": data['release'], "id": 262555, "size": 0 }) else: raise RuntimeError('Unable to find module %s' % path) return builds, packages def _get_modules_by_name(self, tag_name): modules = [] for arch in self._all_arches: for module in self._modules.values(): if module.nvr != tag_name or module.arch != arch: continue modules.append(module) return modules