From fa4640f03eadba8483da35cab3376b3f12baeee6 Mon Sep 17 00:00:00 2001 From: soksanichenko Date: Tue, 14 Mar 2023 18:25:21 +0200 Subject: [PATCH] ALBS-1040: Investigate why Pungi doesn't put modules packages into the final repos - Refactoring - KojiMock extracts all modules which are suitable for the variant's arches --- pungi/phases/gather/__init__.py | 4 +- pungi/phases/pkgset/pkgsets.py | 426 ++---------------- .../phases/pkgset/sources/source_kojimock.py | 12 +- pungi/scripts/create_packages_json.py | 12 + pungi/wrappers/kojimock.py | 80 ++-- pungi/wrappers/kojiwrapper.py | 6 +- tests/test_koji_local_source.py | 6 +- 7 files changed, 108 insertions(+), 438 deletions(-) diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 32e85eef..31986b3e 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -23,6 +23,7 @@ import threading from kobo.rpmlib import parse_nvra from kobo.shortcuts import run from productmd.rpms import Rpms +from pungi.phases.pkgset.common import get_all_arches from six.moves import cPickle as pickle try: @@ -652,7 +653,8 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None): ).koji_module.config.topdir.rstrip("/") + "/", "kojimock": lambda: pungi.wrappers.kojiwrapper.KojiMockWrapper( - compose + compose, + get_all_arches(compose), ).koji_module.config.topdir.rstrip("/") + "/", } diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index 3decf961..a556f139 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -480,7 +480,8 @@ class KojiPackageSet(PackageSetBase): response = None if self.cache_region: - cache_key = "KojiPackageSet.get_latest_rpms_%s_%s_%s" % ( + cache_key = "%s.get_latest_rpms_%s_%s_%s" % ( + str(self.__class__.__name__), str(tag), str(event), str(inherit), @@ -816,22 +817,22 @@ class KojiPackageSet(PackageSetBase): return False -class KojiMockPackageSet(PackageSetBase): +class KojiMockPackageSet(KojiPackageSet): def __init__( - self, - name, - koji_wrapper, - sigkey_ordering, - arches=None, - logger=None, - packages=None, - allow_invalid_sigkeys=False, - populate_only_packages=False, - cache_region=None, - extra_builds=None, - extra_tasks=None, - signed_packages_retries=0, - signed_packages_wait=30, + self, + name, + koji_wrapper, + sigkey_ordering, + arches=None, + logger=None, + packages=None, + allow_invalid_sigkeys=False, + populate_only_packages=False, + cache_region=None, + extra_builds=None, + extra_tasks=None, + signed_packages_retries=0, + signed_packages_wait=30, ): """ Creates new KojiPackageSet. @@ -865,135 +866,21 @@ class KojiMockPackageSet(PackageSetBase): and include in the package set. Useful when building testing compose with RPM scratch builds. """ - super(KojiMockPackageSet , self).__init__( + super(KojiMockPackageSet, self).__init__( name, + koji_wrapper=koji_wrapper, sigkey_ordering=sigkey_ordering, arches=arches, logger=logger, + packages=packages, allow_invalid_sigkeys=allow_invalid_sigkeys, + populate_only_packages=populate_only_packages, + cache_region=cache_region, + extra_builds=extra_builds, + extra_tasks=extra_tasks, + signed_packages_retries=signed_packages_retries, + signed_packages_wait=signed_packages_wait, ) - self.koji_wrapper = koji_wrapper - # Names of packages to look for in the Koji tag. - self.packages = set(packages or []) - self.populate_only_packages = populate_only_packages - self.cache_region = cache_region - self.extra_builds = extra_builds or [] - self.extra_tasks = extra_tasks or [] - self.reuse = None - self.signed_packages_retries = signed_packages_retries - self.signed_packages_wait = signed_packages_wait - - def __getstate__(self): - result = self.__dict__.copy() - del result["koji_wrapper"] - del result["_logger"] - if "cache_region" in result: - del result["cache_region"] - return result - - def __setstate__(self, data): - self._logger = None - self.__dict__.update(data) - - @property - def koji_proxy(self): - return self.koji_wrapper.koji_proxy - - def get_extra_rpms(self): - if not self.extra_builds: - return [], [] - - rpms = [] - builds = [] - - builds = self.koji_wrapper.retrying_multicall_map( - self.koji_proxy, self.koji_proxy.getBuild, list_of_args=self.extra_builds - ) - rpms_in_builds = self.koji_wrapper.retrying_multicall_map( - self.koji_proxy, - self.koji_proxy.listBuildRPMs, - list_of_args=self.extra_builds, - ) - - rpms = [] - for rpms_in_build in rpms_in_builds: - rpms += rpms_in_build - return rpms, builds - - def get_extra_rpms_from_tasks(self): - """ - Returns manually constructed RPM infos from the Koji tasks defined - in `self.extra_tasks`. - - :rtype: list - :return: List with RPM infos defined as dicts with following keys: - - name, version, release, arch, src - as returned by parse_nvra. - - path_from_task - Full path to RPM on /mnt/koji. - - build_id - Always set to None. - """ - if not self.extra_tasks: - return [] - - # Get the IDs of children tasks - these are the tasks containing - # the resulting RPMs. - children_tasks = self.koji_wrapper.retrying_multicall_map( - self.koji_proxy, - self.koji_proxy.getTaskChildren, - list_of_args=self.extra_tasks, - ) - children_task_ids = [] - for tasks in children_tasks: - children_task_ids += [t["id"] for t in tasks] - - # Get the results of these children tasks. - results = self.koji_wrapper.retrying_multicall_map( - self.koji_proxy, - self.koji_proxy.getTaskResult, - list_of_args=children_task_ids, - ) - rpms = [] - for result in results: - rpms += result.get("rpms", []) - rpms += result.get("srpms", []) - - rpm_infos = [] - for rpm in rpms: - rpm_info = kobo.rpmlib.parse_nvra(os.path.basename(rpm)) - rpm_info["path_from_task"] = os.path.join( - self.koji_wrapper.koji_module.pathinfo.work(), rpm - ) - rpm_info["build_id"] = None - rpm_infos.append(rpm_info) - - return rpm_infos - - def get_latest_rpms(self, tag, event, inherit=True): - if not tag: - return [], [] - - response = None - if self.cache_region: - cache_key = "KojiPackageSet.get_latest_rpms_%s_%s_%s" % ( - str(tag), - str(event), - str(inherit), - ) - try: - response = self.cache_region.get(cache_key) - except Exception: - pass - - if not response: - response = self.koji_proxy.listTaggedRPMS( - tag, event=event, inherit=inherit, latest=True - ) - if self.cache_region: - try: - self.cache_region.set(cache_key, response) - except Exception: - pass - - return response def _is_rpm_signed(self, rpm_path) -> bool: ts = rpm.TransactionSet() @@ -1016,8 +903,8 @@ class KojiMockPackageSet(PackageSetBase): def get_package_path(self, queue_item): rpm_info, build_info = queue_item - # Check if this RPM is coming from scratch task. In this case, we already - # know the path. + # Check if this RPM is coming from scratch task. + # In this case, we already know the path. if "path_from_task" in rpm_info: return rpm_info["path_from_task"] @@ -1043,261 +930,14 @@ class KojiMockPackageSet(PackageSetBase): return None def populate(self, tag, event=None, inherit=True, include_packages=None): - """Populate the package set with packages from given tag. - - :param event: the Koji event to query at (or latest if not given) - :param inherit: whether to enable tag inheritance - :param include_packages: an iterable of tuples (package name, arch) that should - be included, all others are skipped. - """ - result_rpms = [] - result_srpms = [] - include_packages = set(include_packages or []) - - if type(event) is dict: - event = event["id"] - - msg = "Getting latest RPMs (tag: %s, event: %s, inherit: %s)" % ( - tag, - event, - inherit, + result = super().populate( + tag=tag, + event=event, + inherit=inherit, + include_packages=include_packages, ) - self.log_info("[BEGIN] %s" % msg) - rpms, builds = self.get_latest_rpms(tag, event, inherit=inherit) - extra_rpms, extra_builds = self.get_extra_rpms() - rpms += extra_rpms - builds += extra_builds - - extra_builds_by_name = {} - for build_info in extra_builds: - extra_builds_by_name[build_info["name"]] = build_info["build_id"] - - builds_by_id = {} - exclude_build_id = [] - for build_info in builds: - build_id, build_name = build_info["build_id"], build_info["name"] - if ( - build_name in extra_builds_by_name - and build_id != extra_builds_by_name[build_name] - ): - exclude_build_id.append(build_id) - else: - builds_by_id.setdefault(build_id, build_info) - - # Get extra RPMs from tasks. - rpms += self.get_extra_rpms_from_tasks() - - skipped_arches = [] - skipped_packages_count = 0 - # We need to process binary packages first, and then source packages. - # If we have a list of packages to use, we need to put all source rpms - # names into it. Otherwise if the SRPM name does not occur on the list, - # it would be missing from the package set. Even if it ultimately does - # not end in the compose, we need it to extract ExcludeArch and - # ExclusiveArch for noarch packages. - for rpm_info in itertools.chain( - (rpm for rpm in rpms if not _is_src(rpm)), - (rpm for rpm in rpms if _is_src(rpm)), - ): - if rpm_info["build_id"] in exclude_build_id: - continue - - if self.arches and rpm_info["arch"] not in self.arches: - if rpm_info["arch"] not in skipped_arches: - self.log_debug("Skipping packages for arch: %s" % rpm_info["arch"]) - skipped_arches.append(rpm_info["arch"]) - continue - - if ( - include_packages - and (rpm_info["name"], rpm_info["arch"]) not in include_packages - and rpm_info["arch"] != "src" - ): - self.log_debug( - "Skipping %(name)s-%(version)s-%(release)s.%(arch)s" % rpm_info - ) - continue - - if ( - self.populate_only_packages - and self.packages - and rpm_info["name"] not in self.packages - ): - skipped_packages_count += 1 - continue - - build_info = builds_by_id.get(rpm_info["build_id"], None) - if _is_src(rpm_info): - result_srpms.append((rpm_info, build_info)) - else: - result_rpms.append((rpm_info, build_info)) - if self.populate_only_packages and self.packages: - # Only add the package if we already have some whitelist. - if build_info: - self.packages.add(build_info["name"]) - else: - # We have no build info and therefore no Koji package name, - # we can only guess that the Koji package name would be the same - # one as the RPM name. - self.packages.add(rpm_info["name"]) - - if skipped_packages_count: - self.log_debug( - "Skipped %d packages, not marked as to be " - "included in a compose." % skipped_packages_count - ) - - result = self.read_packages(result_rpms, result_srpms) - - # Check that after reading the packages, every package that is - # included in a compose has the right sigkey. - if self._invalid_sigkey_rpms: - invalid_sigkey_rpms = [ - rpm for rpm in self._invalid_sigkey_rpms if rpm["name"] in self.packages - ] - if invalid_sigkey_rpms: - self.raise_invalid_sigkeys_exception(invalid_sigkey_rpms) - - self.log_info("[DONE ] %s" % msg) return result - def write_reuse_file(self, compose, include_packages): - """Write data to files for reusing in future. - - :param compose: compose object - :param include_packages: an iterable of tuples (package name, arch) that should - be included. - """ - reuse_file = compose.paths.work.pkgset_reuse_file(self.name) - self.log_info("Writing pkgset reuse file: %s" % reuse_file) - try: - with open(reuse_file, "wb") as f: - pickle.dump( - { - "name": self.name, - "allow_invalid_sigkeys": self._allow_invalid_sigkeys, - "arches": self.arches, - "sigkeys": self.sigkey_ordering, - "packages": self.packages, - "populate_only_packages": self.populate_only_packages, - "rpms_by_arch": self.rpms_by_arch, - "srpms_by_name": self.srpms_by_name, - "extra_builds": self.extra_builds, - "include_packages": include_packages, - }, - f, - protocol=pickle.HIGHEST_PROTOCOL, - ) - except Exception as e: - self.log_warning("Writing pkgset reuse file failed: %s" % str(e)) - - def _get_koji_event_from_file(self, event_file): - with open(event_file, "r") as f: - return json.load(f)["id"] - - def try_to_reuse(self, compose, tag, inherit=True, include_packages=None): - """Try to reuse pkgset data of old compose. - :param compose: compose object - :param str tag: koji tag name - :param inherit: whether to enable tag inheritance - :param include_packages: an iterable of tuples (package name, arch) that should - be included. - """ - if not compose.conf["pkgset_allow_reuse"]: - self.log_info("Reusing pkgset data from old compose is disabled.") - return False - - self.log_info("Trying to reuse pkgset data of old compose") - if not compose.paths.get_old_compose_topdir(): - self.log_debug("No old compose found. Nothing to reuse.") - return False - - event_file = os.path.join( - compose.paths.work.topdir(arch="global", create_dir=False), "koji-event" - ) - old_event_file = compose.paths.old_compose_path(event_file) - - try: - koji_event = self._get_koji_event_from_file(event_file) - old_koji_event = self._get_koji_event_from_file(old_event_file) - except Exception as e: - self.log_debug("Can't read koji event from file: %s" % str(e)) - return False - - if koji_event != old_koji_event: - self.log_debug( - "Koji event doesn't match, querying changes between event %d and %d" - % (old_koji_event, koji_event) - ) - changed = self.koji_proxy.queryHistory( - tables=["tag_listing", "tag_inheritance"], - tag=tag, - afterEvent=min(koji_event, old_koji_event), - beforeEvent=max(koji_event, old_koji_event) + 1, - ) - if changed["tag_listing"]: - self.log_debug("Builds under tag %s changed. Can't reuse." % tag) - return False - if changed["tag_inheritance"]: - self.log_debug("Tag inheritance %s changed. Can't reuse." % tag) - return False - - if inherit: - inherit_tags = self.koji_proxy.getFullInheritance(tag, koji_event) - for t in inherit_tags: - changed = self.koji_proxy.queryHistory( - tables=["tag_listing", "tag_inheritance"], - tag=t["name"], - afterEvent=min(koji_event, old_koji_event), - beforeEvent=max(koji_event, old_koji_event) + 1, - ) - if changed["tag_listing"]: - self.log_debug( - "Builds under inherited tag %s changed. Can't reuse." - % t["name"] - ) - return False - if changed["tag_inheritance"]: - self.log_debug("Tag inheritance %s changed. Can't reuse." % tag) - return False - - repo_dir = compose.paths.work.pkgset_repo(tag, create_dir=False) - old_repo_dir = compose.paths.old_compose_path(repo_dir) - if not old_repo_dir: - self.log_debug("Can't find old repo dir to reuse.") - return False - - old_reuse_file = compose.paths.old_compose_path( - compose.paths.work.pkgset_reuse_file(tag) - ) - - try: - self.log_debug("Loading reuse file: %s" % old_reuse_file) - reuse_data = self.load_old_file_cache(old_reuse_file) - except Exception as e: - self.log_debug("Failed to load reuse file: %s" % str(e)) - return False - - if ( - reuse_data["allow_invalid_sigkeys"] == self._allow_invalid_sigkeys - and reuse_data["packages"] == self.packages - and reuse_data["populate_only_packages"] == self.populate_only_packages - and reuse_data["extra_builds"] == self.extra_builds - and reuse_data["sigkeys"] == self.sigkey_ordering - and reuse_data["include_packages"] == include_packages - ): - self.log_info("Copying repo data for reuse: %s" % old_repo_dir) - copy_all(old_repo_dir, repo_dir) - self.reuse = old_repo_dir - self.rpms_by_arch = reuse_data["rpms_by_arch"] - self.srpms_by_name = reuse_data["srpms_by_name"] - if self.old_file_cache: - self.file_cache = self.old_file_cache - return True - else: - self.log_info("Criteria does not match. Nothing to reuse.") - return False - def _is_src(rpm_info): """Check if rpm info object returned by Koji refers to source packages.""" diff --git a/pungi/phases/pkgset/sources/source_kojimock.py b/pungi/phases/pkgset/sources/source_kojimock.py index 8c2e3701..aa063a18 100644 --- a/pungi/phases/pkgset/sources/source_kojimock.py +++ b/pungi/phases/pkgset/sources/source_kojimock.py @@ -200,7 +200,10 @@ class PkgsetSourceKojiMock(pungi.phases.pkgset.source.PkgsetSourceBase): def __call__(self): compose = self.compose - self.koji_wrapper = pungi.wrappers.kojiwrapper.KojiMockWrapper(compose) + self.koji_wrapper = pungi.wrappers.kojiwrapper.KojiMockWrapper( + compose, + get_all_arches(compose), + ) # path prefix must contain trailing '/' path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip("/") + "/" package_sets = get_pkgset_from_koji( @@ -622,7 +625,6 @@ def _get_modules_from_koji_tags( module_builds = filter_by_whitelist( compose, module_builds, variant_modules, expected_modules ) - # Find the latest builds of all modules. This does following: # - Sorts the module_builds descending by Koji NVR (which maps to NSV # for modules). Split release into modular version and context, and @@ -643,7 +645,11 @@ def _get_modules_from_koji_tags( latest_builds = [] module_builds = sorted(module_builds, key=_key, reverse=True) for ns, ns_builds in groupby( - module_builds, key=lambda x: ":".join([x["name"], x["version"]]) + module_builds, key=lambda x: ":".join([ + x["name"], + x["version"], + x['arch'], + ]) ): for nsv, nsv_builds in groupby( ns_builds, key=lambda x: x["release"].split(".")[0] diff --git a/pungi/scripts/create_packages_json.py b/pungi/scripts/create_packages_json.py index 012cf727..f54b7a02 100644 --- a/pungi/scripts/create_packages_json.py +++ b/pungi/scripts/create_packages_json.py @@ -341,6 +341,18 @@ class PackagesGenerator: ) else: src_package_name = src_package_name[0].name + # TODO: for x86_64 + i686 in one packages.json + # don't remove! + # if package.arch in self.addon_repos[variant_arch]: + # arches = self.addon_repos[variant_arch] + [variant_arch] + # else: + # arches = [variant_arch] + # for arch in arches: + # pkgs_list = packages_json[variant_name][ + # arch][src_package_name] + # added_pkg = f'{package_name}.{package_arch}' + # if added_pkg not in pkgs_list: + # pkgs_list.append(added_pkg) pkgs_list = packages_json[variant_name][ variant_arch][src_package_name] added_pkg = f'{package_name}.{package_arch}' diff --git a/pungi/wrappers/kojimock.py b/pungi/wrappers/kojimock.py index af3ff48b..1267438b 100644 --- a/pungi/wrappers/kojimock.py +++ b/pungi/wrappers/kojimock.py @@ -43,10 +43,11 @@ class KojiMock: Class that acts like real koji (for some needed methods) but uses local storage as data source """ - def __init__(self, packages_dir, modules_dir): + def __init__(self, packages_dir, modules_dir, all_arches): self._modules = self._gather_modules(modules_dir) self._modules_dir = modules_dir self._packages_dir = packages_dir + self._all_arches = all_arches @staticmethod def _gather_modules(modules_dir): @@ -93,6 +94,7 @@ class KojiMock: 'name': module.name, 'id': module.build_id, 'tag_name': tag_name, + 'arch': module.arch, # Following fields are currently not # used but returned by real koji # left them here just for reference @@ -246,15 +248,19 @@ class KojiMock: """ Get list of builds for module and given module tag name. """ - module = self._get_module_by_name(tag_name) - path = os.path.join( - self._modules_dir, - module.arch, - tag_name, - ) + builds = [] + packages = [] + modules = self._get_modules_by_name(tag_name) + for module in modules: + if module is None: + raise ValueError('Module %s is not found' % tag_name) + path = os.path.join( + self._modules_dir, + module.arch, + tag_name, + ) - builds = [ - { + builds.append({ "build_id": module.build_id, "package_name": module.name, "nvr": module.nvr, @@ -280,35 +286,33 @@ class KojiMock: # "volume_id": 0, # "package_id": 104, # "owner_id": 6, - } - ] - if module is None: - raise ValueError('Module %s is not found' % tag_name) + }) - packages = [] - if os.path.exists(path): - info = Modulemd.ModuleStream.read_string(open(path).read(), strict=True) - for art in info.get_rpm_artifacts(): - data = parse_nvra(art) - packages.append({ - "build_id": module.build_id, - "name": data['name'], - "extra": None, - "arch": data['arch'], - "epoch": data['epoch'] or None, - "version": data['version'], - "metadata_only": False, - "release": data['release'], - "id": 262555, - "size": 0 - }) - else: - raise RuntimeError('Unable to find module %s' % path) + if os.path.exists(path): + info = Modulemd.ModuleStream.read_string(open(path).read(), strict=True) + for art in info.get_rpm_artifacts(): + data = parse_nvra(art) + packages.append({ + "build_id": module.build_id, + "name": data['name'], + "extra": None, + "arch": data['arch'], + "epoch": data['epoch'] or None, + "version": data['version'], + "metadata_only": False, + "release": data['release'], + "id": 262555, + "size": 0 + }) + else: + raise RuntimeError('Unable to find module %s' % path) return builds, packages - def _get_module_by_name(self, tag_name): - for module in self._modules.values(): - if module.nvr != tag_name: - continue - return module - return None + def _get_modules_by_name(self, tag_name): + modules = [] + for arch in self._all_arches: + for module in self._modules.values(): + if module.nvr != tag_name or module.arch != arch: + continue + modules.append(module) + return modules diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py index 4348f884..34cdd156 100644 --- a/pungi/wrappers/kojiwrapper.py +++ b/pungi/wrappers/kojiwrapper.py @@ -868,7 +868,8 @@ class KojiWrapper(object): class KojiMockWrapper(object): lock = threading.Lock() - def __init__(self, compose): + def __init__(self, compose, all_arches): + self.all_arches = all_arches self.compose = compose try: self.profile = self.compose.conf["koji_profile"] @@ -898,7 +899,8 @@ class KojiMockWrapper(object): modules_dir=os.path.join( self.koji_module.config.topdir, 'modules', - ) + ), + all_arches=self.all_arches, ) diff --git a/tests/test_koji_local_source.py b/tests/test_koji_local_source.py index 0c64032a..5c5dd2b0 100644 --- a/tests/test_koji_local_source.py +++ b/tests/test_koji_local_source.py @@ -117,7 +117,11 @@ version: '1' os.makedirs(os.path.join(PATH_TO_REPOS, os.path.dirname(filepath)), exist_ok=True) open(os.path.join(PATH_TO_REPOS, filepath), 'w').close() - self._koji = KojiMock(PATH_TO_REPOS, os.path.join(PATH_TO_REPOS, 'modules')) + self._koji = KojiMock( + PATH_TO_REPOS, + os.path.join(PATH_TO_REPOS, 'modules'), + ['x86_64', 'noarch', 'i686'], + ) @ddt.data( [0, {