# -*- coding: utf-8 -*- # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, see . import collections import os import shutil from kobo.shortcuts import run from kobo.threads import run_in_threads from pungi.phases.base import PhaseBase from pungi.phases.gather import write_prepopulate_file from pungi.util import temp_dir from pungi.module_util import iter_module_defaults from pungi.wrappers.comps import CompsWrapper from pungi.wrappers.createrepo import CreaterepoWrapper from pungi.wrappers.scm import get_dir_from_scm, get_file_from_scm class InitPhase(PhaseBase): """INIT is a mandatory phase""" name = "init" def skip(self): # INIT must never be skipped, # because it generates data for LIVEIMAGES return False def run(self): if self.compose.has_comps: # write global comps and arch comps, create comps repos global_comps = write_global_comps(self.compose) validate_comps(global_comps) num_workers = self.compose.conf["createrepo_num_threads"] run_in_threads( _arch_worker, [(self.compose, arch) for arch in self.compose.get_arches()], threads=num_workers, ) # write variant comps run_in_threads( _variant_worker, [ (self.compose, arch, variant) for variant in self.compose.get_variants() for arch in variant.arches ], threads=num_workers, ) # download variants.xml / product.xml? # download module defaults if self.compose.has_module_defaults: write_module_defaults(self.compose) validate_module_defaults( self.compose.paths.work.module_defaults_dir(create_dir=False) ) # write prepopulate file write_prepopulate_file(self.compose) def _arch_worker(_, args, num): compose, arch = args write_arch_comps(compose, arch) create_comps_repo(compose, arch, None) def _variant_worker(_, args, num): compose, arch, variant = args write_variant_comps(compose, arch, variant) create_comps_repo(compose, arch, variant) def write_global_comps(compose): comps_file_global = compose.paths.work.comps(arch="global") scm_dict = compose.conf["comps_file"] if isinstance(scm_dict, dict): comps_name = os.path.basename(scm_dict["file"]) if scm_dict["scm"] == "file": scm_dict["file"] = os.path.join(compose.config_dir, scm_dict["file"]) else: comps_name = os.path.basename(scm_dict) scm_dict = os.path.join(compose.config_dir, scm_dict) compose.log_debug("Writing global comps file: %s", comps_file_global) tmp_dir = compose.mkdtemp(prefix="comps_") get_file_from_scm(scm_dict, tmp_dir, compose=compose) shutil.copy2(os.path.join(tmp_dir, comps_name), comps_file_global) shutil.rmtree(tmp_dir) return comps_file_global def write_arch_comps(compose, arch): comps_file_arch = compose.paths.work.comps(arch=arch) compose.log_debug("Writing comps file for arch '%s': %s", arch, comps_file_arch) run( [ "comps_filter", "--arch=%s" % arch, "--no-cleanup", "--output=%s" % comps_file_arch, compose.paths.work.comps(arch="global"), ] ) UNMATCHED_GROUP_MSG = "Variant %s.%s requires comps group %s which does not match anything in input comps file" # noqa: E501 def get_lookaside_groups(compose, variant): """Find all groups listed in parent variant.""" groups = set() if variant.parent: groups.update(g["name"] for g in variant.parent.groups) for var, lookaside in compose.conf.get("variant_as_lookaside", []): if var == variant.uid: lookaside_variant = compose.all_variants[lookaside] groups.update(g["name"] for g in lookaside_variant.groups) return groups def write_variant_comps(compose, arch, variant): comps_file = compose.paths.work.comps(arch=arch, variant=variant) compose.log_debug( "Writing comps file (arch: %s, variant: %s): %s", arch, variant, comps_file ) cmd = [ "comps_filter", "--arch=%s" % arch, "--keep-empty-group=conflicts", "--keep-empty-group=conflicts-%s" % variant.uid.lower(), "--variant=%s" % variant.uid, "--output=%s" % comps_file, compose.paths.work.comps(arch="global"), ] for group in get_lookaside_groups(compose, variant): cmd.append("--lookaside-group=%s" % group) run(cmd) comps = CompsWrapper(comps_file) if variant.groups or variant.modules is not None or variant.type != "variant": # Filter groups if the variant has some, or it's a modular variant, or # is not a base variant. unmatched = comps.filter_groups(variant.groups) for grp in unmatched: compose.log_warning(UNMATCHED_GROUP_MSG % (variant.uid, arch, grp)) contains_all = not variant.groups and not variant.environments if compose.conf["comps_filter_environments"] and not contains_all: # We only want to filter environments if it's enabled by configuration # and it's a variant with some groups and environements defined. If # there are none, all packages should go in there and also all # environments should be preserved. comps.filter_environments(variant.environments) comps.write_comps() def create_comps_repo(compose, arch, variant): createrepo_c = compose.conf["createrepo_c"] createrepo_checksum = compose.conf["createrepo_checksum"] repo = CreaterepoWrapper(createrepo_c=createrepo_c) comps_repo = compose.paths.work.comps_repo(arch=arch, variant=variant) comps_path = compose.paths.work.comps(arch=arch, variant=variant) msg = "Creating comps repo for arch '%s' variant '%s'" % ( arch, variant.uid if variant else None, ) compose.log_info("[BEGIN] %s" % msg) cmd = repo.get_createrepo_cmd( comps_repo, database=False, outputdir=comps_repo, groupfile=comps_path, checksum=createrepo_checksum, ) logfile = "comps_repo-%s" % variant if variant else "comps_repo" run(cmd, logfile=compose.paths.log.log_file(arch, logfile), show_cmd=True) compose.log_info("[DONE ] %s" % msg) def write_module_defaults(compose): scm_dict = compose.conf["module_defaults_dir"] if isinstance(scm_dict, dict): if scm_dict["scm"] == "file": scm_dict["dir"] = os.path.join(compose.config_dir, scm_dict["dir"]) else: scm_dict = os.path.join(compose.config_dir, scm_dict) with temp_dir(prefix="moduledefaults_") as tmp_dir: get_dir_from_scm(scm_dict, tmp_dir, compose=compose) compose.log_debug("Writing module defaults") shutil.copytree( tmp_dir, compose.paths.work.module_defaults_dir(create_dir=False) ) def validate_module_defaults(path): """Make sure there are no conflicting defaults. Each module name can only have one default stream. :param str path: directory with cloned module defaults """ seen_defaults = collections.defaultdict(set) for module_name, defaults in iter_module_defaults(path): seen_defaults[module_name].add(defaults.get_default_stream()) errors = [] for module_name, defaults in seen_defaults.items(): if len(defaults) > 1: errors.append( "Module %s has multiple defaults: %s" % (module_name, ", ".join(sorted(defaults))) ) if errors: raise RuntimeError( "There are duplicated module defaults:\n%s" % "\n".join(errors) ) def validate_comps(path): """Check that there are whitespace issues in comps.""" wrapper = CompsWrapper(path) wrapper.validate()