cd493bc4c1
After cloning the repository with defaults, open each file and try to check if there is more than one definition for the same module. It's not a problem for the compose process, but consumers of the compose would get confused and possibly explode. Better alert people early. Conceptually this should be part of the test phase, but that would mean waiting for the compose to finish before reporting the error. The earlier the better. Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
206 lines
8.0 KiB
Python
206 lines
8.0 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; version 2 of the License.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Library General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, see <https://gnu.org/licenses/>.
|
|
|
|
|
|
import collections
|
|
import glob
|
|
import os
|
|
import shutil
|
|
|
|
from kobo.shortcuts import run
|
|
|
|
from pungi import Modulemd
|
|
from pungi.phases.base import PhaseBase
|
|
from pungi.phases.gather import write_prepopulate_file
|
|
from pungi.util import temp_dir
|
|
from pungi.wrappers.comps import CompsWrapper
|
|
from pungi.wrappers.createrepo import CreaterepoWrapper
|
|
from pungi.wrappers.scm import get_dir_from_scm, get_file_from_scm
|
|
|
|
|
|
class InitPhase(PhaseBase):
|
|
"""INIT is a mandatory phase"""
|
|
name = "init"
|
|
|
|
def skip(self):
|
|
# INIT must never be skipped,
|
|
# because it generates data for LIVEIMAGES
|
|
return False
|
|
|
|
def run(self):
|
|
if self.compose.has_comps:
|
|
# write global comps and arch comps, create comps repos
|
|
write_global_comps(self.compose)
|
|
for arch in self.compose.get_arches():
|
|
write_arch_comps(self.compose, arch)
|
|
create_comps_repo(self.compose, arch, None)
|
|
|
|
# write variant comps
|
|
for variant in self.compose.get_variants():
|
|
for arch in variant.arches:
|
|
write_variant_comps(self.compose, arch, variant)
|
|
create_comps_repo(self.compose, arch, variant)
|
|
|
|
# download variants.xml / product.xml?
|
|
|
|
# download module defaults
|
|
if self.compose.has_module_defaults:
|
|
write_module_defaults(self.compose)
|
|
validate_module_defaults(
|
|
self.compose.paths.work.module_defaults_dir(create_dir=False)
|
|
)
|
|
|
|
# write prepopulate file
|
|
write_prepopulate_file(self.compose)
|
|
|
|
|
|
def write_global_comps(compose):
|
|
comps_file_global = compose.paths.work.comps(arch="global")
|
|
msg = "Writing global comps file: %s" % comps_file_global
|
|
|
|
if compose.DEBUG and os.path.isfile(comps_file_global):
|
|
compose.log_warning("[SKIP ] %s" % msg)
|
|
else:
|
|
scm_dict = compose.conf["comps_file"]
|
|
if isinstance(scm_dict, dict):
|
|
comps_name = os.path.basename(scm_dict["file"])
|
|
if scm_dict["scm"] == "file":
|
|
scm_dict["file"] = os.path.join(compose.config_dir, scm_dict["file"])
|
|
else:
|
|
comps_name = os.path.basename(scm_dict)
|
|
scm_dict = os.path.join(compose.config_dir, scm_dict)
|
|
|
|
compose.log_debug(msg)
|
|
tmp_dir = compose.mkdtemp(prefix="comps_")
|
|
get_file_from_scm(scm_dict, tmp_dir, logger=compose._logger)
|
|
shutil.copy2(os.path.join(tmp_dir, comps_name), comps_file_global)
|
|
shutil.rmtree(tmp_dir)
|
|
|
|
|
|
def write_arch_comps(compose, arch):
|
|
comps_file_arch = compose.paths.work.comps(arch=arch)
|
|
msg = "Writing comps file for arch '%s': %s" % (arch, comps_file_arch)
|
|
|
|
if compose.DEBUG and os.path.isfile(comps_file_arch):
|
|
compose.log_warning("[SKIP ] %s" % msg)
|
|
return
|
|
|
|
compose.log_debug(msg)
|
|
run(["comps_filter", "--arch=%s" % arch, "--no-cleanup",
|
|
"--output=%s" % comps_file_arch,
|
|
compose.paths.work.comps(arch="global")])
|
|
|
|
|
|
UNMATCHED_GROUP_MSG = 'Variant %s.%s requires comps group %s which does not match anything in input comps file'
|
|
|
|
|
|
def write_variant_comps(compose, arch, variant):
|
|
comps_file = compose.paths.work.comps(arch=arch, variant=variant)
|
|
msg = "Writing comps file (arch: %s, variant: %s): %s" % (arch, variant, comps_file)
|
|
|
|
if compose.DEBUG and os.path.isfile(comps_file):
|
|
# read display_order and groups for environments (needed for live images)
|
|
comps = CompsWrapper(comps_file)
|
|
# groups = variant.groups
|
|
comps.filter_groups(variant.groups)
|
|
if compose.conf["comps_filter_environments"]:
|
|
comps.filter_environments(variant.environments)
|
|
|
|
compose.log_warning("[SKIP ] %s" % msg)
|
|
return
|
|
|
|
compose.log_debug(msg)
|
|
run(["comps_filter", "--arch=%s" % arch, "--keep-empty-group=conflicts",
|
|
"--keep-empty-group=conflicts-%s" % variant.uid.lower(),
|
|
"--variant=%s" % variant.uid,
|
|
"--output=%s" % comps_file, compose.paths.work.comps(arch="global")])
|
|
|
|
comps = CompsWrapper(comps_file)
|
|
if variant.groups or variant.modules is not None or variant.type != 'variant':
|
|
# Filter groups if the variant has some, or it's a modular variant, or
|
|
# is not a base variant.
|
|
unmatched = comps.filter_groups(variant.groups)
|
|
for grp in unmatched:
|
|
compose.log_warning(UNMATCHED_GROUP_MSG % (variant.uid, arch, grp))
|
|
contains_all = not variant.groups and not variant.environments
|
|
if compose.conf["comps_filter_environments"] and not contains_all:
|
|
# We only want to filter environments if it's enabled by configuration
|
|
# and it's a variant with some groups and environements defined. If
|
|
# there are none, all packages should go in there and also all
|
|
# environments should be preserved.
|
|
comps.filter_environments(variant.environments)
|
|
comps.write_comps()
|
|
|
|
|
|
def create_comps_repo(compose, arch, variant):
|
|
createrepo_c = compose.conf["createrepo_c"]
|
|
createrepo_checksum = compose.conf["createrepo_checksum"]
|
|
repo = CreaterepoWrapper(createrepo_c=createrepo_c)
|
|
comps_repo = compose.paths.work.comps_repo(arch=arch, variant=variant)
|
|
comps_path = compose.paths.work.comps(arch=arch, variant=variant)
|
|
msg = "Creating comps repo for arch '%s' variant '%s'" % (arch, variant.uid if variant else None)
|
|
if compose.DEBUG and os.path.isdir(os.path.join(comps_repo, "repodata")):
|
|
compose.log_warning("[SKIP ] %s" % msg)
|
|
else:
|
|
compose.log_info("[BEGIN] %s" % msg)
|
|
cmd = repo.get_createrepo_cmd(comps_repo, database=False,
|
|
outputdir=comps_repo, groupfile=comps_path,
|
|
checksum=createrepo_checksum)
|
|
logfile = 'comps_repo-%s' % variant if variant else 'comps_repo'
|
|
run(cmd, logfile=compose.paths.log.log_file(arch, logfile),
|
|
show_cmd=True)
|
|
compose.log_info("[DONE ] %s" % msg)
|
|
|
|
|
|
def write_module_defaults(compose):
|
|
scm_dict = compose.conf["module_defaults_dir"]
|
|
if isinstance(scm_dict, dict):
|
|
if scm_dict["scm"] == "file":
|
|
scm_dict["dir"] = os.path.join(compose.config_dir, scm_dict["dir"])
|
|
else:
|
|
scm_dict = os.path.join(compose.config_dir, scm_dict)
|
|
|
|
with temp_dir(prefix="moduledefaults_") as tmp_dir:
|
|
get_dir_from_scm(scm_dict, tmp_dir, logger=compose._logger)
|
|
compose.log_debug("Writing module defaults")
|
|
shutil.copytree(tmp_dir, compose.paths.work.module_defaults_dir(create_dir=False))
|
|
|
|
|
|
def validate_module_defaults(path):
|
|
"""Make sure there are no conflicting defaults. Each module name can only
|
|
have one default stream.
|
|
|
|
:param str path: directory with cloned module defaults
|
|
"""
|
|
seen_defaults = collections.defaultdict(set)
|
|
for file in glob.glob(os.path.join(path, "*.yaml")):
|
|
for mmddef in Modulemd.objects_from_file(file):
|
|
if not isinstance(mmddef, Modulemd.Defaults):
|
|
continue
|
|
seen_defaults[mmddef.peek_module_name()].add(mmddef.peek_default_stream())
|
|
|
|
errors = []
|
|
for module_name, defaults in seen_defaults.items():
|
|
if len(defaults) > 1:
|
|
errors.append(
|
|
"Module %s has multiple defaults: %s"
|
|
% (module_name, ", ".join(sorted(defaults)))
|
|
)
|
|
|
|
if errors:
|
|
raise RuntimeError(
|
|
"There are duplicated module defaults:\n%s" % "\n".join(errors)
|
|
)
|