diff --git a/MANIFEST.in b/MANIFEST.in index 3894c459..568e1112 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,9 +1,6 @@ -include Authors -include Changelog +include AUTHORS include COPYING include GPL -include TESTING -include ToDo include pungi.spec -include share/* -include doc/* +recursive-include share/* +recursive-include doc/* diff --git a/TODO b/TODO new file mode 100644 index 00000000..b75bc230 --- /dev/null +++ b/TODO @@ -0,0 +1,75 @@ +Random thoughts on what needs to be done before Pungi 4.0 is completed. + + +Define building blocks and their metadata +========================================= + +* rpms in yum repos +* comps +* kickstart trees +* isos +* kickstart trees +* bootable images +* readme files +* license(s) + + +Compose structure +================= +* topdir + * work, logs, etc. + * compose + * $variant + * $arch + * $content_type (rpms, isos, kickstart trees, etc.) + * actual content + + +Split Pungi into smaller well-defined tools +=========================================== + +* process initial packages + * comps + * json mapping + * ??? + +* grab initial package set + * yum repos + * koji instance (basically what mash does today) + +* resolve deps (gather) + * self-hosting + * fulltree + * multilib + * langpacks + +* create repos + +* create install images + * lorax + * buildinstall + +* create isos + * isos + * bootable + * hybrid + * implant md5sum + * jigdo + * checksums + +* run tests + * just quick sanity tests + +* notification + * email + * messagebus + + +Unsorted +======== +* run any tasks in koji or local host +* support for non-rpm content? (java artifacts, etc.) +* docs! +* unit tests! +* use productmd for metadata: https://github.com/release-engineering/productmd/ +* use next-gen tools: createrepo_c, mergerepo_c, dnf, hawkey, libcomps diff --git a/bin/pungi b/bin/pungi new file mode 100755 index 00000000..97f01bc6 --- /dev/null +++ b/bin/pungi @@ -0,0 +1,340 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + + +import os +import sys +import optparse +import logging +import locale +import datetime +import getpass +import socket +import json +import pipes + +here = sys.path[0] +if here != '/usr/bin': + # Git checkout + sys.path[0] = os.path.dirname(here) + +from pungi import __version__ + + +# force C locales +locale.setlocale(locale.LC_ALL, "C") + + +COMPOSE = None + + +def main(): + global COMPOSE + + parser = optparse.OptionParser() + parser.add_option( + "--target-dir", + metavar="PATH", + help="a compose is created under this directory", + ) + parser.add_option( + "--label", + help="specify compose label (example: Snapshot-1.0); required for production composes" + ) + parser.add_option( + "--no-label", + action="store_true", + default=False, + help="make a production compose without label" + ) + parser.add_option( + "--supported", + action="store_true", + default=False, + help="set supported flag on media (automatically on for 'RC-x.y' labels)" + ) + parser.add_option( + "--old-composes", + metavar="PATH", + dest="old_composes", + default=[], + action="append", + help="Path to directory with old composes. Reuse an existing repodata from the most recent compose.", + ) + parser.add_option( + "--compose-dir", + metavar="PATH", + help="reuse an existing compose directory (DANGEROUS!)", + ) + parser.add_option( + "--debug-mode", + action="store_true", + default=False, + help="run pungi in DEBUG mode (DANGEROUS!)", + ) + parser.add_option( + "--config", + help="Config file" + ) + parser.add_option( + "--skip-phase", + metavar="PHASE", + action="append", + default=[], + help="skip a compose phase", + ) + parser.add_option( + "--just-phase", + metavar="PHASE", + action="append", + default=[], + help="run only a specified compose phase", + ) + parser.add_option( + "--nightly", + action="store_const", + const="nightly", + dest="compose_type", + help="make a nightly compose", + ) + parser.add_option( + "--test", + action="store_const", + const="test", + dest="compose_type", + help="make a test compose", + ) + parser.add_option( + "--koji-event", + metavar="ID", + type="int", + help="specify a koji event for populating package set", + ) + parser.add_option( + "--version", + action="store_true", + help="output version information and exit", + ) + + opts, args = parser.parse_args() + + if opts.version: + print("pungi %s" % __version__) + sys.exit(0) + + if opts.target_dir and opts.compose_dir: + parser.error("cannot specify --target-dir and --compose-dir at once") + + if not opts.target_dir and not opts.compose_dir: + parser.error("please specify a target directory") + + if opts.target_dir and not opts.compose_dir: + opts.target_dir = os.path.abspath(opts.target_dir) + if not os.path.isdir(opts.target_dir): + parser.error("The target directory does not exist or is not a directory: %s" % opts.target_dir) + else: + opts.compose_dir = os.path.abspath(opts.compose_dir) + if not os.path.isdir(opts.compose_dir): + parser.error("The compose directory does not exist or is not a directory: %s" % opts.compose_dir) + + compose_type = opts.compose_type or "production" + if compose_type == "production" and not opts.label and not opts.no_label: + parser.error("must specify label for a production compose") + + if not opts.config: + parser.error("please specify a config") + opts.config = os.path.abspath(opts.config) + + # check if all requirements are met + import pungi.checks + if not pungi.checks.check(): + sys.exit(1) + + import kobo.conf + import kobo.log + import productmd.composeinfo.compose + + if opts.label: + try: + productmd.composeinfo.compose.verify_label(opts.label) + except ValueError as ex: + parser.error(str(ex)) + + from pungi.compose import Compose + + logger = logging.Logger("Pungi") + kobo.log.add_stderr_logger(logger) + + conf = kobo.conf.PyConfigParser() + conf.load_from_file(opts.config) + + if opts.target_dir: + compose_dir = Compose.get_compose_dir(opts.target_dir, conf, compose_type=compose_type, compose_label=opts.label) + else: + compose_dir = opts.compose_dir + + compose = Compose(conf, topdir=compose_dir, debug=opts.debug_mode, skip_phases=opts.skip_phase, just_phases=opts.just_phase, + old_composes=opts.old_composes, koji_event=opts.koji_event, supported=opts.supported, logger=logger) + kobo.log.add_file_logger(logger, compose.paths.log.log_file("global", "pungi.log")) + COMPOSE = compose + run_compose(compose) + + +def run_compose(compose): + import pungi.phases + import pungi.metadata + + compose.write_status("STARTED") + compose.log_info("Host: %s" % socket.gethostname()) + compose.log_info("User name: %s" % getpass.getuser()) + compose.log_info("Working directory: %s" % os.getcwd()) + compose.log_info("Command line: %s" % " ".join([pipes.quote(arg) for arg in sys.argv])) + compose.log_info("Compose top directory: %s" % compose.topdir) + compose.read_variants() + + # dump the config file + date_str = datetime.datetime.strftime(datetime.datetime.now(), "%F_%X").replace(":", "-") + config_dump = compose.paths.log.log_file("global", "config-dump_%s" % date_str) + open(config_dump, "w").write(json.dumps(compose.conf, sort_keys=True, indent=4)) + + # initialize all phases + init_phase = pungi.phases.InitPhase(compose) + pkgset_phase = pungi.phases.PkgsetPhase(compose) + createrepo_phase = pungi.phases.CreaterepoPhase(compose) + buildinstall_phase = pungi.phases.BuildinstallPhase(compose) + productimg_phase = pungi.phases.ProductimgPhase(compose, pkgset_phase) + gather_phase = pungi.phases.GatherPhase(compose, pkgset_phase) + extrafiles_phase = pungi.phases.ExtraFilesPhase(compose, pkgset_phase) + createiso_phase = pungi.phases.CreateisoPhase(compose) + liveimages_phase = pungi.phases.LiveImagesPhase(compose) + test_phase = pungi.phases.TestPhase(compose) + + # check if all config options are set + errors = [] + for phase in (init_phase, pkgset_phase, buildinstall_phase, productimg_phase, gather_phase, createiso_phase, test_phase): + if phase.skip(): + continue + try: + phase.validate() + except ValueError as ex: + for i in str(ex).splitlines(): + errors.append("%s: %s" % (phase.name.upper(), i)) + if errors: + for i in errors: + compose.log_error(i) + print(i) + sys.exit(1) + + # INIT phase + init_phase.start() + init_phase.stop() + + # PKGSET phase + pkgset_phase.start() + pkgset_phase.stop() + + # BUILDINSTALL phase - start + buildinstall_phase.start() + + # GATHER phase + gather_phase.start() + gather_phase.stop() + + # EXTRA_FILES phase + extrafiles_phase.start() + extrafiles_phase.stop() + + # CREATEREPO phase + createrepo_phase.start() + createrepo_phase.stop() + + # BUILDINSTALL phase + # must finish before PRODUCTIMG + # must finish before CREATEISO + buildinstall_phase.stop() + if not buildinstall_phase.skip(): + buildinstall_phase.copy_files() + + # PRODUCTIMG phase + productimg_phase.start() + productimg_phase.stop() + + # write treeinfo before ISOs are created + for variant in compose.get_variants(): + for arch in variant.arches + ["src"]: + pungi.metadata.write_tree_info(compose, arch, variant) + + # write .discinfo and media.repo before ISOs are created + for variant in compose.get_variants(recursive=True): + if variant.type == "addon": + continue + for arch in variant.arches + ["src"]: + timestamp = pungi.metadata.write_discinfo(compose, arch, variant) + pungi.metadata.write_media_repo(compose, arch, variant, timestamp) + + # CREATEISO and LIVEIMAGES phases + createiso_phase.start() + liveimages_phase.start() + + createiso_phase.stop() + liveimages_phase.stop() + + # merge checksum files + for variant in compose.get_variants(types=["variant", "layered-product"]): + for arch in variant.arches + ["src"]: + iso_dir = compose.paths.compose.iso_dir(arch, variant, create_dir=False) + if not iso_dir or not os.path.exists(iso_dir): + continue + for checksum_type in ("md5", "sha1", "sha256"): + checksum_upper = "%sSUM" % checksum_type.upper() + checksums = sorted([i for i in os.listdir(iso_dir) if i.endswith(".%s" % checksum_upper)]) + fo = open(os.path.join(iso_dir, checksum_upper), "w") + for i in checksums: + data = open(os.path.join(iso_dir, i), "r").read() + fo.write(data) + + pungi.metadata.write_compose_info(compose) + compose.im.dump(compose.paths.compose.metadata("images.json") + + # TEST phase + test_phase.start() + test_phase.stop() + + # create a latest symlink + compose_dir = os.path.basename(compose.topdir) + symlink_name = "latest-%s-%s" % (compose.conf["product_short"], ".".join(compose.conf["product_version"].split(".")[:-1])) + if compose.conf["product_is_layered"]: + symlink_name += "-%s-%s" % (compose.conf["base_product_short"], compose.conf["base_product_version"]) + symlink = os.path.join(compose.topdir, "..", symlink_name) + + try: + os.unlink(symlink) + except OSError as ex: + if ex.errno != 2: + raise + try: + os.symlink(compose_dir, symlink) + except Exception as ex: + print("ERROR: couldn't create latest symlink: %s" % ex) + + compose.log_info("Compose finished: %s" % compose.topdir) + compose.write_status("FINISHED") + + +if __name__ == "__main__": + try: + main() + except (Exception, KeyboardInterrupt) as ex: + if COMPOSE: + tb_path = COMPOSE.paths.log.log_file("global", "traceback") + COMPOSE.log_error("Exception: %s" % ex) + COMPOSE.log_error("Extended traceback in: %s" % tb_path) + COMPOSE.log_critical("Compose failed: %s" % COMPOSE.topdir) + COMPOSE.write_status("DOOMED") + import kobo.tback + open(tb_path, "w").write(kobo.tback.Traceback().get_traceback()) + else: + print("Exception: %s" % ex) + sys.stdout.flush() + sys.stderr.flush() + raise diff --git a/pungi/arch.py b/pungi/arch.py index 332b2289..98966766 100644 --- a/pungi/arch.py +++ b/pungi/arch.py @@ -45,7 +45,7 @@ def get_valid_multilib_arches(tree_arch): multilib_arch = get_multilib_arch(yum_arch) if not multilib_arch: return [] - return [ i for i in rpmUtils.arch.getArchList(multilib_arch) if i not in ("noarch", "src") ] + return [i for i in rpmUtils.arch.getArchList(multilib_arch) if i not in ("noarch", "src")] def get_valid_arches(tree_arch, multilib=True, add_noarch=True, add_src=False): diff --git a/pungi/checks.py b/pungi/checks.py new file mode 100644 index 00000000..3b8b0867 --- /dev/null +++ b/pungi/checks.py @@ -0,0 +1,123 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import os.path + + +tools = [ + ("isomd5sum", "/usr/bin/implantisomd5"), + ("isomd5sum", "/usr/bin/checkisomd5"), + ("jigdo", "/usr/bin/jigdo-lite"), + ("genisoimage", "/usr/bin/genisoimage"), + ("gettext", "/usr/bin/msgfmt"), + ("syslinux", "/usr/bin/isohybrid"), + ("yum-utils", "/usr/bin/createrepo"), + ("yum-utils", "/usr/bin/mergerepo"), + ("yum-utils", "/usr/bin/repoquery"), + ("git", "/usr/bin/git"), + ("cvs", "/usr/bin/cvs"), + ("gettext", "/usr/bin/msgfmt"), +] + +imports = [ + ("kobo", "kobo"), + ("kobo-rpmlib", "kobo.rpmlib"), + ("python-lxml", "lxml"), + ("koji", "koji"), + ("productmd", "productmd"), +] + + +def check(): + fail = False + + # Check python modules + for package, module in imports: + try: + __import__(module) + except ImportError: + print("Module '%s' doesn't exist. Install package '%s'." % (module, package)) + fail = True + + # Check tools + for package, path in tools: + if not os.path.exists(path): + print("Program '%s' doesn't exist. Install package '%s'." % (path, package)) + fail = True + + return not fail + + +def validate_options(conf, valid_options): + errors = [] + for i in valid_options: + name = i["name"] + value = conf.get(name) + + if i.get("deprecated", False): + if name in conf: + errors.append("Deprecated config option: %s; %s" % (name, i["comment"])) + continue + + if name not in conf: + if not i.get("optional", False): + errors.append("Config option not set: %s" % name) + continue + + # verify type + if "expected_types" in i: + etypes = i["expected_types"] + if not isinstance(etypes, list) and not isinstance(etypes, tuple): + raise TypeError("The 'expected_types' value must be wrapped in a list: %s" % i) + found = False + for etype in etypes: + if isinstance(value, etype): + found = True + break + if not found: + errors.append("Config option '%s' has invalid type: %s. Expected: %s." % (name, str(type(value)), etypes)) + continue + + # verify value + if "expected_values" in i: + evalues = i["expected_values"] + if not isinstance(evalues, list) and not isinstance(evalues, tuple): + raise TypeError("The 'expected_values' value must be wrapped in a list: %s" % i) + found = False + for evalue in evalues: + if value == evalue: + found = True + break + if not found: + errors.append("Config option '%s' has invalid value: %s. Expected: %s." % (name, value, evalues)) + continue + + if "requires" in i: + for func, requires in i["requires"]: + if func(value): + for req in requires: + if req not in conf: + errors.append("Config option %s=%s requires %s which is not set" % (name, value, req)) + + if "conflicts" in i: + for func, conflicts in i["conflicts"]: + if func(value): + for con in conflicts: + if con in conf: + errors.append("Config option %s=%s conflicts with option %s" % (name, value, con)) + + return errors diff --git a/pungi/compose.py b/pungi/compose.py new file mode 100644 index 00000000..66712312 --- /dev/null +++ b/pungi/compose.py @@ -0,0 +1,241 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +__all__ = ( + "Compose", +) + + +import errno +import os +import time +import tempfile +import shutil + +import kobo.log +from productmd import ComposeInfo, ImageManifest + +from pypungi.wrappers.variants import VariantsXmlParser +from pypungi.paths import Paths +from pypungi.wrappers.scm import get_file_from_scm +from pypungi.util import makedirs +from pypungi.metadata import compose_to_composeinfo + + +def get_compose_dir(topdir, conf, compose_type="production", compose_date=None, compose_respin=None, compose_label=None, already_exists_callbacks=None): + topdir = os.path.abspath(topdir) + already_exists_callbacks = already_exists_callbacks or [] + + # create an incomplete ComposeInfo to generate compose ID + ci = ComposeInfo() + ci.product.name = conf["product_name"] + ci.product.short = conf["product_short"] + ci.product.version = conf["product_version"] + ci.product.is_layered = bool(conf.get("product_is_layered", False)) + if ci.product.is_layered: + ci.base_product.name = conf["base_product_name"] + ci.base_product.short = conf["base_product_short"] + ci.base_product.version = conf["base_product_version"] + + ci.compose.label = compose_label + ci.compose.type = compose_type + ci.compose.date = compose_date or time.strftime("%Y%m%d", time.localtime()) + ci.compose.respin = compose_respin or 0 + + # HACK - add topdir for callbacks + ci.topdir = topdir + + while 1: + ci.compose.id = ci.create_compose_id() + + compose_dir = os.path.join(topdir, ci.compose.id) + + exists = False + # TODO: callbacks to determine if a composeid was already used + # for callback in already_exists_callbacks: + # if callback(data): + # exists = True + # break + + # already_exists_callbacks fallback: does target compose_dir exist? + if not exists: + try: + os.makedirs(compose_dir) + except OSError as ex: + if ex.errno == errno.EEXIST: + exists = True + else: + raise + + if exists: + ci.compose.respin += 1 + continue + break + + open(os.path.join(compose_dir, "COMPOSE_ID"), "w").write(ci.compose.id) + work_dir = os.path.join(compose_dir, "work", "global") + makedirs(work_dir) + ci.dump(os.path.join(work_dir, "composeinfo-base.json")) + return compose_dir + + +class Compose(kobo.log.LoggingBase): + def __init__(self, conf, topdir, debug=False, skip_phases=None, just_phases=None, old_composes=None, koji_event=None, supported=False, logger=None): + kobo.log.LoggingBase.__init__(self, logger) + # TODO: check if minimal conf values are set + self.conf = conf + self.variants = {} + self.topdir = os.path.abspath(topdir) + self.skip_phases = skip_phases or [] + self.just_phases = just_phases or [] + self.old_composes = old_composes or [] + self.koji_event = koji_event + + # intentionally upper-case (visible in the code) + self.DEBUG = debug + + # path definitions + self.paths = Paths(self) + + # to provide compose_id, compose_date and compose_respin + self.ci_base = ComposeInfo() + self.ci_base.load(os.path.join(self.paths.work.topdir(arch="global"), "composeinfo-base.json")) + + self.supported = supported + if self.compose_label and self.compose_label.split("-")[0] == "RC": + self.log_info("Automatically setting 'supported' flag for a Release Candidate (%s) compose." % self.compose_label) + self.supported = True + + self.im = ImageManifest() + if self.DEBUG: + try: + self.im.load(self.paths.compose.metadata("images.json")) + except RuntimeError: + pass + self.im.compose.id = self.compose_id + self.im.compose.type = self.compose_type + self.im.compose.date = self.compose_date + self.im.compose.respin = self.compose_respin + self.im.metadata_path = self.paths.compose.metadata() + + get_compose_dir = staticmethod(get_compose_dir) + + def __getitem__(self, name): + return self.variants[name] + + @property + def compose_id(self): + return self.ci_base.compose.id + + @property + def compose_date(self): + return self.ci_base.compose.date + + @property + def compose_respin(self): + return self.ci_base.compose.respin + + @property + def compose_type(self): + return self.ci_base.compose.type + + @property + def compose_type_suffix(self): + return self.ci_base.compose.type_suffix + + @property + def compose_label(self): + return self.ci_base.compose.label + + @property + def has_comps(self): + return bool(self.conf.get("comps_file", False)) + + @property + def config_dir(self): + return os.path.dirname(self.conf._open_file or "") + + def read_variants(self): + # TODO: move to phases/init ? + variants_file = self.paths.work.variants_file(arch="global") + msg = "Writing variants file: %s" % variants_file + + if self.DEBUG and os.path.isfile(variants_file): + self.log_warning("[SKIP ] %s" % msg) + else: + scm_dict = self.conf["variants_file"] + if isinstance(scm_dict, dict): + file_name = os.path.basename(scm_dict["file"]) + if scm_dict["scm"] == "file": + scm_dict["file"] = os.path.join(self.config_dir, os.path.basename(scm_dict["file"])) + else: + file_name = os.path.basename(scm_dict) + scm_dict = os.path.join(self.config_dir, os.path.basename(scm_dict)) + + self.log_debug(msg) + tmp_dir = tempfile.mkdtemp(prefix="variants_file_") + get_file_from_scm(scm_dict, tmp_dir, logger=self._logger) + shutil.copy2(os.path.join(tmp_dir, file_name), variants_file) + shutil.rmtree(tmp_dir) + + file_obj = open(variants_file, "r") + tree_arches = self.conf.get("tree_arches", None) + self.variants = VariantsXmlParser(file_obj, tree_arches).parse() + + # populate ci_base with variants - needed for layered-products (compose_id) + self.ci_base = compose_to_composeinfo(self) + + def get_variants(self, types=None, arch=None, recursive=False): + result = [] + types = types or ["variant", "optional", "addon", "layered-product"] + for i in self.variants.values(): + if i.type in types: + if arch and arch not in i.arches: + continue + result.append(i) + result.extend(i.get_variants(types=types, arch=arch, recursive=recursive)) + return sorted(set(result)) + + def get_arches(self): + result = set() + tree_arches = self.conf.get("tree_arches", None) + for variant in self.get_variants(): + for arch in variant.arches: + if tree_arches: + if arch in tree_arches: + result.add(arch) + else: + result.add(arch) + return sorted(result) + + def write_status(self, stat_msg): + if stat_msg not in ("STARTED", "FINISHED", "DOOMED"): + self.log_warning("Writing nonstandard compose status: %s" % stat_msg) + old_status = self.get_status() + if stat_msg == old_status: + return + if old_status == "FINISHED": + msg = "Could not modify a FINISHED compose: %s" % self.topdir + self.log_error(msg) + raise RuntimeError(msg) + open(os.path.join(self.topdir, "STATUS"), "w").write(stat_msg + "\n") + + def get_status(self): + path = os.path.join(self.topdir, "STATUS") + if not os.path.isfile(path): + return + return open(path, "r").read().strip() diff --git a/pungi/compose_metadata/__init__.py b/pungi/compose_metadata/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pungi/compose_metadata/discinfo.py b/pungi/compose_metadata/discinfo.py new file mode 100644 index 00000000..fe74fca7 --- /dev/null +++ b/pungi/compose_metadata/discinfo.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +""" +The .discinfo file contains metadata about media. +Following fields are part of the .discinfo file, +one record per line: +- timestamp +- release +- architecture +- disc number (optional) +""" + + +__all__ = ( + "read_discinfo", + "write_discinfo", + "write_media_repo", +) + + +import time + + +def write_discinfo(file_path, description, arch, disc_numbers=None, timestamp=None): + """ + Write a .discinfo file: + """ + disc_numbers = disc_numbers or ["ALL"] + if not isinstance(disc_numbers, list): + raise TypeError("Invalid type: disc_numbers type is %s; expected: " % type(disc_numbers)) + if not timestamp: + timestamp = "%f" % time.time() + f = open(file_path, "w") + f.write("%s\n" % timestamp) + f.write("%s\n" % description) + f.write("%s\n" % arch) + if disc_numbers: + f.write("%s\n" % ",".join([str(i) for i in disc_numbers])) + f.close() + return timestamp + + +def read_discinfo(file_path): + result = {} + f = open(file_path, "r") + result["timestamp"] = f.readline().strip() + result["description"] = f.readline().strip() + result["arch"] = f.readline().strip() + disc_numbers = f.readline().strip() + if not disc_numbers: + result["disc_numbers"] = None + elif disc_numbers == "ALL": + result["disc_numbers"] = ["ALL"] + else: + result["disc_numbers"] = [int(i) for i in disc_numbers.split(",")] + return result + + +def write_media_repo(file_path, description, timestamp=None): + """ + Write media.repo file for the disc to be used on installed system. + PackageKit uses this. + """ + + if not timestamp: + raise + timestamp = "%f" % time.time() + + data = [ + "[InstallMedia]", + "name=%s" % description, + "mediaid=%s" % timestamp, + "metadata_expire=-1", + "gpgcheck=0", + "cost=500", + "", + ] + + repo_file = open(file_path, "w") + repo_file.write("\n".join(data)) + repo_file.close() + return timestamp diff --git a/pungi/config.py b/pungi/config.py index 617ae181..a6a0d755 100644 --- a/pungi/config.py +++ b/pungi/config.py @@ -1,4 +1,6 @@ -#!/usr/bin/python -tt +# -*- coding: utf-8 -*- + + # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. @@ -12,12 +14,14 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + import os import time import yum from ConfigParser import SafeConfigParser + class Config(SafeConfigParser): def __init__(self): SafeConfigParser.__init__(self) diff --git a/pungi/gather.py b/pungi/gather.py index b09b1c69..9d3f7d57 100644 --- a/pungi/gather.py +++ b/pungi/gather.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -tt +# -*- coding: utf-8 -*- # This program is free software; you can redistribute it and/or modify diff --git a/pungi/ks.py b/pungi/ks.py index 53f7520b..ade8d662 100644 --- a/pungi/ks.py +++ b/pungi/ks.py @@ -1,6 +1,20 @@ # -*- coding: utf-8 -*- +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + """ Pungi adds several new sections to kickstarts. diff --git a/pungi/linker.py b/pungi/linker.py new file mode 100644 index 00000000..d739c6dc --- /dev/null +++ b/pungi/linker.py @@ -0,0 +1,315 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import errno +import os +import shutil + +import kobo.log +from kobo.shortcuts import relative_path +from kobo.threads import WorkerThread, ThreadPool + +from pypungi.util import makedirs + + +class LinkerPool(ThreadPool): + def __init__(self, link_type="hardlink-or-copy", logger=None): + ThreadPool.__init__(self, logger) + self.link_type = link_type + self.linker = Linker() + + +class LinkerThread(WorkerThread): + def process(self, item, num): + src, dst = item + + if (num % 100 == 0) or (num == self.pool.queue_total): + self.pool.log_debug("Linked %s out of %s packages" % (num, self.pool.queue_total)) + + self.pool.linker.link(src, dst, link_type=self.pool.link_type) + + +class Linker(kobo.log.LoggingBase): + def __init__(self, ignore_existing=False, always_copy=None, test=False, logger=None): + kobo.log.LoggingBase.__init__(self, logger=logger) + self.ignore_existing = ignore_existing + self.always_copy = always_copy or [] + self.test = test + self._precache = {} + self._inode_map = {} + + def _is_same_type(self, path1, path2): + if not os.path.islink(path1) == os.path.islink(path2): + return False + if not os.path.isdir(path1) == os.path.isdir(path2): + return False + if not os.path.isfile(path1) == os.path.isfile(path2): + return False + return True + + def _is_same(self, path1, path2): + if self.ignore_existing: + return True + if path1 == path2: + return True + if os.path.islink(path2) and not os.path.exists(path2): + return True + if os.path.getsize(path1) != os.path.getsize(path2): + return False + if int(os.path.getmtime(path1)) != int(os.path.getmtime(path2)): + return False + return True + + def symlink(self, src, dst, relative=True): + if src == dst: + return + + old_src = src + if relative: + src = relative_path(src, dst) + + msg = "Symlinking %s -> %s" % (dst, src) + if self.test: + self.log_info("TEST: %s" % msg) + return + self.log_info(msg) + + try: + os.symlink(src, dst) + except OSError as ex: + if ex.errno != errno.EEXIST: + raise + if os.path.islink(dst) and self._is_same(old_src, dst): + if os.readlink(dst) != src: + raise + self.log_debug("The same file already exists, skipping symlink %s -> %s" % (dst, src)) + else: + raise + + def hardlink_on_dest(self, src, dst): + if src == dst: + return + + if os.path.exists(src): + st = os.stat(src) + file_name = os.path.basename(src) + precache_key = (file_name, int(st.st_mtime), st.st_size) + if precache_key in self._precache: + self.log_warning("HIT %s" % str(precache_key)) + cached_path = self._precache[precache_key]["path"] + self.hardlink(cached_path, dst) + return True + return False + + def hardlink(self, src, dst): + if src == dst: + return + + msg = "Hardlinking %s to %s" % (src, dst) + if self.test: + self.log_info("TEST: %s" % msg) + return + self.log_info(msg) + + try: + os.link(src, dst) + except OSError as ex: + if ex.errno != errno.EEXIST: + raise + if self._is_same(src, dst): + if not self._is_same_type(src, dst): + self.log_error("File %s already exists but has different type than %s" % (dst, src)) + raise + self.log_debug("The same file already exists, skipping hardlink %s to %s" % (src, dst)) + else: + raise + + def copy(self, src, dst): + if src == dst: + return True + + if os.path.islink(src): + msg = "Copying symlink %s to %s" % (src, dst) + else: + msg = "Copying file %s to %s" % (src, dst) + + if self.test: + self.log_info("TEST: %s" % msg) + return + self.log_info(msg) + + if os.path.exists(dst): + if self._is_same(src, dst): + if not self._is_same_type(src, dst): + self.log_error("File %s already exists but has different type than %s" % (dst, src)) + raise OSError(errno.EEXIST, "File exists") + self.log_debug("The same file already exists, skipping copy %s to %s" % (src, dst)) + return + else: + raise OSError(errno.EEXIST, "File exists") + + if os.path.islink(src): + if not os.path.islink(dst): + os.symlink(os.readlink(src), dst) + return + return + + src_stat = os.stat(src) + src_key = (src_stat.st_dev, src_stat.st_ino) + if src_key in self._inode_map: + # (st_dev, st_ino) found in the mapping + self.log_debug("Harlink detected, hardlinking in destination %s to %s" % (self._inode_map[src_key], dst)) + os.link(self._inode_map[src_key], dst) + return + + # BEWARE: shutil.copy2 automatically *rewrites* existing files + shutil.copy2(src, dst) + self._inode_map[src_key] = dst + + if not self._is_same(src, dst): + self.log_error("File %s doesn't match the copied file %s" % (src, dst)) + # XXX: + raise OSError(errno.EEXIST, "File exists") + + def _put_into_cache(self, path): + def get_stats(item): + return [item[i] for i in ("st_dev", "st_ino", "st_mtime", "st_size")] + + filename = os.path.basename(path) + st = os.stat(path) + item = { + "st_dev": st.st_dev, + "st_ino": st.st_ino, + "st_mtime": int(st.st_mtime), + "st_size": st.st_size, + "path": path, + } + precache_key = (filename, int(st.st_mtime), st.st_size) + if precache_key in self._precache: + if get_stats(self._precache[precache_key]) != get_stats(item): + # Files have same mtime and size but device + # or/and inode is/are different. + self.log_debug("Caching failed, files are different: %s, %s" + % (path, self._precache[precache_key]["path"])) + return False + self._precache[precache_key] = item + return True + + def scan(self, path): + """Recursively scan a directory and populate the cache.""" + msg = "Scanning directory: %s" % path + self.log_debug("[BEGIN] %s" % msg) + for dirpath, _, filenames in os.walk(path): + for filename in filenames: + path = os.path.join(dirpath, filename) + self._put_into_cache(path) + self.log_debug("[DONE ] %s" % msg) + + def _link_file(self, src, dst, link_type): + if link_type == "hardlink": + if not self.hardlink_on_dest(src, dst): + self.hardlink(src, dst) + elif link_type == "copy": + self.copy(src, dst) + elif link_type in ("symlink", "abspath-symlink"): + if os.path.islink(src): + self.copy(src, dst) + else: + relative = link_type != "abspath-symlink" + self.symlink(src, dst, relative) + elif link_type == "hardlink-or-copy": + if not self.hardlink_on_dest(src, dst): + src_stat = os.stat(src) + dst_stat = os.stat(os.path.dirname(dst)) + if src_stat.st_dev == dst_stat.st_dev: + self.hardlink(src, dst) + else: + self.copy(src, dst) + else: + raise ValueError("Unknown link_type: %s" % link_type) + + def link(self, src, dst, link_type="hardlink-or-copy", scan=True): + """Link directories recursively.""" + if os.path.isfile(src) or os.path.islink(src): + self._link_file(src, dst, link_type) + return + + if os.path.isfile(dst): + raise OSError(errno.EEXIST, "File exists") + + if not self.test: + if not os.path.exists(dst): + makedirs(dst) + shutil.copystat(src, dst) + + for i in os.listdir(src): + src_path = os.path.join(src, i) + dst_path = os.path.join(dst, i) + self.link(src_path, dst_path, link_type) + + return + + if scan: + self.scan(dst) + + self.log_debug("Start linking") + + src = os.path.abspath(src) + for dirpath, dirnames, filenames in os.walk(src): + rel_path = dirpath[len(src):].lstrip("/") + dst_path = os.path.join(dst, rel_path) + + # Dir check and creation + if not os.path.isdir(dst_path): + if os.path.exists(dst_path): + # At destination there is a file with same name but + # it is not a directory. + self.log_error("Cannot create directory %s" % dst_path) + dirnames = [] # noqa + continue + os.mkdir(dst_path) + + # Process all files in directory + for filename in filenames: + path = os.path.join(dirpath, filename) + st = os.stat(path) + # Check cache + # Same file already exists at a destination dir => + # Create the new file by hardlink to the cached one. + precache_key = (filename, int(st.st_mtime), st.st_size) + full_dst_path = os.path.join(dst_path, filename) + if precache_key in self._precache: + # Cache hit + cached_path = self._precache[precache_key]["path"] + self.log_debug("Cache HIT for %s [%s]" % (path, cached_path)) + if cached_path != full_dst_path: + self.hardlink(cached_path, full_dst_path) + else: + self.log_debug("Files are same, skip hardlinking") + continue + # Cache miss + # Copy the new file and put it to the cache. + try: + self.copy(path, full_dst_path) + except Exception as ex: + print(ex) + print(path, open(path, "r").read()) + print(full_dst_path, open(full_dst_path, "r").read()) + print(os.stat(path)) + print(os.stat(full_dst_path)) + os.utime(full_dst_path, (st.st_atime, int(st.st_mtime))) + self._put_into_cache(full_dst_path) diff --git a/pungi/media_split.py b/pungi/media_split.py new file mode 100644 index 00000000..6d3a3a93 --- /dev/null +++ b/pungi/media_split.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import os + + +SIZE_UNITS = { + "b": 1, + "k": 1024, + "M": 1024 ** 2, + "G": 1024 ** 3, +} + + +def convert_media_size(size): + if isinstance(size, str): + if size[-1] in SIZE_UNITS: + num = int(size[:-1]) + units = size[-1] + else: + num = int(size) + units = "b" + result = num * SIZE_UNITS[units] + else: + result = int(size) + + if result <= 0: + raise ValueError("Media size must be a positive number: %s" % size) + + return result + + +def convert_file_size(size, block_size=2048): + """round file size to block""" + blocks = int(size / block_size) + if size % block_size: + blocks += 1 + return blocks * block_size + + +class MediaSplitter(object): + def __init__(self, media_size): + self.media_size = convert_media_size(media_size) + self.files = [] # to preserve order + self.file_sizes = {} + self.sticky_files = set() + + def add_file(self, name, size, sticky=False): + name = os.path.normpath(name) + size = int(size) + old_size = self.file_sizes.get(name, None) + if old_size is None: + self.files.append(name) + self.file_sizes[name] = size + elif old_size != size: + raise ValueError("File size mismatch; file: %s; sizes: %s vs %s" % (name, old_size, size)) + elif size > self.media_size: + raise ValueError("File is larger than media size: %s" % name) + if sticky: + self.sticky_files.add(name) + + ''' + def load(self, file_name): + f = open(file_name, "r") + for line in f: + line = line.strip() + if not line: + continue + name, size = line.split(" ") + self.add_file(name, size) + f.close() + + def scan(self, pattern): + for i in glob.glob(pattern): + self.add_file(i, os.path.getsize(i)) + + def dump(self, file_name): + f = open(file_name, "w") + for name in self.files: + f.write("%s %s\n" % (os.path.basename(name), self.file_sizes[name])) + f.close() + ''' + + @property + def total_size(self): + return sum(self.file_sizes.values()) + + @property + def total_size_in_blocks(self): + return sum([convert_file_size(i) for i in list(self.file_sizes.values())]) + + def split(self, first_disk=0, all_disks=0): + all_files = [] + sticky_files = [] + sticky_files_size = 0 + + for name in self.files: + if name in self.sticky_files: + sticky_files.append(name) + sticky_files_size += convert_file_size(self.file_sizes[name]) + else: + all_files.append(name) + + disks = [] + disk = {} + while all_files: + name = all_files.pop(0) + size = convert_file_size(self.file_sizes[name]) + + if not disks or disk["size"] + size > self.media_size: + disk = {"size": 0, "files": []} + disks.append(disk) + disk["files"].extend(sticky_files) + disk["size"] += sticky_files_size + + disk["files"].append(name) + disk["size"] += convert_file_size(self.file_sizes[name]) + + return disks diff --git a/pungi/metadata.py b/pungi/metadata.py new file mode 100644 index 00000000..d7b63537 --- /dev/null +++ b/pungi/metadata.py @@ -0,0 +1,306 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import os +import time + +import productmd.composeinfo +import productmd.treeinfo +import productmd.treeinfo.product +from productmd import get_major_version +from kobo.shortcuts import relative_path + +from pypungi.compose_metadata.discinfo import write_discinfo as create_discinfo +from pypungi.compose_metadata.discinfo import write_media_repo as create_media_repo + + +def get_description(compose, variant, arch): + if "product_discinfo_description" in compose.conf: + result = compose.conf["product_discinfo_description"] + elif variant.type == "layered-product": + # we need to make sure the layered product behaves as it was composed separately + result = "%s %s for %s %s" % (variant.product_name, variant.product_version, compose.conf["product_name"], get_major_version(compose.conf["product_version"])) + else: + result = "%s %s" % (compose.conf["product_name"], compose.conf["product_version"]) + if compose.conf.get("is_layered", False): + result += "for %s %s" % (compose.conf["base_product_name"], compose.conf["base_product_version"]) + + result = result % {"variant_name": variant.name, "arch": arch} + return result + + +def write_discinfo(compose, arch, variant): + if variant.type == "addon": + return + os_tree = compose.paths.compose.os_tree(arch, variant) + path = os.path.join(os_tree, ".discinfo") + # description = get_volid(compose, arch, variant) + description = get_description(compose, variant, arch) + return create_discinfo(path, description, arch) + + +def write_media_repo(compose, arch, variant, timestamp=None): + if variant.type == "addon": + return + os_tree = compose.paths.compose.os_tree(arch, variant) + path = os.path.join(os_tree, "media.repo") + # description = get_volid(compose, arch, variant) + description = get_description(compose, variant, arch) + return create_media_repo(path, description, timestamp) + + +def compose_to_composeinfo(compose): + ci = productmd.composeinfo.ComposeInfo() + + # compose + ci.compose.id = compose.compose_id + ci.compose.type = compose.compose_type + ci.compose.date = compose.compose_date + ci.compose.respin = compose.compose_respin + ci.compose.label = compose.compose_label + + # product + ci.product.name = compose.conf["product_name"] + ci.product.version = compose.conf["product_version"] + ci.product.short = compose.conf["product_short"] + ci.product.is_layered = compose.conf.get("product_is_layered", False) + + # base product + if ci.product.is_layered: + ci.base_product.name = compose.conf["base_product_name"] + ci.base_product.version = compose.conf["base_product_version"] + ci.base_product.short = compose.conf["base_product_short"] + + def dump_variant(variant, parent=None): + var = productmd.composeinfo.Variant(ci) + + tree_arches = compose.conf.get("tree_arches", None) + if tree_arches and not (set(variant.arches) & set(tree_arches)): + return None + + # variant details + var.id = variant.id + var.uid = variant.uid + var.name = variant.name + var.type = variant.type + var.arches = set(variant.arches) + + if var.type == "layered-product": + var.product.name = variant.product_name + var.product.short = variant.product_short + var.product.version = variant.product_version + var.product.is_layered = True + + for arch in variant.arches: + # paths: binaries + var.os_tree[arch] = relative_path(compose.paths.compose.os_tree(arch=arch, variant=variant, create_dir=False).rstrip("/") + "/", compose.paths.compose.topdir().rstrip("/") + "/").rstrip("/") + var.repository[arch] = relative_path(compose.paths.compose.repository(arch=arch, variant=variant, create_dir=False).rstrip("/") + "/", compose.paths.compose.topdir().rstrip("/") + "/").rstrip("/") + var.packages[arch] = relative_path(compose.paths.compose.packages(arch=arch, variant=variant, create_dir=False).rstrip("/") + "/", compose.paths.compose.topdir().rstrip("/") + "/").rstrip("/") + iso_dir = compose.paths.compose.iso_dir(arch=arch, variant=variant, create_dir=False) or "" + if iso_dir and os.path.isdir(os.path.join(compose.paths.compose.topdir(), iso_dir)): + var.isos[arch] = relative_path(iso_dir, compose.paths.compose.topdir().rstrip("/") + "/").rstrip("/") + jigdo_dir = compose.paths.compose.jigdo_dir(arch=arch, variant=variant, create_dir=False) or "" + if jigdo_dir and os.path.isdir(os.path.join(compose.paths.compose.topdir(), jigdo_dir)): + var.jigdos[arch] = relative_path(jigdo_dir, compose.paths.compose.topdir().rstrip("/") + "/").rstrip("/") + + # paths: sources + var.source_tree[arch] = relative_path(compose.paths.compose.os_tree(arch="source", variant=variant, create_dir=False).rstrip("/") + "/", compose.paths.compose.topdir().rstrip("/") + "/").rstrip("/") + var.source_repository[arch] = relative_path(compose.paths.compose.repository(arch="source", variant=variant, create_dir=False).rstrip("/") + "/", compose.paths.compose.topdir().rstrip("/") + "/").rstrip("/") + var.source_packages[arch] = relative_path(compose.paths.compose.packages(arch="source", variant=variant, create_dir=False).rstrip("/") + "/", compose.paths.compose.topdir().rstrip("/") + "/").rstrip("/") + source_iso_dir = compose.paths.compose.iso_dir(arch="source", variant=variant, create_dir=False) or "" + if source_iso_dir and os.path.isdir(os.path.join(compose.paths.compose.topdir(), source_iso_dir)): + var.source_isos[arch] = relative_path(source_iso_dir, compose.paths.compose.topdir().rstrip("/") + "/").rstrip("/") + source_jigdo_dir = compose.paths.compose.jigdo_dir(arch="source", variant=variant, create_dir=False) or "" + if source_jigdo_dir and os.path.isdir(os.path.join(compose.paths.compose.topdir(), source_jigdo_dir)): + var.source_jigdos[arch] = relative_path(source_jigdo_dir, compose.paths.compose.topdir().rstrip("/") + "/").rstrip("/") + + # paths: debug + var.debug_tree[arch] = relative_path(compose.paths.compose.debug_tree(arch=arch, variant=variant, create_dir=False).rstrip("/") + "/", compose.paths.compose.topdir().rstrip("/") + "/").rstrip("/") + var.debug_repository[arch] = relative_path(compose.paths.compose.debug_repository(arch=arch, variant=variant, create_dir=False).rstrip("/") + "/", compose.paths.compose.topdir().rstrip("/") + "/").rstrip("/") + var.debug_packages[arch] = relative_path(compose.paths.compose.debug_packages(arch=arch, variant=variant, create_dir=False).rstrip("/") + "/", compose.paths.compose.topdir().rstrip("/") + "/").rstrip("/") + ''' + # XXX: not suported (yet?) + debug_iso_dir = compose.paths.compose.debug_iso_dir(arch=arch, variant=variant) or "" + if debug_iso_dir: + var.debug_iso_dir[arch] = relative_path(debug_iso_dir, compose.paths.compose.topdir().rstrip("/") + "/").rstrip("/") + debug_jigdo_dir = compose.paths.compose.debug_jigdo_dir(arch=arch, variant=variant) or "" + if debug_jigdo_dir: + var.debug_jigdo_dir[arch] = relative_path(debug_jigdo_dir, compose.paths.compose.topdir().rstrip("/") + "/").rstrip("/") + ''' + + for v in variant.get_variants(recursive=False): + x = dump_variant(v, parent=variant) + if x is not None: + var.add(x) + return var + + for variant_id in sorted(compose.variants): + variant = compose.variants[variant_id] + v = dump_variant(variant) + if v is not None: + ci.variants.add(v) + return ci + + +def write_compose_info(compose): + ci = compose_to_composeinfo(compose) + + msg = "Writing composeinfo" + compose.log_info("[BEGIN] %s" % msg) + + path = compose.paths.compose.metadata("composeinfo.json") + ci.dump(path) + + compose.log_info("[DONE ] %s" % msg) + + +def write_tree_info(compose, arch, variant, timestamp=None): + if variant.type in ("addon", ): + return + + if not timestamp: + timestamp = int(time.time()) + else: + timestamp = int(timestamp) + + os_tree = compose.paths.compose.os_tree(arch=arch, variant=variant).rstrip("/") + "/" + + ti = productmd.treeinfo.TreeInfo() + # load from buildinstall .treeinfo + + if variant.type == "layered-product": + # we need to make sure the layered product behaves as it was composed separately + + # product + # TODO: read from variants.xml + ti.product.name = variant.product_name + ti.product.version = variant.product_version + ti.product.short = variant.product_short + ti.product.is_layered = True + + # base product + ti.base_product.name = compose.conf["product_name"] + if "." in compose.conf["product_version"]: + # remove minor version if present + ti.base_product.version = get_major_version(compose.conf["product_version"]) + else: + ti.base_product.version = compose.conf["product_version"] + ti.base_product.short = compose.conf["product_short"] + else: + # product + ti.product.name = compose.conf["product_name"] + ti.product.version = compose.conf["product_version"] + ti.product.short = compose.conf["product_short"] + ti.product.is_layered = compose.conf.get("product_is_layered", False) + + # base product + if ti.product.is_layered: + ti.base_product.name = compose.conf["base_product_name"] + ti.base_product.version = compose.conf["base_product_version"] + ti.base_product.short = compose.conf["base_product_short"] + + # tree + ti.tree.arch = arch + ti.tree.build_timestamp = timestamp + # ti.platforms + + # main variant + var = productmd.treeinfo.Variant(ti) + if variant.type == "layered-product": + var.id = variant.parent.id + var.uid = variant.parent.uid + var.name = variant.parent.name + var.type = "variant" + else: + var.id = variant.id + var.uid = variant.uid + var.name = variant.name + var.type = variant.type + + var.packages = relative_path(compose.paths.compose.packages(arch=arch, variant=variant, create_dir=False).rstrip("/") + "/", os_tree).rstrip("/") or "." + var.repository = relative_path(compose.paths.compose.repository(arch=arch, variant=variant, create_dir=False).rstrip("/") + "/", os_tree).rstrip("/") or "." + + ti.variants.add(var) + + repomd_path = os.path.join(var.repository, "repodata", "repomd.xml") + ti.checksums.add(os_tree, repomd_path) + + for i in variant.get_variants(types=["addon"], arch=arch): + addon = productmd.treeinfo.Variant(ti) + addon.id = i.id + addon.uid = i.uid + addon.name = i.name + addon.type = i.type + + os_tree = compose.paths.compose.os_tree(arch=arch, variant=i).rstrip("/") + "/" + addon.packages = relative_path(compose.paths.compose.packages(arch=arch, variant=i, create_dir=False).rstrip("/") + "/", os_tree).rstrip("/") or "." + addon.repository = relative_path(compose.paths.compose.repository(arch=arch, variant=i, create_dir=False).rstrip("/") + "/", os_tree).rstrip("/") or "." + var.add(addon) + + repomd_path = os.path.join(addon.repository, "repodata", "repomd.xml") + ti.checksums.add(os_tree, repomd_path) + + class LoraxProduct(productmd.treeinfo.product.Product): + def _check_short(self): + # HACK: set self.short so .treeinfo produced by lorax can be read + if not self.short: + self.short = compose.conf["product_short"] + + class LoraxTreeInfo(productmd.TreeInfo): + def clear(self): + productmd.TreeInfo.clear(self) + self.product = LoraxProduct(self) + + # images + if variant.type == "variant": + os_tree = compose.paths.compose.os_tree(arch, variant) + + # clone all but 'general' sections from buildinstall .treeinfo + bi_treeinfo = os.path.join(compose.paths.work.buildinstall_dir(arch), ".treeinfo") + if os.path.exists(bi_treeinfo): + bi_ti = LoraxTreeInfo() + bi_ti.load(bi_treeinfo) + + # stage2 - mainimage + if bi_ti.stage2.mainimage: + ti.stage2.mainimage = bi_ti.stage2.mainimage + ti.checksums.add(os_tree, ti.stage2.mainimage) + + # stage2 - instimage + if bi_ti.stage2.instimage: + ti.stage2.instimage = bi_ti.stage2.instimage + ti.checksums.add(os_tree, ti.stage2.instimage) + + # images + for platform in bi_ti.images.images: + ti.images.images[platform] = {} + ti.tree.platforms.add(platform) + for image, path in bi_ti.images.images[platform].items(): + ti.images.images[platform][image] = path + ti.checksums.add(os_tree, path) + + # add product.img to images-$arch + product_img = os.path.join(os_tree, "images", "product.img") + product_img_relpath = relative_path(product_img, os_tree.rstrip("/") + "/") + if os.path.isfile(product_img): + for platform in ti.images.images: + ti.images.images[platform]["product.img"] = product_img_relpath + ti.checksums.add(os_tree, product_img_relpath) + + path = os.path.join(compose.paths.compose.os_tree(arch=arch, variant=variant), ".treeinfo") + compose.log_info("Writing treeinfo: %s" % path) + ti.dump(path) diff --git a/pungi/pathmatch.py b/pungi/pathmatch.py index d37f38d4..08afc8c3 100644 --- a/pungi/pathmatch.py +++ b/pungi/pathmatch.py @@ -1,6 +1,20 @@ # -*- coding: utf-8 -*- +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + import fnmatch diff --git a/pungi/paths.py b/pungi/paths.py new file mode 100644 index 00000000..1c4eee00 --- /dev/null +++ b/pungi/paths.py @@ -0,0 +1,526 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +__all__ = ( + "Paths", +) + + +import errno +import os + +from pypungi.util import makedirs + + +class Paths(object): + def __init__(self, compose): + paths_module_name = compose.conf.get("paths_module", None) + if paths_module_name: + # custom paths + compose.log_info("Using custom paths from module %s" % paths_module_name) + paths_module = __import__(paths_module_name, globals(), locals(), ["LogPaths", "WorkPaths", "ComposePaths"]) + self.compose = paths_module.ComposePaths(compose) + self.log = paths_module.LogPaths(compose) + self.work = paths_module.WorkPaths(compose) + else: + # default paths + self.compose = ComposePaths(compose) + self.log = LogPaths(compose) + self.work = WorkPaths(compose) + # self.metadata ? + + +class LogPaths(object): + def __init__(self, compose): + self.compose = compose + + def topdir(self, arch=None, create_dir=True): + """ + Examples: + log/global + log/x86_64 + """ + arch = arch or "global" + path = os.path.join(self.compose.topdir, "logs", arch) + if create_dir: + makedirs(path) + return path + + def log_file(self, arch, log_name, create_dir=True): + arch = arch or "global" + if log_name.endswith(".log"): + log_name = log_name[:-4] + return os.path.join(self.topdir(arch, create_dir=create_dir), "%s.%s.log" % (log_name, arch)) + + +class WorkPaths(object): + def __init__(self, compose): + self.compose = compose + + def topdir(self, arch=None, create_dir=True): + """ + Examples: + work/global + work/x86_64 + """ + arch = arch or "global" + path = os.path.join(self.compose.topdir, "work", arch) + if create_dir: + makedirs(path) + return path + + def variants_file(self, arch=None, create_dir=True): + """ + Examples: + work/global/variants.xml + """ + arch = "global" + path = os.path.join(self.topdir(arch, create_dir=create_dir), "variants.xml") + return path + + def comps(self, arch=None, variant=None, create_dir=True): + """ + Examples: + work/x86_64/comps/comps-86_64.xml + work/x86_64/comps/comps-Server.x86_64.xml + """ + arch = arch or "global" + if variant is None: + file_name = "comps-%s.xml" % arch + else: + file_name = "comps-%s.%s.xml" % (variant.uid, arch) + path = os.path.join(self.topdir(arch, create_dir=create_dir), "comps") + if create_dir: + makedirs(path) + path = os.path.join(path, file_name) + return path + + def pungi_conf(self, arch=None, variant=None, create_dir=True): + """ + Examples: + work/x86_64/pungi/x86_64.conf + work/x86_64/pungi/Server.x86_64.conf + """ + arch = arch or "global" + if variant is None: + file_name = "%s.conf" % arch + else: + file_name = "%s.%s.conf" % (variant.uid, arch) + path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi") + if create_dir: + makedirs(path) + path = os.path.join(path, file_name) + return path + + def pungi_log(self, arch=None, variant=None, create_dir=True): + """ + Examples: + work/x86_64/pungi/x86_64.log + work/x86_64/pungi/Server.x86_64.log + """ + path = self.pungi_conf(arch, variant, create_dir=create_dir) + path = path[:-5] + ".log" + return path + + def pungi_cache_dir(self, arch, variant=None, create_dir=True): + """ + Examples: + work/global/pungi-cache + """ + # WARNING: Using the same cache dir with repos of the same names may lead to a race condition + # We should use per arch variant cache dirs to workaround this. + path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi-cache") + if variant: + path = os.path.join(path, variant.uid) + if create_dir: + makedirs(path) + return path + + def comps_repo(self, arch=None, create_dir=True): + """ + Examples: + work/x86_64/comps-repo + work/global/comps-repo + """ + arch = arch or "global" + path = os.path.join(self.topdir(arch, create_dir=create_dir), "comps_repo") + if create_dir: + makedirs(path) + return path + + def arch_repo(self, arch=None, create_dir=True): + """ + Examples: + work/x86_64/repo + work/global/repo + """ + arch = arch or "global" + path = os.path.join(self.topdir(arch, create_dir=create_dir), "repo") + if create_dir: + makedirs(path) + return path + + def package_list(self, arch=None, variant=None, pkg_type=None, create_dir=True): + """ + Examples: + work/x86_64/package_list/x86_64.conf + work/x86_64/package_list/Server.x86_64.conf + work/x86_64/package_list/Server.x86_64.rpm.conf + """ + arch = arch or "global" + if variant is not None: + file_name = "%s.%s" % (variant, arch) + else: + file_name = "%s" % arch + if pkg_type is not None: + file_name += ".%s" % pkg_type + file_name += ".conf" + path = os.path.join(self.topdir(arch, create_dir=create_dir), "package_list") + if create_dir: + makedirs(path) + path = os.path.join(path, file_name) + return path + + def pungi_download_dir(self, arch, create_dir=True): + """ + Examples: + work/x86_64/pungi_download + """ + path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi_download") + if create_dir: + makedirs(path) + return path + + def buildinstall_dir(self, arch, create_dir=True): + """ + Examples: + work/x86_64/buildinstall + """ + if arch == "global": + raise RuntimeError("Global buildinstall dir makes no sense.") + path = os.path.join(self.topdir(arch, create_dir=create_dir), "buildinstall") + return path + + def extra_files_dir(self, arch, variant, create_dir=True): + """ + Examples: + work/x86_64/Server/extra-files + """ + if arch == "global": + raise RuntimeError("Global extra files dir makes no sense.") + path = os.path.join(self.topdir(arch, create_dir=create_dir), variant.uid, "extra-files") + if create_dir: + makedirs(path) + return path + + def repo_package_list(self, arch, variant, pkg_type=None, create_dir=True): + """ + Examples: + work/x86_64/repo_package_list/Server.x86_64.rpm.conf + """ + file_name = "%s.%s" % (variant, arch) + if pkg_type is not None: + file_name += ".%s" % pkg_type + file_name += ".conf" + path = os.path.join(self.topdir(arch, create_dir=create_dir), "repo_package_list") + if create_dir: + makedirs(path) + path = os.path.join(path, file_name) + return path + + def product_img(self, variant, create_dir=True): + """ + Examples: + work/global/product-Server.img + """ + file_name = "product-%s.img" % variant + path = self.topdir(arch="global", create_dir=create_dir) + path = os.path.join(path, file_name) + return path + + def iso_dir(self, arch, variant, disc_type="dvd", disc_num=1, create_dir=True): + """ + Examples: + work/x86_64/iso/rhel-7.0-20120127.0-Server-x86_64-dvd1.iso + """ + dir_name = self.compose.paths.compose.iso_path(arch, variant, disc_type, disc_num, create_dir=False) + dir_name = os.path.basename(dir_name) + path = os.path.join(self.topdir(arch, create_dir=create_dir), "iso", dir_name) + if create_dir: + makedirs(path) + return path + + def tmp_dir(self, arch, variant=None, create_dir=True): + """ + Examples: + work/x86_64/tmp + work/x86_64/tmp-Server + """ + dir_name = "tmp" + if variant: + dir_name += "-%s" % variant.uid + path = os.path.join(self.topdir(arch, create_dir=create_dir), dir_name) + if create_dir: + makedirs(path) + return path + + def product_id(self, arch, variant, create_dir=True): + """ + Examples: + work/x86_64/product_id/productid-Server.x86_64.pem/productid + """ + # file_name = "%s.%s.pem" % (variant, arch) + # HACK: modifyrepo doesn't handle renames -> $dir/productid + file_name = "productid" + path = os.path.join(self.topdir(arch, create_dir=create_dir), "product_id", "%s.%s.pem" % (variant, arch)) + if create_dir: + makedirs(path) + path = os.path.join(path, file_name) + return path + + +class ComposePaths(object): + def __init__(self, compose): + self.compose = compose + # TODO: TREES? + + def topdir(self, arch=None, variant=None, create_dir=True, relative=False): + """ + Examples: + compose + compose/Server/x86_64 + """ + if bool(arch) != bool(variant): + raise TypeError("topdir(): either none or 2 arguments are expected") + + path = "" + if not relative: + path = os.path.join(self.compose.topdir, "compose") + + if arch or variant: + if variant.type == "addon": + return self.topdir(arch, variant.parent, create_dir=create_dir, relative=relative) + path = os.path.join(path, variant.uid, arch) + if create_dir and not relative: + makedirs(path) + return path + + def tree_dir(self, arch, variant, create_dir=True, relative=False): + """ + Examples: + compose/Server/x86_64/os + compose/Server-optional/x86_64/os + """ + if arch == "src": + arch = "source" + + if arch == "source": + tree_dir = "tree" + else: + # use 'os' dir due to historical reasons + tree_dir = "os" + + path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), tree_dir) + if create_dir and not relative: + makedirs(path) + return path + + def os_tree(self, arch, variant, create_dir=True, relative=False): + return self.tree_dir(arch, variant, create_dir=create_dir, relative=relative) + + def repository(self, arch, variant, create_dir=True, relative=False): + """ + Examples: + compose/Server/x86_64/os + compose/Server/x86_64/addons/LoadBalancer + """ + if variant.type == "addon": + path = self.packages(arch, variant, create_dir=create_dir, relative=relative) + else: + path = self.tree_dir(arch, variant, create_dir=create_dir, relative=relative) + if create_dir and not relative: + makedirs(path) + return path + + def packages(self, arch, variant, create_dir=True, relative=False): + """ + Examples: + compose/Server/x86_64/os/Packages + compose/Server/x86_64/os/addons/LoadBalancer + compose/Server-optional/x86_64/os/Packages + """ + if variant.type == "addon": + path = os.path.join(self.tree_dir(arch, variant, create_dir=create_dir, relative=relative), "addons", variant.id) + else: + path = os.path.join(self.tree_dir(arch, variant, create_dir=create_dir, relative=relative), "Packages") + if create_dir and not relative: + makedirs(path) + return path + + def debug_topdir(self, arch, variant, create_dir=True, relative=False): + """ + Examples: + compose/Server/x86_64/debug + compose/Server-optional/x86_64/debug + """ + path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), "debug") + if create_dir and not relative: + makedirs(path) + return path + + def debug_tree(self, arch, variant, create_dir=True, relative=False): + """ + Examples: + compose/Server/x86_64/debug/tree + compose/Server-optional/x86_64/debug/tree + """ + path = os.path.join(self.debug_topdir(arch, variant, create_dir=create_dir, relative=relative), "tree") + if create_dir and not relative: + makedirs(path) + return path + + def debug_packages(self, arch, variant, create_dir=True, relative=False): + """ + Examples: + compose/Server/x86_64/debug/tree/Packages + compose/Server/x86_64/debug/tree/addons/LoadBalancer + compose/Server-optional/x86_64/debug/tree/Packages + """ + if arch in ("source", "src"): + return None + if variant.type == "addon": + path = os.path.join(self.debug_tree(arch, variant, create_dir=create_dir, relative=relative), "addons", variant.id) + else: + path = os.path.join(self.debug_tree(arch, variant, create_dir=create_dir, relative=relative), "Packages") + if create_dir and not relative: + makedirs(path) + return path + + def debug_repository(self, arch, variant, create_dir=True, relative=False): + """ + Examples: + compose/Server/x86_64/debug/tree + compose/Server/x86_64/debug/tree/addons/LoadBalancer + compose/Server-optional/x86_64/debug/tree + """ + if arch in ("source", "src"): + return None + if variant.type == "addon": + path = os.path.join(self.debug_tree(arch, variant, create_dir=create_dir, relative=relative), "addons", variant.id) + else: + path = self.debug_tree(arch, variant, create_dir=create_dir, relative=relative) + if create_dir and not relative: + makedirs(path) + return path + + def iso_dir(self, arch, variant, symlink_to=None, create_dir=True, relative=False): + """ + Examples: + compose/Server/x86_64/iso + None + """ + if variant.type == "addon": + return None + if variant.type == "optional": + if not self.compose.conf["create_optional_isos"]: + return None + if arch == "src": + arch = "source" + path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), "iso") + + if symlink_to: + # TODO: create_dir + topdir = self.compose.topdir.rstrip("/") + "/" + relative_dir = path[len(topdir):] + target_dir = os.path.join(symlink_to, self.compose.compose_id, relative_dir) + if create_dir and not relative: + makedirs(target_dir) + try: + os.symlink(target_dir, path) + except OSError as ex: + if ex.errno != errno.EEXIST: + raise + msg = "Symlink pointing to '%s' expected: %s" % (target_dir, path) + if not os.path.islink(path): + raise RuntimeError(msg) + if os.path.abspath(os.readlink(path)) != target_dir: + raise RuntimeError(msg) + else: + if create_dir and not relative: + makedirs(path) + return path + + def iso_path(self, arch, variant, disc_type="dvd", disc_num=1, suffix=".iso", symlink_to=None, create_dir=True, relative=False): + """ + Examples: + compose/Server/x86_64/iso/rhel-7.0-20120127.0-Server-x86_64-dvd1.iso + None + """ + if arch == "src": + arch = "source" + + if disc_type not in ("cd", "dvd", "ec2", "live", "boot"): + raise RuntimeError("Unsupported disc type: %s" % disc_type) + if disc_num: + disc_num = int(disc_num) + else: + disc_num = "" + + path = self.iso_dir(arch, variant, symlink_to=symlink_to, create_dir=create_dir, relative=relative) + if path is None: + return None + + compose_id = self.compose.ci_base[variant.uid].compose_id + if variant.type == "layered-product": + variant_uid = variant.parent.uid + else: + variant_uid = variant.uid + file_name = "%s-%s-%s-%s%s%s" % (compose_id, variant_uid, arch, disc_type, disc_num, suffix) + result = os.path.join(path, file_name) + return result + + def jigdo_dir(self, arch, variant, create_dir=True, relative=False): + """ + Examples: + compose/Server/x86_64/jigdo + None + """ + if variant.type == "addon": + return None + if variant.type == "optional": + if not self.compose.conf["create_optional_isos"]: + return None + if arch == "src": + arch = "source" + path = os.path.join(self.topdir(arch, variant, create_dir=create_dir, relative=relative), "jigdo") + + if create_dir and not relative: + makedirs(path) + return path + + def metadata(self, file_name=None, create_dir=True, relative=False): + """ + Examples: + compose/metadata + compose/metadata/rpms.json + """ + path = os.path.join(self.topdir(create_dir=create_dir, relative=relative), "metadata") + if create_dir and not relative: + makedirs(path) + if file_name: + path = os.path.join(path, file_name) + return path diff --git a/pungi/phases/__init__.py b/pungi/phases/__init__.py new file mode 100644 index 00000000..1f11abe6 --- /dev/null +++ b/pungi/phases/__init__.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +# phases in runtime order +from init import InitPhase # noqa +from pkgset import PkgsetPhase # noqa +from gather import GatherPhase # noqa +from createrepo import CreaterepoPhase # noqa +from product_img import ProductimgPhase # noqa +from buildinstall import BuildinstallPhase # noqa +from extra_files import ExtraFilesPhase # noqa +from createiso import CreateisoPhase # noqa +from live_images import LiveImagesPhase # noqa +from test import TestPhase # noqa diff --git a/pungi/phases/base.py b/pungi/phases/base.py new file mode 100644 index 00000000..cbfcd0b3 --- /dev/null +++ b/pungi/phases/base.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +from pypungi.checks import validate_options + + +class PhaseBase(object): + config_options = () + + def __init__(self, compose): + self.compose = compose + self.msg = "---------- PHASE: %s ----------" % self.name.upper() + self.finished = False + self._skipped = False + + def validate(self): + errors = validate_options(self.compose.conf, self.config_options) + if errors: + raise ValueError("\n".join(errors)) + + def conf_assert_str(self, name): + missing = [] + invalid = [] + if name not in self.compose.conf: + missing.append(name) + elif not isinstance(self.compose.conf[name], str): + invalid.append(name, type(self.compose.conf[name]), str) + return missing, invalid + + def skip(self): + if self._skipped: + return True + if self.compose.just_phases and self.name not in self.compose.just_phases: + return True + if self.name in self.compose.skip_phases: + return True + if self.name in self.compose.conf.get("skip_phases", []): + return True + return False + + def start(self): + self._skipped = self.skip() + if self._skipped: + self.compose.log_warning("[SKIP ] %s" % self.msg) + self.finished = True + return + self.compose.log_info("[BEGIN] %s" % self.msg) + self.run() + + def stop(self): + if self.finished: + return + if hasattr(self, "pool"): + self.pool.stop() + self.finished = True + self.compose.log_info("[DONE ] %s" % self.msg) + + def run(self): + raise NotImplementedError diff --git a/pungi/phases/buildinstall.py b/pungi/phases/buildinstall.py new file mode 100644 index 00000000..758b072c --- /dev/null +++ b/pungi/phases/buildinstall.py @@ -0,0 +1,360 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import errno +import os +import time +import pipes +import tempfile +import shutil +import re +import errno + +from kobo.threads import ThreadPool, WorkerThread +from kobo.shortcuts import run, read_checksum_file, relative_path +from productmd.imagemanifest import Image + +from pypungi.util import get_buildroot_rpms, get_volid +from pypungi.wrappers.lorax import LoraxWrapper +from pypungi.wrappers.kojiwrapper import KojiWrapper +from pypungi.wrappers.iso import IsoWrapper +from pypungi.wrappers.scm import get_file_from_scm +from pypungi.phases.base import PhaseBase + + +class BuildinstallPhase(PhaseBase): + name = "buildinstall" + + config_options = ( + { + "name": "bootable", + "expected_types": [bool], + "expected_values": [True], + }, + { + "name": "buildinstall_method", + "extected_types": [str], + "expected_values": ["lorax", "buildinstall"], + "requires": ( + (lambda x: bool(x) is True, ["bootable"]), + ), + }, + { + "name": "buildinstall_upgrade_image", + "expected_types": [bool], + "optional": True, + }, + { + "name": "buildinstall_kickstart", + "expected_types": [str], + "optional": True, + }, + ) + + def __init__(self, compose): + PhaseBase.__init__(self, compose) + self.pool = ThreadPool(logger=self.compose._logger) + + def skip(self): + if PhaseBase.skip(self): + return True + if not self.compose.conf.get("bootable"): + msg = "Not a bootable product. Skipping buildinstall." + self.compose.log_debug(msg) + return True + return False + + def run(self): + lorax = LoraxWrapper() + product = self.compose.conf["product_name"] + version = self.compose.conf["product_version"] + release = self.compose.conf["product_version"] + noupgrade = not self.compose.conf.get("buildinstall_upgrade_image", False) + buildinstall_method = self.compose.conf["buildinstall_method"] + + for arch in self.compose.get_arches(): + repo_baseurl = self.compose.paths.work.arch_repo(arch) + output_dir = self.compose.paths.work.buildinstall_dir(arch) + volid = get_volid(self.compose, arch) + + if buildinstall_method == "lorax": + cmd = lorax.get_lorax_cmd(product, version, release, repo_baseurl, output_dir, is_final=self.compose.supported, buildarch=arch, volid=volid, nomacboot=True, noupgrade=noupgrade) + elif buildinstall_method == "buildinstall": + cmd = lorax.get_buildinstall_cmd(product, version, release, repo_baseurl, output_dir, is_final=self.compose.supported, buildarch=arch, volid=volid) + else: + raise ValueError("Unsupported buildinstall method: %s" % buildinstall_method) + self.pool.add(BuildinstallThread(self.pool)) + self.pool.queue_put((self.compose, arch, cmd)) + + self.pool.start() + + def copy_files(self): + # copy buildinstall files to the 'os' dir + kickstart_file = get_kickstart_file(self.compose) + for arch in self.compose.get_arches(): + for variant in self.compose.get_variants(arch=arch, types=["self", "variant"]): + buildinstall_dir = self.compose.paths.work.buildinstall_dir(arch) + if not os.path.isdir(buildinstall_dir) or not os.listdir(buildinstall_dir): + continue + + os_tree = self.compose.paths.compose.os_tree(arch, variant) + # TODO: label is not used + label = "" + volid = get_volid(self.compose, arch, variant, escape_spaces=False) + tweak_buildinstall(buildinstall_dir, os_tree, arch, variant.uid, label, volid, kickstart_file) + symlink_boot_iso(self.compose, arch, variant) + + +def get_kickstart_file(compose): + scm_dict = compose.conf.get("buildinstall_kickstart", None) + if not scm_dict: + compose.log_debug("Path to ks.cfg (buildinstall_kickstart) not specified.") + return + + msg = "Getting ks.cfg" + kickstart_path = os.path.join(compose.paths.work.topdir(arch="global"), "ks.cfg") + if os.path.exists(kickstart_path): + compose.log_warn("[SKIP ] %s" % msg) + return kickstart_path + + compose.log_info("[BEGIN] %s" % msg) + if isinstance(scm_dict, dict): + kickstart_name = os.path.basename(scm_dict["file"]) + if scm_dict["scm"] == "file": + scm_dict["file"] = os.path.join(compose.config_dir, scm_dict["file"]) + else: + kickstart_name = os.path.basename(scm_dict) + scm_dict = os.path.join(compose.config_dir, scm_dict) + + tmp_dir = tempfile.mkdtemp(prefix="buildinstall_kickstart_") + get_file_from_scm(scm_dict, tmp_dir, logger=compose._logger) + src = os.path.join(tmp_dir, kickstart_name) + shutil.copy2(src, kickstart_path) + compose.log_info("[DONE ] %s" % msg) + return kickstart_path + + +# HACK: this is a hack! +# * it's quite trivial to replace volids +# * it's not easy to replace menu titles +# * we probably need to get this into lorax +def tweak_buildinstall(src, dst, arch, variant, label, volid, kickstart_file=None): + volid_escaped = volid.replace(" ", r"\x20").replace("\\", "\\\\") + volid_escaped_2 = volid_escaped.replace("\\", "\\\\") + tmp_dir = tempfile.mkdtemp(prefix="tweak_buildinstall_") + + # verify src + if not os.path.isdir(src): + raise OSError(errno.ENOENT, "Directory does not exist: %s" % src) + + # create dst + try: + os.makedirs(dst) + except OSError as ex: + if ex.errno != errno.EEXIST: + raise + + # copy src to temp + # TODO: place temp on the same device as buildinstall dir so we can hardlink + cmd = "cp -av --remove-destination %s/* %s/" % (pipes.quote(src), pipes.quote(tmp_dir)) + run(cmd) + + # tweak configs + configs = [ + "isolinux/isolinux.cfg", + "etc/yaboot.conf", + "ppc/ppc64/yaboot.conf", + "EFI/BOOT/BOOTX64.conf", + "EFI/BOOT/grub.cfg", + ] + for config in configs: + config_path = os.path.join(tmp_dir, config) + if not os.path.exists(config_path): + continue + + data = open(config_path, "r").read() + os.unlink(config_path) # break hadlink by removing file writing a new one + + new_volid = volid_escaped + if "yaboot" in config: + # double-escape volid in yaboot.conf + new_volid = volid_escaped_2 + + ks = "" + if kickstart_file: + shutil.copy2(kickstart_file, os.path.join(dst, "ks.cfg")) + ks = " ks=hd:LABEL=%s:/ks.cfg" % new_volid + + # pre-f18 + data = re.sub(r":CDLABEL=[^ \n]*", r":CDLABEL=%s%s" % (new_volid, ks), data) + # f18+ + data = re.sub(r":LABEL=[^ \n]*", r":LABEL=%s%s" % (new_volid, ks), data) + data = re.sub(r"(search .* -l) '[^'\n]*'", r"\1 '%s'" % volid, data) + + open(config_path, "w").write(data) + + images = [ + os.path.join(tmp_dir, "images", "efiboot.img"), + ] + for image in images: + if not os.path.isfile(image): + continue + mount_tmp_dir = tempfile.mkdtemp(prefix="tweak_buildinstall") + cmd = ["mount", "-o", "loop", image, mount_tmp_dir] + run(cmd) + + for config in configs: + config_path = os.path.join(tmp_dir, config) + config_in_image = os.path.join(mount_tmp_dir, config) + + if os.path.isfile(config_in_image): + cmd = ["cp", "-v", "--remove-destination", config_path, config_in_image] + run(cmd) + + cmd = ["umount", mount_tmp_dir] + run(cmd) + shutil.rmtree(mount_tmp_dir) + + # HACK: make buildinstall files world readable + run("chmod -R a+rX %s" % pipes.quote(tmp_dir)) + + # copy temp to dst + cmd = "cp -av --remove-destination %s/* %s/" % (pipes.quote(tmp_dir), pipes.quote(dst)) + run(cmd) + + shutil.rmtree(tmp_dir) + + +def symlink_boot_iso(compose, arch, variant): + if arch == "src": + return + + symlink_isos_to = compose.conf.get("symlink_isos_to", None) + os_tree = compose.paths.compose.os_tree(arch, variant) + # TODO: find in treeinfo? + boot_iso_path = os.path.join(os_tree, "images", "boot.iso") + if not os.path.isfile(boot_iso_path): + return + + msg = "Symlinking boot.iso (arch: %s, variant: %s)" % (arch, variant) + new_boot_iso_path = compose.paths.compose.iso_path(arch, variant, disc_type="boot", disc_num=None, suffix=".iso", symlink_to=symlink_isos_to) + new_boot_iso_relative_path = compose.paths.compose.iso_path(arch, variant, disc_type="boot", disc_num=None, suffix=".iso", relative=True) + if os.path.exists(new_boot_iso_path): + # TODO: log + compose.log_warning("[SKIP ] %s" % msg) + return + + compose.log_info("[BEGIN] %s" % msg) + # can't make a hardlink - possible cross-device link due to 'symlink_to' argument + symlink_target = relative_path(boot_iso_path, new_boot_iso_path) + os.symlink(symlink_target, new_boot_iso_path) + + iso = IsoWrapper() + implant_md5 = iso.get_implanted_md5(new_boot_iso_path) + # compute md5sum, sha1sum, sha256sum + iso_name = os.path.basename(new_boot_iso_path) + iso_dir = os.path.dirname(new_boot_iso_path) + for cmd in iso.get_checksum_cmds(iso_name): + run(cmd, workdir=iso_dir) + + # create iso manifest + run(iso.get_manifest_cmd(iso_name), workdir=iso_dir) + + img = Image(compose.im) + img.implant_md5 = iso.get_implanted_md5(new_boot_iso_path) + img.path = new_boot_iso_relative_path + img.mtime = int(os.stat(new_boot_iso_path).st_mtime) + img.size = os.path.getsize(new_boot_iso_path) + img.arch = arch + img.type = "boot" + img.format = "iso" + img.disc_number = 1 + img.disc_count = 1 + for checksum_type in ("md5", "sha1", "sha256"): + checksum_path = new_boot_iso_path + ".%sSUM" % checksum_type.upper() + checksum_value = None + if os.path.isfile(checksum_path): + checksum_value, iso_name = read_checksum_file(checksum_path)[0] + if iso_name != os.path.basename(img.path): + # a bit paranoind check - this should never happen + raise ValueError("Image name doesn't match checksum: %s" % checksum_path) + img.add_checksum(compose.paths.compose.topdir(), checksum_type=checksum_type, checksum_value=checksum_value) + img.bootable = True + img.implant_md5 = implant_md5 + try: + img.volume_id = iso.get_volume_id(new_boot_iso_path) + except RuntimeError: + pass + compose.im.add(arch, variant.uid, img) + compose.log_info("[DONE ] %s" % msg) + + +class BuildinstallThread(WorkerThread): + def process(self, item, num): + compose, arch, cmd = item + runroot = compose.conf.get("runroot", False) + buildinstall_method = compose.conf["buildinstall_method"] + log_file = compose.paths.log.log_file(arch, "buildinstall") + + msg = "Runnging buildinstall for arch %s" % arch + + output_dir = compose.paths.work.buildinstall_dir(arch) + if os.path.isdir(output_dir): + if os.listdir(output_dir): + # output dir is *not* empty -> SKIP + self.pool.log_warning("[SKIP ] %s" % msg) + return + else: + # output dir is empty -> remove it and run buildinstall + self.pool.log_debug("Removing existing (but empty) buildinstall dir: %s" % output_dir) + os.rmdir(output_dir) + + self.pool.log_info("[BEGIN] %s" % msg) + + task_id = None + if runroot: + # run in a koji build root + # glibc32 is needed by yaboot on ppc64 + packages = ["glibc32", "strace"] + if buildinstall_method == "lorax": + packages += ["lorax"] + elif buildinstall_method == "buildinstall": + packages += ["anaconda"] + runroot_channel = compose.conf.get("runroot_channel", None) + runroot_tag = compose.conf["runroot_tag"] + + koji_wrapper = KojiWrapper(compose.conf["koji_profile"]) + koji_cmd = koji_wrapper.get_runroot_cmd(runroot_tag, arch, cmd, channel=runroot_channel, use_shell=True, task_id=True, packages=packages, mounts=[compose.topdir]) + + # avoid race conditions? + # Kerberos authentication failed: Permission denied in replay cache code (-1765328215) + time.sleep(num * 3) + + output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file) + task_id = int(output["task_id"]) + if output["retcode"] != 0: + raise RuntimeError("Runroot task failed: %s. See %s for more details." % (output["task_id"], log_file)) + + else: + # run locally + run(cmd, show_cmd=True, logfile=log_file) + + log_file = compose.paths.log.log_file(arch, "buildinstall-RPMs") + rpms = get_buildroot_rpms(compose, task_id) + open(log_file, "w").write("\n".join(rpms)) + + self.pool.log_info("[DONE ] %s" % msg) diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py new file mode 100644 index 00000000..cb439a1d --- /dev/null +++ b/pungi/phases/createiso.py @@ -0,0 +1,421 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import os +import time +import pipes +import random +import shutil + +import koji +import productmd.treeinfo +from productmd.imagemanifest import Image +from kobo.threads import ThreadPool, WorkerThread +from kobo.shortcuts import run, read_checksum_file, relative_path + +from pypungi.wrappers.iso import IsoWrapper +from pypungi.wrappers.createrepo import CreaterepoWrapper +from pypungi.wrappers.kojiwrapper import KojiWrapper +from pypungi.wrappers.jigdo import JigdoWrapper +from pypungi.phases.base import PhaseBase +from pypungi.util import makedirs, get_volid +from pypungi.media_split import MediaSplitter +from pypungi.compose_metadata.discinfo import read_discinfo, write_discinfo + + +class CreateisoPhase(PhaseBase): + name = "createiso" + + def __init__(self, compose): + PhaseBase.__init__(self, compose) + self.pool = ThreadPool(logger=self.compose._logger) + + def run(self): + iso = IsoWrapper(logger=self.compose._logger) + symlink_isos_to = self.compose.conf.get("symlink_isos_to", None) + + commands = [] + for variant in self.compose.get_variants(types=["variant", "layered-product", "optional"], recursive=True): + for arch in variant.arches + ["src"]: + volid = get_volid(self.compose, arch, variant) + os_tree = self.compose.paths.compose.os_tree(arch, variant) + + iso_dir = self.compose.paths.compose.iso_dir(arch, variant, symlink_to=symlink_isos_to) + if not iso_dir: + continue + + found = False + for root, dirs, files in os.walk(os_tree): + if found: + break + for fn in files: + if fn.endswith(".rpm"): + found = True + break + + if not found: + self.compose.log_warning("No RPMs found for %s.%s, skipping ISO" % (variant, arch)) + continue + + split_iso_data = split_iso(self.compose, arch, variant) + disc_count = len(split_iso_data) + + for disc_num, iso_data in enumerate(split_iso_data): + disc_num += 1 + + # XXX: hardcoded disc_type + iso_path = self.compose.paths.compose.iso_path(arch, variant, disc_type="dvd", disc_num=disc_num, symlink_to=symlink_isos_to) + relative_iso_path = self.compose.paths.compose.iso_path(arch, variant, disc_type="dvd", disc_num=disc_num, create_dir=False, relative=True) + if os.path.isfile(iso_path): + self.compose.log_warning("Skipping mkisofs, image already exists: %s" % iso_path) + continue + iso_name = os.path.basename(iso_path) + + graft_points = prepare_iso(self.compose, arch, variant, disc_num=disc_num, disc_count=disc_count, split_iso_data=iso_data) + + bootable = self.compose.conf.get("bootable", False) + if arch == "src": + bootable = False + if variant.type != "variant": + bootable = False + + cmd = { + "arch": arch, + "variant": variant, + "iso_path": iso_path, + "relative_iso_path": relative_iso_path, + "build_arch": arch, + "bootable": bootable, + "cmd": [], + "label": "", # currently not used + "disc_num": disc_num, + "disc_count": disc_count, + } + + if os.path.islink(iso_dir): + cmd["mount"] = os.path.abspath(os.path.join(os.path.dirname(iso_dir), os.readlink(iso_dir))) + + chdir_cmd = "cd %s" % pipes.quote(iso_dir) + cmd["cmd"].append(chdir_cmd) + + mkisofs_kwargs = {} + + if bootable: + buildinstall_method = self.compose.conf["buildinstall_method"] + if buildinstall_method == "lorax": + # TODO: $arch instead of ppc + mkisofs_kwargs["boot_args"] = iso.get_boot_options(arch, "/usr/share/lorax/config_files/ppc") + elif buildinstall_method == "buildinstall": + mkisofs_kwargs["boot_args"] = iso.get_boot_options(arch, "/usr/lib/anaconda-runtime/boot") + + # ppc(64) doesn't seem to support utf-8 + if arch in ("ppc", "ppc64", "ppc64le"): + mkisofs_kwargs["input_charset"] = None + + mkisofs_cmd = iso.get_mkisofs_cmd(iso_name, None, volid=volid, exclude=["./lost+found"], graft_points=graft_points, **mkisofs_kwargs) + mkisofs_cmd = " ".join([pipes.quote(i) for i in mkisofs_cmd]) + cmd["cmd"].append(mkisofs_cmd) + + if bootable and arch == "x86_64": + isohybrid_cmd = "isohybrid --uefi %s" % pipes.quote(iso_name) + cmd["cmd"].append(isohybrid_cmd) + elif bootable and arch == "i386": + isohybrid_cmd = "isohybrid %s" % pipes.quote(iso_name) + cmd["cmd"].append(isohybrid_cmd) + + # implant MD5SUM to iso + isomd5sum_cmd = iso.get_implantisomd5_cmd(iso_name, self.compose.supported) + isomd5sum_cmd = " ".join([pipes.quote(i) for i in isomd5sum_cmd]) + cmd["cmd"].append(isomd5sum_cmd) + + # compute md5sum, sha1sum, sha256sum + cmd["cmd"].extend(iso.get_checksum_cmds(iso_name)) + + # create iso manifest + cmd["cmd"].append(iso.get_manifest_cmd(iso_name)) + + # create jigdo + jigdo = JigdoWrapper(logger=self.compose._logger) + jigdo_dir = self.compose.paths.compose.jigdo_dir(arch, variant) + files = [ + { + "path": os_tree, + "label": None, + "uri": None, + } + ] + jigdo_cmd = jigdo.get_jigdo_cmd(iso_path, files, output_dir=jigdo_dir, no_servers=True, report="noprogress") + jigdo_cmd = " ".join([pipes.quote(i) for i in jigdo_cmd]) + cmd["cmd"].append(jigdo_cmd) + + cmd["cmd"] = " && ".join(cmd["cmd"]) + commands.append(cmd) + + for cmd in commands: + self.pool.add(CreateIsoThread(self.pool)) + self.pool.queue_put((self.compose, cmd)) + + self.pool.start() + + def stop(self, *args, **kwargs): + PhaseBase.stop(self, *args, **kwargs) + if self.skip(): + return + + +class CreateIsoThread(WorkerThread): + def fail(self, compose, cmd): + compose.log_error("CreateISO failed, removing ISO: %s" % cmd["iso_path"]) + try: + # remove incomplete ISO + os.unlink(cmd["iso_path"]) + # TODO: remove jigdo & template & checksums + except OSError: + pass + + def process(self, item, num): + compose, cmd = item + + mounts = [compose.topdir] + if "mount" in cmd: + mounts.append(cmd["mount"]) + + runroot = compose.conf.get("runroot", False) + bootable = compose.conf.get("bootable", False) + log_file = compose.paths.log.log_file(cmd["arch"], "createiso-%s" % os.path.basename(cmd["iso_path"])) + + msg = "Creating ISO (arch: %s, variant: %s): %s" % (cmd["arch"], cmd["variant"], os.path.basename(cmd["iso_path"])) + self.pool.log_info("[BEGIN] %s" % msg) + + if runroot: + # run in a koji build root + packages = ["coreutils", "genisoimage", "isomd5sum", "jigdo", "strace", "lsof"] + if bootable: + buildinstall_method = compose.conf["buildinstall_method"] + if buildinstall_method == "lorax": + packages += ["lorax"] + elif buildinstall_method == "buildinstall": + packages += ["anaconda"] + + runroot_channel = compose.conf.get("runroot_channel", None) + runroot_tag = compose.conf["runroot_tag"] + + # get info about build arches in buildroot_tag + koji_url = compose.conf["pkgset_koji_url"] + koji_proxy = koji.ClientSession(koji_url) + tag_info = koji_proxy.getTag(runroot_tag) + tag_arches = tag_info["arches"].split(" ") + + if not cmd["bootable"]: + if "x86_64" in tag_arches: + # assign non-bootable images to x86_64 if possible + cmd["build_arch"] = "x86_64" + elif cmd["build_arch"] == "src": + # pick random arch from available runroot tag arches + cmd["build_arch"] = random.choice(tag_arches) + + koji_wrapper = KojiWrapper(compose.conf["koji_profile"]) + koji_cmd = koji_wrapper.get_runroot_cmd(runroot_tag, cmd["build_arch"], cmd["cmd"], channel=runroot_channel, use_shell=True, task_id=True, packages=packages, mounts=mounts) + + # avoid race conditions? + # Kerberos authentication failed: Permission denied in replay cache code (-1765328215) + time.sleep(num * 3) + + output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file) + if output["retcode"] != 0: + self.fail(compose, cmd) + raise RuntimeError("Runroot task failed: %s. See %s for more details." % (output["task_id"], log_file)) + + else: + # run locally + try: + run(cmd["cmd"], show_cmd=True, logfile=log_file) + except: + self.fail(compose, cmd) + raise + + iso = IsoWrapper() + + img = Image(compose.im) + img.path = cmd["relative_iso_path"] + img.mtime = int(os.stat(cmd["iso_path"]).st_mtime) + img.size = os.path.getsize(cmd["iso_path"]) + img.arch = cmd["arch"] + # XXX: HARDCODED + img.type = "dvd" + img.format = "iso" + img.disc_number = cmd["disc_num"] + img.disc_count = cmd["disc_count"] + for checksum_type in ("md5", "sha1", "sha256"): + checksum_path = cmd["iso_path"] + ".%sSUM" % checksum_type.upper() + checksum_value = None + if os.path.isfile(checksum_path): + checksum_value, iso_name = read_checksum_file(checksum_path)[0] + if iso_name != os.path.basename(img.path): + # a bit paranoind check - this should never happen + raise ValueError("Image name doesn't match checksum: %s" % checksum_path) + img.add_checksum(compose.paths.compose.topdir(), checksum_type=checksum_type, checksum_value=checksum_value) + img.bootable = cmd["bootable"] + img.implant_md5 = iso.get_implanted_md5(cmd["iso_path"]) + try: + img.volume_id = iso.get_volume_id(cmd["iso_path"]) + except RuntimeError: + pass + compose.im.add(cmd["arch"], cmd["variant"].uid, img) + # TODO: supported_iso_bit + # add: boot.iso + + self.pool.log_info("[DONE ] %s" % msg) + + +def split_iso(compose, arch, variant): + # XXX: hardcoded + media_size = 4700000000 + media_reserve = 10 * 1024 * 1024 + + ms = MediaSplitter(str(media_size - media_reserve)) + + os_tree = compose.paths.compose.os_tree(arch, variant) + extra_files_dir = compose.paths.work.extra_files_dir(arch, variant) + +# ti_path = os.path.join(os_tree, ".treeinfo") +# ti = productmd.treeinfo.TreeInfo() +# ti.load(ti_path) + + # scan extra files to mark them "sticky" -> they'll be on all media after split + extra_files = set() + for root, dirs, files in os.walk(extra_files_dir): + for fn in files: + path = os.path.join(root, fn) + rel_path = relative_path(path, extra_files_dir.rstrip("/") + "/") + extra_files.add(rel_path) + + packages = [] + all_files = [] + all_files_ignore = [] + + ti = productmd.treeinfo.TreeInfo() + ti.load(os.path.join(os_tree, ".treeinfo")) + boot_iso_rpath = ti.images.images.get(arch, {}).get("boot.iso", None) + if boot_iso_rpath: + all_files_ignore.append(boot_iso_rpath) + compose.log_debug("split_iso all_files_ignore = %s" % ", ".join(all_files_ignore)) + + for root, dirs, files in os.walk(os_tree): + for dn in dirs[:]: + repo_dir = os.path.join(root, dn) + if repo_dir == os.path.join(compose.paths.compose.repository(arch, variant), "repodata"): + dirs.remove(dn) + + for fn in files: + path = os.path.join(root, fn) + rel_path = relative_path(path, os_tree.rstrip("/") + "/") + sticky = rel_path in extra_files + if rel_path in all_files_ignore: + compose.log_info("split_iso: Skipping %s" % rel_path) + continue + if root == compose.paths.compose.packages(arch, variant): + packages.append((path, os.path.getsize(path), sticky)) + else: + all_files.append((path, os.path.getsize(path), sticky)) + + for path, size, sticky in all_files + packages: + ms.add_file(path, size, sticky) + + return ms.split() + + +def prepare_iso(compose, arch, variant, disc_num=1, disc_count=None, split_iso_data=None): + tree_dir = compose.paths.compose.os_tree(arch, variant) + iso_dir = compose.paths.work.iso_dir(arch, variant, disc_num=disc_num) + + # modify treeinfo + ti_path = os.path.join(tree_dir, ".treeinfo") + ti = productmd.treeinfo.TreeInfo() + ti.load(ti_path) + ti.media.totaldiscs = disc_count or 1 + ti.media.discnum = disc_num + + # remove boot.iso from all sections + paths = set() + for platform in ti.images.images: + if "boot.iso" in ti.images.images[platform]: + paths.add(ti.images.images[platform].pop("boot.iso")) + + # remove boot.iso from checksums + for i in paths: + if i in ti.checksums.checksums.keys(): + del ti.checksums.checksums[i] + + # make a copy of isolinux/isolinux.bin, images/boot.img - they get modified when mkisofs is called + for i in ("isolinux/isolinux.bin", "images/boot.img"): + src_path = os.path.join(tree_dir, i) + dst_path = os.path.join(iso_dir, i) + if os.path.exists(src_path): + makedirs(os.path.dirname(dst_path)) + shutil.copy2(src_path, dst_path) + + if disc_count > 1: + # remove repodata/repomd.xml from checksums, create a new one later + if "repodata/repomd.xml" in ti.checksums.checksums: + del ti.checksums.checksums["repodata/repomd.xml"] + + # rebuild repodata + createrepo_c = compose.conf.get("createrepo_c", False) + createrepo_checksum = compose.conf.get("createrepo_checksum", None) + repo = CreaterepoWrapper(createrepo_c=createrepo_c) + + file_list = "%s-file-list" % iso_dir + packages_dir = compose.paths.compose.packages(arch, variant) + file_list_content = [] + for i in split_iso_data["files"]: + if not i.endswith(".rpm"): + continue + if not i.startswith(packages_dir): + continue + rel_path = relative_path(i, tree_dir.rstrip("/") + "/") + file_list_content.append(rel_path) + + if file_list_content: + # write modified repodata only if there are packages available + run("cp -a %s/repodata %s/" % (pipes.quote(tree_dir), pipes.quote(iso_dir))) + open(file_list, "w").write("\n".join(file_list_content)) + cmd = repo.get_createrepo_cmd(tree_dir, update=True, database=True, skip_stat=True, pkglist=file_list, outputdir=iso_dir, workers=3, checksum=createrepo_checksum) + run(cmd) + # add repodata/repomd.xml back to checksums + ti.checksums.add(iso_dir, "repodata/repomd.xml") + + new_ti_path = os.path.join(iso_dir, ".treeinfo") + ti.dump(new_ti_path) + + # modify discinfo + di_path = os.path.join(tree_dir, ".discinfo") + data = read_discinfo(di_path) + data["disc_numbers"] = [disc_num] + new_di_path = os.path.join(iso_dir, ".discinfo") + write_discinfo(new_di_path, **data) + + i = IsoWrapper() + if not disc_count or disc_count == 1: + data = i.get_graft_points([tree_dir, iso_dir]) + else: + data = i.get_graft_points([i._paths_from_list(tree_dir, split_iso_data["files"]), iso_dir]) + + # TODO: /content /graft-points + gp = "%s-graft-points" % iso_dir + i.write_graft_points(gp, data, exclude=["*/lost+found", "*/boot.iso"]) + return gp diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py new file mode 100644 index 00000000..861d2d21 --- /dev/null +++ b/pungi/phases/createrepo.py @@ -0,0 +1,205 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +__all__ = ( + "create_variant_repo", +) + + +import os +import glob +import shutil +import tempfile +import threading + +from kobo.threads import ThreadPool, WorkerThread +from kobo.shortcuts import run, relative_path + +from pypungi.wrappers.scm import get_dir_from_scm +from pypungi.wrappers.createrepo import CreaterepoWrapper +from pypungi.phases.base import PhaseBase + + +createrepo_lock = threading.Lock() +createrepo_dirs = set() + + +class CreaterepoPhase(PhaseBase): + name = "createrepo" + + config_options = ( + { + "name": "createrepo_c", + "expected_types": [bool], + "optional": True, + }, + { + "name": "createrepo_checksum", + "expected_types": [bool], + "optional": True, + }, + { + "name": "product_id", + "expected_types": [dict], + "optional": True, + }, + { + "name": "product_id_allow_missing", + "expected_types": [bool], + "optional": True, + }, + ) + + def __init__(self, compose): + PhaseBase.__init__(self, compose) + self.pool = ThreadPool(logger=self.compose._logger) + + def run(self): + get_productids_from_scm(self.compose) + for i in range(3): + self.pool.add(CreaterepoThread(self.pool)) + + for arch in self.compose.get_arches(): + for variant in self.compose.get_variants(arch=arch): + self.pool.queue_put((self.compose, arch, variant, "rpm")) + self.pool.queue_put((self.compose, arch, variant, "debuginfo")) + + for variant in self.compose.get_variants(): + self.pool.queue_put((self.compose, None, variant, "srpm")) + + self.pool.start() + + +def create_variant_repo(compose, arch, variant, pkg_type): + createrepo_c = compose.conf.get("createrepo_c", False) + createrepo_checksum = compose.conf.get("createrepo_checksum", None) + repo = CreaterepoWrapper(createrepo_c=createrepo_c) + if pkg_type == "srpm": + repo_dir_arch = compose.paths.work.arch_repo(arch="global") + else: + repo_dir_arch = compose.paths.work.arch_repo(arch=arch) + + if pkg_type == "rpm": + repo_dir = compose.paths.compose.repository(arch=arch, variant=variant) + package_dir = compose.paths.compose.packages(arch, variant) + elif pkg_type == "srpm": + repo_dir = compose.paths.compose.repository(arch="src", variant=variant) + package_dir = compose.paths.compose.packages("src", variant) + elif pkg_type == "debuginfo": + repo_dir = compose.paths.compose.debug_repository(arch=arch, variant=variant) + package_dir = compose.paths.compose.debug_packages(arch, variant) + else: + raise ValueError("Unknown package type: %s" % pkg_type) + + if not repo_dir: + return + + msg = "Creating repo (arch: %s, variant: %s): %s" % (arch, variant, repo_dir) + + # HACK: using global lock + createrepo_lock.acquire() + if repo_dir in createrepo_dirs: + compose.log_warning("[SKIP ] Already in progress: %s" % msg) + createrepo_lock.release() + return + createrepo_dirs.add(repo_dir) + createrepo_lock.release() + + if compose.DEBUG and os.path.isdir(os.path.join(repo_dir, "repodata")): + compose.log_warning("[SKIP ] %s" % msg) + return + + compose.log_info("[BEGIN] %s" % msg) + + file_list = None + if repo_dir != package_dir: + rel_dir = relative_path(package_dir.rstrip("/") + "/", repo_dir.rstrip("/") + "/") + file_list = compose.paths.work.repo_package_list(arch, variant, pkg_type) + f = open(file_list, "w") + for i in os.listdir(package_dir): + if i.endswith(".rpm"): + f.write("%s\n" % os.path.join(rel_dir, i)) + f.close() + + comps_path = None + if compose.has_comps and pkg_type == "rpm": + comps_path = compose.paths.work.comps(arch=arch, variant=variant) + cmd = repo.get_createrepo_cmd(repo_dir, update=True, database=True, skip_stat=True, pkglist=file_list, outputdir=repo_dir, workers=3, groupfile=comps_path, update_md_path=repo_dir_arch, checksum=createrepo_checksum) +# cmd.append("-vvv") + log_file = compose.paths.log.log_file(arch, "createrepo-%s" % variant) + run(cmd, logfile=log_file, show_cmd=True) + + # call modifyrepo to inject productid + product_id = compose.conf.get("product_id") + if product_id and pkg_type == "rpm": + # add product certificate to base (rpm) repo; skip source and debug + product_id_path = compose.paths.work.product_id(arch, variant) + if os.path.isfile(product_id_path): + cmd = repo.get_modifyrepo_cmd(os.path.join(repo_dir, "repodata"), product_id_path, compress_type="gz") + log_file = compose.paths.log.log_file(arch, "modifyrepo-%s" % variant) + run(cmd, logfile=log_file, show_cmd=True) + # productinfo is not supported by modifyrepo in any way + # this is a HACK to make CDN happy (dmach: at least I think, need to confirm with dgregor) + shutil.copy2(product_id_path, os.path.join(repo_dir, "repodata", "productid")) + + compose.log_info("[DONE ] %s" % msg) + + +class CreaterepoThread(WorkerThread): + def process(self, item, num): + compose, arch, variant, pkg_type = item + create_variant_repo(compose, arch, variant, pkg_type=pkg_type) + + +def get_productids_from_scm(compose): + # product_id is a scm_dict: {scm, repo, branch, dir} + # expected file name format: $variant_uid-$arch-*.pem + product_id = compose.conf.get("product_id") + if not product_id: + compose.log_info("No product certificates specified") + return + + product_id_allow_missing = compose.conf.get("product_id_allow_missing", False) + + msg = "Getting product certificates from SCM..." + compose.log_info("[BEGIN] %s" % msg) + + tmp_dir = tempfile.mkdtemp(prefix="pungi_") + get_dir_from_scm(product_id, tmp_dir) + + for arch in compose.get_arches(): + for variant in compose.get_variants(arch=arch): + # some layered products may use base product name before variant + pem_files = glob.glob("%s/*%s-%s-*.pem" % (tmp_dir, variant.uid, arch)) + # use for development: + # pem_files = glob.glob("%s/*.pem" % tmp_dir)[-1:] + if not pem_files: + msg = "No product certificate found (arch: %s, variant: %s)" % (arch, variant.uid) + if product_id_allow_missing: + compose.log_warning(msg) + continue + else: + shutil.rmtree(tmp_dir) + raise RuntimeError(msg) + if len(pem_files) > 1: + shutil.rmtree(tmp_dir) + raise RuntimeError("Multiple product certificates found (arch: %s, variant: %s): %s" % (arch, variant.uid, ", ".join(sorted([os.path.basename(i) for i in pem_files])))) + product_id_path = compose.paths.work.product_id(arch, variant) + shutil.copy2(pem_files[0], product_id_path) + + shutil.rmtree(tmp_dir) + compose.log_info("[DONE ] %s" % msg) diff --git a/pungi/phases/extra_files.py b/pungi/phases/extra_files.py new file mode 100644 index 00000000..3c73e2a3 --- /dev/null +++ b/pungi/phases/extra_files.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import os +import copy +import fnmatch +import pipes + +from kobo.shortcuts import run + +from pypungi.util import get_arch_variant_data, pkg_is_rpm +from pypungi.arch import split_name_arch +from pypungi.wrappers.scm import get_file_from_scm, get_dir_from_scm +from pypungi.phases.base import PhaseBase + + +class ExtraFilesPhase(PhaseBase): + """EXTRA_FILES""" + name = "extra_files" + + config_options = ( + { + "name": "extra_files", + "expected_types": [list], + "optional": True + }, + ) + + def __init__(self, compose, pkgset_phase): + PhaseBase.__init__(self, compose) + # pkgset_phase provides package_sets and path_prefix + self.pkgset_phase = pkgset_phase + + def run(self): + for arch in self.compose.get_arches() + ["src"]: + for variant in self.compose.get_variants(arch=arch): + copy_extra_files(self.compose, arch, variant, self.pkgset_phase.package_sets) + + +def copy_extra_files(compose, arch, variant, package_sets): + if "extra_files" not in compose.conf: + return + + var_dict = { + "arch": arch, + "variant_id": variant.id, + "variant_id_lower": variant.id.lower(), + "variant_uid": variant.uid, + "variant_uid_lower": variant.uid.lower(), + } + + msg = "Getting extra files (arch: %s, variant: %s)" % (arch, variant) + # no skip (yet?) + compose.log_info("[BEGIN] %s" % msg) + + os_tree = compose.paths.compose.os_tree(arch, variant) + extra_files_dir = compose.paths.work.extra_files_dir(arch, variant) + + for scm_dict in get_arch_variant_data(compose.conf, "extra_files", arch, variant): + scm_dict = copy.deepcopy(scm_dict) + # if scm is "rpm" and repo contains a package name, find the package(s) in package set + if scm_dict["scm"] == "rpm" and not (scm_dict["repo"].startswith("/") or "://" in scm_dict["repo"]): + rpms = [] + for pkgset_file in package_sets[arch]: + pkg_obj = package_sets[arch][pkgset_file] + if not pkg_is_rpm(pkg_obj): + continue + pkg_name, pkg_arch = split_name_arch(scm_dict["repo"] % var_dict) + if fnmatch.fnmatch(pkg_obj.name, pkg_name) and pkg_arch is None or pkg_arch == pkg_obj.arch: + rpms.append(pkg_obj.file_path) + scm_dict["repo"] = rpms + + if "file" in scm_dict: + get_file_from_scm(scm_dict, os.path.join(extra_files_dir, scm_dict.get("target", "").lstrip("/")), logger=compose._logger) + else: + get_dir_from_scm(scm_dict, os.path.join(extra_files_dir, scm_dict.get("target", "").lstrip("/")), logger=compose._logger) + + if os.listdir(extra_files_dir): + cmd = "cp -av --remove-destination %s/* %s/" % (pipes.quote(extra_files_dir), pipes.quote(os_tree)) + run(cmd) + + compose.log_info("[DONE ] %s" % msg) diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py new file mode 100644 index 00000000..d93dbee7 --- /dev/null +++ b/pungi/phases/gather/__init__.py @@ -0,0 +1,515 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import os +import tempfile +import shutil +import json + +from kobo.rpmlib import parse_nvra +from productmd import RpmManifest + +from pypungi.wrappers.scm import get_file_from_scm +from link import link_files + +from pypungi.util import get_arch_variant_data, get_arch_data +from pypungi.phases.base import PhaseBase +from pypungi.arch import split_name_arch, get_compatible_arches + + +def get_gather_source(name): + import pypungi.phases.gather.sources + from source import GatherSourceContainer + GatherSourceContainer.register_module(pypungi.phases.gather.sources) + container = GatherSourceContainer() + return container["GatherSource%s" % name] + + +def get_gather_method(name): + import pypungi.phases.gather.methods + from method import GatherMethodContainer + GatherMethodContainer.register_module(pypungi.phases.gather.methods) + container = GatherMethodContainer() + return container["GatherMethod%s" % name] + + +class GatherPhase(PhaseBase): + """GATHER""" + name = "gather" + + config_options = ( + { + "name": "multilib_arches", + "expected_types": [list], + "optional": True, + }, + { + "name": "gather_lookaside_repos", + "expected_types": [list], + "optional": True, + }, + { + "name": "multilib_methods", + "expected_types": [list], + }, + { + "name": "greedy_method", + "expected_values": ["none", "all", "build"], + "optional": True, + }, + { + "name": "gather_fulltree", + "expected_types": [bool], + "optional": True, + }, + { + "name": "gather_prepopulate", + "expected_types": [str, dict], + "optional": True, + }, + # DEPRECATED OPTIONS + { + "name": "additional_packages_multiarch", + "deprecated": True, + "comment": "Use multilib_whitelist instead", + }, + { + "name": "filter_packages_multiarch", + "deprecated": True, + "comment": "Use multilib_blacklist instead", + }, + ) + + def __init__(self, compose, pkgset_phase): + PhaseBase.__init__(self, compose) + # pkgset_phase provides package_sets and path_prefix + self.pkgset_phase = pkgset_phase + + @staticmethod + def check_deps(): + pass + + def check_config(self): + errors = [] + for i in ["product_name", "product_short", "product_version"]: + errors.append(self.conf_assert_str(i)) + + def run(self): + pkg_map = gather_wrapper(self.compose, self.pkgset_phase.package_sets, self.pkgset_phase.path_prefix) + + manifest_file = self.compose.paths.compose.metadata("rpms.json") + manifest = RpmManifest() + manifest.compose.id = self.compose.compose_id + manifest.compose.type = self.compose.compose_type + manifest.compose.date = self.compose.compose_date + manifest.compose.respin = self.compose.compose_respin + + for arch in self.compose.get_arches(): + for variant in self.compose.get_variants(arch=arch): + link_files(self.compose, arch, variant, pkg_map[arch][variant.uid], self.pkgset_phase.package_sets, manifest=manifest) + + self.compose.log_info("Writing RPM manifest: %s" % manifest_file) + manifest.dump(manifest_file) + + +def get_parent_pkgs(arch, variant, result_dict): + result = { + "rpm": set(), + "srpm": set(), + "debuginfo": set(), + } + if variant.parent is None: + return result + for pkg_type, pkgs in result_dict.get(arch, {}).get(variant.parent.uid, {}).iteritems(): + for pkg in pkgs: + nvra = parse_nvra(pkg["path"]) + result[pkg_type].add((nvra["name"], nvra["arch"])) + return result + + +def gather_packages(compose, arch, variant, package_sets, fulltree_excludes=None): + # multilib is per-arch, common for all variants + multilib_whitelist = get_multilib_whitelist(compose, arch) + multilib_blacklist = get_multilib_blacklist(compose, arch) + GatherMethod = get_gather_method(compose.conf["gather_method"]) + + msg = "Gathering packages (arch: %s, variant: %s)" % (arch, variant) + compose.log_info("[BEGIN] %s" % msg) + + packages, groups, filter_packages = get_variant_packages(compose, arch, variant, package_sets) + prepopulate = get_prepopulate_packages(compose, arch, variant) + fulltree_excludes = fulltree_excludes or set() + + method = GatherMethod(compose) + pkg_map = method(arch, variant, packages, groups, filter_packages, multilib_whitelist, multilib_blacklist, package_sets, fulltree_excludes=fulltree_excludes, prepopulate=prepopulate) + + compose.log_info("[DONE ] %s" % msg) + return pkg_map + + +def write_packages(compose, arch, variant, pkg_map, path_prefix): + msg = "Writing package list (arch: %s, variant: %s)" % (arch, variant) + compose.log_info("[BEGIN] %s" % msg) + + for pkg_type, pkgs in pkg_map.iteritems(): + file_name = compose.paths.work.package_list(arch=arch, variant=variant, pkg_type=pkg_type) + pkg_list = open(file_name, "w") + for pkg in pkgs: + # TODO: flags? + pkg_path = pkg["path"] + if pkg_path.startswith(path_prefix): + pkg_path = pkg_path[len(path_prefix):] + pkg_list.write("%s\n" % pkg_path) + pkg_list.close() + + compose.log_info("[DONE ] %s" % msg) + + +def trim_packages(compose, arch, variant, pkg_map, parent_pkgs=None, remove_pkgs=None): + """Remove parent variant's packages from pkg_map <-- it gets modified in this function""" + # TODO: remove debuginfo and srpm leftovers + + if not variant.parent: + return + + msg = "Trimming package list (arch: %s, variant: %s)" % (arch, variant) + compose.log_info("[BEGIN] %s" % msg) + + remove_pkgs = remove_pkgs or {} + parent_pkgs = parent_pkgs or {} + + addon_pkgs = {} + move_to_parent_pkgs = {} + removed_pkgs = {} + for pkg_type, pkgs in pkg_map.iteritems(): + addon_pkgs.setdefault(pkg_type, set()) + move_to_parent_pkgs.setdefault(pkg_type, []) + removed_pkgs.setdefault(pkg_type, []) + + new_pkgs = [] + for pkg in pkgs: + pkg_path = pkg["path"] + if not pkg_path: + continue + nvra = parse_nvra(pkg_path) + key = ((nvra["name"], nvra["arch"])) + + if nvra["name"] in remove_pkgs.get(pkg_type, set()): + # TODO: make an option to turn this off + if variant.type == "layered-product" and pkg_type in ("srpm", "debuginfo"): + new_pkgs.append(pkg) + # User may not have addons available, therefore we need to + # keep addon SRPMs in layered products in order not to violate GPL. + # The same applies on debuginfo availability. + continue + compose.log_warning("Removed addon package (arch: %s, variant: %s): %s: %s" % (arch, variant, pkg_type, pkg_path)) + removed_pkgs[pkg_type].append(pkg) + elif key not in parent_pkgs.get(pkg_type, set()): + if "input" in pkg["flags"]: + new_pkgs.append(pkg) + addon_pkgs[pkg_type].add(nvra["name"]) + elif "fulltree-exclude" in pkg["flags"]: + # if a package wasn't explicitly included ('input') in an addon, + # move it to parent variant (cannot move it to optional, because addons can't depend on optional) + # this is a workaround for not having $addon-optional + move_to_parent_pkgs[pkg_type].append(pkg) + else: + new_pkgs.append(pkg) + addon_pkgs[pkg_type].add(nvra["name"]) + else: + removed_pkgs[pkg_type].append(pkg) + + pkgs[:] = new_pkgs + compose.log_info("Removed packages (arch: %s, variant: %s): %s: %s" % (arch, variant, pkg_type, len(removed_pkgs[pkg_type]))) + compose.log_info("Moved to parent (arch: %s, variant: %s): %s: %s" % (arch, variant, pkg_type, len(move_to_parent_pkgs[pkg_type]))) + + compose.log_info("[DONE ] %s" % msg) + return addon_pkgs, move_to_parent_pkgs, removed_pkgs + + +def gather_wrapper(compose, package_sets, path_prefix): + result = {} + + # gather packages: variants + for arch in compose.get_arches(): + for variant in compose.get_variants(arch=arch, types=["variant"]): + fulltree_excludes = set() + pkg_map = gather_packages(compose, arch, variant, package_sets, fulltree_excludes=fulltree_excludes) + result.setdefault(arch, {})[variant.uid] = pkg_map + + # gather packages: addons + for arch in compose.get_arches(): + for variant in compose.get_variants(arch=arch, types=["addon"]): + fulltree_excludes = set() + for pkg_name, pkg_arch in get_parent_pkgs(arch, variant, result)["srpm"]: + fulltree_excludes.add(pkg_name) + pkg_map = gather_packages(compose, arch, variant, package_sets, fulltree_excludes=fulltree_excludes) + result.setdefault(arch, {})[variant.uid] = pkg_map + + # gather packages: layered-products + # NOTE: the same code as for addons + for arch in compose.get_arches(): + for variant in compose.get_variants(arch=arch, types=["layered-product"]): + fulltree_excludes = set() + for pkg_name, pkg_arch in get_parent_pkgs(arch, variant, result)["srpm"]: + fulltree_excludes.add(pkg_name) + pkg_map = gather_packages(compose, arch, variant, package_sets, fulltree_excludes=fulltree_excludes) + result.setdefault(arch, {})[variant.uid] = pkg_map + + # gather packages: optional + # NOTE: the same code as for variants + for arch in compose.get_arches(): + for variant in compose.get_variants(arch=arch, types=["optional"]): + fulltree_excludes = set() + pkg_map = gather_packages(compose, arch, variant, package_sets, fulltree_excludes=fulltree_excludes) + result.setdefault(arch, {})[variant.uid] = pkg_map + + # trim packages: addons + all_addon_pkgs = {} + for arch in compose.get_arches(): + for variant in compose.get_variants(arch=arch, types=["addon"]): + pkg_map = result[arch][variant.uid] + parent_pkgs = get_parent_pkgs(arch, variant, result) + addon_pkgs, move_to_parent_pkgs, removed_pkgs = trim_packages(compose, arch, variant, pkg_map, parent_pkgs) + + # update all_addon_pkgs + for pkg_type, pkgs in addon_pkgs.iteritems(): + all_addon_pkgs.setdefault(pkg_type, set()).update(pkgs) + + # move packages to parent + parent_pkg_map = result[arch][variant.parent.uid] + for pkg_type, pkgs in move_to_parent_pkgs.iteritems(): + for pkg in pkgs: + compose.log_debug("Moving package to parent (arch: %s, variant: %s, pkg_type: %s): %s" % (arch, variant.uid, pkg_type, os.path.basename(pkg["path"]))) + if pkg not in parent_pkg_map[pkg_type]: + parent_pkg_map[pkg_type].append(pkg) + + # trim packages: layered-products + all_lp_pkgs = {} + for arch in compose.get_arches(): + for variant in compose.get_variants(arch=arch, types=["layered-product"]): + pkg_map = result[arch][variant.uid] + parent_pkgs = get_parent_pkgs(arch, variant, result) + lp_pkgs, move_to_parent_pkgs, removed_pkgs = trim_packages(compose, arch, variant, pkg_map, parent_pkgs, remove_pkgs=all_addon_pkgs) + + # update all_addon_pkgs + for pkg_type, pkgs in lp_pkgs.iteritems(): + all_lp_pkgs.setdefault(pkg_type, set()).update(pkgs) + + # move packages to parent + # XXX: do we really want this? + parent_pkg_map = result[arch][variant.parent.uid] + for pkg_type, pkgs in move_to_parent_pkgs.iteritems(): + for pkg in pkgs: + compose.log_debug("Moving package to parent (arch: %s, variant: %s, pkg_type: %s): %s" % (arch, variant.uid, pkg_type, os.path.basename(pkg["path"]))) + if pkg not in parent_pkg_map[pkg_type]: + parent_pkg_map[pkg_type].append(pkg) + + # merge all_addon_pkgs with all_lp_pkgs + for pkg_type in set(all_addon_pkgs.keys()) | set(all_lp_pkgs.keys()): + all_addon_pkgs.setdefault(pkg_type, set()).update(all_lp_pkgs.get(pkg_type, set())) + + # trim packages: variants + for arch in compose.get_arches(): + for variant in compose.get_variants(arch=arch, types=["optional"]): + pkg_map = result[arch][variant.uid] + addon_pkgs, move_to_parent_pkgs, removed_pkgs = trim_packages(compose, arch, variant, pkg_map, remove_pkgs=all_addon_pkgs) + + # trim packages: optional + for arch in compose.get_arches(): + for variant in compose.get_variants(arch=arch, types=["optional"]): + pkg_map = result[arch][variant.uid] + parent_pkgs = get_parent_pkgs(arch, variant, result) + addon_pkgs, move_to_parent_pkgs, removed_pkgs = trim_packages(compose, arch, variant, pkg_map, parent_pkgs, remove_pkgs=all_addon_pkgs) + + # write packages (package lists) for all variants + for arch in compose.get_arches(): + for variant in compose.get_variants(arch=arch, recursive=True): + pkg_map = result[arch][variant.uid] + write_packages(compose, arch, variant, pkg_map, path_prefix=path_prefix) + + return result + + +def write_prepopulate_file(compose): + if not compose.conf.get("gather_prepopulate", None): + return + + prepopulate_file = os.path.join(compose.paths.work.topdir(arch="global"), "prepopulate.json") + msg = "Writing prepopulate file: %s" % prepopulate_file + + if compose.DEBUG and os.path.isfile(prepopulate_file): + compose.log_warning("[SKIP ] %s" % msg) + else: + scm_dict = compose.conf["gather_prepopulate"] + if isinstance(scm_dict, dict): + file_name = os.path.basename(scm_dict["file"]) + if scm_dict["scm"] == "file": + scm_dict["file"] = os.path.join(compose.config_dir, os.path.basename(scm_dict["file"])) + else: + file_name = os.path.basename(scm_dict) + scm_dict = os.path.join(compose.config_dir, os.path.basename(scm_dict)) + + compose.log_debug(msg) + tmp_dir = tempfile.mkdtemp(prefix="prepopulate_file_") + get_file_from_scm(scm_dict, tmp_dir, logger=compose._logger) + shutil.copy2(os.path.join(tmp_dir, file_name), prepopulate_file) + shutil.rmtree(tmp_dir) + + +def get_prepopulate_packages(compose, arch, variant): + result = set() + + prepopulate_file = os.path.join(compose.paths.work.topdir(arch="global"), "prepopulate.json") + if not os.path.isfile(prepopulate_file): + return result + + prepopulate_data = json.load(open(prepopulate_file, "r")) + + if variant: + variants = [variant.uid] + else: + # ALL variants + variants = prepopulate_data.keys() + + for var in variants: + for build, packages in prepopulate_data.get(var, {}).get(arch, {}).iteritems(): + for i in packages: + pkg_name, pkg_arch = split_name_arch(i) + if pkg_arch not in get_compatible_arches(arch, multilib=True): + raise ValueError("Incompatible package arch '%s' for tree arch '%s'" % (pkg_arch, arch)) + result.add(i) + return result + + +def get_additional_packages(compose, arch, variant): + result = set() + for i in get_arch_variant_data(compose.conf, "additional_packages", arch, variant): + pkg_name, pkg_arch = split_name_arch(i) + if pkg_arch is not None and pkg_arch not in get_compatible_arches(arch, multilib=True): + raise ValueError("Incompatible package arch '%s' for tree arch '%s'" % (pkg_arch, arch)) + result.add((pkg_name, pkg_arch)) + return result + + +def get_filter_packages(compose, arch, variant): + result = set() + for i in get_arch_variant_data(compose.conf, "filter_packages", arch, variant): + result.add(split_name_arch(i)) + return result + + +def get_multilib_whitelist(compose, arch): + return set(get_arch_data(compose.conf, "multilib_whitelist", arch)) + + +def get_multilib_blacklist(compose, arch): + return set(get_arch_data(compose.conf, "multilib_blacklist", arch)) + + +def get_lookaside_repos(compose, arch, variant): + return get_arch_variant_data(compose.conf, "gather_lookaside_repos", arch, variant) + + +def get_variant_packages(compose, arch, variant, package_sets=None): + GatherSource = get_gather_source(compose.conf["gather_source"]) + source = GatherSource(compose) + packages, groups = source(arch, variant) +# if compose.conf["gather_source"] == "comps": +# packages = set() + filter_packages = set() + + # no variant -> no parent -> we have everything we need + # doesn't make sense to do any package filtering + if variant is None: + return packages, groups, filter_packages + + packages |= get_additional_packages(compose, arch, variant) + filter_packages |= get_filter_packages(compose, arch, variant) + + system_release_packages, system_release_filter_packages = get_system_release_packages(compose, arch, variant, package_sets) + packages |= system_release_packages + filter_packages |= system_release_filter_packages + + # if the variant is "optional", include all groups and packages + # from the main "variant" and all "addons" + if variant.type == "optional": + for var in variant.parent.get_variants(arch=arch, types=["self", "variant", "addon", "layered-product"]): + var_packages, var_groups, var_filter_packages = get_variant_packages(compose, arch, var, package_sets=package_sets) + packages |= var_packages + groups |= var_groups + # we don't always want automatical inheritance of filtered packages from parent to child variants + # filter_packages |= var_filter_packages + + if variant.type in ["addon", "layered-product"]: + var_packages, var_groups, var_filter_packages = get_variant_packages(compose, arch, variant.parent, package_sets=package_sets) + packages |= var_packages + groups |= var_groups + # filter_packages |= var_filter_packages + + return packages, groups, filter_packages + + +def get_system_release_packages(compose, arch, variant, package_sets): + packages = set() + filter_packages = set() + + if not variant: + # include all system-release-* (gathering for a package superset) + return packages, filter_packages + + if not package_sets or not package_sets.get(arch, None): + return packages, filter_packages + + package_set = package_sets[arch] + + system_release_packages = set() + for i in package_set: + pkg = package_set[i] + + if pkg.is_system_release: + system_release_packages.add(pkg) + + if not system_release_packages: + return packages, filter_packages + elif len(system_release_packages) == 1: + # always include system-release package if available + pkg = list(system_release_packages)[0] + packages.add((pkg.name, None)) + else: + if variant.type == "variant": + # search for best match + best_match = None + for pkg in system_release_packages: + if pkg.name.endswith("release-%s" % variant.uid.lower()) or pkg.name.startswith("%s-release" % variant.uid.lower()): + best_match = pkg + break + else: + # addons: return release packages from parent variant + return get_system_release_packages(compose, arch, variant.parent, package_sets) + + if not best_match: + # no package matches variant name -> pick the first one + best_match = sorted(system_release_packages)[0] + + packages.add((best_match.name, None)) + for pkg in system_release_packages: + if pkg.name == best_match.name: + continue + filter_packages.add((pkg.name, None)) + + return packages, filter_packages diff --git a/pungi/phases/gather/link.py b/pungi/phases/gather/link.py new file mode 100644 index 00000000..d5ae0db9 --- /dev/null +++ b/pungi/phases/gather/link.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + + +import os + +import kobo.rpmlib + +from pypungi.linker import LinkerThread, LinkerPool + + +# TODO: global Linker instance - to keep hardlinks on dest? +# DONE: show overall progress, not each file +# TODO: (these should be logged separately) + +def _get_src_nevra(compose, pkg_obj, srpm_map): + """Return source N-E:V-R.A.rpm; guess if necessary.""" + result = srpm_map.get(pkg_obj.sourcerpm, None) + if not result: + nvra = kobo.rpmlib.parse_nvra(pkg_obj.sourcerpm) + nvra["epoch"] = pkg_obj.epoch + result = kobo.rpmlib.make_nvra(nvra, add_rpm=True, force_epoch=True) + compose.log_warning("Package %s has no SRPM available, guessing epoch: %s" % (pkg_obj.nevra, result)) + return result + + +def link_files(compose, arch, variant, pkg_map, pkg_sets, manifest, srpm_map={}): + # srpm_map instance is shared between link_files() runs + pkg_set = pkg_sets[arch] + + msg = "Linking packages (arch: %s, variant: %s)" % (arch, variant) + compose.log_info("[BEGIN] %s" % msg) + link_type = compose.conf.get("link_type", "hardlink-or-copy") + + pool = LinkerPool(link_type, logger=compose._logger) + for i in range(10): + pool.add(LinkerThread(pool)) + + packages_dir = compose.paths.compose.packages("src", variant) + packages_dir_relpath = compose.paths.compose.packages("src", variant, relative=True) + for pkg in pkg_map["srpm"]: + dst = os.path.join(packages_dir, os.path.basename(pkg["path"])) + dst_relpath = os.path.join(packages_dir_relpath, os.path.basename(pkg["path"])) + + # link file + pool.queue_put((pkg["path"], dst)) + + # update rpm manifest + pkg_obj = pkg_set[pkg["path"]] + nevra = pkg_obj.nevra + manifest.add("src", variant.uid, nevra, path=dst_relpath, sigkey=pkg_obj.signature, rpm_type="source") + + # update srpm_map + srpm_map.setdefault(pkg_obj.file_name, nevra) + + packages_dir = compose.paths.compose.packages(arch, variant) + packages_dir_relpath = compose.paths.compose.packages(arch, variant, relative=True) + for pkg in pkg_map["rpm"]: + dst = os.path.join(packages_dir, os.path.basename(pkg["path"])) + dst_relpath = os.path.join(packages_dir_relpath, os.path.basename(pkg["path"])) + + # link file + pool.queue_put((pkg["path"], dst)) + + # update rpm manifest + pkg_obj = pkg_set[pkg["path"]] + nevra = pkg_obj.nevra + src_nevra = _get_src_nevra(compose, pkg_obj, srpm_map) + manifest.add(arch, variant.uid, nevra, path=dst_relpath, sigkey=pkg_obj.signature, rpm_type="package", srpm_nevra=src_nevra) + + packages_dir = compose.paths.compose.debug_packages(arch, variant) + packages_dir_relpath = compose.paths.compose.debug_packages(arch, variant, relative=True) + for pkg in pkg_map["debuginfo"]: + dst = os.path.join(packages_dir, os.path.basename(pkg["path"])) + dst_relpath = os.path.join(packages_dir_relpath, os.path.basename(pkg["path"])) + + # link file + pool.queue_put((pkg["path"], dst)) + + # update rpm manifest + pkg_obj = pkg_set[pkg["path"]] + nevra = pkg_obj.nevra + src_nevra = _get_src_nevra(compose, pkg_obj, srpm_map) + manifest.add(arch, variant.uid, nevra, path=dst_relpath, sigkey=pkg_obj.signature, rpm_type="debug", srpm_nevra=src_nevra) + + pool.start() + pool.stop() + compose.log_info("[DONE ] %s" % msg) diff --git a/pungi/phases/gather/method.py b/pungi/phases/gather/method.py new file mode 100644 index 00000000..7cbda6bd --- /dev/null +++ b/pungi/phases/gather/method.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import kobo.plugins + +from pypungi.checks import validate_options + + +class GatherMethodBase(kobo.plugins.Plugin): + config_options = () + + def __init__(self, compose): + self.compose = compose + + def validate(self): + errors = validate_options(self.compose.conf, self.config_options) + if errors: + raise ValueError("\n".join(errors)) + + +class GatherMethodContainer(kobo.plugins.PluginContainer): + @classmethod + def normalize_name(cls, name): + return name.lower() diff --git a/pungi/phases/gather/methods/__init__.py b/pungi/phases/gather/methods/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pungi/phases/gather/methods/method_deps.py b/pungi/phases/gather/methods/method_deps.py new file mode 100644 index 00000000..0f79e62f --- /dev/null +++ b/pungi/phases/gather/methods/method_deps.py @@ -0,0 +1,174 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import os +import tempfile + +from kobo.shortcuts import run + +from pypungi.util import rmtree +from pypungi.wrappers.pungi import PungiWrapper + +from pypungi.arch import tree_arch_to_yum_arch +import pypungi.phases.gather + + +import pypungi.phases.gather.method + + +class GatherMethodDeps(pypungi.phases.gather.method.GatherMethodBase): + enabled = True + config_options = ( + { + "name": "gather_method", + "expected_types": [str], + "expected_values": ["deps"], + }, + { + "name": "check_deps", + "expected_types": [bool], + }, + { + "name": "gather_fulltree", + "expected_types": [bool], + "optional": True, + }, + { + "name": "gather_selfhosting", + "expected_types": [bool], + "optional": True, + }, + ) + + def __call__(self, arch, variant, packages, groups, filter_packages, multilib_whitelist, multilib_blacklist, package_sets, path_prefix=None, fulltree_excludes=None, prepopulate=None): + # result = { + # "rpm": [], + # "srpm": [], + # "debuginfo": [], + # } + + write_pungi_config(self.compose, arch, variant, packages, groups, filter_packages, multilib_whitelist, multilib_blacklist, package_set=package_sets[arch], fulltree_excludes=fulltree_excludes, prepopulate=prepopulate) + result = resolve_deps(self.compose, arch, variant) + check_deps(self.compose, arch, variant) + return result + + +def write_pungi_config(compose, arch, variant, packages, groups, filter_packages, multilib_whitelist, multilib_blacklist, repos=None, comps_repo=None, package_set=None, fulltree_excludes=None, prepopulate=None): + """write pungi config (kickstart) for arch/variant""" + pungi = PungiWrapper() + pungi_cfg = compose.paths.work.pungi_conf(variant=variant, arch=arch) + msg = "Writing pungi config (arch: %s, variant: %s): %s" % (arch, variant, pungi_cfg) + + if compose.DEBUG and os.path.isfile(pungi_cfg): + compose.log_warning("[SKIP ] %s" % msg) + return + + compose.log_info(msg) + + if not repos: + repo_path = compose.paths.work.arch_repo(arch=arch) + repos = {"pungi-repo": repo_path} + + lookaside_repos = {} + for i, repo_url in enumerate(pypungi.phases.gather.get_lookaside_repos(compose, arch, variant)): + lookaside_repos["lookaside-repo-%s" % i] = repo_url + + packages_str = [] + for pkg_name, pkg_arch in sorted(packages): + if pkg_arch: + packages_str.append("%s.%s" % (pkg_name, pkg_arch)) + else: + packages_str.append(pkg_name) + + filter_packages_str = [] + for pkg_name, pkg_arch in sorted(filter_packages): + if pkg_arch: + filter_packages_str.append("%s.%s" % (pkg_name, pkg_arch)) + else: + filter_packages_str.append(pkg_name) + + pungi.write_kickstart(ks_path=pungi_cfg, repos=repos, groups=groups, packages=packages_str, exclude_packages=filter_packages_str, comps_repo=comps_repo, lookaside_repos=lookaside_repos, fulltree_excludes=fulltree_excludes, multilib_whitelist=multilib_whitelist, multilib_blacklist=multilib_blacklist, prepopulate=prepopulate) + + +def resolve_deps(compose, arch, variant): + pungi = PungiWrapper() + pungi_log = compose.paths.work.pungi_log(arch, variant) + + msg = "Running pungi (arch: %s, variant: %s)" % (arch, variant) + if compose.DEBUG and os.path.exists(pungi_log): + compose.log_warning("[SKIP ] %s" % msg) + return pungi.get_packages(open(pungi_log, "r").read()) + + compose.log_info("[BEGIN] %s" % msg) + pungi_conf = compose.paths.work.pungi_conf(arch, variant) + + multilib_methods = compose.conf.get("multilib_methods", None) + multilib_methods = compose.conf.get("multilib_methods", None) + is_multilib = arch in compose.conf["multilib_arches"] + if not is_multilib: + multilib_methods = None + + greedy_method = compose.conf.get("greedy_method", "none") + + # variant + fulltree = compose.conf.get("gather_fulltree", False) + selfhosting = compose.conf.get("gather_selfhosting", False) + + # optional + if variant.type == "optional": + fulltree = True + selfhosting = True + + # addon + if variant.type in ["addon", "layered-product"]: + # packages having SRPM in parent variant are excluded from fulltree (via %fulltree-excludes) + fulltree = True + selfhosting = False + + lookaside_repos = {} + for i, repo_url in enumerate(pypungi.phases.gather.get_lookaside_repos(compose, arch, variant)): + lookaside_repos["lookaside-repo-%s" % i] = repo_url + + yum_arch = tree_arch_to_yum_arch(arch) + tmp_dir = compose.paths.work.tmp_dir(arch, variant) + cache_dir = compose.paths.work.pungi_cache_dir(arch, variant) + cmd = pungi.get_pungi_cmd(pungi_conf, destdir=tmp_dir, name=variant.uid, selfhosting=selfhosting, fulltree=fulltree, arch=yum_arch, full_archlist=True, greedy=greedy_method, cache_dir=cache_dir, lookaside_repos=lookaside_repos, multilib_methods=multilib_methods) + # Use temp working directory directory as workaround for + # https://bugzilla.redhat.com/show_bug.cgi?id=795137 + tmp_dir = tempfile.mkdtemp(prefix="pungi_") + try: + run(cmd, logfile=pungi_log, show_cmd=True, workdir=tmp_dir) + finally: + rmtree(tmp_dir) + result = pungi.get_packages(open(pungi_log, "r").read()) + + compose.log_info("[DONE ] %s" % msg) + return result + + +def check_deps(compose, arch, variant): + check_deps = compose.conf.get("check_deps", True) + if not check_deps: + return + + pungi = PungiWrapper() + pungi_log = compose.paths.work.pungi_log(arch, variant) + missing_deps = pungi.get_missing_deps(open(pungi_log, "r").read()) + if missing_deps: + for pkg in sorted(missing_deps): + compose.log_error("Unresolved dependencies in package %s: %s" % (pkg, sorted(missing_deps[pkg]))) + raise RuntimeError("Unresolved dependencies detected") diff --git a/pungi/phases/gather/methods/method_nodeps.py b/pungi/phases/gather/methods/method_nodeps.py new file mode 100644 index 00000000..667817be --- /dev/null +++ b/pungi/phases/gather/methods/method_nodeps.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import pypungi.arch +from pypungi.util import pkg_is_rpm, pkg_is_srpm, pkg_is_debug + +import pypungi.phases.gather.method + + +class GatherMethodNodeps(pypungi.phases.gather.method.GatherMethodBase): + enabled = True + config_options = ( + { + "name": "gather_method", + "expected_types": [str], + "expected_values": ["nodeps"], + }, + ) + + def __call__(self, arch, variant, packages, groups, filter_packages, multilib_whitelist, multilib_blacklist, package_sets, path_prefix=None, fulltree_excludes=None, prepopulate=None): + global_pkgset = package_sets["global"] + result = { + "rpm": [], + "srpm": [], + "debuginfo": [], + } + + seen_rpms = {} + seen_srpms = {} + + valid_arches = pypungi.arch.get_valid_arches(arch, multilib=True) + compatible_arches = {} + for i in valid_arches: + compatible_arches[i] = pypungi.arch.get_compatible_arches(i) + + for i in global_pkgset: + pkg = global_pkgset[i] + if not pkg_is_rpm(pkg): + continue + for pkg_name, pkg_arch in packages: + if pkg.arch not in valid_arches: + continue + if pkg.name != pkg_name: + continue + if pkg_arch is not None and pkg.arch != pkg_arch: + continue + result["rpm"].append({ + "path": pkg.file_path, + "flags": ["input"], + }) + seen_rpms.setdefault(pkg.name, set()).add(pkg.arch) + seen_srpms.setdefault(pkg.sourcerpm, set()).add(pkg.arch) + + for i in global_pkgset: + pkg = global_pkgset[i] + if not pkg_is_srpm(pkg): + continue + if pkg.file_name in seen_srpms: + result["srpm"].append({ + "path": pkg.file_path, + "flags": ["input"], + }) + + for i in global_pkgset: + pkg = global_pkgset[i] + if pkg.arch not in valid_arches: + continue + if not pkg_is_debug(pkg): + continue + if pkg.sourcerpm not in seen_srpms: + continue + if not set(compatible_arches[pkg.arch]) & set(seen_srpms[pkg.sourcerpm]): + # this handles stuff like i386 debuginfo in a i686 package + continue + result["debuginfo"].append({ + "path": pkg.file_path, + "flags": ["input"], + }) + + return result diff --git a/pungi/phases/gather/source.py b/pungi/phases/gather/source.py new file mode 100644 index 00000000..747ac9b6 --- /dev/null +++ b/pungi/phases/gather/source.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import kobo.plugins + +from pypungi.checks import validate_options + + +class GatherSourceBase(kobo.plugins.Plugin): + config_options = () + + def __init__(self, compose): + self.compose = compose + + def validate(self): + errors = validate_options(self.compose.conf, self.config_options) + if errors: + raise ValueError("\n".join(errors)) + + +class GatherSourceContainer(kobo.plugins.PluginContainer): + @classmethod + def normalize_name(cls, name): + return name.lower() diff --git a/pungi/phases/gather/sources/__init__.py b/pungi/phases/gather/sources/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pungi/phases/gather/sources/source_comps.py b/pungi/phases/gather/sources/source_comps.py new file mode 100644 index 00000000..f101455e --- /dev/null +++ b/pungi/phases/gather/sources/source_comps.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +""" +Get a package list based on comps.xml. + +Input format: +see comps.dtd + +Output: +set([(rpm_name, rpm_arch or None)]) +""" + + +from pypungi.wrappers.comps import CompsWrapper +import pypungi.phases.gather.source + + +class GatherSourceComps(pypungi.phases.gather.source.GatherSourceBase): + enabled = True + config_options = ( + { + "name": "gather_source", + "expected_types": [str], + "expected_values": ["comps"], + }, + { + "name": "comps_file", + "expected_types": [str, dict], + }, + ) + + def __call__(self, arch, variant): + groups = set() + comps = CompsWrapper(self.compose.paths.work.comps(arch=arch)) + + if variant is not None: + # get packages for a particular variant + comps.filter_groups(variant.groups) + + for i in comps.get_comps_groups(): + groups.add(i.groupid) + return set(), groups diff --git a/pungi/phases/gather/sources/source_json.py b/pungi/phases/gather/sources/source_json.py new file mode 100644 index 00000000..c6cf1642 --- /dev/null +++ b/pungi/phases/gather/sources/source_json.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +""" +Get a package list based on a JSON mapping. + +Input format: +{ + variant: { + tree_arch: { + rpm_name: [rpm_arch, rpm_arch, ... (or None for any/best arch)], + } + } +} + +Output: +set([(rpm_name, rpm_arch or None)]) +""" + + +import json + +import pypungi.phases.gather.source + + +class GatherSourceJson(pypungi.phases.gather.source.GatherSourceBase): + enabled = True + config_options = ( + { + "name": "gather_source", + "expected_types": [str], + "expected_values": ["json"], + }, + { + "name": "gather_source_mapping", + "expected_types": [str], + }, + ) + + def __call__(self, arch, variant): + json_path = self.compose.conf["gather_source_mapping"] + data = open(json_path, "r").read() + mapping = json.loads(data) + + packages = set() + if variant is None: + # get all packages for all variants + for variant_uid in mapping: + for pkg_name, pkg_arches in mapping[variant_uid][arch].iteritems(): + for pkg_arch in pkg_arches: + packages.add((pkg_name, pkg_arch)) + else: + # get packages for a particular variant + for pkg_name, pkg_arches in mapping[variant.uid][arch].iteritems(): + for pkg_arch in pkg_arches: + packages.add((pkg_name, pkg_arch)) + return packages, set() diff --git a/pungi/phases/gather/sources/source_none.py b/pungi/phases/gather/sources/source_none.py new file mode 100644 index 00000000..0e0855f7 --- /dev/null +++ b/pungi/phases/gather/sources/source_none.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +""" +Get an empty package list. + +Input: +none + +Output: +set() +""" + + +import pypungi.phases.gather.source + + +class GatherSourceNone(pypungi.phases.gather.source.GatherSourceBase): + enabled = True + + config_options = ( + { + "name": "gather_source", + "expected_types": [str], + "expected_values": ["none"], + }, + ) + + def __call__(self, arch, variant): + return set(), set() diff --git a/pungi/phases/init.py b/pungi/phases/init.py new file mode 100644 index 00000000..f0f82266 --- /dev/null +++ b/pungi/phases/init.py @@ -0,0 +1,267 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import os +import tempfile +import shutil + +from kobo.shortcuts import run + +from pypungi.phases.base import PhaseBase +from pypungi.phases.gather import write_prepopulate_file +from pypungi.wrappers.createrepo import CreaterepoWrapper +from pypungi.wrappers.comps import CompsWrapper +from pypungi.wrappers.scm import get_file_from_scm + + +class InitPhase(PhaseBase): + """INIT is a mandatory phase""" + name = "init" + + config_options = ( + # PRODUCT INFO + { + "name": "product_name", + "expected_types": [str], + }, + { + "name": "product_short", + "expected_types": [str], + }, + { + "name": "product_version", + "expected_types": [str], + }, + { + # override description in .discinfo; accepts %(variant_name)s and %(arch)s variables + "name": "product_discinfo_description", + "expected_types": [str], + "optional": True, + }, + { + "name": "product_is_layered", + "expected_types": [bool], + "requires": ( + (lambda x: bool(x), ["base_product_name", "base_product_short", "base_product_version"]), + ), + "conflicts": ( + (lambda x: not bool(x), ["base_product_name", "base_product_short", "base_product_version"]), + ), + }, + + # BASE PRODUCT INFO (FOR A LAYERED PRODUCT ONLY) + { + "name": "base_product_name", + "expected_types": [str], + "optional": True, + }, + { + "name": "base_product_short", + "expected_types": [str], + "optional": True, + }, + { + "name": "base_product_version", + "expected_types": [str], + "optional": True, + }, + + { + "name": "comps_file", + "expected_types": [str, dict], + "optional": True, + }, + { + "name": "comps_filter_environments", # !!! default is True !!! + "expected_types": [bool], + "optional": True, + }, + { + "name": "variants_file", + "expected_types": [str, dict], + }, + { + "name": "sigkeys", + "expected_types": [list], + }, + + { + "name": "tree_arches", + "expected_types": [list], + "optional": True, + }, + { + "name": "tree_variants", + "expected_types": [list], + "optional": True, + }, + { + "name": "multilib_arches", + "expected_types": [list], + "optional": True, + }, + + # CREATEREPO SETTINGS + { + "name": "createrepo_c", + "expected_types": [bool], + "optional": True, + }, + { + "name": "createrepo_checksum", + "expected_types": [str], + "expected_values": ["sha256", "sha"], + "optional": True, + }, + + # RUNROOT SETTINGS + { + "name": "runroot", + "expected_types": [bool], + "requires": ( + (lambda x: bool(x), ["runroot_tag", "runroot_channel"]), + ), + "conflicts": ( + (lambda x: not bool(x), ["runroot_tag", "runroot_channel"]), + ), + }, + { + "name": "runroot_tag", + "expected_types": [str], + "optional": True, + }, + { + "name": "runroot_channel", + "expected_types": [str], + "optional": True, + }, + + + ) + + def skip(self): + # INIT must never be skipped, + # because it generates data for LIVEIMAGES + return False + + def run(self): + # write global comps and arch comps + write_global_comps(self.compose) + for arch in self.compose.get_arches(): + write_arch_comps(self.compose, arch) + + # create comps repos + for arch in self.compose.get_arches(): + create_comps_repo(self.compose, arch) + + # write variant comps + for variant in self.compose.get_variants(): + for arch in variant.arches: + write_variant_comps(self.compose, arch, variant) + + # download variants.xml / product.xml? + + # write prepopulate file + write_prepopulate_file(self.compose) + + +def write_global_comps(compose): + if not compose.has_comps: + return + + comps_file_global = compose.paths.work.comps(arch="global") + msg = "Writing global comps file: %s" % comps_file_global + + if compose.DEBUG and os.path.isfile(comps_file_global): + compose.log_warning("[SKIP ] %s" % msg) + else: + scm_dict = compose.conf["comps_file"] + if isinstance(scm_dict, dict): + comps_name = os.path.basename(scm_dict["file"]) + if scm_dict["scm"] == "file": + scm_dict["file"] = os.path.join(compose.config_dir, scm_dict["file"]) + else: + comps_name = os.path.basename(scm_dict) + scm_dict = os.path.join(compose.config_dir, scm_dict) + + compose.log_debug(msg) + tmp_dir = tempfile.mkdtemp(prefix="comps_") + get_file_from_scm(scm_dict, tmp_dir, logger=compose._logger) + shutil.copy2(os.path.join(tmp_dir, comps_name), comps_file_global) + shutil.rmtree(tmp_dir) + + +def write_arch_comps(compose, arch): + if not compose.has_comps: + return + + comps_file_arch = compose.paths.work.comps(arch=arch) + msg = "Writing comps file for arch '%s': %s" % (arch, comps_file_arch) + + if compose.DEBUG and os.path.isfile(comps_file_arch): + compose.log_warning("[SKIP ] %s" % msg) + return + + compose.log_debug(msg) + run(["comps_filter", "--arch=%s" % arch, "--no-cleanup", "--output=%s" % comps_file_arch, compose.paths.work.comps(arch="global")]) + + +def write_variant_comps(compose, arch, variant): + if not compose.has_comps: + return + + comps_file = compose.paths.work.comps(arch=arch, variant=variant) + msg = "Writing comps file (arch: %s, variant: %s): %s" % (arch, variant, comps_file) + + if compose.DEBUG and os.path.isfile(comps_file): + # read display_order and groups for environments (needed for live images) + comps = CompsWrapper(comps_file) + # groups = variant.groups + comps.filter_groups(variant.groups) + if compose.conf.get("comps_filter_environments", True): + comps.filter_environments(variant.environments) + + compose.log_warning("[SKIP ] %s" % msg) + return + + compose.log_debug(msg) + run(["comps_filter", "--arch=%s" % arch, "--keep-empty-group=conflicts", "--keep-empty-group=conflicts-%s" % variant.uid.lower(), "--output=%s" % comps_file, compose.paths.work.comps(arch="global")]) + + comps = CompsWrapper(comps_file) + comps.filter_groups(variant.groups) + if compose.conf.get("comps_filter_environments", True): + comps.filter_environments(variant.environments) + comps.write_comps() + + +def create_comps_repo(compose, arch): + if not compose.has_comps: + return + + createrepo_c = compose.conf.get("createrepo_c", False) + createrepo_checksum = compose.conf.get("createrepo_checksum", None) + repo = CreaterepoWrapper(createrepo_c=createrepo_c) + comps_repo = compose.paths.work.comps_repo(arch=arch) + comps_path = compose.paths.work.comps(arch=arch) + msg = "Creating comps repo for arch '%s'" % arch + if compose.DEBUG and os.path.isdir(os.path.join(comps_repo, "repodata")): + compose.log_warning("[SKIP ] %s" % msg) + else: + compose.log_info("[BEGIN] %s" % msg) + cmd = repo.get_createrepo_cmd(comps_repo, update=True, database=True, skip_stat=True, outputdir=comps_repo, groupfile=comps_path, checksum=createrepo_checksum) + run(cmd, logfile=compose.paths.log.log_file("global", "arch_repo"), show_cmd=True) + compose.log_info("[DONE ] %s" % msg) diff --git a/pungi/phases/live_images.py b/pungi/phases/live_images.py new file mode 100644 index 00000000..f439fb4e --- /dev/null +++ b/pungi/phases/live_images.py @@ -0,0 +1,224 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import os +import sys +import copy +import time +import pipes +import shutil +import re +import tempfile + +from kobo.threads import ThreadPool, WorkerThread +from kobo.shortcuts import run + +from pypungi.wrappers.kojiwrapper import KojiWrapper +from pypungi.wrappers.iso import IsoWrapper +from pypungi.wrappers.scm import get_file_from_scm +from pypungi.phases.base import PhaseBase +from pypungi.util import get_arch_variant_data + + +# HACK: define cmp in python3 +if sys.version_info[0] == 3: + def cmp(a, b): + return (a > b) - (a < b) + + +class LiveImagesPhase(PhaseBase): + name = "liveimages" + + config_options = ( + { + "name": "live_target", + "expected_types": [str], + "optional": True, + }, + ) + + def __init__(self, compose): + PhaseBase.__init__(self, compose) + self.pool = ThreadPool(logger=self.compose._logger) + + def skip(self): + if PhaseBase.skip(self): + return True + if not self.compose.conf.get("live_images"): + return True + return False + + def run(self): + symlink_isos_to = self.compose.conf.get("symlink_isos_to", None) + iso = IsoWrapper() + commands = [] + + for variant in self.compose.variants.values(): + for arch in variant.arches + ["src"]: + ks_in = get_ks_in(self.compose, arch, variant) + if not ks_in: + continue + + ks_file = tweak_ks(self.compose, arch, variant, ks_in) + + iso_dir = self.compose.paths.compose.iso_dir(arch, variant, symlink_to=symlink_isos_to) + if not iso_dir: + continue + + # XXX: hardcoded disc_type and disc_num + iso_path = self.compose.paths.compose.iso_path(arch, variant, disc_type="live", disc_num=None, symlink_to=symlink_isos_to) + if os.path.isfile(iso_path): + self.compose.log_warning("Skipping creating live image, it already exists: %s" % iso_path) + continue + + iso_name = os.path.basename(iso_path) + + cmd = { + "arch": arch, + "variant": variant, + "iso_path": iso_path, + "build_arch": arch, + "ks_file": ks_file, + "cmd": [], + "label": "", # currently not used + } + repo = self.compose.paths.compose.repository(arch, variant) + # HACK: + repo = re.sub(r"^/mnt/koji/", "https://kojipkgs.fedoraproject.org/", repo) + cmd["repos"] = [repo] + + # additional repos + data = get_arch_variant_data(self.compose.conf, "live_images", arch, variant) + cmd["repos"].extend(data[0].get("additional_repos", [])) + + chdir_cmd = "cd %s" % pipes.quote(iso_dir) + cmd["cmd"].append(chdir_cmd) + + # compute md5sum, sha1sum, sha256sum + cmd["cmd"].extend(iso.get_checksum_cmds(iso_name)) + + # create iso manifest + cmd["cmd"].append(iso.get_manifest_cmd(iso_name)) + + cmd["cmd"] = " && ".join(cmd["cmd"]) + commands.append(cmd) + + for cmd in commands: + self.pool.add(CreateLiveImageThread(self.pool)) + self.pool.queue_put((self.compose, cmd)) + + self.pool.start() + + def stop(self, *args, **kwargs): + PhaseBase.stop(self, *args, **kwargs) + if self.skip(): + return + + +class CreateLiveImageThread(WorkerThread): + def fail(self, compose, cmd): + compose.log_error("LiveImage failed, removing ISO: %s" % cmd["iso_path"]) + try: + # remove (possibly?) incomplete ISO + os.unlink(cmd["iso_path"]) + # TODO: remove checksums + except OSError: + pass + + def process(self, item, num): + compose, cmd = item + + runroot = compose.conf.get("runroot", False) + log_file = compose.paths.log.log_file(cmd["arch"], "createiso-%s" % os.path.basename(cmd["iso_path"])) + + msg = "Creating ISO (arch: %s, variant: %s): %s" % (cmd["arch"], cmd["variant"], os.path.basename(cmd["iso_path"])) + self.pool.log_info("[BEGIN] %s" % msg) + + if runroot: + # run in a koji build root + + koji_wrapper = KojiWrapper(compose.conf["koji_profile"]) + name, version = compose.compose_id.rsplit("-", 1) + target = compose.conf["live_target"] + koji_cmd = koji_wrapper.get_create_image_cmd(name, version, target, cmd["build_arch"], cmd["ks_file"], cmd["repos"], image_type="live", wait=True, archive=False) + + # avoid race conditions? + # Kerberos authentication failed: Permission denied in replay cache code (-1765328215) + time.sleep(num * 3) + + output = koji_wrapper.run_create_image_cmd(koji_cmd, log_file=log_file) + if output["retcode"] != 0: + self.fail(compose, cmd) + raise RuntimeError("LiveImage task failed: %s. See %s for more details." % (output["task_id"], log_file)) + + # copy finished image to isos/ + image_path = koji_wrapper.get_image_path(output["task_id"]) + # TODO: assert len == 1 + image_path = image_path[0] + shutil.copy2(image_path, cmd["iso_path"]) + + # write checksum and manifest + run(cmd["cmd"]) + + else: + raise RuntimeError("NOT IMPLEMENTED") + + self.pool.log_info("[DONE ] %s" % msg) + + +def get_ks_in(compose, arch, variant): + data = get_arch_variant_data(compose.conf, "live_images", arch, variant) + if not data: + return + scm_dict = data[0]["kickstart"] + + if isinstance(scm_dict, dict): + if scm_dict["scm"] == "file": + file_name = os.path.basename(os.path.basename(scm_dict["file"])) + scm_dict["file"] = os.path.join(compose.config_dir, os.path.basename(scm_dict["file"])) + else: + file_name = os.path.basename(os.path.basename(scm_dict)) + scm_dict = os.path.join(compose.config_dir, os.path.basename(scm_dict)) + + tmp_dir = tempfile.mkdtemp(prefix="ks_in_") + get_file_from_scm(scm_dict, tmp_dir, logger=compose._logger) + ks_in = os.path.join(compose.paths.work.topdir(arch), "liveimage-%s.%s.ks.in" % (variant.uid, arch)) + shutil.copy2(os.path.join(tmp_dir, file_name), ks_in) + shutil.rmtree(tmp_dir) + return ks_in + + +def tweak_ks(compose, arch, variant, ks_in): + if variant.environments: + # get groups from default environment (with lowest display_order) + envs = copy.deepcopy(variant.environments) + envs.sort(lambda x, y: cmp(x["display_order"], y["display_order"])) + env = envs[0] + groups = sorted(env["groups"]) + else: + # no environments -> get default groups + groups = [] + for i in variant.groups: + if i["default"]: + groups.append(i["name"]) + groups.sort() + + ks_file = os.path.join(compose.paths.work.topdir(arch), "liveimage-%s.%s.ks" % (variant.uid, arch)) + contents = open(ks_in, "r").read() + contents = contents.replace("__GROUPS__", "\n".join(["@%s" % i for i in groups])) + open(ks_file, "w").write(contents) + return ks_file diff --git a/pungi/phases/pkgset/__init__.py b/pungi/phases/pkgset/__init__.py new file mode 100644 index 00000000..5dfba416 --- /dev/null +++ b/pungi/phases/pkgset/__init__.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import os + +from kobo.shortcuts import force_list + +import pypungi.phases.pkgset.pkgsets +from pypungi.arch import get_valid_arches +from pypungi.phases.base import PhaseBase + + +class PkgsetPhase(PhaseBase): + """PKGSET""" + name = "pkgset" + + config_options = ( + { + "name": "pkgset_source", + "expected_types": [str], + }, + { + "name": "multilib_arches", + "expected_types": [list], + }, + ) + + def run(self): + pkgset_source = "PkgsetSource%s" % self.compose.conf["pkgset_source"] + from source import PkgsetSourceContainer + import sources + PkgsetSourceContainer.register_module(sources) + container = PkgsetSourceContainer() + SourceClass = container[pkgset_source] + self.package_sets, self.path_prefix = SourceClass(self.compose)() + + +# TODO: per arch? +def populate_arch_pkgsets(compose, path_prefix, global_pkgset): + result = {} + for arch in compose.get_arches(): + compose.log_info("Populating package set for arch: %s" % arch) + is_multilib = arch in compose.conf["multilib_arches"] + arches = get_valid_arches(arch, is_multilib, add_src=True) + pkgset = pypungi.phases.pkgset.pkgsets.PackageSetBase(compose.conf["sigkeys"], logger=compose._logger, arches=arches) + pkgset.merge(global_pkgset, arch, arches) + pkgset.save_file_list(compose.paths.work.package_list(arch=arch), remove_path_prefix=path_prefix) + result[arch] = pkgset + return result + + +def find_old_compose(old_compose_dirs, shortname=None, version=None): + composes = [] + + for compose_dir in force_list(old_compose_dirs): + if not os.path.isdir(compose_dir): + continue + + # get all finished composes + for i in os.listdir(compose_dir): + # TODO: read .composeinfo + if shortname and not i.startswith(shortname): + continue + + if shortname and version and not i.startswith("%s-%s" % (shortname, version)): + continue + + path = os.path.join(compose_dir, i) + if not os.path.isdir(path): + continue + + if os.path.islink(path): + continue + + status_path = os.path.join(path, "STATUS") + if not os.path.isfile(status_path): + continue + + try: + if open(status_path, "r").read().strip() in ("FINISHED", "DOOMED"): + composes.append((i, os.path.abspath(path))) + except: + continue + + if not composes: + return None + + return sorted(composes)[-1][1] diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py new file mode 100644 index 00000000..6ae32bf2 --- /dev/null +++ b/pungi/phases/pkgset/common.py @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import os + +from kobo.shortcuts import run, force_list, relative_path + +import pypungi.phases.pkgset.pkgsets +from pypungi.arch import get_valid_arches +from pypungi.wrappers.createrepo import CreaterepoWrapper + + +# TODO: per arch? +def populate_arch_pkgsets(compose, path_prefix, global_pkgset): + result = {} + for arch in compose.get_arches(): + compose.log_info("Populating package set for arch: %s" % arch) + is_multilib = arch in compose.conf["multilib_arches"] + arches = get_valid_arches(arch, is_multilib, add_src=True) + pkgset = pypungi.phases.pkgset.pkgsets.PackageSetBase(compose.conf["sigkeys"], logger=compose._logger, arches=arches) + pkgset.merge(global_pkgset, arch, arches) + pkgset.save_file_list(compose.paths.work.package_list(arch=arch), remove_path_prefix=path_prefix) + result[arch] = pkgset + return result + + +def create_global_repo(compose, path_prefix): + createrepo_c = compose.conf.get("createrepo_c", False) + createrepo_checksum = compose.conf.get("createrepo_checksum", None) + repo = CreaterepoWrapper(createrepo_c=createrepo_c) + repo_dir_global = compose.paths.work.arch_repo(arch="global") + msg = "Running createrepo for the global package set" + + if compose.DEBUG and os.path.isdir(os.path.join(repo_dir_global, "repodata")): + compose.log_warning("[SKIP ] %s" % msg) + return + + compose.log_info("[BEGIN] %s" % msg) + + # find an old compose suitable for repodata reuse + old_compose_path = None + update_md_path = None + if compose.old_composes: + old_compose_path = find_old_compose(compose.old_composes, compose.conf["product_short"], compose.conf["product_version"], compose.conf.get("base_product_short", None), compose.conf.get("base_product_version", None)) + if old_compose_path is None: + compose.log_info("No suitable old compose found in: %s" % compose.old_composes) + else: + repo_dir = compose.paths.work.arch_repo(arch="global") + rel_path = relative_path(repo_dir, os.path.abspath(compose.topdir).rstrip("/") + "/") + old_repo_dir = os.path.join(old_compose_path, rel_path) + if os.path.isdir(old_repo_dir): + compose.log_info("Using old repodata from: %s" % old_repo_dir) + update_md_path = old_repo_dir + + # IMPORTANT: must not use --skip-stat here -- to make sure that correctly signed files are pulled in + cmd = repo.get_createrepo_cmd(path_prefix, update=True, database=True, skip_stat=False, pkglist=compose.paths.work.package_list(arch="global"), outputdir=repo_dir_global, baseurl="file://%s" % path_prefix, workers=5, update_md_path=update_md_path, checksum=createrepo_checksum) + run(cmd, logfile=compose.paths.log.log_file("global", "arch_repo"), show_cmd=True) + compose.log_info("[DONE ] %s" % msg) + + +def create_arch_repos(compose, arch, path_prefix): + createrepo_c = compose.conf.get("createrepo_c", False) + createrepo_checksum = compose.conf.get("createrepo_checksum", None) + repo = CreaterepoWrapper(createrepo_c=createrepo_c) + repo_dir_global = compose.paths.work.arch_repo(arch="global") + repo_dir = compose.paths.work.arch_repo(arch=arch) + msg = "Running createrepo for arch '%s'" % arch + + if compose.DEBUG and os.path.isdir(os.path.join(repo_dir, "repodata")): + compose.log_warning("[SKIP ] %s" % msg) + return + + compose.log_info("[BEGIN] %s" % msg) + comps_path = None + if compose.has_comps: + comps_path = compose.paths.work.comps(arch=arch) + cmd = repo.get_createrepo_cmd(path_prefix, update=True, database=True, skip_stat=True, pkglist=compose.paths.work.package_list(arch=arch), outputdir=repo_dir, baseurl="file://%s" % path_prefix, workers=5, groupfile=comps_path, update_md_path=repo_dir_global, checksum=createrepo_checksum) + run(cmd, logfile=compose.paths.log.log_file(arch, "arch_repo"), show_cmd=True) + compose.log_info("[DONE ] %s" % msg) + + +def find_old_compose(old_compose_dirs, product_short, product_version, base_product_short=None, base_product_version=None): + composes = [] + + for compose_dir in force_list(old_compose_dirs): + if not os.path.isdir(compose_dir): + continue + + # get all finished composes + for i in os.listdir(compose_dir): + # TODO: read .composeinfo + + pattern = "%s-%s" % (product_short, product_version) + if base_product_short: + pattern += "-%s" % base_product_short + if base_product_version: + pattern += "-%s" % base_product_version + + if not i.startswith(pattern): + continue + + path = os.path.join(compose_dir, i) + if not os.path.isdir(path): + continue + + if os.path.islink(path): + continue + + status_path = os.path.join(path, "STATUS") + if not os.path.isfile(status_path): + continue + + try: + if open(status_path, "r").read().strip() in ("FINISHED", "DOOMED"): + composes.append((i, os.path.abspath(path))) + except: + continue + + if not composes: + return None + + return sorted(composes)[-1][1] diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py new file mode 100644 index 00000000..1c2ea6ca --- /dev/null +++ b/pungi/phases/pkgset/pkgsets.py @@ -0,0 +1,285 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +""" +The KojiPackageSet object obtains the latest RPMs from a Koji tag. +It automatically finds a signed copies according to *sigkey_ordering*. +""" + + +import os + +import kobo.log +import kobo.pkgset +import kobo.rpmlib + +from kobo.threads import WorkerThread, ThreadPool + +from pypungi.util import pkg_is_srpm +from pypungi.arch import get_valid_arches + + +class ReaderPool(ThreadPool): + def __init__(self, package_set, logger=None): + ThreadPool.__init__(self, logger) + self.package_set = package_set + + +class ReaderThread(WorkerThread): + def process(self, item, num): + # rpm_info, build_info = item + + if (num % 100 == 0) or (num == self.pool.queue_total): + self.pool.package_set.log_debug("Processed %s out of %s packages" % (num, self.pool.queue_total)) + + rpm_path = self.pool.package_set.get_package_path(item) + rpm_obj = self.pool.package_set.file_cache.add(rpm_path) + self.pool.package_set.rpms_by_arch.setdefault(rpm_obj.arch, []).append(rpm_obj) + + if pkg_is_srpm(rpm_obj): + self.pool.package_set.srpms_by_name[rpm_obj.file_name] = rpm_obj + elif rpm_obj.arch == "noarch": + srpm = self.pool.package_set.srpms_by_name.get(rpm_obj.sourcerpm, None) + if srpm: + # HACK: copy {EXCLUDE,EXCLUSIVE}ARCH from SRPM to noarch RPMs + rpm_obj.excludearch = srpm.excludearch + rpm_obj.exclusivearch = srpm.exclusivearch + else: + self.pool.log_warning("Can't find a SRPM for %s" % rpm_obj.file_name) + + +class PackageSetBase(kobo.log.LoggingBase): + def __init__(self, sigkey_ordering, arches=None, logger=None): + kobo.log.LoggingBase.__init__(self, logger=logger) + self.file_cache = kobo.pkgset.FileCache(kobo.pkgset.SimpleRpmWrapper) + self.sigkey_ordering = sigkey_ordering or [None] + self.arches = arches + self.rpms_by_arch = {} + self.srpms_by_name = {} + + def __getitem__(self, name): + return self.file_cache[name] + + def __len__(self): + return len(self.file_cache) + + def __iter__(self): + for i in self.file_cache: + yield i + + def __getstate__(self): + result = self.__dict__.copy() + del result["_logger"] + return result + + def __setstate__(self, data): + self._logger = None + self.__dict__.update(data) + + def read_packages(self, rpms, srpms): + srpm_pool = ReaderPool(self, self._logger) + rpm_pool = ReaderPool(self, self._logger) + + for i in rpms: + rpm_pool.queue_put(i) + + for i in srpms: + srpm_pool.queue_put(i) + + thread_count = 10 + for i in range(thread_count): + srpm_pool.add(ReaderThread(srpm_pool)) + rpm_pool.add(ReaderThread(rpm_pool)) + + # process SRC and NOSRC packages first (see ReaderTread for the EXCLUDEARCH/EXCLUSIVEARCH hack for noarch packages) + self.log_debug("Package set: spawning %s worker threads (SRPMs)" % thread_count) + srpm_pool.start() + srpm_pool.stop() + self.log_debug("Package set: worker threads stopped (SRPMs)") + + self.log_debug("Package set: spawning %s worker threads (RPMs)" % thread_count) + rpm_pool.start() + rpm_pool.stop() + self.log_debug("Package set: worker threads stopped (RPMs)") + + return self.rpms_by_arch + + def merge(self, other, primary_arch, arch_list): + msg = "Merging package sets for %s: %s" % (primary_arch, arch_list) + self.log_debug("[BEGIN] %s" % msg) + + # if "src" is present, make sure "nosrc" is included too + if "src" in arch_list and "nosrc" not in arch_list: + arch_list.append("nosrc") + + # make sure sources are processed last + for i in ("nosrc", "src"): + if i in arch_list: + arch_list.remove(i) + arch_list.append(i) + + seen_sourcerpms = set() + # {Exclude,Exclusive}Arch must match *tree* arch + compatible native arches (excluding multilib arches) + exclusivearch_list = get_valid_arches(primary_arch, multilib=False, add_noarch=False, add_src=False) + for arch in arch_list: + self.rpms_by_arch.setdefault(arch, []) + for i in other.rpms_by_arch.get(arch, []): + if i.file_path in self.file_cache: + # TODO: test if it really works + continue + if arch == "noarch": + if i.excludearch and set(i.excludearch) & set(exclusivearch_list): + self.log_debug("Excluding (EXCLUDEARCH: %s): %s" % (sorted(set(i.excludearch)), i.file_name)) + continue + if i.exclusivearch and not (set(i.exclusivearch) & set(exclusivearch_list)): + self.log_debug("Excluding (EXCLUSIVEARCH: %s): %s " % (sorted(set(i.exclusivearch)), i.file_name)) + continue + + if arch in ("nosrc", "src"): + # include only sources having binary packages + if i.name not in seen_sourcerpms: + continue + else: + sourcerpm_name = kobo.rpmlib.parse_nvra(i.sourcerpm)["name"] + seen_sourcerpms.add(sourcerpm_name) + + self.file_cache.file_cache[i.file_path] = i + self.rpms_by_arch[arch].append(i) + + self.log_debug("[DONE ] %s" % msg) + + def save_file_list(self, file_path, remove_path_prefix=None): + f = open(file_path, "w") + for arch in sorted(self.rpms_by_arch): + for i in self.rpms_by_arch[arch]: + rpm_path = i.file_path + if remove_path_prefix and rpm_path.startswith(remove_path_prefix): + rpm_path = rpm_path[len(remove_path_prefix):] + f.write("%s\n" % rpm_path) + f.close() + + +class FilelistPackageSet(PackageSetBase): + def get_package_path(self, queue_item): + # TODO: sigkey checking + rpm_path = os.path.abspath(queue_item) + return rpm_path + + def populate(self, file_list): + result_rpms = [] + result_srpms = [] + msg = "Getting RPMs from file list" + self.log_info("[BEGIN] %s" % msg) + for i in file_list: + if i.endswith(".src.rpm") or i.endswith(".nosrc.rpm"): + result_srpms.append(i) + else: + result_rpms.append(i) + result = self.read_packages(result_rpms, result_srpms) + self.log_info("[DONE ] %s" % msg) + return result + + +class KojiPackageSet(PackageSetBase): + def __init__(self, koji_proxy, sigkey_ordering, arches=None, logger=None): + PackageSetBase.__init__(self, sigkey_ordering=sigkey_ordering, arches=arches, logger=logger) + self.koji_proxy = koji_proxy + self.koji_pathinfo = getattr(__import__(koji_proxy.__module__, {}, {}, []), "pathinfo") + + def __getstate__(self): + result = self.__dict__.copy() + result["koji_class"] = self.koji_proxy.__class__.__name__ + result["koji_module"] = self.koji_proxy.__class__.__module__ + result["koji_baseurl"] = self.koji_proxy.baseurl + result["koji_opts"] = self.koji_proxy.opts + del result["koji_proxy"] + del result["koji_pathinfo"] + del result["_logger"] + return result + + def __setstate__(self, data): + class_name = data.pop("koji_class") + module_name = data.pop("koji_module") + module = __import__(module_name, {}, {}, [class_name]) + cls = getattr(module, class_name) + self.koji_proxy = cls(data.pop("koji_baseurl"), data.pop("koji_opts")) + self._logger = None + self.__dict__.update(data) + + def get_latest_rpms(self, tag, event, inherit=True): + return self.koji_proxy.listTaggedRPMS(tag, event=event, inherit=inherit, latest=True) + + def get_package_path(self, queue_item): + rpm_info, build_info = queue_item + rpm_path = None + found = False + pathinfo = self.koji_pathinfo + for sigkey in self.sigkey_ordering: + if sigkey is None: + # we're looking for *signed* copies here + continue + sigkey = sigkey.lower() + rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.signed(rpm_info, sigkey)) + if os.path.isfile(rpm_path): + found = True + break + + if not found: + if None in self.sigkey_ordering: + # use an unsigned copy (if allowed) + rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info)) + if os.path.isfile(rpm_path): + found = True + else: + # or raise an exception + raise RuntimeError("RPM not found for sigs: %s" % self.sigkey_ordering) + + if not found: + raise RuntimeError("Package not found: %s" % rpm_info) + return rpm_path + + def populate(self, tag, event=None, inherit=True): + result_rpms = [] + result_srpms = [] + + if type(event) is dict: + event = event["id"] + + msg = "Getting latest RPMs (tag: %s, event: %s, inherit: %s)" % (tag, event, inherit) + self.log_info("[BEGIN] %s" % msg) + rpms, builds = self.get_latest_rpms(tag, event) + + builds_by_id = {} + for build_info in builds: + builds_by_id.setdefault(build_info["build_id"], build_info) + + skipped_arches = [] + for rpm_info in rpms: + if self.arches and rpm_info["arch"] not in self.arches: + if rpm_info["arch"] not in skipped_arches: + self.log_debug("Skipping packages for arch: %s" % rpm_info["arch"]) + skipped_arches.append(rpm_info["arch"]) + continue + + build_info = builds_by_id[rpm_info["build_id"]] + if rpm_info["arch"] in ("src", "nosrc"): + result_srpms.append((rpm_info, build_info)) + else: + result_rpms.append((rpm_info, build_info)) + result = self.read_packages(result_rpms, result_srpms) + self.log_info("[DONE ] %s" % msg) + return result diff --git a/pungi/phases/pkgset/source.py b/pungi/phases/pkgset/source.py new file mode 100644 index 00000000..7ddf3343 --- /dev/null +++ b/pungi/phases/pkgset/source.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import kobo.plugins + +from pypungi.checks import validate_options + + +class PkgsetSourceBase(kobo.plugins.Plugin): + config_options = () + + def __init__(self, compose): + self.compose = compose + + def validate(self): + errors = validate_options(self.compose.conf, self.config_options) + if errors: + raise ValueError("\n".join(errors)) + + +class PkgsetSourceContainer(kobo.plugins.PluginContainer): + @classmethod + def normalize_name(cls, name): + return name.lower() diff --git a/pungi/phases/pkgset/sources/__init__.py b/pungi/phases/pkgset/sources/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py new file mode 100644 index 00000000..a393ba98 --- /dev/null +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import os +import cPickle as pickle +import json + +import koji + +import pypungi.phases.pkgset.pkgsets +from pypungi.arch import get_valid_arches + +from pypungi.phases.pkgset.common import create_arch_repos, create_global_repo, populate_arch_pkgsets + + +import pypungi.phases.pkgset.source + + +class PkgsetSourceKoji(pypungi.phases.pkgset.source.PkgsetSourceBase): + enabled = True + config_options = ( + { + "name": "pkgset_source", + "expected_types": [str], + "expected_values": "koji", + }, + { + "name": "pkgset_koji_url", + "expected_types": [str], + }, + { + "name": "pkgset_koji_tag", + "expected_types": [str], + }, + { + "name": "pkgset_koji_inherit", + "expected_types": [bool], + "optional": True, + }, + { + "name": "pkgset_koji_path_prefix", + "expected_types": [str], + }, + ) + + def __call__(self): + compose = self.compose + koji_url = compose.conf["pkgset_koji_url"] + # koji_tag = compose.conf["pkgset_koji_tag"] + path_prefix = compose.conf["pkgset_koji_path_prefix"].rstrip("/") + "/" # must contain trailing '/' + + koji_proxy = koji.ClientSession(koji_url) + package_sets = get_pkgset_from_koji(self.compose, koji_proxy, path_prefix) + return (package_sets, path_prefix) + + +''' +class PkgsetKojiPhase(PhaseBase): + """PKGSET""" + name = "pkgset" + + def __init__(self, compose): + PhaseBase.__init__(self, compose) + self.package_sets = None + self.path_prefix = None + + def run(self): + path_prefix = self.compose.conf["koji_path_prefix"] + path_prefix = path_prefix.rstrip("/") + "/" # must contain trailing '/' + koji_url = self.compose.conf["koji_url"] + koji_proxy = koji.ClientSession(koji_url) + self.package_sets = get_pkgset_from_koji(self.compose, koji_proxy, path_prefix) + self.path_prefix = path_prefix +''' + + +def get_pkgset_from_koji(compose, koji_proxy, path_prefix): + event_info = get_koji_event_info(compose, koji_proxy) + tag_info = get_koji_tag_info(compose, koji_proxy) + + pkgset_global = populate_global_pkgset(compose, koji_proxy, path_prefix, tag_info, event_info) +# get_extra_packages(compose, pkgset_global) + package_sets = populate_arch_pkgsets(compose, path_prefix, pkgset_global) + package_sets["global"] = pkgset_global + + create_global_repo(compose, path_prefix) + for arch in compose.get_arches(): + # TODO: threads? runroot? + create_arch_repos(compose, arch, path_prefix) + + return package_sets + + +def populate_global_pkgset(compose, koji_proxy, path_prefix, compose_tag, event_id): + ALL_ARCHES = set(["src"]) + for arch in compose.get_arches(): + is_multilib = arch in compose.conf["multilib_arches"] + arches = get_valid_arches(arch, is_multilib) + ALL_ARCHES.update(arches) + + compose_tag = compose.conf["pkgset_koji_tag"] + inherit = compose.conf.get("pkgset_koji_inherit", True) + msg = "Populating the global package set from tag '%s'" % compose_tag + global_pkgset_path = os.path.join(compose.paths.work.topdir(arch="global"), "pkgset_global.pickle") + if compose.DEBUG and os.path.isfile(global_pkgset_path): + compose.log_warning("[SKIP ] %s" % msg) + pkgset = pickle.load(open(global_pkgset_path, "r")) + else: + compose.log_info(msg) + pkgset = pypungi.phases.pkgset.pkgsets.KojiPackageSet(koji_proxy, compose.conf["sigkeys"], logger=compose._logger, arches=ALL_ARCHES) + pkgset.populate(compose_tag, event_id, inherit=inherit) + f = open(global_pkgset_path, "w") + data = pickle.dumps(pkgset) + f.write(data) + f.close() + + # write global package list + pkgset.save_file_list(compose.paths.work.package_list(arch="global"), remove_path_prefix=path_prefix) + return pkgset + + +def get_koji_event_info(compose, koji_proxy): + event_file = os.path.join(compose.paths.work.topdir(arch="global"), "koji-event") + + if compose.koji_event: + koji_event = koji_proxy.getEvent(compose.koji_event) + compose.log_info("Setting koji event to a custom value: %s" % compose.koji_event) + json.dump(koji_event, open(event_file, "w")) + + msg = "Getting koji event" + if compose.DEBUG and os.path.exists(event_file): + compose.log_warning("[SKIP ] %s" % msg) + result = json.load(open(event_file, "r")) + else: + compose.log_info(msg) + result = koji_proxy.getLastEvent() + json.dump(result, open(event_file, "w")) + compose.log_info("Koji event: %s" % result["id"]) + return result + + +def get_koji_tag_info(compose, koji_proxy): + tag_file = os.path.join(compose.paths.work.topdir(arch="global"), "koji-tag") + msg = "Getting a koji tag info" + if compose.DEBUG and os.path.exists(tag_file): + compose.log_warning("[SKIP ] %s" % msg) + result = json.load(open(tag_file, "r")) + else: + compose.log_info(msg) + tag_name = compose.conf["pkgset_koji_tag"] + result = koji_proxy.getTag(tag_name) + if result is None: + raise ValueError("Unknown koji tag: %s" % tag_name) + result["name"] = tag_name + json.dump(result, open(tag_file, "w")) + compose.log_info("Koji compose tag: %(name)s (ID: %(id)s)" % result) + return result diff --git a/pungi/phases/pkgset/sources/source_repos.py b/pungi/phases/pkgset/sources/source_repos.py new file mode 100644 index 00000000..d40ccd06 --- /dev/null +++ b/pungi/phases/pkgset/sources/source_repos.py @@ -0,0 +1,185 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import os +import cPickle as pickle + +from kobo.shortcuts import run + +import pypungi.phases.pkgset.pkgsets +from pypungi.arch import get_valid_arches +from pypungi.util import makedirs +from pypungi.wrappers.pungi import PungiWrapper + +from pypungi.phases.pkgset.common import create_global_repo, create_arch_repos, populate_arch_pkgsets +from pypungi.phases.gather import get_prepopulate_packages +from pypungi.linker import LinkerThread, LinkerPool + + +import pypungi.phases.pkgset.source + + +class PkgsetSourceRepos(pypungi.phases.pkgset.source.PkgsetSourceBase): + enabled = True + config_options = ( + { + "name": "pkgset_source", + "expected_types": [str], + "expected_values": "repos", + }, + { + "name": "pkgset_repos", + "expected_types": [dict], + }, + ) + + def __call__(self): + package_sets, path_prefix = get_pkgset_from_repos(self.compose) + return (package_sets, path_prefix) + + +def get_pkgset_from_repos(compose): + # populate pkgset from yum repos + # TODO: noarch hack - secondary arches, use x86_64 noarch where possible + flist = [] + + link_type = compose.conf.get("link_type", "hardlink-or-copy") + pool = LinkerPool(link_type, logger=compose._logger) + for i in range(10): + pool.add(LinkerThread(pool)) + + seen_packages = set() + for arch in compose.get_arches(): + # write a pungi config for remote repos and a local comps repo + repos = {} + for num, repo in enumerate(compose.conf["pkgset_repos"][arch]): + repo_path = repo + if "://" not in repo_path: + repo_path = os.path.join(compose.config_dir, repo) + repos["repo-%s" % num] = repo_path + + comps_repo = None + if compose.has_comps: + repos["comps"] = compose.paths.work.comps_repo(arch=arch) + comps_repo = "comps" + write_pungi_config(compose, arch, None, repos=repos, comps_repo=comps_repo) + + pungi = PungiWrapper() + pungi_conf = compose.paths.work.pungi_conf(arch=arch) + pungi_log = compose.paths.log.log_file(arch, "fooo") + pungi_dir = compose.paths.work.pungi_download_dir(arch) + cmd = pungi.get_pungi_cmd(pungi_conf, destdir=pungi_dir, name="FOO", selfhosting=True, fulltree=True, multilib_methods=["all"], nodownload=False, full_archlist=True, arch=arch, cache_dir=compose.paths.work.pungi_cache_dir(arch=arch)) + cmd.append("--force") + + # TODO: runroot + run(cmd, logfile=pungi_log, show_cmd=True, stdout=False) + + path_prefix = os.path.join(compose.paths.work.topdir(arch="global"), "download") + "/" + makedirs(path_prefix) + for root, dirs, files in os.walk(pungi_dir): + for fn in files: + if not fn.endswith(".rpm"): + continue + if fn in seen_packages: + continue + seen_packages.add(fn) + src = os.path.join(root, fn) + dst = os.path.join(path_prefix, os.path.basename(src)) + flist.append(dst) + pool.queue_put((src, dst)) + + msg = "Linking downloaded pkgset packages" + compose.log_info("[BEGIN] %s" % msg) + pool.start() + pool.stop() + compose.log_info("[DONE ] %s" % msg) + + flist = sorted(set(flist)) + pkgset_global = populate_global_pkgset(compose, flist, path_prefix) +# get_extra_packages(compose, pkgset_global) + package_sets = populate_arch_pkgsets(compose, path_prefix, pkgset_global) + + create_global_repo(compose, path_prefix) + for arch in compose.get_arches(): + # TODO: threads? runroot? + create_arch_repos(compose, arch, path_prefix) + + package_sets["global"] = pkgset_global + return package_sets, path_prefix + + +def populate_global_pkgset(compose, file_list, path_prefix): + ALL_ARCHES = set(["src"]) + for arch in compose.get_arches(): + is_multilib = arch in compose.conf["multilib_arches"] + arches = get_valid_arches(arch, is_multilib) + ALL_ARCHES.update(arches) + + msg = "Populating the global package set from a file list" + global_pkgset_path = os.path.join(compose.paths.work.topdir(arch="global"), "packages.pickle") + if compose.DEBUG and os.path.isfile(global_pkgset_path): + compose.log_warning("[SKIP ] %s" % msg) + pkgset = pickle.load(open(global_pkgset_path, "r")) + else: + compose.log_info(msg) + pkgset = pypungi.phases.pkgset.pkgsets.FilelistPackageSet(compose.conf["sigkeys"], logger=compose._logger, arches=ALL_ARCHES) + pkgset.populate(file_list) + f = open(global_pkgset_path, "w") + data = pickle.dumps(pkgset) + f.write(data) + f.close() + + # write global package list + pkgset.save_file_list(compose.paths.work.package_list(arch="global"), remove_path_prefix=path_prefix) + return pkgset + + +def write_pungi_config(compose, arch, variant, repos=None, comps_repo=None, package_set=None): + """write pungi config (kickstart) for arch/variant""" + pungi = PungiWrapper() + pungi_cfg = compose.paths.work.pungi_conf(variant=variant, arch=arch) + msg = "Writing pungi config (arch: %s, variant: %s): %s" % (arch, variant, pungi_cfg) + + if compose.DEBUG and os.path.isfile(pungi_cfg): + compose.log_warning("[SKIP ] %s" % msg) + return + + compose.log_info(msg) + + # TODO move to a function + gather_source = "GatherSource%s" % compose.conf["gather_source"] + from pypungi.phases.gather.source import GatherSourceContainer + import pypungi.phases.gather.sources + GatherSourceContainer.register_module(pypungi.phases.gather.sources) + container = GatherSourceContainer() + SourceClass = container[gather_source] + src = SourceClass(compose) + + packages = [] + pkgs, grps = src(arch, variant) + for pkg_name, pkg_arch in pkgs: + if pkg_arch is None: + packages.append(pkg_name) + else: + packages.append("%s.%s" % (pkg_name, pkg_arch)) + + # include *all* packages providing system-release + if "system-release" not in packages: + packages.append("system-release") + + prepopulate = get_prepopulate_packages(compose, arch, None) + pungi.write_kickstart(ks_path=pungi_cfg, repos=repos, groups=grps, packages=packages, exclude_packages=[], comps_repo=None, prepopulate=prepopulate) diff --git a/pungi/phases/product_img.py b/pungi/phases/product_img.py new file mode 100644 index 00000000..5334abec --- /dev/null +++ b/pungi/phases/product_img.py @@ -0,0 +1,279 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +""" +Expected product.img paths +========================== + +RHEL 6 +------ +installclasses/$variant.py +locale/$lang/LC_MESSAGES/comps.mo + +RHEL 7 +------ +run/install/product/installclasses/$variant.py +run/install/product/locale/$lang/LC_MESSAGES/comps.mo + +Compatibility symlinks +---------------------- +installclasses -> run/install/product/installclasses +locale -> run/install/product/locale +run/install/product/pyanaconda/installclasses -> ../installclasses +""" + + +import os +import fnmatch +import tempfile +import shutil +import pipes + +from kobo.shortcuts import run + +from pypungi.arch import split_name_arch +from pypungi.util import makedirs, pkg_is_rpm +from pypungi.phases.base import PhaseBase +from pypungi.wrappers.iso import IsoWrapper +from pypungi.wrappers.scm import get_file_from_scm, get_dir_from_scm + + +class ProductimgPhase(PhaseBase): + """PRODUCTIMG""" + name = "productimg" + + config_options = ( + { + "name": "bootable", + "expected_types": [bool], + "expected_values": [True], + }, + ) + + def __init__(self, compose, pkgset_phase): + PhaseBase.__init__(self, compose) + # pkgset_phase provides package_sets and path_prefix + self.pkgset_phase = pkgset_phase + + def skip(self): + if PhaseBase.skip(self): + return True + if not self.compose.conf.get("bootable"): + msg = "Not a bootable product. Skipping creating product images." + self.compose.log_debug(msg) + return True + return False + + def run(self): + # create PRODUCT.IMG + for variant in self.compose.get_variants(): + if variant.type != "variant": + continue + create_product_img(self.compose, "global", variant) + + # copy PRODUCT.IMG + for arch in self.compose.get_arches(): + for variant in self.compose.get_variants(arch=arch): + if variant.type != "variant": + continue + image = self.compose.paths.work.product_img(variant) + os_tree = self.compose.paths.compose.os_tree(arch, variant) + target_dir = os.path.join(os_tree, "images") + target_path = os.path.join(target_dir, "product.img") + if not os.path.isfile(target_path): + makedirs(target_dir) + shutil.copy2(image, target_path) + + for arch in self.compose.get_arches(): + for variant in self.compose.get_variants(arch=arch): + if variant.type != "variant": + continue + rebuild_boot_iso(self.compose, arch, variant, self.pkgset_phase.package_sets) + + +def create_product_img(compose, arch, variant): + # product.img is noarch (at least on rhel6 and rhel7) + arch = "global" + + msg = "Creating product.img (arch: %s, variant: %s)" % (arch, variant) + image = compose.paths.work.product_img(variant) + if os.path.exists(image): + compose.log_warning("[SKIP ] %s" % msg) + return + + compose.log_info("[BEGIN] %s" % msg) + + product_tmp = tempfile.mkdtemp(prefix="product_img_") + install_class = compose.conf["install_class"].copy() + install_class["file"] = install_class["file"] % {"variant_id": variant.id.lower()} + install_dir = os.path.join(product_tmp, "installclasses") + makedirs(install_dir) + get_file_from_scm(install_class, target_path=install_dir, logger=None) + + po_files = compose.conf["po_files"] + po_tmp = tempfile.mkdtemp(prefix="pofiles_") + get_dir_from_scm(po_files, po_tmp, logger=compose._logger) + for po_file in os.listdir(po_tmp): + if not po_file.endswith(".po"): + continue + lang = po_file[:-3] + target_dir = os.path.join(product_tmp, "locale", lang, "LC_MESSAGES") + makedirs(target_dir) + run(["msgfmt", "--output-file", os.path.join(target_dir, "comps.mo"), os.path.join(po_tmp, po_file)]) + + shutil.rmtree(po_tmp) + + mount_tmp = tempfile.mkdtemp(prefix="product_img_mount_") + cmds = [ + # allocate image + "dd if=/dev/zero of=%s bs=1k count=5760" % pipes.quote(image), + # create file system + "mke2fs -F %s" % pipes.quote(image), + "mount -o loop %s %s" % (pipes.quote(image), pipes.quote(mount_tmp)), + "mkdir -p %s/run/install/product" % pipes.quote(mount_tmp), + "cp -rp %s/* %s/run/install/product/" % (pipes.quote(product_tmp), pipes.quote(mount_tmp)), + "mkdir -p %s/run/install/product/pyanaconda" % pipes.quote(mount_tmp), + # compat symlink: installclasses -> run/install/product/installclasses + "ln -s run/install/product/installclasses %s" % pipes.quote(mount_tmp), + # compat symlink: locale -> run/install/product/locale + "ln -s run/install/product/locale %s" % pipes.quote(mount_tmp), + # compat symlink: run/install/product/pyanaconda/installclasses -> ../installclasses + "ln -s ../installclasses %s/run/install/product/pyanaconda/installclasses" % pipes.quote(mount_tmp), + "umount %s" % pipes.quote(mount_tmp), + # tweak last mount path written in the image + "tune2fs -M /run/install/product %s" % pipes.quote(image), + ] + run(" && ".join(cmds)) + shutil.rmtree(mount_tmp) + + shutil.rmtree(product_tmp) + + compose.log_info("[DONE ] %s" % msg) + + +def rebuild_boot_iso(compose, arch, variant, package_sets): + os_tree = compose.paths.compose.os_tree(arch, variant) + buildinstall_dir = compose.paths.work.buildinstall_dir(arch) + boot_iso = os.path.join(os_tree, "images", "boot.iso") + product_img = compose.paths.work.product_img(variant) + buildinstall_boot_iso = os.path.join(buildinstall_dir, "images", "boot.iso") + buildinstall_method = compose.conf["buildinstall_method"] + log_file = compose.paths.log.log_file(arch, "rebuild_boot_iso-%s.%s" % (variant, arch)) + + msg = "Rebuilding boot.iso (arch: %s, variant: %s)" % (arch, variant) + + if not os.path.isfile(boot_iso): + # nothing to do + compose.log_warning("[SKIP ] %s" % msg) + return + + compose.log_info("[BEGIN] %s" % msg) + + iso = IsoWrapper() + + # read the original volume id + volume_id = iso.get_volume_id(boot_iso) + + # remove the original boot.iso (created during buildinstall) from the os dir + os.remove(boot_iso) + + tmp_dir = tempfile.mkdtemp(prefix="boot_iso_") + mount_dir = tempfile.mkdtemp(prefix="boot_iso_mount_") + + cmd = "mount -o loop %s %s" % (pipes.quote(buildinstall_boot_iso), pipes.quote(mount_dir)) + run(cmd, logfile=log_file, show_cmd=True) + + images_dir = os.path.join(tmp_dir, "images") + os.makedirs(images_dir) + shutil.copy2(product_img, os.path.join(images_dir, "product.img")) + + if os.path.isfile(os.path.join(mount_dir, "isolinux", "isolinux.bin")): + os.makedirs(os.path.join(tmp_dir, "isolinux")) + shutil.copy2(os.path.join(mount_dir, "isolinux", "isolinux.bin"), os.path.join(tmp_dir, "isolinux")) + + graft_points = iso.get_graft_points([mount_dir, tmp_dir]) + graft_points_path = os.path.join(compose.paths.work.topdir(arch=arch), "boot-%s.%s.iso-graft-points" % (variant, arch)) + iso.write_graft_points(graft_points_path, graft_points, exclude=["*/TRANS.TBL", "*/boot.cat"]) + + mkisofs_kwargs = {} + boot_files = None + if buildinstall_method == "lorax": + # TODO: $arch instead of ppc + mkisofs_kwargs["boot_args"] = iso.get_boot_options(arch, "/usr/share/lorax/config_files/ppc") + elif buildinstall_method == "buildinstall": + boot_files = explode_anaconda(compose, arch, variant, package_sets) + mkisofs_kwargs["boot_args"] = iso.get_boot_options(arch, boot_files) + + # ppc(64) doesn't seem to support utf-8 + if arch in ("ppc", "ppc64"): + mkisofs_kwargs["input_charset"] = None + + mkisofs_cmd = iso.get_mkisofs_cmd(boot_iso, None, volid=volume_id, exclude=["./lost+found"], graft_points=graft_points_path, **mkisofs_kwargs) + run(mkisofs_cmd, logfile=log_file, show_cmd=True) + + cmd = "umount %s" % pipes.quote(mount_dir) + run(cmd, logfile=log_file, show_cmd=True) + + if arch == "x86_64": + isohybrid_cmd = "isohybrid --uefi %s" % pipes.quote(boot_iso) + run(isohybrid_cmd, logfile=log_file, show_cmd=True) + elif arch == "i386": + isohybrid_cmd = "isohybrid %s" % pipes.quote(boot_iso) + run(isohybrid_cmd, logfile=log_file, show_cmd=True) + + # implant MD5SUM to iso + isomd5sum_cmd = iso.get_implantisomd5_cmd(boot_iso, compose.supported) + isomd5sum_cmd = " ".join([pipes.quote(i) for i in isomd5sum_cmd]) + run(isomd5sum_cmd, logfile=log_file, show_cmd=True) + + if boot_files: + shutil.rmtree(boot_files) + shutil.rmtree(tmp_dir) + shutil.rmtree(mount_dir) + + # .treeinfo is written after productimg phase + # -> checksums should match + # -> no need to write/modify it here + compose.log_info("[DONE ] %s" % msg) + + +def explode_anaconda(compose, arch, variant, package_sets): + tmp_dir = tempfile.mkdtemp(prefix="anaconda_") + scm_dict = { + "scm": "rpm", + "repo": "anaconda.%s" % arch, + "file": [ + "/usr/lib/anaconda-runtime/boot/*", + ] + } + # if scm is "rpm" and repo contains a package name, find the package(s) in package set + if scm_dict["scm"] == "rpm" and not (scm_dict["repo"].startswith("/") or "://" in scm_dict["repo"]): + rpms = [] + for pkgset_file in package_sets[arch]: + pkg_obj = package_sets[arch][pkgset_file] + if not pkg_is_rpm(pkg_obj): + continue + pkg_name, pkg_arch = split_name_arch(scm_dict["repo"]) + if fnmatch.fnmatch(pkg_obj.name, pkg_name) and (pkg_arch is None or pkg_arch == pkg_obj.arch): + compose.log_critical("%s %s %s" % (pkg_obj.name, pkg_name, pkg_arch)) + rpms.append(pkg_obj.file_path) + scm_dict["repo"] = rpms + + if not rpms: + return None + get_file_from_scm(scm_dict, tmp_dir, logger=compose._logger) + return tmp_dir diff --git a/pungi/phases/test.py b/pungi/phases/test.py new file mode 100644 index 00000000..38770f3b --- /dev/null +++ b/pungi/phases/test.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import tempfile + +from kobo.shortcuts import run + +from pypungi.wrappers.repoclosure import RepoclosureWrapper +from pypungi.arch import get_valid_arches +from pypungi.phases.base import PhaseBase +from pypungi.phases.gather import get_lookaside_repos +from pypungi.util import rmtree + + +class TestPhase(PhaseBase): + name = "test" + + def run(self): + run_repoclosure(self.compose) + + +def run_repoclosure(compose): + repoclosure = RepoclosureWrapper() + + # TODO: Special handling for src packages (use repoclosure param builddeps) + + msg = "Running repoclosure" + compose.log_info("[BEGIN] %s" % msg) + + # Arch repos + for arch in compose.get_arches(): + is_multilib = arch in compose.conf["multilib_arches"] + arches = get_valid_arches(arch, is_multilib) + repo_id = "repoclosure-%s" % arch + repo_dir = compose.paths.work.arch_repo(arch=arch) + + lookaside = {} + if compose.conf.get("product_is_layered", False): + for i, lookaside_url in enumerate(get_lookaside_repos(compose, arch, None)): + lookaside["lookaside-%s-%s" % (arch, i)] = lookaside_url + + cmd = repoclosure.get_repoclosure_cmd(repos={repo_id: repo_dir}, lookaside=lookaside, arch=arches) + # Use temp working directory directory as workaround for + # https://bugzilla.redhat.com/show_bug.cgi?id=795137 + tmp_dir = tempfile.mkdtemp(prefix="repoclosure_") + try: + run(cmd, logfile=compose.paths.log.log_file(arch, "repoclosure"), show_cmd=True, can_fail=True, workdir=tmp_dir) + finally: + rmtree(tmp_dir) + + # Variant repos + all_repos = {} # to be used as lookaside for the self-hosting check + all_arches = set() + for arch in compose.get_arches(): + is_multilib = arch in compose.conf["multilib_arches"] + arches = get_valid_arches(arch, is_multilib) + all_arches.update(arches) + for variant in compose.get_variants(arch=arch): + lookaside = {} + if variant.parent: + repo_id = "repoclosure-%s.%s" % (variant.parent.uid, arch) + repo_dir = compose.paths.compose.repository(arch=arch, variant=variant.parent) + lookaside[repo_id] = repo_dir + + repos = {} + repo_id = "repoclosure-%s.%s" % (variant.uid, arch) + repo_dir = compose.paths.compose.repository(arch=arch, variant=variant) + repos[repo_id] = repo_dir + + if compose.conf.get("product_is_layered", False): + for i, lookaside_url in enumerate(get_lookaside_repos(compose, arch, variant)): + lookaside["lookaside-%s.%s-%s" % (variant.uid, arch, i)] = lookaside_url + + cmd = repoclosure.get_repoclosure_cmd(repos=repos, lookaside=lookaside, arch=arches) + # Use temp working directory directory as workaround for + # https://bugzilla.redhat.com/show_bug.cgi?id=795137 + tmp_dir = tempfile.mkdtemp(prefix="repoclosure_") + try: + run(cmd, logfile=compose.paths.log.log_file(arch, "repoclosure-%s" % variant), show_cmd=True, can_fail=True, workdir=tmp_dir) + finally: + rmtree(tmp_dir) + + all_repos.update(repos) + all_repos.update(lookaside) + repo_id = "repoclosure-%s.%s" % (variant.uid, "src") + repo_dir = compose.paths.compose.repository(arch="src", variant=variant) + all_repos[repo_id] = repo_dir + + # A SRPM can be built on any arch and is always rebuilt before building on the target arch. + # This means the deps can't be always satisfied within one tree arch. + # As a workaround, let's run the self-hosting check across all repos. + + # XXX: This doesn't solve a situation, when a noarch package is excluded due to ExcludeArch/ExclusiveArch and it's still required on that arch. + # In this case, it's an obvious bug in the test. + + # check BuildRequires (self-hosting) + cmd = repoclosure.get_repoclosure_cmd(repos=all_repos, arch=all_arches, builddeps=True) + # Use temp working directory directory as workaround for + # https://bugzilla.redhat.com/show_bug.cgi?id=795137 + tmp_dir = tempfile.mkdtemp(prefix="repoclosure_") + try: + run(cmd, logfile=compose.paths.log.log_file("global", "repoclosure-builddeps"), show_cmd=True, can_fail=True, workdir=tmp_dir) + finally: + rmtree(tmp_dir) + + compose.log_info("[DONE ] %s" % msg) diff --git a/pungi/util.py b/pungi/util.py index 0a2ea111..77486a1f 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -1,4 +1,6 @@ -#!/usr/bin/python -tt +# -*- coding: utf-8 -*- + + # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. @@ -12,11 +14,19 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + import subprocess import os import shutil import sys import hashlib +import errno +import pipes +import re + +from kobo.shortcuts import run +from productmd import get_major_version + def _doRunCommand(command, logger, rundir='/tmp', output=subprocess.PIPE, error=subprocess.PIPE, env=None): """Run a command and log the output. Error out if we get something on stderr""" @@ -121,3 +131,179 @@ def _doCheckSum(path, hash, logger): myfile.close() return '%s:%s' % (hash, sum.hexdigest()) + + +def makedirs(path, mode=0o775): + mask = os.umask(0) + try: + os.makedirs(path, mode=mode) + except OSError as ex: + if ex.errno != errno.EEXIST: + raise + os.umask(mask) + + +def rmtree(path, ignore_errors=False, onerror=None): + """shutil.rmtree ENOENT (ignoring no such file or directory) errors""" + try: + shutil.rmtree(path, ignore_errors, onerror) + except OSError as ex: + if ex.errno != errno.ENOENT: + raise + + +def explode_rpm_package(pkg_path, target_dir): + """Explode a rpm package into target_dir.""" + pkg_path = os.path.abspath(pkg_path) + makedirs(target_dir) + run("rpm2cpio %s | cpio -iuvmd && chmod -R a+rX ." % pipes.quote(pkg_path), workdir=target_dir) + + +def pkg_is_rpm(pkg_obj): + if pkg_is_srpm(pkg_obj): + return False + if pkg_is_debug(pkg_obj): + return False + return True + + +def pkg_is_srpm(pkg_obj): + if isinstance(pkg_obj, str): + # string, probably N.A, N-V-R.A, N-V-R.A.rpm + for i in (".src", ".nosrc", ".src.rpm", ".nosrc.rpm"): + if pkg_obj.endswith(i): + return True + else: + # package object + if pkg_obj.arch in ("src", "nosrc"): + return True + return False + + +def pkg_is_debug(pkg_obj): + if pkg_is_srpm(pkg_obj): + return False + if isinstance(pkg_obj, str): + # string + if "-debuginfo" in pkg_obj: + return True + else: + # package object + if "-debuginfo" in pkg_obj.name: + return True + return False + + +# fomat: [(variant_uid_regex, {arch|*: [data]})] +def get_arch_variant_data(conf, var_name, arch, variant): + result = [] + for conf_variant, conf_data in conf.get(var_name, []): + if variant is not None and not re.match(conf_variant, variant.uid): + continue + for conf_arch in conf_data: + if conf_arch != "*" and conf_arch != arch: + continue + if conf_arch == "*" and arch == "src": + # src is excluded from '*' and needs to be explicitly added to the mapping + continue + if isinstance(conf_data[conf_arch], list): + result.extend(conf_data[conf_arch]) + else: + result.append(conf_data[conf_arch]) + return result + + +# fomat: {arch|*: [data]} +def get_arch_data(conf, var_name, arch): + result = [] + for conf_arch, conf_data in conf.get(var_name, {}).items(): + if conf_arch != "*" and conf_arch != arch: + continue + if conf_arch == "*" and arch == "src": + # src is excluded from '*' and needs to be explicitly added to the mapping + continue + if isinstance(conf_data, list): + result.extend(conf_data) + else: + result.append(conf_data) + return result + + +def get_buildroot_rpms(compose, task_id): + """Get build root RPMs - either from runroot or local""" + result = [] + if task_id: + # runroot + import koji + koji_url = compose.conf["pkgset_koji_url"] + koji_proxy = koji.ClientSession(koji_url) + buildroot_infos = koji_proxy.listBuildroots(taskID=task_id) + buildroot_info = buildroot_infos[-1] + data = koji_proxy.listRPMs(componentBuildrootID=buildroot_info["id"]) + for rpm_info in data: + fmt = "%(nvr)s.%(arch)s" + result.append(fmt % rpm_info) + else: + # local + retcode, output = run("rpm -qa --qf='%{name}-%{version}-%{release}.%{arch}\n'") + for i in output.splitlines(): + if not i: + continue + result.append(i) + result.sort() + return result + + +def get_volid(compose, arch, variant=None, escape_spaces=False): + """Get ISO volume ID for arch and variant""" + if variant and variant.type == "addon": + # addons are part of parent variant media + return None + + if variant and variant.type == "layered-product": + product_short = variant.product_short + product_version = variant.product_version + product_is_layered = True + base_product_short = compose.conf["product_short"] + base_product_version = get_major_version(compose.conf["product_version"]) + variant_uid = variant.parent.uid + else: + product_short = compose.conf["product_short"] + product_version = compose.conf["product_version"] + product_is_layered = compose.conf["product_is_layered"] + base_product_short = compose.conf.get("base_product_short", "") + base_product_version = compose.conf.get("base_product_version", "") + variant_uid = variant and variant.uid or None + + products = [ + "%(product_short)s-%(product_version)s %(variant_uid)s.%(arch)s", + "%(product_short)s-%(product_version)s %(arch)s", + ] + layered_products = [ + "%(product_short)s-%(product_version)s %(base_product_short)s-%(base_product_version)s %(variant_uid)s.%(arch)s", + "%(product_short)s-%(product_version)s %(base_product_short)s-%(base_product_version)s %(arch)s", + ] + + volid = None + if product_is_layered: + all_products = layered_products + products + else: + all_products = products + + for i in all_products: + if not variant_uid and "%(variant_uid)s" in i: + continue + volid = i % locals() + if len(volid) <= 32: + break + + # from wrappers.iso import IsoWrapper + # iso = IsoWrapper(logger=compose._logger) + # volid = iso._truncate_volid(volid) + + if len(volid) > 32: + raise ValueError("Could not create volume ID <= 32 characters") + + if escape_spaces: + volid = volid.replace(" ", r"\x20") + return volid diff --git a/pungi/wrappers/__init__.py b/pungi/wrappers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pungi/wrappers/comps.py b/pungi/wrappers/comps.py new file mode 100644 index 00000000..3e511a3e --- /dev/null +++ b/pungi/wrappers/comps.py @@ -0,0 +1,427 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import sys +import fnmatch +import xml.dom.minidom +import yum.comps + + +if sys.version_info[:2] < (2, 7): + # HACK: remove spaces from text elements on py < 2.7 + OldElement = xml.dom.minidom.Element + + class Element(OldElement): + def writexml(self, writer, indent="", addindent="", newl=""): + if len(self.childNodes) == 1 and self.firstChild.nodeType == 3: + writer.write(indent) + OldElement.writexml(self, writer) + writer.write(newl) + else: + OldElement.writexml(self, writer, indent, addindent, newl) + + xml.dom.minidom.Element = Element + + +class CompsWrapper(object): + """Class for reading and retreiving information from comps XML files""" + + def __init__(self, comps_file): + self.comps = yum.comps.Comps() + self.comps.add(comps_file) + self.comps_file = comps_file + + def get_comps_packages(self): + """Returns a dictionary containing all packages in comps""" + + packages = set() + for group in self.comps.get_groups(): + packages.update(group.packages) + return list(packages) + + def get_comps_groups(self): + return self.comps.get_groups() + + def write_comps(self, comps_obj=None, target_file=None): + if not comps_obj: + comps_obj = self.generate_comps() + if not target_file: + target_file = self.comps_file + stream = open(target_file, "w") + # comps_obj.writexml(stream, addindent=" ", newl="\n") # no encoding -> use toprettyxml() + stream.write(comps_obj.toprettyxml(indent=" ", encoding="UTF-8")) + stream.close() + + def generate_comps(self): + impl = xml.dom.minidom.getDOMImplementation() + doctype = impl.createDocumentType("comps", "-//Red Hat, Inc.//DTD Comps info//EN", "comps.dtd") + doc = impl.createDocument(None, "comps", doctype) + msg_elem = doc.documentElement + + groups = {} + for group_obj in self.comps.get_groups(): + groupid = group_obj.groupid + groups[groupid] = {"group_obj": group_obj} + + group_names = groups.keys() + group_names.sort() + for group_key in group_names: + group = groups[group_key]["group_obj"] + group_node = doc.createElement("group") + msg_elem.appendChild(group_node) + + id_node = doc.createElement("id") + id_node.appendChild(doc.createTextNode(group.groupid)) + group_node.appendChild(id_node) + + name_node = doc.createElement("name") + name_node.appendChild(doc.createTextNode(group.name)) + group_node.appendChild(name_node) + + langs = group.translated_name.keys() + langs.sort() + + for lang in langs: + text = group.translated_name[lang].decode("UTF-8") + node = doc.createElement("name") + node.setAttribute("xml:lang", lang) + node.appendChild(doc.createTextNode(text)) + group_node.appendChild(node) + + node = doc.createElement("description") + group_node.appendChild(node) + if group.description and group.description != "": + node.appendChild(doc.createTextNode(group.description)) + langs = group.translated_description.keys() + langs.sort() + + for lang in langs: + text = group.translated_description[lang].decode("UTF-8") + node = doc.createElement("description") + node.setAttribute("xml:lang", lang) + node.appendChild(doc.createTextNode(text)) + group_node.appendChild(node) + + node = doc.createElement("default") + + if group.default: + node.appendChild(doc.createTextNode("true")) + else: + node.appendChild(doc.createTextNode("false")) + group_node.appendChild(node) + + node = doc.createElement("uservisible") + + if group.user_visible: + node.appendChild(doc.createTextNode("true")) + else: + node.appendChild(doc.createTextNode("false")) + + group_node.appendChild(node) + + if group.langonly: + node = doc.createElement("langonly") + node.appendChild(doc.createTextNode(group.langonly)) + group_node.appendChild(node) + + packagelist = doc.createElement("packagelist") + + for package_type in ("mandatory", "default", "optional", "conditional"): + packages = getattr(group, package_type + "_packages").keys() + packages.sort() + for package in packages: + node = doc.createElement("packagereq") + node.appendChild(doc.createTextNode(package)) + node.setAttribute("type", package_type) + packagelist.appendChild(node) + if package_type == "conditional": + node.setAttribute("requires", group.conditional_packages[package]) + + group_node.appendChild(packagelist) + + categories = self.comps.get_categories() + for category in categories: + groups = set(category.groups) & set([i.groupid for i in self.comps.get_groups()]) + if not groups: + continue + cat_node = doc.createElement("category") + msg_elem.appendChild(cat_node) + + id_node = doc.createElement("id") + id_node.appendChild(doc.createTextNode(category.categoryid)) + cat_node.appendChild(id_node) + + name_node = doc.createElement("name") + name_node.appendChild(doc.createTextNode(category.name)) + cat_node.appendChild(name_node) + + langs = category.translated_name.keys() + langs.sort() + + for lang in langs: + text = category.translated_name[lang].decode("UTF-8") + node = doc.createElement("name") + node.setAttribute("xml:lang", lang) + node.appendChild(doc.createTextNode(text)) + cat_node.appendChild(node) + + if category.description and category.description != "": + node = doc.createElement("description") + node.appendChild(doc.createTextNode(category.description)) + cat_node.appendChild(node) + langs = category.translated_description.keys() + langs.sort() + + for lang in langs: + text = category.translated_description[lang].decode("UTF-8") + node = doc.createElement("description") + node.setAttribute("xml:lang", lang) + node.appendChild(doc.createTextNode(text)) + cat_node.appendChild(node) + + if category.display_order is not None: + display_node = doc.createElement("display_order") + display_node.appendChild(doc.createTextNode("%s" % category.display_order)) + cat_node.appendChild(display_node) + + grouplist_node = doc.createElement("grouplist") + groupids = sorted(groups) + + for groupid in groupids: + node = doc.createElement("groupid") + node.appendChild(doc.createTextNode(groupid)) + grouplist_node.appendChild(node) + + cat_node.appendChild(grouplist_node) + + # XXX + environments = self.comps.get_environments() + if environments: + for environment in environments: + groups = set(environment.groups) & set([i.groupid for i in self.comps.get_groups()]) + if not groups: + continue + env_node = doc.createElement("environment") + msg_elem.appendChild(env_node) + + id_node = doc.createElement("id") + id_node.appendChild(doc.createTextNode(environment.environmentid)) + env_node.appendChild(id_node) + + name_node = doc.createElement("name") + name_node.appendChild(doc.createTextNode(environment.name)) + env_node.appendChild(name_node) + + langs = environment.translated_name.keys() + langs.sort() + + for lang in langs: + text = environment.translated_name[lang].decode("UTF-8") + node = doc.createElement("name") + node.setAttribute("xml:lang", lang) + node.appendChild(doc.createTextNode(text)) + env_node.appendChild(node) + + if environment.description: + node = doc.createElement("description") + node.appendChild(doc.createTextNode(environment.description)) + env_node.appendChild(node) + + langs = environment.translated_description.keys() + langs.sort() + + for lang in langs: + text = environment.translated_description[lang].decode("UTF-8") + node = doc.createElement("description") + node.setAttribute("xml:lang", lang) + node.appendChild(doc.createTextNode(text)) + env_node.appendChild(node) + + if environment.display_order is not None: + display_node = doc.createElement("display_order") + display_node.appendChild(doc.createTextNode("%s" % environment.display_order)) + env_node.appendChild(display_node) + + grouplist_node = doc.createElement("grouplist") + groupids = sorted(groups) + for groupid in groupids: + node = doc.createElement("groupid") + node.appendChild(doc.createTextNode(groupid)) + grouplist_node.appendChild(node) + env_node.appendChild(grouplist_node) + + optionids = sorted(environment.options) + if optionids: + optionlist_node = doc.createElement("optionlist") + for optionid in optionids: + node = doc.createElement("groupid") + node.appendChild(doc.createTextNode(optionid)) + optionlist_node.appendChild(node) + env_node.appendChild(optionlist_node) + + # XXX + langpacks = self.comps.get_langpacks() + if langpacks: + lang_node = doc.createElement("langpacks") + msg_elem.appendChild(lang_node) + + for langpack in langpacks: + match_node = doc.createElement("match") + match_node.setAttribute("name", langpack["name"]) + match_node.setAttribute("install", langpack["install"]) + lang_node.appendChild(match_node) + + return doc + + def _tweak_group(self, group_obj, group_dict): + if group_dict["default"] is not None: + group_obj.default = group_dict["default"] + if group_dict["uservisible"] is not None: + group_obj.uservisible = group_dict["uservisible"] + + def _tweak_env(self, env_obj, env_dict): + if env_dict["display_order"] is not None: + env_obj.display_order = env_dict["display_order"] + else: + # write actual display order back to env_dict + env_dict["display_order"] = env_obj.display_order + # write group list back to env_dict + env_dict["groups"] = env_obj.groups[:] + + def filter_groups(self, group_dicts): + """Filter groups according to group definitions in group_dicts. + group_dicts = [{ + "name": group ID, + "glob": True/False -- is "name" a glob? + "default: True/False/None -- if not None, set "default" accordingly + "uservisible": True/False/None -- if not None, set "uservisible" accordingly + }] + """ + to_remove = [] + for group_obj in self.comps.groups: + found = False + for group_dict in group_dicts: + if group_dict["glob"]: + if fnmatch.fnmatch(group_obj.groupid, group_dict["name"]): + found = True + self._tweak_group(group_obj, group_dict) + break + else: + if group_obj.groupid == group_dict["name"]: + self._tweak_group(group_obj, group_dict) + found = True + break + + if not found: + to_remove.append(group_obj.groupid) + + if to_remove: + for key, value in self.comps._groups.items(): + if key in to_remove: + del self.comps._groups[key] + + def filter_packages(self, pkglist): + rv = [] + for group_obj in self.comps.get_groups(): + for package_type in ("mandatory", "default", "optional", "conditional"): + group_pkgs = getattr(group_obj, "%s_packages" % package_type) + pkg_names = group_pkgs.keys() + pkg_names.sort() + for pkg in pkg_names: + if pkg not in pkglist: + rv.append((pkg, group_obj.name)) + del group_pkgs[pkg] + rv.sort() + return rv + + def filter_categories(self, catlist=None, include_empty=False): + rv = [] + if catlist is not None: + for categoryobj in self.comps.get_categories(): + if categoryobj.categoryid not in catlist: + rv.append(categoryobj.categoryid) + del self.comps._categories[categoryobj.categoryid] + if not include_empty: + comps_groups = [group.groupid for group in self.comps.get_groups()] + for categoryobj in self.comps.get_categories(): + matched = False + groupids = categoryobj.groups + groupids.sort() + for groupid in groupids: + if groupid not in comps_groups: + del categoryobj._groups[groupid] + else: + matched = True + if not matched: + rv.append(categoryobj.categoryid) + del self.comps._categories[categoryobj.categoryid] + rv.sort() + return rv + + def filter_environments(self, env_dicts): + """Filter environments according to group definitions in group_dicts. + env_dicts = [{ + "name": environment ID, + "display_order: /None -- if not None, set "display_order" accordingly + }] + """ + to_remove = [] + for env_obj in self.comps.environments: + found = False + for env_dict in env_dicts: + if env_obj.environmentid == env_dict["name"]: + self._tweak_env(env_obj, env_dict) + found = True + break + + if not found: + to_remove.append(env_obj.environmentid) + + if to_remove: + for key, value in self.comps._environments.items(): + if key in to_remove: + del self.comps._environments[key] + + def injectpackages(self, pkglist): + def getgroup(pkgname): + if pkgname.endswith("-devel"): + return "compat-arch-development" + elif pkgname.endswith("libs"): + return "compat-arch-libs" + else: + return "compat-arch-support" + + groups_dict = {} + for group_obj in self.comps.get_groups(): + groupid = group_obj.groupid + groups_dict[groupid] = {"group_obj": group_obj} + + pkggroup_dict = { + "compat-arch-development": [], + "compat-arch-libs": [], + "compat-arch-support": [], + } + + for pkgname in pkglist: + group = getgroup(pkgname) + pkggroup_dict[group].append(pkgname) + + for group_obj in self.comps.get_groups(): + groupid = group_obj.groupid + for pkg in pkggroup_dict[groupid]: + if pkg not in group_obj.packages: + group_obj.default_packages[pkg] = 1 diff --git a/pungi/wrappers/createrepo.py b/pungi/wrappers/createrepo.py new file mode 100644 index 00000000..7b24c14f --- /dev/null +++ b/pungi/wrappers/createrepo.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +from kobo.shortcuts import force_list + + +class CreaterepoWrapper(object): + def __init__(self, createrepo_c=False): + if createrepo_c: + self.createrepo = "createrepo_c" + self.mergerepo = "mergerepo_c" + else: + self.createrepo = "createrepo" + self.mergerepo = "mergerepo" + self.modifyrepo = "modifyrepo" + + def get_createrepo_cmd(self, directory, baseurl=None, outputdir=None, excludes=None, pkglist=None, groupfile=None, cachedir=None, + update=True, update_md_path=None, skip_stat=False, checkts=False, split=False, pretty=True, database=True, checksum=None, + unique_md_filenames=True, distro=None, content=None, repo=None, revision=None, deltas=False, oldpackagedirs=None, + num_deltas=None, workers=None): + # groupfile = /path/to/comps.xml + + cmd = [self.createrepo] + + cmd.append(directory) + + if baseurl: + cmd.append("--baseurl=%s" % baseurl) + + if outputdir: + cmd.append("--outputdir=%s" % outputdir) + + if excludes: + for i in force_list(excludes): + cmd.append("--excludes=%s" % i) + + if pkglist: + cmd.append("--pkglist=%s" % pkglist) + + if groupfile: + cmd.append("--groupfile=%s" % groupfile) + + if cachedir: + cmd.append("--cachedir=%s" % cachedir) + + if update: + cmd.append("--update") + + if update_md_path: + cmd.append("--update-md-path=%s" % update_md_path) + + if skip_stat: + cmd.append("--skip-stat") + + if checkts: + cmd.append("--checkts") + + if split: + cmd.append("--split") + + # HACK: + if "createrepo_c" in self.createrepo: + pretty = False + if pretty: + cmd.append("--pretty") + + if database: + cmd.append("--database") + else: + cmd.append("--no-database") + + if checksum: + cmd.append("--checksum=%s" % checksum) + + if unique_md_filenames: + cmd.append("--unique-md-filenames") + else: + cmd.append("--simple-md-filenames") + + if distro: + for i in force_list(distro): + cmd.append("--distro=%s" % i) + + if content: + for i in force_list(content): + cmd.append("--content=%s" % i) + + if repo: + for i in force_list(repo): + cmd.append("--repo=%s" % i) + + if revision: + cmd.append("--revision=%s" % revision) + + if deltas: + cmd.append("--deltas=%s" % deltas) + + if oldpackagedirs: + for i in force_list(oldpackagedirs): + cmd.append("--oldpackagedirs=%s" % i) + + if num_deltas: + cmd.append("--num-deltas=%d" % int(num_deltas)) + + if workers: + cmd.append("--workers=%d" % int(workers)) + + return cmd + + def get_mergerepo_cmd(self, outputdir, repos, database=True, pkglist=None, nogroups=False, noupdateinfo=None): + cmd = [self.mergerepo] + + cmd.append("--outputdir=%s" % outputdir) + + for repo in repos: + if "://" not in repo: + repo = "file://" + repo + cmd.append("--repo=%s" % repo) + + if database: + cmd.append("--database") + else: + cmd.append("--nodatabase") + + # XXX: a custom mergerepo hack, not in upstream git repo + if pkglist: + cmd.append("--pkglist=%s" % pkglist) + + if nogroups: + cmd.append("--nogroups") + + if noupdateinfo: + cmd.append("--noupdateinfo") + + return cmd + + def get_modifyrepo_cmd(self, repo_path, file_path, mdtype=None, compress_type=None, remove=False): + cmd = [self.modifyrepo] + + cmd.append(file_path) + cmd.append(repo_path) + + if mdtype: + cmd.append("--mdtype=%s" % mdtype) + + if remove: + cmd.append("--remove") + + if compress_type: + cmd.append("--compress") + cmd.append("--compress-type=%s" % compress_type) + + return cmd + + def get_repoquery_cmd(self, repos, whatrequires=False, alldeps=False, packages=None, tempcache=True): + cmd = ["/usr/bin/repoquery"] + + if tempcache: + cmd.append("--tempcache") + + # a dict is expected: {repo_name: repo_path} + for repo_name in sorted(repos): + repo_path = repos[repo_name] + if "://" not in repo_path: + repo_path = "file://" + repo_path + cmd.append("--repofrompath=%s,%s" % (repo_name, repo_path)) + cmd.append("--enablerepo=%s" % repo_name) + + if whatrequires: + cmd.append("--whatrequires") + + if alldeps: + cmd.append("--alldeps") + + if packages: + for pkg in packages: + cmd.append(pkg) + + return cmd diff --git a/pungi/wrappers/iso.py b/pungi/wrappers/iso.py new file mode 100644 index 00000000..63afce38 --- /dev/null +++ b/pungi/wrappers/iso.py @@ -0,0 +1,378 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import os +import sys +import pipes +from fnmatch import fnmatch + +import kobo.log +from kobo.shortcuts import force_list, relative_path, run + + +# HACK: define cmp in python3 +if sys.version_info[0] == 3: + def cmp(a, b): + return (a > b) - (a < b) + + +class IsoWrapper(kobo.log.LoggingBase): + + def get_boot_options(self, arch, createfrom, efi=True): + """Checks to see what we need as the -b option for mkisofs""" + + if arch in ("aarch64", ): + result = [ + '-eltorito-alt-boot', + '-e', 'images/efiboot.img', + '-no-emul-boot', + ] + return result + + if arch in ("i386", "i686", "x86_64"): + result = [ + '-b', 'isolinux/isolinux.bin', + '-c', 'isolinux/boot.cat', + '-no-emul-boot', + '-boot-load-size', '4', + '-boot-info-table', + ] + + # EFI args + if arch == "x86_64": + result.extend([ + '-eltorito-alt-boot', + '-e', 'images/efiboot.img', + '-no-emul-boot', + ]) + return result + + if arch == "ia64": + result = [ + '-b', 'images/boot.img', + '-no-emul-boot', + ] + return result + + if arch in ("ppc", "ppc64", "ppc64le"): + result = [ + '-part', + '-hfs', + '-r', + '-l', + '-sysid', 'PPC', + '-no-desktop', + '-allow-multidot', + '-chrp-boot', + "-map", os.path.join(createfrom, 'mapping'), # -map %s/ppc/mapping + "-magic", os.path.join(createfrom, 'magic'), # -magic %s/ppc/magic + '-hfs-bless', "/ppc/mac", # must be the last + ] + return result + + if arch == "sparc": + result = [ + '-G', '/boot/isofs.b', + '-B', '...', + '-s', '/boot/silo.conf', + '-sparc-label', '"sparc"', + ] + return result + + if arch in ("s390", "s390x"): + result = [ + # "-no-emul-boot", + # "-b", "images/cdboot.img", + # "-c", "boot.cat", + ] + return result + + raise ValueError("Unknown arch: %s" % arch) + + def _truncate_volid(self, volid): + if len(volid) > 32: + old_volid = volid + volid = volid.replace("-", "") + self.log_warning("Truncating volume ID from '%s' to '%s'" % (old_volid, volid)) + + if len(volid) > 32: + old_volid = volid + volid = volid.replace(" ", "") + self.log_warning("Truncating volume ID from '%s' to '%s'" % (old_volid, volid)) + + if len(volid) > 32: + old_volid = volid + volid = volid.replace("Supplementary", "Supp") + self.log_warning("Truncating volume ID from '%s' to '%s'" % (old_volid, volid)) + + if len(volid) > 32: + raise ValueError("Volume ID must be less than 32 character: %s" % volid) + + return volid + + def get_mkisofs_cmd(self, iso, paths, appid=None, volid=None, volset=None, exclude=None, verbose=False, boot_args=None, input_charset="utf-8", graft_points=None): + # following options are always enabled + untranslated_filenames = True + translation_table = True + joliet = True + joliet_long = True + rock = True + + cmd = ["/usr/bin/genisoimage"] + if appid: + cmd.extend(["-appid", appid]) + + if untranslated_filenames: + cmd.append("-untranslated-filenames") + + if volid: + cmd.extend(["-volid", self._truncate_volid(volid)]) + + if joliet: + cmd.append("-J") + + if joliet_long: + cmd.append("-joliet-long") + + if volset: + cmd.extend(["-volset", volset]) + + if rock: + cmd.append("-rational-rock") + + if verbose: + cmd.append("-verbose") + + if translation_table: + cmd.append("-translation-table") + + if input_charset: + cmd.extend(["-input-charset", input_charset]) + + if exclude: + for i in force_list(exclude): + cmd.extend(["-x", i]) + + if boot_args: + cmd.extend(boot_args) + + cmd.extend(["-o", iso]) + + if graft_points: + cmd.append("-graft-points") + cmd.extend(["-path-list", graft_points]) + else: + # we're either using graft points or file lists, not both + cmd.extend(force_list(paths)) + + return cmd + + def get_implantisomd5_cmd(self, iso_path, supported=False): + cmd = ["/usr/bin/implantisomd5"] + if supported: + cmd.append("--supported-iso") + cmd.append(iso_path) + return cmd + + def get_checkisomd5_cmd(self, iso_path, just_print=False): + cmd = ["/usr/bin/checkisomd5"] + if just_print: + cmd.append("--md5sumonly") + cmd.append(iso_path) + return cmd + + def get_implanted_md5(self, iso_path): + cmd = self.get_checkisomd5_cmd(iso_path, just_print=True) + retcode, output = run(cmd) + line = output.splitlines()[0] + result = line.rsplit(":")[-1].strip() + return result + + def get_checksum_cmds(self, iso_name, checksum_types=None): + checksum_types = checksum_types or ["md5", "sha1", "sha256"] + result = [] + for checksum_type in checksum_types: + cmd = "%ssum -b %s > %s.%sSUM" % (checksum_type.lower(), pipes.quote(iso_name), pipes.quote(iso_name), checksum_type.upper()) + result.append(cmd) + return result + + def get_manifest_cmd(self, iso_name): + return "isoinfo -R -f -i %s | grep -v '/TRANS.TBL$' | sort >> %s.manifest" % (pipes.quote(iso_name), pipes.quote(iso_name)) + + def get_volume_id(self, path): + cmd = ["isoinfo", "-d", "-i", path] + retcode, output = run(cmd) + + for line in output.splitlines(): + line = line.strip() + if line.startswith("Volume id:"): + return line[11:].strip() + + raise RuntimeError("Could not read Volume ID") + + def get_graft_points(self, paths, exclusive_paths=None, exclude=None): + # path priority in ascending order (1st = lowest prio) + # paths merge according to priority + # exclusive paths override whole dirs + + result = {} + exclude = exclude or [] + exclusive_paths = exclusive_paths or [] + + for i in paths: + if isinstance(i, dict): + tree = i + else: + tree = self._scan_tree(i) + result = self._merge_trees(result, tree) + + for i in exclusive_paths: + tree = self._scan_tree(i) + result = self._merge_trees(result, tree, exclusive=True) + + # TODO: exclude + return result + + def _paths_from_list(self, root, paths): + root = os.path.abspath(root).rstrip("/") + "/" + result = {} + for i in paths: + i = os.path.normpath(os.path.join(root, i)) + key = i[len(root):] + result[key] = i + return result + + def _scan_tree(self, path): + path = os.path.abspath(path) + result = {} + for root, dirs, files in os.walk(path): + for f in files: + abspath = os.path.join(root, f) + relpath = relative_path(abspath, path.rstrip("/") + "/") + result[relpath] = abspath + + # include empty dirs + if root != path: + abspath = os.path.join(root, "") + relpath = relative_path(abspath, path.rstrip("/") + "/") + result[relpath] = abspath + + return result + + def _merge_trees(self, tree1, tree2, exclusive=False): + # tree2 has higher priority + result = tree2.copy() + all_dirs = set([os.path.dirname(i).rstrip("/") for i in result if os.path.dirname(i) != ""]) + + for i in tree1: + dn = os.path.dirname(i) + if exclusive: + match = False + for a in all_dirs: + if dn == a or dn.startswith("%s/" % a): + match = True + break + if match: + continue + + if i in result: + continue + + result[i] = tree1[i] + return result + + def write_graft_points(self, file_name, h, exclude=None): + exclude = exclude or [] + result = {} + seen_dirs = set() + for i in sorted(h, reverse=True): + dn = os.path.dirname(i) + + if not i.endswith("/"): + result[i] = h[i] + seen_dirs.add(dn) + continue + + found = False + for j in seen_dirs: + if j.startswith(dn): + found = True + break + if not found: + result[i] = h[i] + seen_dirs.add(dn) + + f = open(file_name, "w") + for i in sorted(result, cmp=cmp_graft_points): + # make sure all files required for boot come first, + # otherwise there may be problems with booting (large LBA address, etc.) + found = False + for excl in exclude: + if fnmatch(i, excl): + found = True + break + if found: + continue + f.write("%s=%s\n" % (i, h[i])) + f.close() + + +def _is_rpm(path): + if path.endswith(".rpm"): + return True + return False + + +def _is_image(path): + if path.startswith("images/"): + return True + if path.startswith("isolinux/"): + return True + if path.startswith("EFI/"): + return True + if path.startswith("etc/"): + return True + if path.startswith("ppc/"): + return True + if path.endswith(".img"): + return True + if path.endswith(".ins"): + return True + return False + + +def cmp_graft_points(x, y): + x_is_rpm = _is_rpm(x) + y_is_rpm = _is_rpm(y) + x_is_image = _is_image(x) + y_is_image = _is_image(y) + + if x_is_rpm and y_is_rpm: + return cmp(x, y) + if x_is_rpm: + return 1 + if y_is_rpm: + return -1 + + if x_is_image and y_is_image: + return cmp(x, y) + if x_is_image: + return -1 + if y_is_image: + return 1 + + return cmp(x, y) diff --git a/pungi/wrappers/jigdo.py b/pungi/wrappers/jigdo.py new file mode 100644 index 00000000..6aa28ff7 --- /dev/null +++ b/pungi/wrappers/jigdo.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import os + +import kobo.log +from kobo.shortcuts import force_list + + +class JigdoWrapper(kobo.log.LoggingBase): + def get_jigdo_cmd(self, image, files, output_dir, cache=None, no_servers=False, report=None): + """ + files: [{"path", "label", "uri"}] + """ + cmd = ["jigdo-file", "make-template"] + + cmd.append("--force") # overrides existing template + image = os.path.abspath(image) + cmd.append("--image=%s" % image) + + output_dir = os.path.abspath(output_dir) + jigdo_file = os.path.join(output_dir, os.path.basename(image)) + ".jigdo" + cmd.append("--jigdo=%s" % jigdo_file) + + template_file = os.path.join(output_dir, os.path.basename(image)) + ".template" + cmd.append("--template=%s" % template_file) + + if cache: + cache = os.path.abspath(cache) + cmd.append("--cache=%s" % cache) + + if no_servers: + cmd.append("--no-servers-section") + + if report: + cmd.append("--report=%s" % report) + + for i in force_list(files): + # double-slash magic; read man jigdo-file + if isinstance(i, str): + i = {"path": i} + path = os.path.abspath(i["path"]).rstrip("/") + "//" + cmd.append(path) + + label = i.get("label", None) + if label is not None: + cmd.append("--label=%s=%s" % (label, path.rstrip("/"))) + + uri = i.get("uri", None) + if uri is not None: + cmd.append("--uri=%s=%s" % (label, uri)) + + return cmd diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py new file mode 100644 index 00000000..db377387 --- /dev/null +++ b/pungi/wrappers/kojiwrapper.py @@ -0,0 +1,206 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import os +import pipes +import re + +import koji +import rpmUtils.arch +from kobo.shortcuts import run + + +class KojiWrapper(object): + def __init__(self, profile): + self.profile = profile + # assumption: profile name equals executable name (it's a symlink -> koji) + self.executable = self.profile.replace("_", "-") + self.koji_module = __import__(self.profile) + + def get_runroot_cmd(self, target, arch, command, quiet=False, use_shell=True, channel=None, packages=None, mounts=None, weight=None, task_id=True): + cmd = [self.executable, "runroot"] + + if quiet: + cmd.append("--quiet") + + if use_shell: + cmd.append("--use-shell") + + if task_id: + cmd.append("--task-id") + + if channel: + cmd.append("--channel-override=%s" % channel) + else: + cmd.append("--channel-override=runroot-local") + + if weight: + cmd.append("--weight=%s" % int(weight)) + + if packages: + for package in packages: + cmd.append("--package=%s" % package) + + if mounts: + for mount in mounts: + # directories are *not* created here + cmd.append("--mount=%s" % mount) + + # IMPORTANT: all --opts have to be provided *before* args + + cmd.append(target) + + # i686 -> i386 etc. + arch = rpmUtils.arch.getBaseArch(arch) + cmd.append(arch) + + if isinstance(command, list): + command = " ".join([pipes.quote(i) for i in command]) + + # HACK: remove rpmdb and yum cache + command = "rm -f /var/lib/rpm/__db*; rm -rf /var/cache/yum/*; set -x; " + command + cmd.append(command) + + return cmd + + def run_runroot_cmd(self, command, log_file=None): + # runroot is blocking -> you probably want to run it in a thread + + task_id = None + retcode, output = run(command, can_fail=True, logfile=log_file) + if "--task-id" in command: + task_id = int(output.splitlines()[0]) + output_ends_with_eol = output.endswith("\n") + output = "\n".join(output.splitlines()[1:]) + if output_ends_with_eol: + output += "\n" + + result = { + "retcode": retcode, + "output": output, + "task_id": task_id, + } + return result + + def get_create_image_cmd(self, name, version, target, arch, ks_file, repos, image_type="live", image_format=None, release=None, wait=True, archive=False): + # Usage: koji spin-livecd [options] + # Usage: koji spin-appliance [options] + # Examples: + # * name: RHEL-7.0 + # * name: Satellite-6.0.1-RHEL-6 + # ** -. + # * version: YYYYMMDD[.n|.t].X + # * release: 1 + + cmd = [self.executable] + + if image_type == "live": + cmd.append("spin-livecd") + elif image_type == "appliance": + cmd.append("spin-appliance") + else: + raise ValueError("Invalid image type: %s" % image_type) + + if not archive: + cmd.append("--scratch") + + cmd.append("--noprogress") + + if wait: + cmd.append("--wait") + else: + cmd.append("--nowait") + + if isinstance(repos, list): + for repo in repos: + cmd.append("--repo=%s" % repo) + else: + cmd.append("--repo=%s" % repos) + + if image_format: + if image_type != "appliance": + raise ValueError("Format can be specified only for appliance images'") + supported_formats = ["raw", "qcow", "qcow2", "vmx"] + if image_format not in supported_formats: + raise ValueError("Format is not supported: %s. Supported formats: %s" % (image_format, " ".join(sorted(supported_formats)))) + cmd.append("--format=%s" % image_format) + + if release is not None: + cmd.append("--release=%s" % release) + + # IMPORTANT: all --opts have to be provided *before* args + # Usage: koji spin-livecd [options] + + cmd.append(name) + cmd.append(version) + cmd.append(target) + + # i686 -> i386 etc. + arch = rpmUtils.arch.getBaseArch(arch) + cmd.append(arch) + + cmd.append(ks_file) + + return cmd + + def run_create_image_cmd(self, command, log_file=None): + # spin-{livecd,appliance} is blocking by default -> you probably want to run it in a thread + + retcode, output = run(command, can_fail=True, logfile=log_file) + + match = re.search(r"Created task: (\d+)", output) + if not match: + raise RuntimeError("Could not find task ID in output") + + result = { + "retcode": retcode, + "output": output, + "task_id": int(match.groups()[0]), + } + return result + + def get_image_path(self, task_id): + result = [] + # XXX: hardcoded URL + koji_proxy = self.koji_module.ClientSession(self.koji_module.config.server) + task_info_list = [] + task_info_list.append(koji_proxy.getTaskInfo(task_id, request=True)) + task_info_list.extend(koji_proxy.getTaskChildren(task_id, request=True)) + + # scan parent and child tasks for certain methods + task_info = None + for i in task_info_list: + if i["method"] in ("createAppliance", "createLiveCD"): + task_info = i + break + + scratch = task_info["request"][-1].get("scratch", False) + task_result = koji_proxy.getTaskResult(task_info["id"]) + task_result.pop("rpmlist", None) + + if scratch: + topdir = os.path.join(self.koji_module.pathinfo.work(), self.koji_module.pathinfo.taskrelpath(task_info["id"])) + else: + build = koji_proxy.getImageBuild("%(name)s-%(version)s-%(release)s" % task_result) + build["name"] = task_result["name"] + build["version"] = task_result["version"] + build["release"] = task_result["release"] + build["arch"] = task_result["arch"] + topdir = self.koji_module.pathinfo.imagebuild(build) + for i in task_result["files"]: + result.append(os.path.join(topdir, i)) + return result diff --git a/pungi/wrappers/lorax.py b/pungi/wrappers/lorax.py new file mode 100644 index 00000000..80d7f9f8 --- /dev/null +++ b/pungi/wrappers/lorax.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import os + +from kobo.shortcuts import force_list + + +class LoraxWrapper(object): + def get_lorax_cmd(self, product, version, release, repo_baseurl, output_dir, variant=None, bugurl=None, nomacboot=False, noupgrade=False, is_final=False, buildarch=None, volid=None): + cmd = ["lorax"] + cmd.append("--product=%s" % product) + cmd.append("--version=%s" % version) + cmd.append("--release=%s" % release) + + for i in force_list(repo_baseurl): + if "://" not in i: + i = "file://%s" % os.path.abspath(i) + cmd.append("--source=%s" % i) + + if variant is not None: + cmd.append("--variant=%s" % variant) + + if bugurl is not None: + cmd.append("--bugurl=%s" % variant) + + if nomacboot: + cmd.append("--nomacboot") + + if noupgrade: + cmd.append("--noupgrade") + + if is_final: + cmd.append("--isfinal") + + if buildarch: + cmd.append("--buildarch=%s" % buildarch) + + if volid: + cmd.append("--volid=%s" % volid) + + output_dir = os.path.abspath(output_dir) + cmd.append(output_dir) + + # TODO: workdir + + return cmd + + def get_buildinstall_cmd(self, product, version, release, repo_baseurl, output_dir, variant=None, bugurl=None, nomacboot=False, noupgrade=False, is_final=False, buildarch=None, volid=None, brand=None): + # RHEL 6 compatibility + # Usage: buildinstall [--debug] --version --brand --product --release --final [--output outputdir] [--discs ] + + brand = brand or "redhat" + # HACK: ignore provided release + release = "%s %s" % (brand, version) + bugurl = bugurl or "https://bugzilla.redhat.com" + + cmd = ["/usr/lib/anaconda-runtime/buildinstall"] + + cmd.append("--debug") + + cmd.extend(["--version", version]) + cmd.extend(["--brand", brand]) + cmd.extend(["--product", product]) + cmd.extend(["--release", release]) + + if is_final: + cmd.append("--final") + + if buildarch: + cmd.extend(["--buildarch", buildarch]) + + if bugurl: + cmd.extend(["--bugurl", bugurl]) + + output_dir = os.path.abspath(output_dir) + cmd.extend(["--output", output_dir]) + + for i in force_list(repo_baseurl): + if "://" not in i: + i = "file://%s" % os.path.abspath(i) + cmd.append(i) + + return cmd diff --git a/pungi/wrappers/pungi.py b/pungi/wrappers/pungi.py new file mode 100644 index 00000000..064687d6 --- /dev/null +++ b/pungi/wrappers/pungi.py @@ -0,0 +1,203 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import errno +import os +import re + + +PACKAGES_RE = { + "rpm": re.compile(r"^RPM(\((?P[^\)]+)\))?: (?:file://)?(?P/?[^ ]+)$"), + "srpm": re.compile(r"^SRPM(\((?P[^\)]+)\))?: (?:file://)?(?P/?[^ ]+)$"), + "debuginfo": re.compile(r"^DEBUGINFO(\((?P[^\)]+)\))?: (?:file://)?(?P/?[^ ]+)$"), +} + + +UNRESOLVED_DEPENDENCY_RE = re.compile(r"^.*Unresolvable dependency (.+) in ([^ ]+).*$") + + +class PungiWrapper(object): + + def write_kickstart(self, ks_path, repos, groups, packages, exclude_packages=None, comps_repo=None, lookaside_repos=None, fulltree_excludes=None, multilib_blacklist=None, multilib_whitelist=None, prepopulate=None): + groups = groups or [] + exclude_packages = exclude_packages or {} + lookaside_repos = lookaside_repos or {} + # repos = {name: url} + fulltree_excludes = fulltree_excludes or set() + multilib_blacklist = multilib_blacklist or set() + multilib_whitelist = multilib_whitelist or set() + ks_path = os.path.abspath(ks_path) + + ks_dir = os.path.dirname(ks_path) + try: + os.makedirs(ks_dir) + except OSError as ex: + if ex.errno != errno.EEXIST: + raise + + kickstart = open(ks_path, "w") + + # repos + for repo_name, repo_url in repos.items() + lookaside_repos.items(): + if "://" not in repo_url: + repo_url = "file://" + os.path.abspath(repo_url) + repo_str = "repo --name=%s --baseurl=%s" % (repo_name, repo_url) + # TODO: make sure pungi works when there are no comps in repodata + # XXX: if groups are ignored, langpacks are ignored too + if comps_repo and repo_name != comps_repo: + repo_str += " --ignoregroups=true" + kickstart.write(repo_str + "\n") + + # %packages + kickstart.write("\n") + kickstart.write("%packages\n") + + for group in sorted(groups): + kickstart.write("@%s --optional\n" % group) + + for package in sorted(packages): + kickstart.write("%s\n" % package) + + for package in sorted(exclude_packages): + kickstart.write("-%s\n" % package) + + kickstart.write("%end\n") + + # %fulltree-excludes + if fulltree_excludes: + kickstart.write("\n") + kickstart.write("%fulltree-excludes\n") + for i in sorted(fulltree_excludes): + kickstart.write("%s\n" % i) + kickstart.write("%end\n") + + # %multilib-blacklist + if multilib_blacklist: + kickstart.write("\n") + kickstart.write("%multilib-blacklist\n") + for i in sorted(multilib_blacklist): + kickstart.write("%s\n" % i) + kickstart.write("%end\n") + + # %multilib-whitelist + if multilib_whitelist: + kickstart.write("\n") + kickstart.write("%multilib-whitelist\n") + for i in sorted(multilib_whitelist): + kickstart.write("%s\n" % i) + kickstart.write("%end\n") + + # %prepopulate + if prepopulate: + kickstart.write("\n") + kickstart.write("%prepopulate\n") + for i in sorted(prepopulate): + kickstart.write("%s\n" % i) + kickstart.write("%end\n") + + kickstart.close() + + def get_pungi_cmd(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None): + cmd = ["pungi-gather"] + + # Gather stage + cmd.append("-G") + + # path to a kickstart file + cmd.append("--config=%s" % config) + + # destdir is optional in Pungi (defaults to current dir), but want it mandatory here + cmd.append("--destdir=%s" % destdir) + + # name + cmd.append("--name=%s" % name) + + # version; optional, defaults to datestamp + if version: + cmd.append("--ver=%s" % version) + + # rhel variant; optional + if flavor: + cmd.append("--flavor=%s" % flavor) + + # turn selfhosting on + if selfhosting: + cmd.append("--selfhosting") + + # NPLB + if fulltree: + cmd.append("--fulltree") + + greedy = greedy or "none" + cmd.append("--greedy=%s" % greedy) + + if nodeps: + cmd.append("--nodeps") + + # don't download packages, just print paths + if nodownload: + cmd.append("--nodownload") + + if full_archlist: + cmd.append("--full-archlist") + + if arch: + cmd.append("--arch=%s" % arch) + + if multilib_methods: + for i in multilib_methods: + cmd.append("--multilib=%s" % i) + + if cache_dir: + cmd.append("--cachedir=%s" % cache_dir) + + if lookaside_repos: + for i in lookaside_repos: + cmd.append("--lookaside-repo=%s" % i) + + return cmd + + def get_packages(self, output): + global PACKAGES_RE + result = dict(((i, []) for i in PACKAGES_RE)) + + for line in output.splitlines(): + for file_type, pattern in PACKAGES_RE.iteritems(): + match = pattern.match(line) + if match: + item = {} + item["path"] = match.groupdict()["path"] + flags = match.groupdict()["flags"] or "" + flags = sorted([i.strip() for i in flags.split(",") if i.strip()]) + item["flags"] = flags + result[file_type].append(item) + break + + # no packages are filtered + + return result + + def get_missing_deps(self, output): + global UNRESOLVED_DEPENDENCY_RE + result = {} + + for line in output.splitlines(): + match = UNRESOLVED_DEPENDENCY_RE.match(line) + if match: + result.setdefault(match.group(2), set()).add(match.group(1)) + + return result diff --git a/pungi/wrappers/repoclosure.py b/pungi/wrappers/repoclosure.py new file mode 100644 index 00000000..9d2e518e --- /dev/null +++ b/pungi/wrappers/repoclosure.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import os + +from kobo.shortcuts import force_list + + +class RepoclosureWrapper(object): + + def __init__(self): + self.actual_id = 0 + + def get_repoclosure_cmd(self, config=None, arch=None, basearch=None, builddeps=False, + repos=None, lookaside=None, tempcache=False, quiet=False, newest=False, pkg=None, group=None): + + cmd = ["/usr/bin/repoclosure"] + + if config: + cmd.append("--config=%s" % config) + + if arch: + for i in force_list(arch): + cmd.append("--arch=%s" % i) + + if basearch: + cmd.append("--basearch=%s" % basearch) + + if builddeps: + cmd.append("--builddeps") + + if tempcache: + cmd.append("--tempcache") + + if quiet: + cmd.append("--quiet") + + if newest: + cmd.append("--newest") + + repos = repos or {} + for repo_id, repo_path in repos.iteritems(): + if "://" not in repo_path: + repo_path = "file://%s" % os.path.abspath(repo_path) + cmd.append("--repofrompath=%s,%s" % (repo_id, repo_path)) + cmd.append("--repoid=%s" % repo_id) + + lookaside = lookaside or {} + for repo_id, repo_path in lookaside.iteritems(): + if "://" not in repo_path: + repo_path = "file://%s" % os.path.abspath(repo_path) + cmd.append("--repofrompath=%s,%s" % (repo_id, repo_path)) + cmd.append("--lookaside=%s" % repo_id) + + if pkg: + cmd.append("--pkg=%s" % pkg) + + if group: + cmd.append("--group=%s" % group) + + return cmd diff --git a/pungi/wrappers/scm.py b/pungi/wrappers/scm.py new file mode 100644 index 00000000..0c484a13 --- /dev/null +++ b/pungi/wrappers/scm.py @@ -0,0 +1,262 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import os +import tempfile +import shutil +import pipes +import glob +import time + +import kobo.log +from kobo.shortcuts import run, force_list +from pypungi.util import explode_rpm_package, makedirs + + +class ScmBase(kobo.log.LoggingBase): + def __init__(self, logger=None): + kobo.log.LoggingBase.__init__(self, logger=logger) + + def _create_temp_dir(self, tmp_dir=None): + if tmp_dir is not None: + makedirs(tmp_dir) + return tempfile.mkdtemp(prefix="cvswrapper_", dir=tmp_dir) + + def _delete_temp_dir(self, tmp_dir): + self.log_debug("Removing %s" % tmp_dir) + try: + shutil.rmtree(tmp_dir) + except OSError as ex: + self.log_warning("Error removing %s: %s" % (tmp_dir, ex)) + + def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, tmp_dir=None, log_file=None): + raise NotImplemented + + def retry_run(self, cmd, retries=5, timeout=60, **kwargs): + """ + @param cmd - cmd passed to kobo.shortcuts.run() + @param retries=5 - attempt to execute n times + @param timeout=60 - seconds before next try + @param **kwargs - args passed to kobo.shortcuts.run() + """ + + for n in range(1, retries + 1): + try: + self.log_debug("Retrying execution %s/%s of '%s'" % (n, retries, cmd)) + return run(cmd, **kwargs) + except RuntimeError as ex: + if n == retries: + raise ex + self.log_debug("Waiting %s seconds to retry execution of '%s'" % (timeout, cmd)) + time.sleep(timeout) + + raise RuntimeError("Something went wrong during execution of '%s'" % cmd) + + +class FileWrapper(ScmBase): + def export_dir(self, scm_root, scm_dir, target_dir, scm_branch=None, tmp_dir=None, log_file=None): + if scm_root: + raise ValueError("FileWrapper: 'scm_root' should be empty.") + dirs = glob.glob(scm_dir) + for i in dirs: + run("cp -a %s/* %s/" % (pipes.quote(i), pipes.quote(target_dir))) + + def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, tmp_dir=None, log_file=None): + if scm_root: + raise ValueError("FileWrapper: 'scm_root' should be empty.") + files = glob.glob(scm_file) + for i in files: + target_path = os.path.join(target_dir, os.path.basename(i)) + shutil.copy2(i, target_path) + + +class CvsWrapper(ScmBase): + def export_dir(self, scm_root, scm_dir, target_dir, scm_branch=None, tmp_dir=None, log_file=None): + scm_dir = scm_dir.lstrip("/") + scm_branch = scm_branch or "HEAD" + tmp_dir = self._create_temp_dir(tmp_dir=tmp_dir) + + self.log_debug("Exporting directory %s from CVS %s (branch %s)..." % (scm_dir, scm_root, scm_branch)) + self.retry_run(["/usr/bin/cvs", "-q", "-d", scm_root, "export", "-r", scm_branch, scm_dir], workdir=tmp_dir, show_cmd=True, logfile=log_file) + # TODO: hidden files + run("cp -a %s/* %s/" % (pipes.quote(os.path.join(tmp_dir, scm_dir)), pipes.quote(target_dir))) + self._delete_temp_dir(tmp_dir) + + def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, tmp_dir=None, log_file=None): + scm_file = scm_file.lstrip("/") + scm_branch = scm_branch or "HEAD" + tmp_dir = self._create_temp_dir(tmp_dir=tmp_dir) + + target_path = os.path.join(target_dir, os.path.basename(scm_file)) + self.log_debug("Exporting file %s from CVS %s (branch %s)..." % (scm_file, scm_root, scm_branch)) + self.retry_run(["/usr/bin/cvs", "-q", "-d", scm_root, "export", "-r", scm_branch, scm_file], workdir=tmp_dir, show_cmd=True, logfile=log_file) + + makedirs(target_dir) + shutil.copy2(os.path.join(tmp_dir, scm_file), target_path) + self._delete_temp_dir(tmp_dir) + + +class GitWrapper(ScmBase): + def export_dir(self, scm_root, scm_dir, target_dir, scm_branch=None, tmp_dir=None, log_file=None): + scm_dir = scm_dir.lstrip("/") + scm_branch = scm_branch or "master" + tmp_dir = self._create_temp_dir(tmp_dir=tmp_dir) + + if "://" not in scm_root: + scm_root = "file://%s" % scm_root + + self.log_debug("Exporting directory %s from git %s (branch %s)..." % (scm_dir, scm_root, scm_branch)) + cmd = "/usr/bin/git archive --remote=%s %s %s | tar xf -" % (pipes.quote(scm_root), pipes.quote(scm_branch), pipes.quote(scm_dir)) + self.retry_run(cmd, workdir=tmp_dir, show_cmd=True, logfile=log_file) + + run("cp -a %s/* %s/" % (pipes.quote(os.path.join(tmp_dir, scm_dir)), pipes.quote(target_dir))) + self._delete_temp_dir(tmp_dir) + + def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, tmp_dir=None, log_file=None): + scm_file = scm_file.lstrip("/") + scm_branch = scm_branch or "master" + tmp_dir = self._create_temp_dir(tmp_dir=tmp_dir) + + target_path = os.path.join(target_dir, os.path.basename(scm_file)) + + if "://" not in scm_root: + scm_root = "file://%s" % scm_root + + self.log_debug("Exporting file %s from git %s (branch %s)..." % (scm_file, scm_root, scm_branch)) + cmd = "/usr/bin/git archive --remote=%s %s %s | tar xf -" % (pipes.quote(scm_root), pipes.quote(scm_branch), pipes.quote(scm_file)) + self.retry_run(cmd, workdir=tmp_dir, show_cmd=True, logfile=log_file) + + makedirs(target_dir) + shutil.copy2(os.path.join(tmp_dir, scm_file), target_path) + self._delete_temp_dir(tmp_dir) + + +class RpmScmWrapper(ScmBase): + def export_dir(self, scm_root, scm_dir, target_dir, scm_branch=None, tmp_dir=None, log_file=None): + # if scm_root is a list, recursively process all RPMs + if isinstance(scm_root, list): + for i in scm_root: + self.export_dir(i, scm_dir, target_dir, scm_branch, tmp_dir, log_file) + return + + # if scm_root is a glob, recursively process all RPMs + rpms = glob.glob(scm_root) + if len(rpms) > 1 or (rpms and rpms[0] != scm_root): + for i in rpms: + self.export_dir(i, scm_dir, target_dir, scm_branch, tmp_dir, log_file) + return + + scm_dir = scm_dir.lstrip("/") + tmp_dir = self._create_temp_dir(tmp_dir=tmp_dir) + self.log_debug("Extracting directory %s from RPM package %s..." % (scm_dir, scm_root)) + explode_rpm_package(scm_root, tmp_dir) + + makedirs(target_dir) + # "dir" includes the whole directory while "dir/" includes it's content + if scm_dir.endswith("/"): + run("cp -a %s/* %s/" % (pipes.quote(os.path.join(tmp_dir, scm_dir)), pipes.quote(target_dir))) + else: + run("cp -a %s %s/" % (pipes.quote(os.path.join(tmp_dir, scm_dir)), pipes.quote(target_dir))) + self._delete_temp_dir(tmp_dir) + + def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, tmp_dir=None, log_file=None): + # if scm_root is a list, recursively process all RPMs + if isinstance(scm_root, list): + for i in scm_root: + self.export_file(i, scm_file, target_dir, scm_branch, tmp_dir, log_file) + return + + # if scm_root is a glob, recursively process all RPMs + rpms = glob.glob(scm_root) + if len(rpms) > 1 or (rpms and rpms[0] != scm_root): + for i in rpms: + self.export_file(i, scm_file, target_dir, scm_branch, tmp_dir, log_file) + return + + scm_file = scm_file.lstrip("/") + tmp_dir = self._create_temp_dir(tmp_dir=tmp_dir) + + self.log_debug("Exporting file %s from RPM file %s..." % (scm_file, scm_root)) + explode_rpm_package(scm_root, tmp_dir) + + makedirs(target_dir) + for src in glob.glob(os.path.join(tmp_dir, scm_file)): + dst = os.path.join(target_dir, os.path.basename(src)) + shutil.copy2(src, dst) + self._delete_temp_dir(tmp_dir) + + +def get_file_from_scm(scm_dict, target_path, logger=None): + if isinstance(scm_dict, str): + scm_type = "file" + scm_repo = None + scm_file = os.path.abspath(scm_dict) + scm_branch = None + else: + scm_type = scm_dict["scm"] + scm_repo = scm_dict["repo"] + scm_file = scm_dict["file"] + scm_branch = scm_dict.get("branch", None) + + if scm_type == "file": + scm = FileWrapper(logger=logger) + elif scm_type == "cvs": + scm = CvsWrapper(logger=logger) + elif scm_type == "git": + scm = GitWrapper(logger=logger) + elif scm_type == "rpm": + scm = RpmScmWrapper(logger=logger) + else: + raise ValueError("Unknown SCM type: %s" % scm_type) + + for i in force_list(scm_file): + tmp_dir = tempfile.mkdtemp(prefix="scm_checkout_") + scm.export_file(scm_repo, i, scm_branch=scm_branch, target_dir=tmp_dir) + makedirs(target_path) + run("cp -a %s/* %s/" % (pipes.quote(tmp_dir), pipes.quote(target_path))) + shutil.rmtree(tmp_dir) + + +def get_dir_from_scm(scm_dict, target_path, logger=None): + if isinstance(scm_dict, str): + scm_type = "file" + scm_repo = None + scm_dir = os.path.abspath(scm_dict) + scm_branch = None + else: + scm_type = scm_dict["scm"] + scm_repo = scm_dict.get("repo", None) + scm_dir = scm_dict["dir"] + scm_branch = scm_dict.get("branch", None) + + if scm_type == "file": + scm = FileWrapper(logger=logger) + elif scm_type == "cvs": + scm = CvsWrapper(logger=logger) + elif scm_type == "git": + scm = GitWrapper(logger=logger) + elif scm_type == "rpm": + scm = RpmScmWrapper(logger=logger) + else: + raise ValueError("Unknown SCM type: %s" % scm_type) + + tmp_dir = tempfile.mkdtemp(prefix="scm_checkout_") + scm.export_dir(scm_repo, scm_dir, scm_branch=scm_branch, target_dir=tmp_dir) + # TODO: hidden files + makedirs(target_path) + run("cp -a %s/* %s/" % (pipes.quote(tmp_dir), pipes.quote(target_path))) + shutil.rmtree(tmp_dir) diff --git a/pungi/wrappers/variants.py b/pungi/wrappers/variants.py new file mode 100755 index 00000000..ff18bcc3 --- /dev/null +++ b/pungi/wrappers/variants.py @@ -0,0 +1,304 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +from __future__ import print_function +import os +import sys +import copy +import lxml.etree + + +# HACK: define cmp in python3 +if sys.version_info[0] == 3: + def cmp(a, b): + return (a > b) - (a < b) + + +VARIANTS_DTD = "/usr/share/pungi/variants.dtd" +if not os.path.isfile(VARIANTS_DTD): + DEVEL_VARIANTS_DTD = os.path.normpath(os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "..", "share", "variants.dtd"))) + msg = "Variants DTD not found: %s" % VARIANTS_DTD + if os.path.isfile(DEVEL_VARIANTS_DTD): + sys.stderr.write("%s\n" % msg) + sys.stderr.write("Using alternative DTD: %s\n" % DEVEL_VARIANTS_DTD) + VARIANTS_DTD = DEVEL_VARIANTS_DTD + else: + raise RuntimeError(msg) + + +class VariantsXmlParser(object): + def __init__(self, file_obj, tree_arches=None): + self.tree = lxml.etree.parse(file_obj) + self.dtd = lxml.etree.DTD(open(VARIANTS_DTD, "r")) + self.addons = {} + self.layered_products = {} + self.tree_arches = tree_arches + self.validate() + + def _is_true(self, value): + if value == "true": + return True + if value == "false": + return False + raise ValueError("Invalid boolean value in variants XML: %s" % value) + + def validate(self): + if not self.dtd.validate(self.tree): + errors = [str(i) for i in self.dtd.error_log.filter_from_errors()] + raise ValueError("Variants XML doesn't validate:\n%s" % "\n".join(errors)) + + def parse_variant_node(self, variant_node): + variant_dict = { + "id": str(variant_node.attrib["id"]), + "name": str(variant_node.attrib["name"]), + "name": str(variant_node.attrib["name"]), + "type": str(variant_node.attrib["type"]), + "arches": [str(i) for i in variant_node.xpath("arches/arch/text()")], + "groups": [], + "environments": [], + } + if self.tree_arches: + variant_dict["arches"] = [i for i in variant_dict["arches"] if i in self.tree_arches] + + for grouplist_node in variant_node.xpath("groups"): + for group_node in grouplist_node.xpath("group"): + group = { + "name": str(group_node.text), + "glob": self._is_true(group_node.attrib.get("glob", "false")), + "default": None, + "uservisible": None, + } + + default = group_node.attrib.get("default") + if default is not None: + group["default"] = self._is_true(default) + + uservisible = group_node.attrib.get("uservisible") + if uservisible is not None: + group["uservisible"] = self._is_true(uservisible) + + variant_dict["groups"].append(group) + + for environments_node in variant_node.xpath("environments"): + for environment_node in environments_node.xpath("environment"): + environment = { + "name": str(environment_node.text), + "display_order": None, + } + + display_order = environment_node.attrib.get("display_order") + if display_order is not None: + environment["display_order"] = int(display_order) + + variant_dict["environments"].append(environment) + + variant = Variant(**variant_dict) + if variant.type == "layered-product": + product_node = variant_node.xpath("product")[0] + variant.product_name = str(product_node.attrib["name"]) + variant.product_version = str(product_node.attrib["version"]) + variant.product_short = str(product_node.attrib["short"]) + + contains_optional = False + for child_node in variant_node.xpath("variants/variant"): + child_variant = self.parse_variant_node(child_node) + variant.add_variant(child_variant) + if child_variant.type == "optional": + contains_optional = True + + has_optional = self._is_true(variant_node.attrib.get("has_optional", "false")) + if has_optional and not contains_optional: + optional = Variant(id="optional", name="optional", type="optional", arches=variant.arches, groups=[]) + variant.add_variant(optional) + + for ref in variant_node.xpath("variants/ref/@id"): + child_variant = self.parse_variant_node(self.addons[ref]) + variant.add_variant(child_variant) + +# XXX: top-level optional +# for ref in variant_node.xpath("variants/ref/@id"): +# variant["variants"].append(copy.deepcopy(addons[ref])) + + return variant + + def parse(self): + # we allow top-level addon definitions which can be referenced in variants + for variant_node in self.tree.xpath("/variants/variant[@type='addon']"): + variant_id = str(variant_node.attrib["id"]) + self.addons[variant_id] = variant_node + + for variant_node in self.tree.xpath("/variants/variant[@type='layered-product']"): + variant_id = str(variant_node.attrib["id"]) + self.addons[variant_id] = variant_node + + result = {} + for variant_node in self.tree.xpath("/variants/variant[@type='variant']"): + variant = self.parse_variant_node(variant_node) + result[variant.id] = variant + + for variant_node in self.tree.xpath("/variants/variant[not(@type='variant' or @type='addon' or @type='layered-product')]"): + raise RuntimeError("Invalid variant type at the top-level: %s" % variant_node.attrib["type"]) + + return result + + +class Variant(object): + def __init__(self, id, name, type, arches, groups, environments=None): + if not id.isalnum(): + raise ValueError("Variant ID must contain only alphanumeric characters: %s" % id) + + environments = environments or [] + + self.id = id + self.name = name + self.type = type + self.arches = sorted(copy.deepcopy(arches)) + self.groups = sorted(copy.deepcopy(groups), lambda x, y: cmp(x["name"], y["name"])) + self.environments = sorted(copy.deepcopy(environments), lambda x, y: cmp(x["name"], y["name"])) + self.variants = {} + self.parent = None + + def __getitem__(self, name): + return self.variants[name] + + def __str__(self): + return self.uid + + def __cmp__(self, other): + # variant < addon, layered-product < optional + if self.type == other.type: + return cmp(self.uid, other.uid) + if self.type == "variant": + return -1 + if other.type == "variant": + return 1 + if self.type == "optional": + return 1 + if other.type == "optional": + return -1 + return cmp(self.uid, other.uid) + + @property + def uid(self): + if self.parent: + return "%s-%s" % (self.parent, self.id) + return self.id + + def add_variant(self, variant): + """Add a variant object to the child variant list.""" + if variant.id in self.variants: + return + if self.type != "variant": + raise RuntimeError("Only 'variant' can contain another variants.") + if variant.id == self.id: + # due to os/ path -- addon id would conflict with parent variant id + raise RuntimeError("Child variant id must be different than parent variant id: %s" % variant.id) + # sometimes an addon or layered product can be part of multiple variants with different set of arches + arches = sorted(set(self.arches).intersection(set(variant.arches))) + if self.arches and not arches: + raise RuntimeError("%s: arch list %s does not intersect with parent arch list: %s" % (variant, variant.arches, self.arches)) + variant.arches = arches + self.variants[variant.id] = variant + variant.parent = self + + def get_groups(self, arch=None, types=None, recursive=False): + """Return list of groups, default types is ["self"]""" + + types = types or ["self"] + result = copy.deepcopy(self.groups) + for variant in self.get_variants(arch=arch, types=types, recursive=recursive): + if variant == self: + # XXX + continue + for group in variant.get_groups(arch=arch, types=types, recursive=recursive): + if group not in result: + result.append(group) + return result + + def get_variants(self, arch=None, types=None, recursive=False): + """ + Return all variants of given arch and types. + + Supported variant types: + self - include the top-level ("self") variant as well + addon + variant + optional + """ + types = types or [] + result = [] + + if arch and arch not in self.arches + ["src"]: + return result + + if "self" in types: + result.append(self) + + for variant in self.variants.values(): + if types and variant.type not in types: + continue + if arch and arch not in variant.arches + ["src"]: + continue + result.append(variant) + if recursive: + result.extend(variant.get_variants(types=[i for i in types if i != "self"], recursive=True)) + + return result + + def get_addons(self, arch=None): + """Return all 'addon' child variants. No recursion.""" + return self.get_variants(arch=arch, types=["addon"], recursive=False) + + def get_layered_products(self, arch=None): + """Return all 'layered-product' child variants. No recursion.""" + return self.get_variants(arch=arch, types=["layered-product"], recursive=False) + + def get_optional(self, arch=None): + """Return all 'optional' child variants. No recursion.""" + return self.get_variants(arch=arch, types=["optional"], recursive=False) + + +def main(argv): + import optparse + + parser = optparse.OptionParser("%prog ") + opts, args = parser.parse_args(argv) + + if len(args) != 1: + parser.error("Please provide a file.") + + file_path = args[0] + try: + file_obj = open(file_path, "r") + except Exception as ex: + print(str(ex), file=sys.stderr) + sys.exit(1) + + for top_level_variant in list(VariantsXmlParser(file_obj).parse().values()): + for i in top_level_variant.get_variants(types=["self", "variant", "addon", "layered-product", "optional"], recursive=True): + print("ID: %-30s NAME: %-40s TYPE: %-12s UID: %s" % (i.id, i.name, i.type, i)) + print(" ARCHES: %s" % ", ".join(sorted(i.arches))) + for group in i.groups: + print(" GROUP: %(name)-40s GLOB: %(glob)-5s DEFAULT: %(default)-5s USERVISIBLE: %(uservisible)-5s" % group) + for env in i.environments: + print(" ENV: %(name)-40s DISPLAY_ORDER: %(display_order)s" % env) + print() + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/share/variants.dtd b/share/variants.dtd new file mode 100644 index 00000000..40236ffc --- /dev/null +++ b/share/variants.dtd @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + +