pungi/bin/pungi-koji

#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function

import argparse
import getpass
import json
import locale
import logging
import os
import socket
import signal
import sys
import traceback
import shutil

from six.moves import shlex_quote

here = sys.path[0]
if here != '/usr/bin':
    # Git checkout
    sys.path[0] = os.path.dirname(here)

from pungi.phases import PHASES_NAMES
from pungi import get_full_version, util


# force C locales
try:
    locale.setlocale(locale.LC_ALL, "C.UTF-8")
except locale.Error:
    # RHEL < 8 does not have C.UTF-8 locale...
    locale.setlocale(locale.LC_ALL, "C")


COMPOSE = None


def main():
    global COMPOSE

    parser = argparse.ArgumentParser()
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument(
        "--target-dir",
        metavar="PATH",
        help="a compose is created under this directory",
    )
    group.add_argument(
        "--compose-dir",
        metavar="PATH",
        help="reuse an existing compose directory (DANGEROUS!)",
    )
    parser.add_argument(
        "--label",
        help="specify compose label (example: Snapshot-1.0); required for production composes"
    )
    parser.add_argument(
        "--no-label",
        action="store_true",
        default=False,
        help="make a production compose without label"
    )
    parser.add_argument(
        "--supported",
        action="store_true",
        default=False,
        help="set supported flag on media (automatically on for 'RC-x.y' labels)"
    )
    parser.add_argument(
        "--old-composes",
        metavar="PATH",
        dest="old_composes",
        default=[],
        action="append",
        help="Path to directory with old composes. Reuse an existing repodata from the most recent compose.",
    )
    parser.add_argument(
        "--config",
        help="Config file",
        required=True
    )
    parser.add_argument(
        "--skip-phase",
        metavar="PHASE",
        choices=PHASES_NAMES,
        action="append",
        default=[],
        help="skip a compose phase",
    )
    parser.add_argument(
        "--just-phase",
        metavar="PHASE",
        choices=PHASES_NAMES,
        action="append",
        default=[],
        help="run only a specified compose phase",
    )
    parser.add_argument(
        "--nightly",
        action="store_const",
        const="nightly",
        dest="compose_type",
        help="make a nightly compose",
    )
    parser.add_argument(
        "--test",
        action="store_const",
        const="test",
        dest="compose_type",
        help="make a test compose",
    )
    parser.add_argument(
        "--ci",
        action="store_const",
        const="ci",
        dest="compose_type",
        help="make a CI compose",
    )
    parser.add_argument(
        "--production",
        action="store_const",
        const="production",
        dest="compose_type",
        help="make production compose (default unless config specifies otherwise)",
    )
    parser.add_argument(
        "--koji-event",
        metavar="ID",
        type=util.parse_koji_event,
        help="specify a koji event for populating package set, either as event ID "
             "or a path to a compose from which to reuse the event",
    )
    parser.add_argument(
        "--version",
        action="version",
        version=get_full_version(),
        help="output version information and exit",
    )
    parser.add_argument(
        "--notification-script",
        action="append",
        default=[],
        help="script for sending progress notification messages"
    )
    parser.add_argument(
        "--no-latest-link",
        action="store_true",
        default=False,
        dest="no_latest_link",
        help="don't create latest symbol link to this compose"
    )
    parser.add_argument(
        "--latest-link-status",
        metavar="STATUS",
        action="append",
        default=[],
        help="only create latest symbol link to this compose when compose status matches specified status",
    )
    parser.add_argument(
        "--print-output-dir",
        action="store_true",
        default=False,
        help="print the compose directory"
    )
    parser.add_argument(
        "--quiet",
        action="store_true",
        default=False,
        help="quiet mode, don't print log on screen"
    )

    opts = parser.parse_args()
    import pungi.notifier
    notifier = pungi.notifier.PungiNotifier(opts.notification_script)

    def fail_to_start(msg, **kwargs):
        notifier.send('fail-to-start', workdir=opts.target_dir,
                      command=sys.argv, target_dir=opts.target_dir,
                      config=opts.config, detail=msg, **kwargs)

    def abort(msg):
        fail_to_start(msg)
        parser.error(msg)

    if opts.target_dir and not opts.compose_dir:
        opts.target_dir = os.path.abspath(opts.target_dir)
        if not os.path.isdir(opts.target_dir):
            abort("The target directory does not exist or is not a directory: %s" % opts.target_dir)
    else:
        opts.compose_dir = os.path.abspath(opts.compose_dir)
        if not os.path.isdir(opts.compose_dir):
            abort("The compose directory does not exist or is not a directory: %s" % opts.compose_dir)

    opts.config = os.path.abspath(opts.config)

    create_latest_link = not opts.no_latest_link
    latest_link_status = opts.latest_link_status or None

    import kobo.conf
    import kobo.log
    import productmd.composeinfo

    if opts.label:
        try:
            productmd.composeinfo.verify_label(opts.label)
        except ValueError as ex:
            abort(str(ex))

    from pungi.compose import Compose

    logger = logging.getLogger("pungi")
    logger.setLevel(logging.DEBUG)
    if not opts.quiet:
        kobo.log.add_stderr_logger(logger)

    conf = util.load_config(opts.config)

    compose_type = opts.compose_type or conf.get('compose_type', 'production')
    if compose_type == "production" and not opts.label and not opts.no_label:
        abort("must specify label for a production compose")

    # check if all requirements are met
    import pungi.checks
    if not pungi.checks.check(conf):
        sys.exit(1)
    pungi.checks.check_umask(logger)
    errors, warnings = pungi.checks.validate(conf)
    if not opts.quiet:
        for warning in warnings:
            print(warning, file=sys.stderr)
    if errors:
        for error in errors:
            print(error, file=sys.stderr)
        fail_to_start('Config validation failed', errors=errors)
        sys.exit(1)

    if opts.target_dir:
        compose_dir = Compose.get_compose_dir(opts.target_dir, conf, compose_type=compose_type, compose_label=opts.label)
    else:
        compose_dir = opts.compose_dir

    if opts.print_output_dir:
        print('Compose dir: %s' % compose_dir)

    compose = Compose(conf,
                      topdir=compose_dir,
                      skip_phases=opts.skip_phase,
                      just_phases=opts.just_phase,
                      old_composes=opts.old_composes,
                      koji_event=opts.koji_event,
                      supported=opts.supported,
                      logger=logger,
                      notifier=notifier)
    notifier.compose = compose
    COMPOSE = compose
    run_compose(compose, create_latest_link=create_latest_link, latest_link_status=latest_link_status)


def run_compose(compose, create_latest_link=True, latest_link_status=None):
    import pungi.phases
    import pungi.metadata
    import pungi.util

    errors = []

    compose.write_status("STARTED")
    compose.log_info("Host: %s" % socket.gethostname())
    compose.log_info("Pungi version: %s" % get_full_version())
    compose.log_info("User name: %s" % getpass.getuser())
    compose.log_info("Working directory: %s" % os.getcwd())
    compose.log_info("Command line: %s" % " ".join([shlex_quote(arg) for arg in sys.argv]))
    compose.log_info("Compose top directory: %s" % compose.topdir)
    compose.log_info("Current timezone offset: %s" % pungi.util.get_tz_offset())
    compose.read_variants()

    # dump the config file
    config_copy_path = os.path.join(compose.paths.log.topdir(), "config-copy")
    if not os.path.exists(config_copy_path):
        os.makedirs(config_copy_path)
    for config_file in compose.conf.opened_files:
        shutil.copy2(config_file, config_copy_path)
    config_dump_full = compose.paths.log.log_file("global", "config-dump")
    with open(config_dump_full, "w") as f:
        json.dump(compose.conf, f, sort_keys=True, indent=4)

    # initialize all phases
    init_phase = pungi.phases.InitPhase(compose)
    pkgset_phase = pungi.phases.PkgsetPhase(compose)
    buildinstall_phase = pungi.phases.BuildinstallPhase(compose)
    gather_phase = pungi.phases.GatherPhase(compose, pkgset_phase)
    extrafiles_phase = pungi.phases.ExtraFilesPhase(compose, pkgset_phase)
    createrepo_phase = pungi.phases.CreaterepoPhase(compose)
    ostree_installer_phase = pungi.phases.OstreeInstallerPhase(compose, buildinstall_phase)
    ostree_phase = pungi.phases.OSTreePhase(compose)
    productimg_phase = pungi.phases.ProductimgPhase(compose, pkgset_phase)
    createiso_phase = pungi.phases.CreateisoPhase(compose, buildinstall_phase)
    extra_isos_phase = pungi.phases.ExtraIsosPhase(compose)
    liveimages_phase = pungi.phases.LiveImagesPhase(compose)
    livemedia_phase = pungi.phases.LiveMediaPhase(compose)
    image_build_phase = pungi.phases.ImageBuildPhase(compose)
    osbs_phase = pungi.phases.OSBSPhase(compose)
    image_checksum_phase = pungi.phases.ImageChecksumPhase(compose)
    test_phase = pungi.phases.TestPhase(compose)

    # check if all config options are set
    for phase in (init_phase, pkgset_phase, createrepo_phase,
                  buildinstall_phase, productimg_phase, gather_phase,
                  extrafiles_phase, createiso_phase, liveimages_phase,
                  livemedia_phase, image_build_phase, image_checksum_phase,
                  test_phase, ostree_phase, ostree_installer_phase,
                  extra_isos_phase, osbs_phase):
        if phase.skip():
            continue
        try:
            phase.validate()
        except ValueError as ex:
            for i in str(ex).splitlines():
                errors.append("%s: %s" % (phase.name.upper(), i))
    if errors:
        for i in errors:
            compose.log_error(i)
            print(i)
        raise RuntimeError('Configuration is not valid')

    # PREP

    # Note: This may be put into a new method of phase classes (e.g. .prep())
    # in same way as .validate() or .run()

    # Prep for liveimages - Obtain a password for signing rpm wrapped images
    if ("signing_key_password_file" in compose.conf
            and "signing_command" in compose.conf
            and "%(signing_key_password)s" in compose.conf["signing_command"]
            and not liveimages_phase.skip()):
        # TODO: Don't require key if signing is turned off
        # Obtain signing key password
        signing_key_password = None

        # Use appropriate method
        if compose.conf["signing_key_password_file"] == "-":
            # Use stdin (by getpass module)
            try:
                signing_key_password = getpass.getpass("Signing key password: ")
            except EOFError:
                compose.log_debug("Ignoring signing key password")
                pass
        else:
            # Use text file with password
            try:
                signing_key_password = open(compose.conf["signing_key_password_file"], "r").readline().rstrip('\n')
            except IOError:
                # Filename is not print intentionally in case someone puts password directly into the option
                err_msg = "Cannot load password from file specified by 'signing_key_password_file' option"
                compose.log_error(err_msg)
                print(err_msg)
                raise RuntimeError(err_msg)

        if signing_key_password:
            # Store the password
            compose.conf["signing_key_password"] = signing_key_password

    init_phase.start()
    init_phase.stop()

    pkgset_phase.start()
    pkgset_phase.stop()

    # WEAVER phase - launches other phases which can safely run in parallel
    essentials_schema = (
        buildinstall_phase,
        (gather_phase, extrafiles_phase, createrepo_phase),
        (ostree_phase, ostree_installer_phase),
    )
    essentials_phase = pungi.phases.WeaverPhase(compose, essentials_schema)
    essentials_phase.start()
    essentials_phase.stop()

    productimg_phase.start()
    productimg_phase.stop()

    # write treeinfo before ISOs are created
    for variant in compose.get_variants():
        for arch in variant.arches + ["src"]:
            pungi.metadata.write_tree_info(compose, arch, variant, bi=buildinstall_phase)

    # write .discinfo and media.repo before ISOs are created
    for variant in compose.get_variants():
        if variant.type == "addon" or variant.is_empty:
            continue
        for arch in variant.arches + ["src"]:
            timestamp = pungi.metadata.write_discinfo(compose, arch, variant)
            pungi.metadata.write_media_repo(compose, arch, variant, timestamp)

    # Start all phases for image artifacts
    compose_images_schema = (
        createiso_phase,
        extra_isos_phase,
        liveimages_phase,
        image_build_phase,
        livemedia_phase,
        osbs_phase,
    )
    compose_images_phase = pungi.phases.WeaverPhase(compose, compose_images_schema)
    compose_images_phase.start()
    compose_images_phase.stop()

    image_checksum_phase.start()
    image_checksum_phase.stop()

    pungi.metadata.write_compose_info(compose)
    compose.im.dump(compose.paths.compose.metadata("images.json"))
    osbs_phase.dump_metadata()

    test_phase.start()
    test_phase.stop()

    compose.write_status("FINISHED")
    osbs_phase.request_push()
    latest_link = False
    if create_latest_link:
        if latest_link_status is None:
            # create latest symbol link by default if latest_link_status is not specified
            latest_link = True
        else:
            latest_link_status = [s.upper() for s in latest_link_status]
            if compose.get_status() in [s.upper() for s in latest_link_status]:
                latest_link = True
            else:
                compose.log_warning("Compose status (%s) doesn't match with specified latest-link-status (%s), not create latest link."
                                    % (compose.get_status(), str(latest_link_status)))

    if latest_link:
        compose_dir = os.path.basename(compose.topdir)
        if len(compose.conf["release_version"].split(".")) == 1:
            symlink_name = "latest-%s-%s" % (compose.conf["release_short"], compose.conf["release_version"])
        else:
            symlink_name = "latest-%s-%s" % (compose.conf["release_short"], ".".join(compose.conf["release_version"].split(".")[:-1]))
        if compose.conf.get("base_product_name", ""):
            symlink_name += "-%s-%s" % (compose.conf["base_product_short"], compose.conf["base_product_version"])
        symlink = os.path.join(compose.topdir, "..", symlink_name)

        try:
            os.unlink(symlink)
        except OSError as ex:
            if ex.errno != 2:
                raise
        try:
            os.symlink(compose_dir, symlink)
        except Exception as ex:
            compose.log_error("Couldn't create latest symlink: %s" % ex)
            raise

    compose.log_info("Compose finished: %s" % compose.topdir)


def sigterm_handler(signum, frame):
    if COMPOSE:
        COMPOSE.log_error("Compose run failed: signal %s" % signum)
        COMPOSE.log_error("Traceback:\n%s"
                          % '\n'.join(traceback.format_stack(frame)))
        COMPOSE.log_critical("Compose failed: %s" % COMPOSE.topdir)
        COMPOSE.write_status("TERMINATED")
    else:
        print("Signal %s captured" % signum)
    sys.stdout.flush()
    sys.stderr.flush()
    sys.exit(1)


if __name__ == "__main__":
    signal.signal(signal.SIGTERM, sigterm_handler)

    try:
        main()
    except (Exception, KeyboardInterrupt) as ex:
        if COMPOSE:
            tb_path = COMPOSE.paths.log.log_file("global", "traceback")
            COMPOSE.log_error("Compose run failed: %s" % ex)
            COMPOSE.log_error("Extended traceback in: %s" % tb_path)
            COMPOSE.log_critical("Compose failed: %s" % COMPOSE.topdir)
            COMPOSE.write_status("DOOMED")
            import kobo.tback
            with open(tb_path, "wb") as f:
                f.write(kobo.tback.Traceback().get_traceback())
        else:
            print("Exception: %s" % ex)
            raise
        sys.stdout.flush()
        sys.stderr.flush()
        sys.exit(1)