pungi/bin/pungi-koji
Lubomír Sedlář 066855a039 Add ability to send messages about progress
With this patch, Pungi can invoke an arbitrary command on various
moments of the compose process. The invoked command can the decide on
what message to send (and using what messaging system).

The actual command is specified in the config file.

There is a script provided that sends the messages via fedmsg.

The documentation is updated to have details about the new config option
as well as the interface for the messaging script.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
2015-11-25 15:46:50 +01:00

367 lines
11 KiB
Python
Executable File

#!/usr/bin/python
# -*- coding: utf-8 -*-
import os
import sys
import optparse
import logging
import locale
import datetime
import getpass
import socket
import json
import pipes
here = sys.path[0]
if here != '/usr/bin':
# Git checkout
sys.path[0] = os.path.dirname(here)
from pungi import __version__
# force C locales
locale.setlocale(locale.LC_ALL, "C")
COMPOSE = None
NOTIFIER = None
def main():
global COMPOSE
parser = optparse.OptionParser()
parser.add_option(
"--target-dir",
metavar="PATH",
help="a compose is created under this directory",
)
parser.add_option(
"--label",
help="specify compose label (example: Snapshot-1.0); required for production composes"
)
parser.add_option(
"--no-label",
action="store_true",
default=False,
help="make a production compose without label"
)
parser.add_option(
"--supported",
action="store_true",
default=False,
help="set supported flag on media (automatically on for 'RC-x.y' labels)"
)
parser.add_option(
"--old-composes",
metavar="PATH",
dest="old_composes",
default=[],
action="append",
help="Path to directory with old composes. Reuse an existing repodata from the most recent compose.",
)
parser.add_option(
"--compose-dir",
metavar="PATH",
help="reuse an existing compose directory (DANGEROUS!)",
)
parser.add_option(
"--debug-mode",
action="store_true",
default=False,
help="run pungi in DEBUG mode (DANGEROUS!)",
)
parser.add_option(
"--config",
help="Config file"
)
parser.add_option(
"--skip-phase",
metavar="PHASE",
action="append",
default=[],
help="skip a compose phase",
)
parser.add_option(
"--just-phase",
metavar="PHASE",
action="append",
default=[],
help="run only a specified compose phase",
)
parser.add_option(
"--nightly",
action="store_const",
const="nightly",
dest="compose_type",
help="make a nightly compose",
)
parser.add_option(
"--test",
action="store_const",
const="test",
dest="compose_type",
help="make a test compose",
)
parser.add_option(
"--koji-event",
metavar="ID",
type="int",
help="specify a koji event for populating package set",
)
parser.add_option(
"--version",
action="store_true",
help="output version information and exit",
)
opts, args = parser.parse_args()
if opts.version:
print("pungi %s" % __version__)
sys.exit(0)
if opts.target_dir and opts.compose_dir:
parser.error("cannot specify --target-dir and --compose-dir at once")
if not opts.target_dir and not opts.compose_dir:
parser.error("please specify a target directory")
if opts.target_dir and not opts.compose_dir:
opts.target_dir = os.path.abspath(opts.target_dir)
if not os.path.isdir(opts.target_dir):
parser.error("The target directory does not exist or is not a directory: %s" % opts.target_dir)
else:
opts.compose_dir = os.path.abspath(opts.compose_dir)
if not os.path.isdir(opts.compose_dir):
parser.error("The compose directory does not exist or is not a directory: %s" % opts.compose_dir)
compose_type = opts.compose_type or "production"
if compose_type == "production" and not opts.label and not opts.no_label:
parser.error("must specify label for a production compose")
if not opts.config:
parser.error("please specify a config")
opts.config = os.path.abspath(opts.config)
# check if all requirements are met
import pungi.checks
if not pungi.checks.check():
sys.exit(1)
import kobo.conf
import kobo.log
import productmd.composeinfo
if opts.label:
try:
productmd.composeinfo.verify_label(opts.label)
except ValueError as ex:
parser.error(str(ex))
from pungi.compose import Compose
logger = logging.Logger("Pungi")
kobo.log.add_stderr_logger(logger)
conf = kobo.conf.PyConfigParser()
conf.load_from_file(opts.config)
if opts.target_dir:
compose_dir = Compose.get_compose_dir(opts.target_dir, conf, compose_type=compose_type, compose_label=opts.label)
else:
compose_dir = opts.compose_dir
compose = Compose(conf, topdir=compose_dir, debug=opts.debug_mode, skip_phases=opts.skip_phase, just_phases=opts.just_phase,
old_composes=opts.old_composes, koji_event=opts.koji_event, supported=opts.supported, logger=logger)
kobo.log.add_file_logger(logger, compose.paths.log.log_file("global", "pungi.log"))
COMPOSE = compose
try:
run_compose(compose)
except Exception, ex:
compose.log_error("Compose run failed: %s" % ex)
raise
def run_compose(compose):
import pungi.phases
import pungi.metadata
import pungi.notifier
errors = []
# initializer notifier
compose.notifier = pungi.notifier.PungiNotifier(compose)
try:
compose.notifier.validate()
except ValueError as ex:
errors.extend(["NOTIFIER: %s" % m for m in ex.message.split('\n')])
compose.write_status("STARTED")
compose.notifier.send('start')
compose.log_info("Host: %s" % socket.gethostname())
compose.log_info("User name: %s" % getpass.getuser())
compose.log_info("Working directory: %s" % os.getcwd())
compose.log_info("Command line: %s" % " ".join([pipes.quote(arg) for arg in sys.argv]))
compose.log_info("Compose top directory: %s" % compose.topdir)
compose.read_variants()
# dump the config file
date_str = datetime.datetime.strftime(datetime.datetime.now(), "%F_%X").replace(":", "-")
config_dump = compose.paths.log.log_file("global", "config-dump_%s" % date_str)
open(config_dump, "w").write(json.dumps(compose.conf, sort_keys=True, indent=4))
# initialize all phases
init_phase = pungi.phases.InitPhase(compose)
pkgset_phase = pungi.phases.PkgsetPhase(compose)
buildinstall_phase = pungi.phases.BuildinstallPhase(compose)
gather_phase = pungi.phases.GatherPhase(compose, pkgset_phase)
extrafiles_phase = pungi.phases.ExtraFilesPhase(compose, pkgset_phase)
createrepo_phase = pungi.phases.CreaterepoPhase(compose)
productimg_phase = pungi.phases.ProductimgPhase(compose, pkgset_phase)
createiso_phase = pungi.phases.CreateisoPhase(compose)
liveimages_phase = pungi.phases.LiveImagesPhase(compose)
image_build_phase = pungi.phases.ImageBuildPhase(compose)
test_phase = pungi.phases.TestPhase(compose)
# check if all config options are set
for phase in (init_phase, pkgset_phase, createrepo_phase,
buildinstall_phase, productimg_phase, gather_phase,
extrafiles_phase, createiso_phase, liveimages_phase,
image_build_phase, test_phase):
if phase.skip():
continue
try:
phase.validate()
except ValueError as ex:
for i in str(ex).splitlines():
errors.append("%s: %s" % (phase.name.upper(), i))
if errors:
for i in errors:
compose.notifier.send('abort')
compose.log_error(i)
print(i)
sys.exit(1)
# INIT phase
init_phase.start()
init_phase.stop()
# PKGSET phase
pkgset_phase.start()
pkgset_phase.stop()
# BUILDINSTALL phase - start
buildinstall_phase.start()
# GATHER phase
gather_phase.start()
gather_phase.stop()
# EXTRA_FILES phase
extrafiles_phase.start()
extrafiles_phase.stop()
# CREATEREPO phase
createrepo_phase.start()
createrepo_phase.stop()
# BUILDINSTALL phase
# must finish before PRODUCTIMG
# must finish before CREATEISO
buildinstall_phase.stop()
if not buildinstall_phase.skip():
buildinstall_phase.copy_files()
# PRODUCTIMG phase
productimg_phase.start()
productimg_phase.stop()
# write treeinfo before ISOs are created
for variant in compose.get_variants():
for arch in variant.arches + ["src"]:
pungi.metadata.write_tree_info(compose, arch, variant)
# write .discinfo and media.repo before ISOs are created
for variant in compose.get_variants(recursive=True):
if variant.type == "addon":
continue
for arch in variant.arches + ["src"]:
timestamp = pungi.metadata.write_discinfo(compose, arch, variant)
pungi.metadata.write_media_repo(compose, arch, variant, timestamp)
# CREATEISO and LIVEIMAGES phases
createiso_phase.start()
liveimages_phase.start()
image_build_phase.start()
createiso_phase.stop()
liveimages_phase.stop()
image_build_phase.stop()
# merge checksum files
for variant in compose.get_variants(types=["variant", "layered-product"]):
for arch in variant.arches + ["src"]:
iso_dir = compose.paths.compose.iso_dir(arch, variant, create_dir=False)
if not iso_dir or not os.path.exists(iso_dir):
continue
for checksum_type in ("md5", "sha1", "sha256"):
checksum_upper = "%sSUM" % checksum_type.upper()
checksums = sorted([i for i in os.listdir(iso_dir) if i.endswith(".%s" % checksum_upper)])
fo = open(os.path.join(iso_dir, checksum_upper), "w")
for i in checksums:
data = open(os.path.join(iso_dir, i), "r").read()
fo.write(data)
pungi.metadata.write_compose_info(compose)
compose.im.dump(compose.paths.compose.metadata("images.json"))
# TEST phase
test_phase.start()
test_phase.stop()
# create a latest symlink
compose_dir = os.path.basename(compose.topdir)
symlink_name = "latest-%s-%s" % (compose.conf["release_short"], ".".join(compose.conf["release_version"].split(".")[:-1]))
if compose.conf["release_is_layered"]:
symlink_name += "-%s-%s" % (compose.conf["base_product_short"], compose.conf["base_product_version"])
symlink = os.path.join(compose.topdir, "..", symlink_name)
try:
os.unlink(symlink)
except OSError as ex:
if ex.errno != 2:
raise
try:
os.symlink(compose_dir, symlink)
except Exception as ex:
compose.log_error("Couldn't create latest symlink: %s" % ex)
compose.log_info("Compose finished: %s" % compose.topdir)
compose.write_status("FINISHED")
compose.notifier.send('finish')
if __name__ == "__main__":
try:
main()
except (Exception, KeyboardInterrupt) as ex:
if COMPOSE:
tb_path = COMPOSE.paths.log.log_file("global", "traceback")
COMPOSE.log_error("Exception: %s" % ex)
COMPOSE.log_error("Extended traceback in: %s" % tb_path)
COMPOSE.log_critical("Compose failed: %s" % COMPOSE.topdir)
COMPOSE.write_status("DOOMED")
import kobo.tback
open(tb_path, "w").write(kobo.tback.Traceback().get_traceback())
if COMPOSE.notifier:
COMPOSE.notifier.send('doomed')
else:
print("Exception: %s" % ex)
raise
sys.stdout.flush()
sys.stderr.flush()
sys.exit(1)