orchestrator: Monitor status of parts

When a phase is started or stopped, add a line to the to output. This
should help users keep track of what is happening in case the part takes
a long time to run.

JIRA: COMPOSE-3287
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2019-03-20 10:44:57 +01:00
parent d7ef86293e
commit 31ef7736aa
4 changed files with 64 additions and 0 deletions

View File

@ -0,0 +1,25 @@
#!/usr/bin/env python
from __future__ import print_function
import argparse
import json
import os
import sys
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('cmd')
opts = parser.parse_args()
data = json.load(sys.stdin)
compose = data["location"]
dest = os.environ["_PUNGI_ORCHESTRATOR_PROGRESS_MONITOR"]
with open(dest, "a") as f:
if opts.cmd == "phase-start":
print("%s: phase %s started" % (compose, data["phase_name"]), file=f)
elif opts.cmd == "phase-stop":
print("%s: phase %s finished" % (compose, data["phase_name"]), file=f)

View File

@ -99,6 +99,7 @@ rm -rf %{buildroot}
%{_bindir}/%{name}-config-dump %{_bindir}/%{name}-config-dump
%{_bindir}/%{name}-config-validate %{_bindir}/%{name}-config-validate
%{_bindir}/%{name}-fedmsg-notification %{_bindir}/%{name}-fedmsg-notification
%{_bindir}/%{name}-notification-report-progress
%{_bindir}/%{name}-orchestrate %{_bindir}/%{name}-orchestrate
%{_bindir}/%{name}-patch-iso %{_bindir}/%{name}-patch-iso
%{_bindir}/%{name}-compare-depsolving %{_bindir}/%{name}-compare-depsolving

View File

@ -13,6 +13,8 @@ import shutil
import subprocess import subprocess
import sys import sys
import tempfile import tempfile
import time
import threading
from collections import namedtuple from collections import namedtuple
import kobo.conf import kobo.conf
@ -555,6 +557,39 @@ def send_notification(compose_dir, command, parts):
notifier.send("status-change", workdir=compose_dir, status=status, **data) notifier.send("status-change", workdir=compose_dir, status=status, **data)
def setup_progress_monitor(global_config, parts):
"""Update configuration so that each part send notifications about its
progress to the orchestrator.
There is a file to which the notification is written. The orchestrator is
reading it and mapping the entries to particular parts. The path to this
file is stored in an environment variable.
"""
tmp_file = tempfile.NamedTemporaryFile(prefix="pungi-progress-monitor_")
os.environ["_PUNGI_ORCHESTRATOR_PROGRESS_MONITOR"] = tmp_file.name
atexit.register(os.remove, tmp_file.name)
global_config.extra_args.append(
"--notification-script=pungi-notification-report-progress"
)
def reader():
while True:
line = tmp_file.readline()
if not line:
time.sleep(0.1)
continue
path, msg = line.split(":", 1)
for part in parts:
if parts[part].path == os.path.dirname(path):
log.debug("%s: %s", part, msg.strip())
break
monitor = threading.Thread(target=reader)
monitor.daemon = True
monitor.start()
def run(work_dir, main_config_file, args): def run(work_dir, main_config_file, args):
config_dir = os.path.join(work_dir, "config") config_dir = os.path.join(work_dir, "config")
shutil.copytree(os.path.dirname(main_config_file), config_dir) shutil.copytree(os.path.dirname(main_config_file), config_dir)
@ -616,6 +651,8 @@ def run(work_dir, main_config_file, args):
if hasattr(args, "part"): if hasattr(args, "part"):
setup_for_restart(global_config, parts, args.part) setup_for_restart(global_config, parts, args.part)
setup_progress_monitor(global_config, parts)
send_notification(target_dir, parser.get("general", "notification_script"), parts) send_notification(target_dir, parser.get("general", "notification_script"), parts)
retcode = run_all(global_config, parts) retcode = run_all(global_config, parts)

View File

@ -43,6 +43,7 @@ setup(
'bin/pungi-gather', 'bin/pungi-gather',
'bin/pungi-koji', 'bin/pungi-koji',
'bin/pungi-make-ostree', 'bin/pungi-make-ostree',
'bin/pungi-notification-report-progress',
'bin/pungi-orchestrate', 'bin/pungi-orchestrate',
'bin/pungi-patch-iso', 'bin/pungi-patch-iso',
'bin/pungi-wait-for-signed-ostree-handler', 'bin/pungi-wait-for-signed-ostree-handler',