orchestrator: Monitor status of parts

When a phase is started or stopped, add a line to the to output. This
should help users keep track of what is happening in case the part takes
a long time to run.

JIRA: COMPOSE-3287
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

bin/pungi-notification-report-progress

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import argparse
+import json
+import os
+import sys
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('cmd')
+    opts = parser.parse_args()
+
+    data = json.load(sys.stdin)
+    compose = data["location"]
+
+    dest = os.environ["_PUNGI_ORCHESTRATOR_PROGRESS_MONITOR"]
+
+    with open(dest, "a") as f:
+        if opts.cmd == "phase-start":
+            print("%s: phase %s started" % (compose, data["phase_name"]), file=f)
+        elif opts.cmd == "phase-stop":
+            print("%s: phase %s finished" % (compose, data["phase_name"]), file=f)

pungi.spec

@@ -99,6 +99,7 @@ rm -rf %{buildroot}
 %{_bindir}/%{name}-config-dump
 %{_bindir}/%{name}-config-validate
 %{_bindir}/%{name}-fedmsg-notification
+%{_bindir}/%{name}-notification-report-progress
 %{_bindir}/%{name}-orchestrate
 %{_bindir}/%{name}-patch-iso
 %{_bindir}/%{name}-compare-depsolving

pungi_utils/orchestrator.py

@@ -13,6 +13,8 @@ import shutil
 import subprocess
 import sys
 import tempfile
+import time
+import threading
 from collections import namedtuple
 
 import kobo.conf
@@ -555,6 +557,39 @@ def send_notification(compose_dir, command, parts):
     notifier.send("status-change", workdir=compose_dir, status=status, **data)
 
 
+def setup_progress_monitor(global_config, parts):
+    """Update configuration so that each part send notifications about its
+    progress to the orchestrator.
+
+    There is a file to which the notification is written. The orchestrator is
+    reading it and mapping the entries to particular parts. The path to this
+    file is stored in an environment variable.
+    """
+    tmp_file = tempfile.NamedTemporaryFile(prefix="pungi-progress-monitor_")
+    os.environ["_PUNGI_ORCHESTRATOR_PROGRESS_MONITOR"] = tmp_file.name
+    atexit.register(os.remove, tmp_file.name)
+
+    global_config.extra_args.append(
+        "--notification-script=pungi-notification-report-progress"
+    )
+
+    def reader():
+        while True:
+            line = tmp_file.readline()
+            if not line:
+                time.sleep(0.1)
+                continue
+            path, msg = line.split(":", 1)
+            for part in parts:
+                if parts[part].path == os.path.dirname(path):
+                    log.debug("%s: %s", part, msg.strip())
+                    break
+
+    monitor = threading.Thread(target=reader)
+    monitor.daemon = True
+    monitor.start()
+
+
 def run(work_dir, main_config_file, args):
     config_dir = os.path.join(work_dir, "config")
     shutil.copytree(os.path.dirname(main_config_file), config_dir)
@@ -616,6 +651,8 @@ def run(work_dir, main_config_file, args):
     if hasattr(args, "part"):
         setup_for_restart(global_config, parts, args.part)
 
+    setup_progress_monitor(global_config, parts)
+
     send_notification(target_dir, parser.get("general", "notification_script"), parts)
 
     retcode = run_all(global_config, parts)

setup.py

@@ -43,6 +43,7 @@ setup(
         'bin/pungi-gather',
         'bin/pungi-koji',
         'bin/pungi-make-ostree',
+        'bin/pungi-notification-report-progress',
         'bin/pungi-orchestrate',
         'bin/pungi-patch-iso',
         'bin/pungi-wait-for-signed-ostree-handler',