orchestrator: Send messages about the main compose

Only start/finish messages will be sent if a handler is configured.

JIRA: COMPOSE-3288
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

bin/pungi-config-validate

@@ -78,14 +78,7 @@ def read_variants(compose, config):
 
 
 def run(config, topdir, has_old, offline):
-    conf = kobo.conf.PyConfigParser()
+    conf = pungi.util.load_config(config)
-    if config.endswith(".json"):
-        with open(config) as f:
-            conf.load_from_dict(json.load(f))
-        conf.opened_files = [config]
-        conf._open_file = config
-    else:
-        conf.load_from_file(config)
 
     errors, warnings = pungi.checks.validate(conf, offline=offline)
     if errors or warnings:

bin/pungi-koji

@@ -217,14 +217,7 @@ def main():
     if not opts.quiet:
         kobo.log.add_stderr_logger(logger)
 
-    conf = kobo.conf.PyConfigParser()
+    conf = util.load_config(opts.config)
-    if opts.config.endswith(".json"):
-        with open(opts.config) as f:
-            conf.load_from_dict(json.load(f))
-        conf.opened_files = [opts.config]
-        conf._open_file = opts.config
-    else:
-        conf.load_from_file(opts.config)
 
     compose_type = opts.compose_type or conf.get('compose_type', 'production')
     if compose_type == "production" and not opts.label and not opts.no_label:

doc/multi_compose.rst

@@ -77,6 +77,16 @@ General settings
     * ``BASE_PRODUCT_VERSION`` – only set for layered products
     * ``BASE_PRODUCT_TYPE`` – only set for layered products
 
+**notification_script**
+   Executable name (or path to a script) that will be used to send a message
+   once the compose is finished. In order for a valid URL to be included in the
+   message, at least one part must configure path translation that would apply
+   to location of main compose.
+
+   Only two messages will be sent, one for start and one for finish (either
+   successful or not).
+
+
 Partial compose settings
 ------------------------
 

pungi/util.py

@@ -32,6 +32,7 @@ import time
 import functools
 from six.moves import urllib, range, shlex_quote
 
+import kobo.conf
 from kobo.shortcuts import run, force_list
 from productmd.common import get_major_version
 
@@ -711,14 +712,8 @@ def run_unmount_cmd(cmd, max_retries=10, path=None, logger=None):
     raise RuntimeError('Failed to run %r: Device or resource busy.' % cmd)
 
 
-def translate_path(compose, path):
+def translate_path_raw(mapping, path):
-    """
-    @param compose - required for access to config
-    @param path
-    """
     normpath = os.path.normpath(path)
-    mapping = compose.conf["translate_paths"]
-
     for prefix, newvalue in mapping:
         prefix = os.path.normpath(prefix)
         # Strip trailing slashes: the prefix has them stripped by `normpath`.
@@ -728,10 +723,18 @@ def translate_path(compose, path):
             # a path - http:// would get changed to  http:/ and so on.
             # Only the first occurance should be replaced.
             return normpath.replace(prefix, newvalue, 1)
-
     return normpath
 
 
+def translate_path(compose, path):
+    """
+    @param compose - required for access to config
+    @param path
+    """
+    mapping = compose.conf["translate_paths"]
+    return translate_path_raw(mapping, path)
+
+
 def get_repo_url(compose, repo, arch='$basearch'):
     """
     Convert repo to repo URL.
@@ -923,3 +926,16 @@ def iter_module_defaults(path):
         for mmddef in Modulemd.objects_from_file(file):
             if isinstance(mmddef, Modulemd.Defaults):
                 yield mmddef
+
+
+def load_config(file_path):
+    """Open and load configuration file form .conf or .json file."""
+    conf = kobo.conf.PyConfigParser()
+    if file_path.endswith(".json"):
+        with open(file_path) as f:
+            conf.load_from_dict(json.load(f))
+        conf.opened_files = [file_path]
+        conf._open_file = file_path
+    else:
+        conf.load_from_file(file_path)
+    return conf

pungi_utils/orchestrator.py

@@ -21,6 +21,7 @@ import productmd
 from kobo import shortcuts
 from six.moves import configparser, shlex_quote
 
+import pungi.util
 from pungi.compose import get_compose_dir
 from pungi.linker import linker_pool
 from pungi.phases.pkgset.sources.source_koji import get_koji_event_raw
@@ -114,8 +115,7 @@ class ComposePart(object):
         )
         substitutions["configdir"] = global_config.config_dir
 
-        config = kobo.conf.PyConfigParser()
+        config = pungi.util.load_config(self.config)
-        config.load_from_file(self.config)
 
         for f in config.opened_files:
             # apply substitutions
@@ -417,6 +417,8 @@ def prepare_compose_dir(config, args, main_config_file, compose_info):
             except OSError as exc:
                 if exc.errno != errno.EEXIST:
                     raise
+        with open(os.path.join(target_dir, "STATUS"), "w") as fh:
+            fh.write("STARTED")
         # Copy initial composeinfo for new compose
         shutil.copy(
             os.path.join(target_dir, "work/global/composeinfo-base.json"),
@@ -484,33 +486,42 @@ def run_kinit(config):
     atexit.register(os.remove, fname)
 
 
+def get_compose_data(compose_path):
+    try:
+        compose = productmd.compose.Compose(compose_path)
+        data = {
+            "compose_id": compose.info.compose.id,
+            "compose_date": compose.info.compose.date,
+            "compose_type": compose.info.compose.type,
+            "compose_respin": str(compose.info.compose.respin),
+            "compose_label": compose.info.compose.label,
+            "release_id": compose.info.release_id,
+            "release_name": compose.info.release.name,
+            "release_short": compose.info.release.short,
+            "release_version": compose.info.release.version,
+            "release_type": compose.info.release.type,
+            "release_is_layered": compose.info.release.is_layered,
+        }
+        if compose.info.release.is_layered:
+            data.update({
+                "base_product_name": compose.info.base_product.name,
+                "base_product_short": compose.info.base_product.short,
+                "base_product_version": compose.info.base_product.version,
+                "base_product_type": compose.info.base_product.type,
+            })
+        return data
+    except Exception as exc:
+        return {}
+
+
 def get_script_env(compose_path):
     env = os.environ.copy()
     env["COMPOSE_PATH"] = compose_path
-    try:
+    for key, value in get_compose_data(compose_path).items():
-        compose = productmd.compose.Compose(compose_path)
+        if isinstance(value, bool):
-        env.update({
+            env[key.upper()] = "YES" if value else ""
-            "COMPOSE_ID": compose.info.compose.id,
+        else:
-            "COMPOSE_DATE": compose.info.compose.date,
+            env[key.upper()] = str(value) if value else ""
-            "COMPOSE_TYPE": compose.info.compose.type,
-            "COMPOSE_RESPIN": str(compose.info.compose.respin),
-            "COMPOSE_LABEL": compose.info.compose.label or "",
-            "RELEASE_ID": compose.info.release_id,
-            "RELEASE_NAME": compose.info.release.name,
-            "RELEASE_SHORT": compose.info.release.short,
-            "RELEASE_VERSION": compose.info.release.version,
-            "RELEASE_TYPE": compose.info.release.type,
-            "RELEASE_IS_LAYERED": "YES" if compose.info.release.is_layered else "",
-        })
-        if compose.info.release.is_layered:
-            env.update({
-                "BASE_PRODUCT_NAME": compose.info.base_product.name,
-                "BASE_PRODUCT_SHORT": compose.info.base_product.short,
-                "BASE_PRODUCT_VERSION": compose.info.base_product.version,
-                "BASE_PRODUCT_TYPE": compose.info.base_product.type,
-            })
-    except Exception as exc:
-        pass
     return env
 
 
@@ -524,6 +535,26 @@ def run_scripts(prefix, compose_dir, scripts):
         shortcuts.run(command, env=env, logfile=logfile)
 
 
+def try_translate_path(parts, path):
+    translation = []
+    for part in parts.values():
+        conf = pungi.util.load_config(part.config)
+        translation.extend(conf.get("translate_paths", []))
+    return pungi.util.translate_path_raw(translation, path)
+
+
+def send_notification(compose_dir, command, parts):
+    if not command:
+        return
+    from pungi.notifier import PungiNotifier
+    data = get_compose_data(compose_dir)
+    data["location"] = try_translate_path(parts, compose_dir)
+    notifier = PungiNotifier([command])
+    with open(os.path.join(compose_dir, "STATUS")) as f:
+        status = f.read().strip()
+    notifier.send("status-change", workdir=compose_dir, status=status, **data)
+
+
 def run(work_dir, main_config_file, args):
     config_dir = os.path.join(work_dir, "config")
     shutil.copytree(os.path.dirname(main_config_file), config_dir)
@@ -534,6 +565,7 @@ def run(work_dir, main_config_file, args):
             "kerberos": "false",
             "pre_compose_script": "",
             "post_compose_script": "",
+            "notification_script": "",
         }
     )
     parser.read(main_config_file)
@@ -584,6 +616,8 @@ def run(work_dir, main_config_file, args):
     if hasattr(args, "part"):
         setup_for_restart(global_config, parts, args.part)
 
+    send_notification(target_dir, parser.get("general", "notification_script"), parts)
+
     retcode = run_all(global_config, parts)
 
     if retcode:
@@ -592,6 +626,8 @@ def run(work_dir, main_config_file, args):
             "post_compose_", target_dir, parser.get("general", "post_compose_script")
         )
 
+    send_notification(target_dir, parser.get("general", "notification_script"), parts)
+
     return retcode
 
 

tests/fixtures/DP-1.0-20161013.t.4/STATUS

@@ -0,0 +1 @@
+FINISHED

tests/test_orchestrator.py

@@ -730,6 +730,9 @@ class TestPrepareComposeDir(PungiTestCase):
         self.assertTrue(os.path.isdir(os.path.join(self.topdir, "logs")))
         self.assertTrue(os.path.isdir(os.path.join(self.topdir, "parts")))
         self.assertTrue(os.path.isdir(os.path.join(self.topdir, "work/global")))
+        self.assertFileContent(
+            os.path.join(self.topdir, "STATUS"), "STARTED"
+        )
 
     def test_restarting_compose(self, gtd):
         args = mock.Mock(name="args", spec=["label", "compose_path"])
@@ -890,3 +893,45 @@ class TestRunScripts(BaseTestCase):
                 ),
             ],
         )
+
+
+@mock.patch("pungi.notifier.PungiNotifier")
+class TestSendNotification(BaseTestCase):
+    def test_no_command(self, notif):
+        o.send_notification("/foobar", None, None)
+        self.assertEqual(notif.mock_calls, [])
+
+    @mock.patch("pungi.util.load_config")
+    def test_with_command_and_translate(self, load_config, notif):
+        compose_dir = os.path.join(FIXTURE_DIR, "DP-1.0-20161013.t.4")
+        load_config.return_value = {
+            "translate_paths": [(os.path.dirname(compose_dir), "http://example.com")],
+        }
+        parts = {"foo": mock.Mock()}
+
+        o.send_notification(compose_dir, "handler", parts)
+
+        self.assertEqual(len(notif.mock_calls), 2)
+        self.assertEqual(notif.mock_calls[0], mock.call(["handler"]))
+        _, args, kwargs = notif.mock_calls[1]
+        self.assertEqual(args, ("status-change", ))
+        self.assertEqual(
+            kwargs,
+            {
+                "status": "FINISHED",
+                "workdir": compose_dir,
+                "location": "http://example.com/DP-1.0-20161013.t.4",
+                "compose_id": "DP-1.0-20161013.t.4",
+                "compose_date": "20161013",
+                "compose_type": "test",
+                "compose_respin": "4",
+                "compose_label": None,
+                "release_id": "DP-1.0",
+                "release_name": "Dummy Product",
+                "release_short": "DP",
+                "release_version": "1.0",
+                "release_type": "ga",
+                "release_is_layered": False,
+            },
+        )
+        self.assertEqual(load_config.call_args_list, [mock.call(parts["foo"].config)])