orchestrator: Support generic pre- and post- scripts

Run arbitrary commands before and after the compose.

The example config is updated to generate latest symlink with a
post-compose script. The pre compose script runs always, post compose
runs only if the compose is not doomed.

JIRA: COMPOSE-3288
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

doc/multi_compose.rst

@@ -47,6 +47,35 @@ General settings
 **kerberos_principal**
    Kerberos principal for the ticket
 
+**pre_compose_script**
+   Commands to execute before first part is started. Can contain multiple
+   commands on separate lines.
+**post_compose_script**
+   Commands to execute after the last part finishes and final status is
+   updated. Can contain multiple commands on separate lines. ::
+
+      post_compose_script =
+          compose-latest-symlink $COMPOSE_PATH
+          custom-post-compose-script.sh
+
+   Multiple environment variables are defined for the scripts:
+
+    * ``COMPOSE_PATH``
+    * ``COMPOSE_ID``
+    * ``COMPOSE_DATE``
+    * ``COMPOSE_TYPE``
+    * ``COMPOSE_RESPIN``
+    * ``COMPOSE_LABEL``
+    * ``RELEASE_ID``
+    * ``RELEASE_NAME``
+    * ``RELEASE_SHORT``
+    * ``RELEASE_VERSION``
+    * ``RELEASE_TYPE``
+    * ``RELEASE_IS_LAYERED`` – ``YES`` for layered products, empty otherwise
+    * ``BASE_PRODUCT_NAME`` – only set for layered products
+    * ``BASE_PRODUCT_SHORT`` – only set for layered products
+    * ``BASE_PRODUCT_VERSION`` – only set for layered products
+    * ``BASE_PRODUCT_TYPE`` – only set for layered products
 
 Partial compose settings
 ------------------------

pungi_utils/orchestrator.py

@@ -484,12 +484,58 @@ def run_kinit(config):
     atexit.register(os.remove, fname)
 
 
+def get_script_env(compose_path):
+    env = os.environ.copy()
+    env["COMPOSE_PATH"] = compose_path
+    try:
+        compose = productmd.compose.Compose(compose_path)
+        env.update({
+            "COMPOSE_ID": compose.info.compose.id,
+            "COMPOSE_DATE": compose.info.compose.date,
+            "COMPOSE_TYPE": compose.info.compose.type,
+            "COMPOSE_RESPIN": str(compose.info.compose.respin),
+            "COMPOSE_LABEL": compose.info.compose.label or "",
+            "RELEASE_ID": compose.info.release_id,
+            "RELEASE_NAME": compose.info.release.name,
+            "RELEASE_SHORT": compose.info.release.short,
+            "RELEASE_VERSION": compose.info.release.version,
+            "RELEASE_TYPE": compose.info.release.type,
+            "RELEASE_IS_LAYERED": "YES" if compose.info.release.is_layered else "",
+        })
+        if compose.info.release.is_layered:
+            env.update({
+                "BASE_PRODUCT_NAME": compose.info.base_product.name,
+                "BASE_PRODUCT_SHORT": compose.info.base_product.short,
+                "BASE_PRODUCT_VERSION": compose.info.base_product.version,
+                "BASE_PRODUCT_TYPE": compose.info.base_product.type,
+            })
+    except Exception as exc:
+        pass
+    return env
+
+
+def run_scripts(prefix, compose_dir, scripts):
+    env = get_script_env(compose_dir)
+    for idx, script in enumerate(scripts.strip().splitlines()):
+        command = script.strip()
+        logfile = os.path.join(compose_dir, "logs", "%s%s.log" % (prefix, idx))
+        log.debug("Running command: %r", command)
+        log.debug("See output in %s", logfile)
+        shortcuts.run(command, env=env, logfile=logfile)
+
+
 def run(work_dir, main_config_file, args):
     config_dir = os.path.join(work_dir, "config")
     shutil.copytree(os.path.dirname(main_config_file), config_dir)
 
     # Read main config
-    parser = configparser.RawConfigParser(defaults={"kerberos": "false"})
+    parser = configparser.RawConfigParser(
+        defaults={
+            "kerberos": "false",
+            "pre_compose_script": "",
+            "post_compose_script": "",
+        }
+    )
     parser.read(main_config_file)
 
     # Create kerberos ticket
@@ -502,6 +548,8 @@ def run(work_dir, main_config_file, args):
     kobo.log.add_file_logger(log, os.path.join(target_dir, "logs", "orchestrator.log"))
     log.info("Composing %s", target_dir)
 
+    run_scripts("pre_compose_", target_dir, parser.get("general", "pre_compose_script"))
+
     old_compose = find_old_compose(
         os.path.dirname(target_dir),
         compose_info["release_short"],
@@ -536,7 +584,15 @@ def run(work_dir, main_config_file, args):
     if hasattr(args, "part"):
         setup_for_restart(global_config, parts, args.part)
 
-    return run_all(global_config, parts)
+    retcode = run_all(global_config, parts)
+
+    if retcode:
+        # Only run the script if we are not doomed.
+        run_scripts(
+            "post_compose_", target_dir, parser.get("general", "post_compose_script")
+        )
+
+    return retcode
 
 
 def parse_args(argv):

tests/data/multi-compose.conf

@@ -7,6 +7,9 @@ compose_type = nightly
 target = ../_composes/
 extra_args = --quiet
 
+post_compose_script =
+    compose-latest-symlink $COMPOSE_PATH
+
 [server]
 config = server.conf
 

tests/test_orchestrator.py

@@ -833,3 +833,60 @@ class TestRunKinit(BaseTestCase):
         self.assertEqual(
             register.call_args_list, [mock.call(os.remove, os.environ["KRB5CCNAME"])]
         )
+
+
+@mock.patch.dict("os.environ", {}, clear=True)
+class TestGetScriptEnv(BaseTestCase):
+    def test_without_metadata(self):
+        env = o.get_script_env("/foobar")
+        self.assertEqual(env, {"COMPOSE_PATH": "/foobar"})
+
+    def test_with_metadata(self):
+        compose_dir = os.path.join(FIXTURE_DIR, "DP-1.0-20161013.t.4")
+        env = o.get_script_env(compose_dir)
+        self.maxDiff = None
+        self.assertEqual(
+            env,
+            {
+                "COMPOSE_PATH": compose_dir,
+                "COMPOSE_ID": "DP-1.0-20161013.t.4",
+                "COMPOSE_DATE": "20161013",
+                "COMPOSE_TYPE": "test",
+                "COMPOSE_RESPIN": "4",
+                "COMPOSE_LABEL": "",
+                "RELEASE_ID": "DP-1.0",
+                "RELEASE_NAME": "Dummy Product",
+                "RELEASE_SHORT": "DP",
+                "RELEASE_VERSION": "1.0",
+                "RELEASE_TYPE": "ga",
+                "RELEASE_IS_LAYERED": "",
+            },
+        )
+
+
+class TestRunScripts(BaseTestCase):
+    @mock.patch("pungi_utils.orchestrator.get_script_env")
+    @mock.patch("kobo.shortcuts.run")
+    def test_run_scripts(self, run, get_env):
+        commands = """
+           date
+           env
+           """
+
+        o.run_scripts("pref_", "/tmp/compose", commands)
+
+        self.assertEqual(
+            run.call_args_list,
+            [
+                mock.call(
+                    "date",
+                    logfile="/tmp/compose/logs/pref_0.log",
+                    env=get_env.return_value,
+                ),
+                mock.call(
+                    "env",
+                    logfile="/tmp/compose/logs/pref_1.log",
+                    env=get_env.return_value,
+                ),
+            ],
+        )