From 45cdbb2fafe6291f441af0e1f4192d6d42dc835a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 6 Mar 2019 13:35:04 +0100 Subject: [PATCH] orchestrator: Send messages about the main compose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only start/finish messages will be sent if a handler is configured. JIRA: COMPOSE-3288 Signed-off-by: Lubomír Sedlář --- bin/pungi-config-validate | 9 +-- bin/pungi-koji | 9 +-- doc/multi_compose.rst | 10 +++ pungi/util.py | 32 ++++++--- pungi_utils/orchestrator.py | 88 ++++++++++++++++------- tests/fixtures/DP-1.0-20161013.t.4/STATUS | 1 + tests/test_orchestrator.py | 45 ++++++++++++ 7 files changed, 144 insertions(+), 50 deletions(-) create mode 100644 tests/fixtures/DP-1.0-20161013.t.4/STATUS diff --git a/bin/pungi-config-validate b/bin/pungi-config-validate index d46b00f9..e54990fb 100755 --- a/bin/pungi-config-validate +++ b/bin/pungi-config-validate @@ -78,14 +78,7 @@ def read_variants(compose, config): def run(config, topdir, has_old, offline): - conf = kobo.conf.PyConfigParser() - if config.endswith(".json"): - with open(config) as f: - conf.load_from_dict(json.load(f)) - conf.opened_files = [config] - conf._open_file = config - else: - conf.load_from_file(config) + conf = pungi.util.load_config(config) errors, warnings = pungi.checks.validate(conf, offline=offline) if errors or warnings: diff --git a/bin/pungi-koji b/bin/pungi-koji index 32e4c447..e48686eb 100755 --- a/bin/pungi-koji +++ b/bin/pungi-koji @@ -217,14 +217,7 @@ def main(): if not opts.quiet: kobo.log.add_stderr_logger(logger) - conf = kobo.conf.PyConfigParser() - if opts.config.endswith(".json"): - with open(opts.config) as f: - conf.load_from_dict(json.load(f)) - conf.opened_files = [opts.config] - conf._open_file = opts.config - else: - conf.load_from_file(opts.config) + conf = util.load_config(opts.config) compose_type = opts.compose_type or conf.get('compose_type', 'production') if compose_type == "production" and not opts.label and not opts.no_label: diff --git a/doc/multi_compose.rst b/doc/multi_compose.rst index 3f9ddca4..f809d5f0 100644 --- a/doc/multi_compose.rst +++ b/doc/multi_compose.rst @@ -77,6 +77,16 @@ General settings * ``BASE_PRODUCT_VERSION`` – only set for layered products * ``BASE_PRODUCT_TYPE`` – only set for layered products +**notification_script** + Executable name (or path to a script) that will be used to send a message + once the compose is finished. In order for a valid URL to be included in the + message, at least one part must configure path translation that would apply + to location of main compose. + + Only two messages will be sent, one for start and one for finish (either + successful or not). + + Partial compose settings ------------------------ diff --git a/pungi/util.py b/pungi/util.py index fa7dd9d7..86f35d81 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -32,6 +32,7 @@ import time import functools from six.moves import urllib, range, shlex_quote +import kobo.conf from kobo.shortcuts import run, force_list from productmd.common import get_major_version @@ -711,14 +712,8 @@ def run_unmount_cmd(cmd, max_retries=10, path=None, logger=None): raise RuntimeError('Failed to run %r: Device or resource busy.' % cmd) -def translate_path(compose, path): - """ - @param compose - required for access to config - @param path - """ +def translate_path_raw(mapping, path): normpath = os.path.normpath(path) - mapping = compose.conf["translate_paths"] - for prefix, newvalue in mapping: prefix = os.path.normpath(prefix) # Strip trailing slashes: the prefix has them stripped by `normpath`. @@ -728,10 +723,18 @@ def translate_path(compose, path): # a path - http:// would get changed to http:/ and so on. # Only the first occurance should be replaced. return normpath.replace(prefix, newvalue, 1) - return normpath +def translate_path(compose, path): + """ + @param compose - required for access to config + @param path + """ + mapping = compose.conf["translate_paths"] + return translate_path_raw(mapping, path) + + def get_repo_url(compose, repo, arch='$basearch'): """ Convert repo to repo URL. @@ -923,3 +926,16 @@ def iter_module_defaults(path): for mmddef in Modulemd.objects_from_file(file): if isinstance(mmddef, Modulemd.Defaults): yield mmddef + + +def load_config(file_path): + """Open and load configuration file form .conf or .json file.""" + conf = kobo.conf.PyConfigParser() + if file_path.endswith(".json"): + with open(file_path) as f: + conf.load_from_dict(json.load(f)) + conf.opened_files = [file_path] + conf._open_file = file_path + else: + conf.load_from_file(file_path) + return conf diff --git a/pungi_utils/orchestrator.py b/pungi_utils/orchestrator.py index 6a3444b8..e40f3f51 100644 --- a/pungi_utils/orchestrator.py +++ b/pungi_utils/orchestrator.py @@ -21,6 +21,7 @@ import productmd from kobo import shortcuts from six.moves import configparser, shlex_quote +import pungi.util from pungi.compose import get_compose_dir from pungi.linker import linker_pool from pungi.phases.pkgset.sources.source_koji import get_koji_event_raw @@ -114,8 +115,7 @@ class ComposePart(object): ) substitutions["configdir"] = global_config.config_dir - config = kobo.conf.PyConfigParser() - config.load_from_file(self.config) + config = pungi.util.load_config(self.config) for f in config.opened_files: # apply substitutions @@ -417,6 +417,8 @@ def prepare_compose_dir(config, args, main_config_file, compose_info): except OSError as exc: if exc.errno != errno.EEXIST: raise + with open(os.path.join(target_dir, "STATUS"), "w") as fh: + fh.write("STARTED") # Copy initial composeinfo for new compose shutil.copy( os.path.join(target_dir, "work/global/composeinfo-base.json"), @@ -484,33 +486,42 @@ def run_kinit(config): atexit.register(os.remove, fname) +def get_compose_data(compose_path): + try: + compose = productmd.compose.Compose(compose_path) + data = { + "compose_id": compose.info.compose.id, + "compose_date": compose.info.compose.date, + "compose_type": compose.info.compose.type, + "compose_respin": str(compose.info.compose.respin), + "compose_label": compose.info.compose.label, + "release_id": compose.info.release_id, + "release_name": compose.info.release.name, + "release_short": compose.info.release.short, + "release_version": compose.info.release.version, + "release_type": compose.info.release.type, + "release_is_layered": compose.info.release.is_layered, + } + if compose.info.release.is_layered: + data.update({ + "base_product_name": compose.info.base_product.name, + "base_product_short": compose.info.base_product.short, + "base_product_version": compose.info.base_product.version, + "base_product_type": compose.info.base_product.type, + }) + return data + except Exception as exc: + return {} + + def get_script_env(compose_path): env = os.environ.copy() env["COMPOSE_PATH"] = compose_path - try: - compose = productmd.compose.Compose(compose_path) - env.update({ - "COMPOSE_ID": compose.info.compose.id, - "COMPOSE_DATE": compose.info.compose.date, - "COMPOSE_TYPE": compose.info.compose.type, - "COMPOSE_RESPIN": str(compose.info.compose.respin), - "COMPOSE_LABEL": compose.info.compose.label or "", - "RELEASE_ID": compose.info.release_id, - "RELEASE_NAME": compose.info.release.name, - "RELEASE_SHORT": compose.info.release.short, - "RELEASE_VERSION": compose.info.release.version, - "RELEASE_TYPE": compose.info.release.type, - "RELEASE_IS_LAYERED": "YES" if compose.info.release.is_layered else "", - }) - if compose.info.release.is_layered: - env.update({ - "BASE_PRODUCT_NAME": compose.info.base_product.name, - "BASE_PRODUCT_SHORT": compose.info.base_product.short, - "BASE_PRODUCT_VERSION": compose.info.base_product.version, - "BASE_PRODUCT_TYPE": compose.info.base_product.type, - }) - except Exception as exc: - pass + for key, value in get_compose_data(compose_path).items(): + if isinstance(value, bool): + env[key.upper()] = "YES" if value else "" + else: + env[key.upper()] = str(value) if value else "" return env @@ -524,6 +535,26 @@ def run_scripts(prefix, compose_dir, scripts): shortcuts.run(command, env=env, logfile=logfile) +def try_translate_path(parts, path): + translation = [] + for part in parts.values(): + conf = pungi.util.load_config(part.config) + translation.extend(conf.get("translate_paths", [])) + return pungi.util.translate_path_raw(translation, path) + + +def send_notification(compose_dir, command, parts): + if not command: + return + from pungi.notifier import PungiNotifier + data = get_compose_data(compose_dir) + data["location"] = try_translate_path(parts, compose_dir) + notifier = PungiNotifier([command]) + with open(os.path.join(compose_dir, "STATUS")) as f: + status = f.read().strip() + notifier.send("status-change", workdir=compose_dir, status=status, **data) + + def run(work_dir, main_config_file, args): config_dir = os.path.join(work_dir, "config") shutil.copytree(os.path.dirname(main_config_file), config_dir) @@ -534,6 +565,7 @@ def run(work_dir, main_config_file, args): "kerberos": "false", "pre_compose_script": "", "post_compose_script": "", + "notification_script": "", } ) parser.read(main_config_file) @@ -584,6 +616,8 @@ def run(work_dir, main_config_file, args): if hasattr(args, "part"): setup_for_restart(global_config, parts, args.part) + send_notification(target_dir, parser.get("general", "notification_script"), parts) + retcode = run_all(global_config, parts) if retcode: @@ -592,6 +626,8 @@ def run(work_dir, main_config_file, args): "post_compose_", target_dir, parser.get("general", "post_compose_script") ) + send_notification(target_dir, parser.get("general", "notification_script"), parts) + return retcode diff --git a/tests/fixtures/DP-1.0-20161013.t.4/STATUS b/tests/fixtures/DP-1.0-20161013.t.4/STATUS new file mode 100644 index 00000000..067b935d --- /dev/null +++ b/tests/fixtures/DP-1.0-20161013.t.4/STATUS @@ -0,0 +1 @@ +FINISHED \ No newline at end of file diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 2a37ea28..2b07528d 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -730,6 +730,9 @@ class TestPrepareComposeDir(PungiTestCase): self.assertTrue(os.path.isdir(os.path.join(self.topdir, "logs"))) self.assertTrue(os.path.isdir(os.path.join(self.topdir, "parts"))) self.assertTrue(os.path.isdir(os.path.join(self.topdir, "work/global"))) + self.assertFileContent( + os.path.join(self.topdir, "STATUS"), "STARTED" + ) def test_restarting_compose(self, gtd): args = mock.Mock(name="args", spec=["label", "compose_path"]) @@ -890,3 +893,45 @@ class TestRunScripts(BaseTestCase): ), ], ) + + +@mock.patch("pungi.notifier.PungiNotifier") +class TestSendNotification(BaseTestCase): + def test_no_command(self, notif): + o.send_notification("/foobar", None, None) + self.assertEqual(notif.mock_calls, []) + + @mock.patch("pungi.util.load_config") + def test_with_command_and_translate(self, load_config, notif): + compose_dir = os.path.join(FIXTURE_DIR, "DP-1.0-20161013.t.4") + load_config.return_value = { + "translate_paths": [(os.path.dirname(compose_dir), "http://example.com")], + } + parts = {"foo": mock.Mock()} + + o.send_notification(compose_dir, "handler", parts) + + self.assertEqual(len(notif.mock_calls), 2) + self.assertEqual(notif.mock_calls[0], mock.call(["handler"])) + _, args, kwargs = notif.mock_calls[1] + self.assertEqual(args, ("status-change", )) + self.assertEqual( + kwargs, + { + "status": "FINISHED", + "workdir": compose_dir, + "location": "http://example.com/DP-1.0-20161013.t.4", + "compose_id": "DP-1.0-20161013.t.4", + "compose_date": "20161013", + "compose_type": "test", + "compose_respin": "4", + "compose_label": None, + "release_id": "DP-1.0", + "release_name": "Dummy Product", + "release_short": "DP", + "release_version": "1.0", + "release_type": "ga", + "release_is_layered": False, + }, + ) + self.assertEqual(load_config.call_args_list, [mock.call(parts["foo"].config)])