From 066855a0393680786ac62ad17765121ca3c3797c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Mon, 23 Nov 2015 16:09:01 +0100 Subject: [PATCH] Add ability to send messages about progress MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With this patch, Pungi can invoke an arbitrary command on various moments of the compose process. The invoked command can the decide on what message to send (and using what messaging system). The actual command is specified in the config file. There is a script provided that sends the messages via fedmsg. The documentation is updated to have details about the new config option as well as the interface for the messaging script. Signed-off-by: Lubomír Sedlář --- bin/pungi-fedmsg-notification | 20 +++++++++ bin/pungi-koji | 17 +++++++- doc/configuration.rst | 36 +++++++++++++++ pungi/compose.py | 1 + pungi/notifier.py | 74 +++++++++++++++++++++++++++++++ pungi/phases/base.py | 3 +- setup.py | 1 + tests/test_notifier.py | 82 +++++++++++++++++++++++++++++++++++ 8 files changed, 232 insertions(+), 2 deletions(-) create mode 100755 bin/pungi-fedmsg-notification create mode 100644 pungi/notifier.py create mode 100755 tests/test_notifier.py diff --git a/bin/pungi-fedmsg-notification b/bin/pungi-fedmsg-notification new file mode 100755 index 00000000..0e4630bf --- /dev/null +++ b/bin/pungi-fedmsg-notification @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import argparse +import fedmsg +import json +import sys + + +def send(cmd, data): + topic = 'compose.%s' % cmd.replace('-', '.').lower() + fedmsg.publish(topic=topic, modname='pungi', msg=data) + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('cmd') + + opts = parser.parse_args() + data = json.load(sys.stdin) + send(opts.cmd, data) diff --git a/bin/pungi-koji b/bin/pungi-koji index 359d4373..43f177c9 100755 --- a/bin/pungi-koji +++ b/bin/pungi-koji @@ -26,6 +26,7 @@ locale.setlocale(locale.LC_ALL, "C") COMPOSE = None +NOTIFIER = None def main(): @@ -187,8 +188,19 @@ def main(): def run_compose(compose): import pungi.phases import pungi.metadata + import pungi.notifier + + errors = [] + + # initializer notifier + compose.notifier = pungi.notifier.PungiNotifier(compose) + try: + compose.notifier.validate() + except ValueError as ex: + errors.extend(["NOTIFIER: %s" % m for m in ex.message.split('\n')]) compose.write_status("STARTED") + compose.notifier.send('start') compose.log_info("Host: %s" % socket.gethostname()) compose.log_info("User name: %s" % getpass.getuser()) compose.log_info("Working directory: %s" % os.getcwd()) @@ -215,7 +227,6 @@ def run_compose(compose): test_phase = pungi.phases.TestPhase(compose) # check if all config options are set - errors = [] for phase in (init_phase, pkgset_phase, createrepo_phase, buildinstall_phase, productimg_phase, gather_phase, extrafiles_phase, createiso_phase, liveimages_phase, @@ -229,6 +240,7 @@ def run_compose(compose): errors.append("%s: %s" % (phase.name.upper(), i)) if errors: for i in errors: + compose.notifier.send('abort') compose.log_error(i) print(i) sys.exit(1) @@ -329,6 +341,7 @@ def run_compose(compose): compose.log_info("Compose finished: %s" % compose.topdir) compose.write_status("FINISHED") + compose.notifier.send('finish') if __name__ == "__main__": @@ -343,6 +356,8 @@ if __name__ == "__main__": COMPOSE.write_status("DOOMED") import kobo.tback open(tb_path, "w").write(kobo.tback.Traceback().get_traceback()) + if COMPOSE.notifier: + COMPOSE.notifier.send('doomed') else: print("Exception: %s" % ex) raise diff --git a/doc/configuration.rst b/doc/configuration.rst index 6418b1b6..a505e66c 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -548,3 +548,39 @@ Example usage >>> from pungi.paths import translate_paths >>> print translate_paths(compose_object_with_mapping, "/mnt/a/c/somefile") http://b/dir/c/somefile + + +Progress notification +===================== + +*Pungi* has the ability to emit notification messages about progress and +status. These can be used to e.g. send messages to *fedmsg*. This is +implemented by actually calling a separate script. + +The script will be called with one argument describing action that just +happened. A JSON-encoded object will be passed to standard input to provide +more information about the event. At least, the object will contain a +``compose_id`` key. + +Currently these messages are sent: + + * ``start`` -- when composing starts + * ``abort`` -- when compose is aborted due to incorrect configuration + * ``finish`` -- on successful finish of compose + * ``doomed`` -- when an error happens + * ``phase-start`` -- on start of a phase + * ``phase-stop`` -- when phase is finished + +For phase related messages ``phase_name`` key is provided as well. + +The script is invoked in compose directory and can read other information +there. + +A ``pungi-fedmsg-notification`` script is provided and understands this +interface. + +Config options +-------------- + +**notification_script** + (*str*) -- executable to be invoked to send the message diff --git a/pungi/compose.py b/pungi/compose.py index 6b18d3af..017080a1 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -102,6 +102,7 @@ class Compose(kobo.log.LoggingBase): self.just_phases = just_phases or [] self.old_composes = old_composes or [] self.koji_event = koji_event + self.notifier = None # intentionally upper-case (visible in the code) self.DEBUG = debug diff --git a/pungi/notifier.py b/pungi/notifier.py new file mode 100644 index 00000000..5fa20928 --- /dev/null +++ b/pungi/notifier.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import json +import threading + +from kobo import shortcuts +from pungi.checks import validate_options + + +class PungiNotifier(object): + """Wrapper around an external script for sending messages. + + If no script is configured, the messages are just silently ignored. If the + script fails, a warning will be logged, but the compose process will not be + interrupted. + """ + config_options = ( + { + "name": "notification_script", + "expected_types": [str], + "optional": True + }, + ) + + def __init__(self, compose): + self.compose = compose + self.lock = threading.Lock() + + def validate(self): + errors = validate_options(self.compose.conf, self.config_options) + if errors: + raise ValueError("\n".join(errors)) + + def _update_args(self, data): + """Add compose related information to the data.""" + data.setdefault('compose_id', self.compose.compose_id) + + def send(self, msg, **kwargs): + """Send a message. + + The actual meaning of ``msg`` depends on what the notification script + will be doing. The keyword arguments will be JSON-encoded and passed on + to standard input of the notification process. + + Unless you specify it manually, a ``compose_id`` key with appropriate + value will be automatically added. + """ + script = self.compose.conf.get('notification_script', None) + if not script: + return + + self._update_args(kwargs) + + with self.lock: + ret, _ = shortcuts.run((script, msg), + stdin_data=json.dumps(kwargs), + can_fail=True, + workdir=self.compose.paths.compose.topdir(), + return_stdout=False) + if ret != 0: + self.compose.log_warning('Failed to invoke notification script.') diff --git a/pungi/phases/base.py b/pungi/phases/base.py index 2fea9242..6c83bdf7 100644 --- a/pungi/phases/base.py +++ b/pungi/phases/base.py @@ -14,7 +14,6 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - from pungi.checks import validate_options @@ -59,6 +58,7 @@ class PhaseBase(object): self.finished = True return self.compose.log_info("[BEGIN] %s" % self.msg) + self.compose.notifier.send('phase-start', phase_name=self.name) self.run() def stop(self): @@ -68,6 +68,7 @@ class PhaseBase(object): self.pool.stop() self.finished = True self.compose.log_info("[DONE ] %s" % self.msg) + self.compose.notifier.send('phase-stop', phase_name=self.name) def run(self): raise NotImplementedError diff --git a/setup.py b/setup.py index d6606f22..5b8d7504 100755 --- a/setup.py +++ b/setup.py @@ -37,6 +37,7 @@ setup( 'bin/pungi', 'bin/pungi-koji', 'bin/comps_filter', + 'bin/pungi-fedmsg-notifier', ], data_files = [ ('/usr/share/pungi', glob.glob('share/*.xsl')), diff --git a/tests/test_notifier.py b/tests/test_notifier.py new file mode 100755 index 00000000..12515b88 --- /dev/null +++ b/tests/test_notifier.py @@ -0,0 +1,82 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import json +import mock +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from pungi.notifier import PungiNotifier + + +class TestNotifier(unittest.TestCase): + def test_incorrect_config(self): + compose = mock.Mock( + conf={'notification_script': [1, 2]} + ) + + n = PungiNotifier(compose) + with self.assertRaises(ValueError) as err: + n.validate() + self.assertIn('notification_script', err.message) + + @mock.patch('kobo.shortcuts.run') + def test_invokes_script(self, run): + compose = mock.Mock( + compose_id='COMPOSE_ID', + conf={'notification_script': 'run-notify'}, + paths=mock.Mock( + compose=mock.Mock( + topdir=mock.Mock(return_value='/a/b') + ) + ) + ) + + run.return_value = (0, None) + + n = PungiNotifier(compose) + data = {'foo': 'bar', 'baz': 'quux'} + n.send('cmd', **data) + + data['compose_id'] = 'COMPOSE_ID' + run.assert_called_once_with(('run-notify', 'cmd'), + stdin_data=json.dumps(data), + can_fail=True, return_stdout=False, workdir='/a/b') + + @mock.patch('kobo.shortcuts.run') + def test_does_not_run_without_config(self, run): + compose = mock.Mock(conf={}) + + n = PungiNotifier(compose) + n.send('cmd', foo='bar', baz='quux') + self.assertFalse(run.called) + + @mock.patch('kobo.shortcuts.run') + def test_logs_warning_on_failure(self, run): + compose = mock.Mock( + compose_id='COMPOSE_ID', + log_warning=mock.Mock(), + conf={'notification_script': 'run-notify'}, + paths=mock.Mock( + compose=mock.Mock( + topdir=mock.Mock(return_value='/a/b') + ) + ) + ) + + run.return_value = (1, None) + + n = PungiNotifier(compose) + n.send('cmd') + + run.assert_called_once_with(('run-notify', 'cmd'), + stdin_data=json.dumps({'compose_id': 'COMPOSE_ID'}), + can_fail=True, return_stdout=False, workdir='/a/b') + self.assertTrue(compose.log_warning.called) + + +if __name__ == "__main__": + unittest.main()