almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

orchestrator: Send messages about the main compose

Browse Source 
Only start/finish messages will be sent if a handler is configured.

JIRA: COMPOSE-3288
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2019-03-06 13:35:04 +01:00
parent 088ea7fe37
commit 45cdbb2faf
7 changed files with 144 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
bin/pungi-config-validate
View File

@ -78,14 +78,7 @@ def read_variants(compose, config):
def run(config, topdir, has_old, offline):
conf = kobo.conf.PyConfigParser()
if config.endswith(".json"):
with open(config) as f:
conf.load_from_dict(json.load(f))
conf.opened_files = [config]
conf._open_file = config
else:
conf.load_from_file(config)
conf = pungi.util.load_config(config)
errors, warnings = pungi.checks.validate(conf, offline=offline)
if errors or warnings:

9
bin/pungi-koji
View File

@ -217,14 +217,7 @@ def main():
if not opts.quiet:
kobo.log.add_stderr_logger(logger)
conf = kobo.conf.PyConfigParser()
if opts.config.endswith(".json"):
with open(opts.config) as f:
conf.load_from_dict(json.load(f))
conf.opened_files = [opts.config]
conf._open_file = opts.config
else:
conf.load_from_file(opts.config)
conf = util.load_config(opts.config)
compose_type = opts.compose_type or conf.get('compose_type', 'production')
if compose_type == "production" and not opts.label and not opts.no_label:

10
doc/multi_compose.rst
View File

@ -77,6 +77,16 @@ General settings
* ``BASE_PRODUCT_VERSION`` – only set for layered products
* ``BASE_PRODUCT_TYPE`` – only set for layered products
**notification_script**
Executable name (or path to a script) that will be used to send a message
once the compose is finished. In order for a valid URL to be included in the
message, at least one part must configure path translation that would apply
to location of main compose.
Only two messages will be sent, one for start and one for finish (either
successful or not).
Partial compose settings
------------------------

32
pungi/util.py
View File

@ -32,6 +32,7 @@ import time
import functools
from six.moves import urllib, range, shlex_quote
import kobo.conf
from kobo.shortcuts import run, force_list
from productmd.common import get_major_version
@ -711,14 +712,8 @@ def run_unmount_cmd(cmd, max_retries=10, path=None, logger=None):
raise RuntimeError('Failed to run %r: Device or resource busy.' % cmd)
def translate_path(compose, path):
"""
@param compose - required for access to config
@param path
"""
def translate_path_raw(mapping, path):
normpath = os.path.normpath(path)
mapping = compose.conf["translate_paths"]
for prefix, newvalue in mapping:
prefix = os.path.normpath(prefix)
# Strip trailing slashes: the prefix has them stripped by `normpath`.
@ -728,10 +723,18 @@ def translate_path(compose, path):
# a path - http:// would get changed to http:/ and so on.
# Only the first occurance should be replaced.
return normpath.replace(prefix, newvalue, 1)
return normpath
def translate_path(compose, path):
"""
@param compose - required for access to config
@param path
"""
mapping = compose.conf["translate_paths"]
return translate_path_raw(mapping, path)
def get_repo_url(compose, repo, arch='$basearch'):
"""
Convert repo to repo URL.
@ -923,3 +926,16 @@ def iter_module_defaults(path):
for mmddef in Modulemd.objects_from_file(file):
if isinstance(mmddef, Modulemd.Defaults):
yield mmddef
def load_config(file_path):
"""Open and load configuration file form .conf or .json file."""
conf = kobo.conf.PyConfigParser()
if file_path.endswith(".json"):
with open(file_path) as f:
conf.load_from_dict(json.load(f))
conf.opened_files = [file_path]
conf._open_file = file_path
else:
conf.load_from_file(file_path)
return conf

88
pungi_utils/orchestrator.py
View File

@ -21,6 +21,7 @@ import productmd
from kobo import shortcuts
from six.moves import configparser, shlex_quote
import pungi.util
from pungi.compose import get_compose_dir
from pungi.linker import linker_pool
from pungi.phases.pkgset.sources.source_koji import get_koji_event_raw
@ -114,8 +115,7 @@ class ComposePart(object):
)
substitutions["configdir"] = global_config.config_dir
config = kobo.conf.PyConfigParser()
config.load_from_file(self.config)
config = pungi.util.load_config(self.config)
for f in config.opened_files:
# apply substitutions
@ -417,6 +417,8 @@ def prepare_compose_dir(config, args, main_config_file, compose_info):
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
with open(os.path.join(target_dir, "STATUS"), "w") as fh:
fh.write("STARTED")
# Copy initial composeinfo for new compose
shutil.copy(
os.path.join(target_dir, "work/global/composeinfo-base.json"),
@ -484,33 +486,42 @@ def run_kinit(config):
atexit.register(os.remove, fname)
def get_compose_data(compose_path):
try:
compose = productmd.compose.Compose(compose_path)
data = {
"compose_id": compose.info.compose.id,
"compose_date": compose.info.compose.date,
"compose_type": compose.info.compose.type,
"compose_respin": str(compose.info.compose.respin),
"compose_label": compose.info.compose.label,
"release_id": compose.info.release_id,
"release_name": compose.info.release.name,
"release_short": compose.info.release.short,
"release_version": compose.info.release.version,
"release_type": compose.info.release.type,
"release_is_layered": compose.info.release.is_layered,
}
if compose.info.release.is_layered:
data.update({
"base_product_name": compose.info.base_product.name,
"base_product_short": compose.info.base_product.short,
"base_product_version": compose.info.base_product.version,
"base_product_type": compose.info.base_product.type,
})
return data
except Exception as exc:
return {}
def get_script_env(compose_path):
env = os.environ.copy()
env["COMPOSE_PATH"] = compose_path
try:
compose = productmd.compose.Compose(compose_path)
env.update({
"COMPOSE_ID": compose.info.compose.id,
"COMPOSE_DATE": compose.info.compose.date,
"COMPOSE_TYPE": compose.info.compose.type,
"COMPOSE_RESPIN": str(compose.info.compose.respin),
"COMPOSE_LABEL": compose.info.compose.label or "",
"RELEASE_ID": compose.info.release_id,
"RELEASE_NAME": compose.info.release.name,
"RELEASE_SHORT": compose.info.release.short,
"RELEASE_VERSION": compose.info.release.version,
"RELEASE_TYPE": compose.info.release.type,
"RELEASE_IS_LAYERED": "YES" if compose.info.release.is_layered else "",
})
if compose.info.release.is_layered:
env.update({
"BASE_PRODUCT_NAME": compose.info.base_product.name,
"BASE_PRODUCT_SHORT": compose.info.base_product.short,
"BASE_PRODUCT_VERSION": compose.info.base_product.version,
"BASE_PRODUCT_TYPE": compose.info.base_product.type,
})
except Exception as exc:
pass
for key, value in get_compose_data(compose_path).items():
if isinstance(value, bool):
env[key.upper()] = "YES" if value else ""
else:
env[key.upper()] = str(value) if value else ""
return env
@ -524,6 +535,26 @@ def run_scripts(prefix, compose_dir, scripts):
shortcuts.run(command, env=env, logfile=logfile)
def try_translate_path(parts, path):
translation = []
for part in parts.values():
conf = pungi.util.load_config(part.config)
translation.extend(conf.get("translate_paths", []))
return pungi.util.translate_path_raw(translation, path)
def send_notification(compose_dir, command, parts):
if not command:
return
from pungi.notifier import PungiNotifier
data = get_compose_data(compose_dir)
data["location"] = try_translate_path(parts, compose_dir)
notifier = PungiNotifier([command])
with open(os.path.join(compose_dir, "STATUS")) as f:
status = f.read().strip()
notifier.send("status-change", workdir=compose_dir, status=status, **data)
def run(work_dir, main_config_file, args):
config_dir = os.path.join(work_dir, "config")
shutil.copytree(os.path.dirname(main_config_file), config_dir)
@ -534,6 +565,7 @@ def run(work_dir, main_config_file, args):
"kerberos": "false",
"pre_compose_script": "",
"post_compose_script": "",
"notification_script": "",
}
)
parser.read(main_config_file)
@ -584,6 +616,8 @@ def run(work_dir, main_config_file, args):
if hasattr(args, "part"):
setup_for_restart(global_config, parts, args.part)
send_notification(target_dir, parser.get("general", "notification_script"), parts)
retcode = run_all(global_config, parts)
if retcode:
@ -592,6 +626,8 @@ def run(work_dir, main_config_file, args):
"post_compose_", target_dir, parser.get("general", "post_compose_script")
)
send_notification(target_dir, parser.get("general", "notification_script"), parts)
return retcode

1
tests/fixtures/DP-1.0-20161013.t.4/STATUS vendored Normal file
View File

@ -0,0 +1 @@
FINISHED

45
tests/test_orchestrator.py
View File

@ -730,6 +730,9 @@ class TestPrepareComposeDir(PungiTestCase):
self.assertTrue(os.path.isdir(os.path.join(self.topdir, "logs")))
self.assertTrue(os.path.isdir(os.path.join(self.topdir, "parts")))
self.assertTrue(os.path.isdir(os.path.join(self.topdir, "work/global")))
self.assertFileContent(
os.path.join(self.topdir, "STATUS"), "STARTED"
)
def test_restarting_compose(self, gtd):
args = mock.Mock(name="args", spec=["label", "compose_path"])
@ -890,3 +893,45 @@ class TestRunScripts(BaseTestCase):
),
],
)
@mock.patch("pungi.notifier.PungiNotifier")
class TestSendNotification(BaseTestCase):
def test_no_command(self, notif):
o.send_notification("/foobar", None, None)
self.assertEqual(notif.mock_calls, [])
@mock.patch("pungi.util.load_config")
def test_with_command_and_translate(self, load_config, notif):
compose_dir = os.path.join(FIXTURE_DIR, "DP-1.0-20161013.t.4")
load_config.return_value = {
"translate_paths": [(os.path.dirname(compose_dir), "http://example.com")],
}
parts = {"foo": mock.Mock()}
o.send_notification(compose_dir, "handler", parts)
self.assertEqual(len(notif.mock_calls), 2)
self.assertEqual(notif.mock_calls[0], mock.call(["handler"]))
_, args, kwargs = notif.mock_calls[1]
self.assertEqual(args, ("status-change", ))
self.assertEqual(
kwargs,
{
"status": "FINISHED",
"workdir": compose_dir,
"location": "http://example.com/DP-1.0-20161013.t.4",
"compose_id": "DP-1.0-20161013.t.4",
"compose_date": "20161013",
"compose_type": "test",
"compose_respin": "4",
"compose_label": None,
"release_id": "DP-1.0",
"release_name": "Dummy Product",
"release_short": "DP",
"release_version": "1.0",
"release_type": "ga",
"release_is_layered": False,
},
)
self.assertEqual(load_config.call_args_list, [mock.call(parts["foo"].config)])