diff --git a/doc/configuration.rst b/doc/configuration.rst index d9cccf16..54532060 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -176,6 +176,16 @@ Options Please note that when ``dnf`` is used, the build dependencies check is skipped. On Python 3, only ``dnf`` backend is available. +**cts_url** + (*str*) -- URL to Compose Tracking Service. If defined, Pungi will add + the compose to Compose Tracking Service and ge the compose ID from it. + For example ``https://cts.localhost.tld/`` + +**cts_keytab** + (*str*) -- Path to Kerberos keytab which will be used for Compose + Tracking Service Kerberos authentification. If not defined, the default + Kerberos principal is used. + **compose_type** (*str*) -- Allows to set default compose type. Type set via a command-line option overwrites this. diff --git a/pungi/checks.py b/pungi/checks.py index b7fd3c93..c91760bf 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -765,6 +765,8 @@ def make_schema(): "pdc_url": {"deprecated": "Koji is queried instead"}, "pdc_develop": {"deprecated": "Koji is queried instead"}, "pdc_insecure": {"deprecated": "Koji is queried instead"}, + "cts_url": {"type": "string"}, + "cts_keytab": {"type": "string"}, "koji_profile": {"type": "string"}, "koji_event": {"type": "number"}, "pkgset_koji_tag": {"$ref": "#/definitions/strings"}, diff --git a/pungi/compose.py b/pungi/compose.py index 624484d7..aa682815 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -51,18 +51,16 @@ except ImportError: SUPPORTED_MILESTONES = ["RC", "Update", "SecurityFix"] -def get_compose_dir( - topdir, +def get_compose_info( conf, compose_type="production", compose_date=None, compose_respin=None, compose_label=None, - already_exists_callbacks=None, ): - already_exists_callbacks = already_exists_callbacks or [] - - # create an incomplete composeinfo to generate compose ID + """ + Creates inncomplete ComposeInfo to generate Compose ID + """ ci = ComposeInfo() ci.release.name = conf["release_name"] ci.release.short = conf["release_short"] @@ -81,37 +79,112 @@ def get_compose_dir( ci.compose.date = compose_date or time.strftime("%Y%m%d", time.localtime()) ci.compose.respin = compose_respin or 0 - while 1: + cts_url = conf.get("cts_url", None) + if cts_url: + # Import requests and requests-kerberos here so it is not needed + # if running without Compose Tracking Service. + import requests + from requests_kerberos import HTTPKerberosAuth + + # Requests-kerberos cannot accept custom keytab, we need to use + # environment variable for this. But we need to change environment + # only temporarily just for this single requests.post. + # So at first backup the current environment and revert to it + # after the requests.post call. + cts_keytab = conf.get("cts_keytab", None) + if cts_keytab: + environ_copy = dict(os.environ) + os.environ["KRB5_CLIENT_KTNAME"] = cts_keytab + + try: + # Create compose in CTS and get the reserved compose ID. + ci.compose.id = ci.create_compose_id() + url = os.path.join(cts_url, "api/1/composes/") + data = {"compose_info": json.loads(ci.dumps())} + rv = requests.post(url, json=data, auth=HTTPKerberosAuth()) + rv.raise_for_status() + finally: + if cts_keytab: + os.environ.clear() + os.environ.update(environ_copy) + + # Update local ComposeInfo with received ComposeInfo. + cts_ci = ComposeInfo() + cts_ci.loads(rv.text) + ci.compose.respin = cts_ci.compose.respin + ci.compose.id = cts_ci.compose.id + else: ci.compose.id = ci.create_compose_id() - compose_dir = os.path.join(topdir, ci.compose.id) + return ci - exists = False - # TODO: callbacks to determine if a composeid was already used - # for callback in already_exists_callbacks: - # if callback(data): - # exists = True - # break - - # already_exists_callbacks fallback: does target compose_dir exist? - try: - os.makedirs(compose_dir) - except OSError as ex: - if ex.errno == errno.EEXIST: - exists = True - else: - raise - - if exists: - ci.compose.respin += 1 - continue - break +def write_compose_info(compose_dir, ci): + """ + Write ComposeInfo `ci` to `compose_dir` subdirectories. + """ + makedirs(compose_dir) with open(os.path.join(compose_dir, "COMPOSE_ID"), "w") as f: f.write(ci.compose.id) work_dir = os.path.join(compose_dir, "work", "global") makedirs(work_dir) ci.dump(os.path.join(work_dir, "composeinfo-base.json")) + + +def get_compose_dir( + topdir, + conf, + compose_type="production", + compose_date=None, + compose_respin=None, + compose_label=None, + already_exists_callbacks=None, +): + already_exists_callbacks = already_exists_callbacks or [] + + ci = get_compose_info( + conf, compose_type, compose_date, compose_respin, compose_label + ) + + cts_url = conf.get("cts_url", None) + if cts_url: + # Create compose directory. + compose_dir = os.path.join(topdir, ci.compose.id) + os.makedirs(compose_dir) + else: + while 1: + ci.compose.id = ci.create_compose_id() + + compose_dir = os.path.join(topdir, ci.compose.id) + + exists = False + # TODO: callbacks to determine if a composeid was already used + # for callback in already_exists_callbacks: + # if callback(data): + # exists = True + # break + + # already_exists_callbacks fallback: does target compose_dir exist? + try: + os.makedirs(compose_dir) + except OSError as ex: + if ex.errno == errno.EEXIST: + exists = True + else: + raise + + if exists: + ci = get_compose_info( + conf, + compose_type, + compose_date, + ci.compose.respin + 1, + compose_label, + ) + continue + break + + write_compose_info(compose_dir, ci) return compose_dir @@ -221,6 +294,8 @@ class Compose(kobo.log.LoggingBase): else: self.cache_region = make_region().configure("dogpile.cache.null") + get_compose_info = staticmethod(get_compose_info) + write_compose_info = staticmethod(write_compose_info) get_compose_dir = staticmethod(get_compose_dir) def __getitem__(self, name): diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index 1fab656c..35764c24 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -47,7 +47,8 @@ def main(): group.add_argument( "--compose-dir", metavar="PATH", - help="reuse an existing compose directory (DANGEROUS!)", + help="specify compose directory in which the compose will be generated." + "If directory already exists, Pungi will reuse it (DANGEROUS!).", ) parser.add_argument( "--label", @@ -199,11 +200,8 @@ def main(): ) else: opts.compose_dir = os.path.abspath(opts.compose_dir) - if not os.path.isdir(opts.compose_dir): - abort( - "The compose directory does not exist or is not a directory: %s" - % opts.compose_dir - ) + if os.path.exists(opts.compose_dir) and not os.path.isdir(opts.compose_dir): + abort("The compose directory is not a directory: %s" % opts.compose_dir) opts.config = os.path.abspath(opts.config) @@ -275,6 +273,11 @@ def main(): ) else: compose_dir = opts.compose_dir + if not os.path.exists(compose_dir): + ci = Compose.get_compose_info( + conf, compose_type=compose_type, compose_label=opts.label + ) + Compose.write_compose_info(compose_dir, ci) if opts.print_output_dir: print("Compose dir: %s" % compose_dir) diff --git a/tests/test_compose.py b/tests/test_compose.py index 158da3bf..8379e9cb 100644 --- a/tests/test_compose.py +++ b/tests/test_compose.py @@ -11,6 +11,7 @@ import os import six import tempfile import shutil +import json from pungi.compose import Compose @@ -27,6 +28,27 @@ class ComposeTestCase(unittest.TestCase): def setUp(self): self.tmp_dir = tempfile.mkdtemp() + # Basic ComposeInfo metadata used in tests. + self.ci_json = { + "header": {"type": "productmd.composeinfo", "version": mock.ANY}, + "payload": { + "compose": { + "date": "20200526", + "id": "test-1.0-20200526.0", + "respin": 0, + "type": "production", + }, + "release": { + "internal": False, + "name": "Test", + "short": "test", + "type": "ga", + "version": "1.0", + }, + "variants": {}, + }, + } + def tearDown(self): shutil.rmtree(self.tmp_dir) @@ -571,6 +593,61 @@ class ComposeTestCase(unittest.TestCase): d = compose.mkdtemp(prefix="tweak_buildinstall") self.assertTrue(os.path.isdir(d)) + def test_get_compose_info(self): + conf = ConfigWrapper( + release_name="Test", + release_version="1.0", + release_short="test", + release_type="ga", + release_internal=False, + ) + + ci = Compose.get_compose_info(conf) + ci_json = json.loads(ci.dumps()) + self.assertEqual(ci_json, self.ci_json) + + def test_get_compose_info_cts(self): + conf = ConfigWrapper( + release_name="Test", + release_version="1.0", + release_short="test", + release_type="ga", + release_internal=False, + cts_url="https://cts.localhost.tld/", + cts_keytab="/tmp/some.keytab", + ) + + # The `mock.ANY` in ["header"]["version"] cannot be serialized, + # so for this test, we replace it with real version. + ci_copy = dict(self.ci_json) + ci_copy["header"]["version"] = "1.2" + mocked_response = mock.MagicMock() + mocked_response.text = json.dumps(self.ci_json) + mocked_requests = mock.MagicMock() + mocked_requests.post.return_value = mocked_response + + mocked_requests_kerberos = mock.MagicMock() + + # The `requests` and `requests_kerberos` modules are imported directly + # in the `get_compose_info` function. To patch them, we need to patch + # the `sys.modules` directly so the patched modules are returned by + # `import`. + with mock.patch.dict( + "sys.modules", + requests=mocked_requests, + requests_kerberos=mocked_requests_kerberos, + ): + ci = Compose.get_compose_info(conf) + ci_json = json.loads(ci.dumps()) + self.assertEqual(ci_json, self.ci_json) + + mocked_response.raise_for_status.assert_called_once() + mocked_requests.post.assert_called_once_with( + "https://cts.localhost.tld/api/1/composes/", + auth=mock.ANY, + json={"compose_info": self.ci_json}, + ) + class StatusTest(unittest.TestCase): def setUp(self):