Allow getting the compose id from CTS (Compose Tracking Service).

Signed-off-by: Jan Kaluza <jkaluza@redhat.com>

doc/configuration.rst

@@ -176,6 +176,16 @@ Options
     Please note that when ``dnf`` is used, the build dependencies check is
     skipped. On Python 3, only ``dnf`` backend is available.
 
+**cts_url**
+    (*str*) -- URL to Compose Tracking Service. If defined, Pungi will add
+    the compose to Compose Tracking Service and ge the compose ID from it.
+    For example ``https://cts.localhost.tld/``
+
+**cts_keytab**
+    (*str*) -- Path to Kerberos keytab which will be used for Compose
+    Tracking Service Kerberos authentification. If not defined, the default
+    Kerberos principal is used.
+
 **compose_type**
     (*str*) -- Allows to set default compose type. Type set via a command-line
     option overwrites this.

pungi/checks.py

@@ -765,6 +765,8 @@ def make_schema():
             "pdc_url": {"deprecated": "Koji is queried instead"},
             "pdc_develop": {"deprecated": "Koji is queried instead"},
             "pdc_insecure": {"deprecated": "Koji is queried instead"},
+            "cts_url": {"type": "string"},
+            "cts_keytab": {"type": "string"},
             "koji_profile": {"type": "string"},
             "koji_event": {"type": "number"},
             "pkgset_koji_tag": {"$ref": "#/definitions/strings"},

pungi/compose.py

@@ -51,18 +51,16 @@ except ImportError:
     SUPPORTED_MILESTONES = ["RC", "Update", "SecurityFix"]
 
 
-def get_compose_dir(
+def get_compose_info(
-    topdir,
     conf,
     compose_type="production",
     compose_date=None,
     compose_respin=None,
     compose_label=None,
-    already_exists_callbacks=None,
 ):
-    already_exists_callbacks = already_exists_callbacks or []
+    """
-
+    Creates inncomplete ComposeInfo to generate Compose ID
-    # create an incomplete composeinfo to generate compose ID
+    """
     ci = ComposeInfo()
     ci.release.name = conf["release_name"]
     ci.release.short = conf["release_short"]
@@ -81,37 +79,112 @@ def get_compose_dir(
     ci.compose.date = compose_date or time.strftime("%Y%m%d", time.localtime())
     ci.compose.respin = compose_respin or 0
 
-    while 1:
+    cts_url = conf.get("cts_url", None)
+    if cts_url:
+        # Import requests and requests-kerberos here so it is not needed
+        # if running without Compose Tracking Service.
+        import requests
+        from requests_kerberos import HTTPKerberosAuth
+
+        # Requests-kerberos cannot accept custom keytab, we need to use
+        # environment variable for this. But we need to change environment
+        # only temporarily just for this single requests.post.
+        # So at first backup the current environment and revert to it
+        # after the requests.post call.
+        cts_keytab = conf.get("cts_keytab", None)
+        if cts_keytab:
+            environ_copy = dict(os.environ)
+            os.environ["KRB5_CLIENT_KTNAME"] = cts_keytab
+
+        try:
+            # Create compose in CTS and get the reserved compose ID.
+            ci.compose.id = ci.create_compose_id()
+            url = os.path.join(cts_url, "api/1/composes/")
+            data = {"compose_info": json.loads(ci.dumps())}
+            rv = requests.post(url, json=data, auth=HTTPKerberosAuth())
+            rv.raise_for_status()
+        finally:
+            if cts_keytab:
+                os.environ.clear()
+                os.environ.update(environ_copy)
+
+        # Update local ComposeInfo with received ComposeInfo.
+        cts_ci = ComposeInfo()
+        cts_ci.loads(rv.text)
+        ci.compose.respin = cts_ci.compose.respin
+        ci.compose.id = cts_ci.compose.id
+    else:
         ci.compose.id = ci.create_compose_id()
 
-        compose_dir = os.path.join(topdir, ci.compose.id)
+    return ci
 
-        exists = False
-        # TODO: callbacks to determine if a composeid was already used
-        # for callback in already_exists_callbacks:
-        #     if callback(data):
-        #         exists = True
-        #         break
-
-        # already_exists_callbacks fallback: does target compose_dir exist?
-        try:
-            os.makedirs(compose_dir)
-        except OSError as ex:
-            if ex.errno == errno.EEXIST:
-                exists = True
-            else:
-                raise
-
-        if exists:
-            ci.compose.respin += 1
-            continue
-        break
 
+def write_compose_info(compose_dir, ci):
+    """
+    Write ComposeInfo `ci` to `compose_dir` subdirectories.
+    """
+    makedirs(compose_dir)
     with open(os.path.join(compose_dir, "COMPOSE_ID"), "w") as f:
         f.write(ci.compose.id)
     work_dir = os.path.join(compose_dir, "work", "global")
     makedirs(work_dir)
     ci.dump(os.path.join(work_dir, "composeinfo-base.json"))
+
+
+def get_compose_dir(
+    topdir,
+    conf,
+    compose_type="production",
+    compose_date=None,
+    compose_respin=None,
+    compose_label=None,
+    already_exists_callbacks=None,
+):
+    already_exists_callbacks = already_exists_callbacks or []
+
+    ci = get_compose_info(
+        conf, compose_type, compose_date, compose_respin, compose_label
+    )
+
+    cts_url = conf.get("cts_url", None)
+    if cts_url:
+        # Create compose directory.
+        compose_dir = os.path.join(topdir, ci.compose.id)
+        os.makedirs(compose_dir)
+    else:
+        while 1:
+            ci.compose.id = ci.create_compose_id()
+
+            compose_dir = os.path.join(topdir, ci.compose.id)
+
+            exists = False
+            # TODO: callbacks to determine if a composeid was already used
+            # for callback in already_exists_callbacks:
+            #     if callback(data):
+            #         exists = True
+            #         break
+
+            # already_exists_callbacks fallback: does target compose_dir exist?
+            try:
+                os.makedirs(compose_dir)
+            except OSError as ex:
+                if ex.errno == errno.EEXIST:
+                    exists = True
+                else:
+                    raise
+
+            if exists:
+                ci = get_compose_info(
+                    conf,
+                    compose_type,
+                    compose_date,
+                    ci.compose.respin + 1,
+                    compose_label,
+                )
+                continue
+            break
+
+    write_compose_info(compose_dir, ci)
     return compose_dir
 
 
@@ -221,6 +294,8 @@ class Compose(kobo.log.LoggingBase):
         else:
             self.cache_region = make_region().configure("dogpile.cache.null")
 
+    get_compose_info = staticmethod(get_compose_info)
+    write_compose_info = staticmethod(write_compose_info)
     get_compose_dir = staticmethod(get_compose_dir)
 
     def __getitem__(self, name):

pungi/scripts/pungi_koji.py

@@ -47,7 +47,8 @@ def main():
     group.add_argument(
         "--compose-dir",
         metavar="PATH",
-        help="reuse an existing compose directory (DANGEROUS!)",
+        help="specify compose directory in which the compose will be generated."
+        "If directory already exists, Pungi will reuse it (DANGEROUS!).",
     )
     parser.add_argument(
         "--label",
@@ -199,11 +200,8 @@ def main():
             )
     else:
         opts.compose_dir = os.path.abspath(opts.compose_dir)
-        if not os.path.isdir(opts.compose_dir):
+        if os.path.exists(opts.compose_dir) and not os.path.isdir(opts.compose_dir):
-            abort(
+            abort("The compose directory is not a directory: %s" % opts.compose_dir)
-                "The compose directory does not exist or is not a directory: %s"
-                % opts.compose_dir
-            )
 
     opts.config = os.path.abspath(opts.config)
 
@@ -275,6 +273,11 @@ def main():
         )
     else:
         compose_dir = opts.compose_dir
+        if not os.path.exists(compose_dir):
+            ci = Compose.get_compose_info(
+                conf, compose_type=compose_type, compose_label=opts.label
+            )
+            Compose.write_compose_info(compose_dir, ci)
 
     if opts.print_output_dir:
         print("Compose dir: %s" % compose_dir)

tests/test_compose.py

@@ -11,6 +11,7 @@ import os
 import six
 import tempfile
 import shutil
+import json
 
 from pungi.compose import Compose
 
@@ -27,6 +28,27 @@ class ComposeTestCase(unittest.TestCase):
     def setUp(self):
         self.tmp_dir = tempfile.mkdtemp()
 
+        # Basic ComposeInfo metadata used in tests.
+        self.ci_json = {
+            "header": {"type": "productmd.composeinfo", "version": mock.ANY},
+            "payload": {
+                "compose": {
+                    "date": "20200526",
+                    "id": "test-1.0-20200526.0",
+                    "respin": 0,
+                    "type": "production",
+                },
+                "release": {
+                    "internal": False,
+                    "name": "Test",
+                    "short": "test",
+                    "type": "ga",
+                    "version": "1.0",
+                },
+                "variants": {},
+            },
+        }
+
     def tearDown(self):
         shutil.rmtree(self.tmp_dir)
 
@@ -571,6 +593,61 @@ class ComposeTestCase(unittest.TestCase):
         d = compose.mkdtemp(prefix="tweak_buildinstall")
         self.assertTrue(os.path.isdir(d))
 
+    def test_get_compose_info(self):
+        conf = ConfigWrapper(
+            release_name="Test",
+            release_version="1.0",
+            release_short="test",
+            release_type="ga",
+            release_internal=False,
+        )
+
+        ci = Compose.get_compose_info(conf)
+        ci_json = json.loads(ci.dumps())
+        self.assertEqual(ci_json, self.ci_json)
+
+    def test_get_compose_info_cts(self):
+        conf = ConfigWrapper(
+            release_name="Test",
+            release_version="1.0",
+            release_short="test",
+            release_type="ga",
+            release_internal=False,
+            cts_url="https://cts.localhost.tld/",
+            cts_keytab="/tmp/some.keytab",
+        )
+
+        # The `mock.ANY` in ["header"]["version"] cannot be serialized,
+        # so for this test, we replace it with real version.
+        ci_copy = dict(self.ci_json)
+        ci_copy["header"]["version"] = "1.2"
+        mocked_response = mock.MagicMock()
+        mocked_response.text = json.dumps(self.ci_json)
+        mocked_requests = mock.MagicMock()
+        mocked_requests.post.return_value = mocked_response
+
+        mocked_requests_kerberos = mock.MagicMock()
+
+        # The `requests` and `requests_kerberos` modules are imported directly
+        # in the `get_compose_info` function. To patch them, we need to patch
+        # the `sys.modules` directly so the patched modules are returned by
+        # `import`.
+        with mock.patch.dict(
+            "sys.modules",
+            requests=mocked_requests,
+            requests_kerberos=mocked_requests_kerberos,
+        ):
+            ci = Compose.get_compose_info(conf)
+            ci_json = json.loads(ci.dumps())
+            self.assertEqual(ci_json, self.ci_json)
+
+            mocked_response.raise_for_status.assert_called_once()
+            mocked_requests.post.assert_called_once_with(
+                "https://cts.localhost.tld/api/1/composes/",
+                auth=mock.ANY,
+                json={"compose_info": self.ci_json},
+            )
+
 
 class StatusTest(unittest.TestCase):
     def setUp(self):