almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Allow getting the compose id from CTS (Compose Tracking Service).

Browse Source 
Signed-off-by: Jan Kaluza <jkaluza@redhat.com>
This commit is contained in:
Jan Kaluza 2020-05-22 07:40:02 +02:00 committed by lsedlar
parent 59e2aa9607
commit f1eea0b5a6
5 changed files with 201 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
doc/configuration.rst
View File

@ -176,6 +176,16 @@ Options
Please note that when ``dnf`` is used, the build dependencies check is
skipped. On Python 3, only ``dnf`` backend is available.
**cts_url**
(*str*) -- URL to Compose Tracking Service. If defined, Pungi will add
the compose to Compose Tracking Service and ge the compose ID from it.
For example ``https://cts.localhost.tld/``
**cts_keytab**
(*str*) -- Path to Kerberos keytab which will be used for Compose
Tracking Service Kerberos authentification. If not defined, the default
Kerberos principal is used.
**compose_type**
(*str*) -- Allows to set default compose type. Type set via a command-line
option overwrites this.

2
pungi/checks.py
View File

@ -765,6 +765,8 @@ def make_schema():
"pdc_url": {"deprecated": "Koji is queried instead"},
"pdc_develop": {"deprecated": "Koji is queried instead"},
"pdc_insecure": {"deprecated": "Koji is queried instead"},
"cts_url": {"type": "string"},
"cts_keytab": {"type": "string"},
"koji_profile": {"type": "string"},
"koji_event": {"type": "number"},
"pkgset_koji_tag": {"$ref": "#/definitions/strings"},

131
pungi/compose.py
View File

@ -51,18 +51,16 @@ except ImportError:
SUPPORTED_MILESTONES = ["RC", "Update", "SecurityFix"]
def get_compose_dir(
topdir,
def get_compose_info(
conf,
compose_type="production",
compose_date=None,
compose_respin=None,
compose_label=None,
already_exists_callbacks=None,
):
already_exists_callbacks = already_exists_callbacks or []
# create an incomplete composeinfo to generate compose ID
"""
Creates inncomplete ComposeInfo to generate Compose ID
"""
ci = ComposeInfo()
ci.release.name = conf["release_name"]
ci.release.short = conf["release_short"]
@ -81,37 +79,112 @@ def get_compose_dir(
ci.compose.date = compose_date or time.strftime("%Y%m%d", time.localtime())
ci.compose.respin = compose_respin or 0
while 1:
cts_url = conf.get("cts_url", None)
if cts_url:
# Import requests and requests-kerberos here so it is not needed
# if running without Compose Tracking Service.
import requests
from requests_kerberos import HTTPKerberosAuth
# Requests-kerberos cannot accept custom keytab, we need to use
# environment variable for this. But we need to change environment
# only temporarily just for this single requests.post.
# So at first backup the current environment and revert to it
# after the requests.post call.
cts_keytab = conf.get("cts_keytab", None)
if cts_keytab:
environ_copy = dict(os.environ)
os.environ["KRB5_CLIENT_KTNAME"] = cts_keytab
try:
# Create compose in CTS and get the reserved compose ID.
ci.compose.id = ci.create_compose_id()
url = os.path.join(cts_url, "api/1/composes/")
data = {"compose_info": json.loads(ci.dumps())}
rv = requests.post(url, json=data, auth=HTTPKerberosAuth())
rv.raise_for_status()
finally:
if cts_keytab:
os.environ.clear()
os.environ.update(environ_copy)
# Update local ComposeInfo with received ComposeInfo.
cts_ci = ComposeInfo()
cts_ci.loads(rv.text)
ci.compose.respin = cts_ci.compose.respin
ci.compose.id = cts_ci.compose.id
else:
ci.compose.id = ci.create_compose_id()
compose_dir = os.path.join(topdir, ci.compose.id)
return ci
exists = False
# TODO: callbacks to determine if a composeid was already used
# for callback in already_exists_callbacks:
# if callback(data):
# exists = True
# break
# already_exists_callbacks fallback: does target compose_dir exist?
try:
os.makedirs(compose_dir)
except OSError as ex:
if ex.errno == errno.EEXIST:
exists = True
else:
raise
if exists:
ci.compose.respin += 1
continue
break
def write_compose_info(compose_dir, ci):
"""
Write ComposeInfo `ci` to `compose_dir` subdirectories.
"""
makedirs(compose_dir)
with open(os.path.join(compose_dir, "COMPOSE_ID"), "w") as f:
f.write(ci.compose.id)
work_dir = os.path.join(compose_dir, "work", "global")
makedirs(work_dir)
ci.dump(os.path.join(work_dir, "composeinfo-base.json"))
def get_compose_dir(
topdir,
conf,
compose_type="production",
compose_date=None,
compose_respin=None,
compose_label=None,
already_exists_callbacks=None,
):
already_exists_callbacks = already_exists_callbacks or []
ci = get_compose_info(
conf, compose_type, compose_date, compose_respin, compose_label
)
cts_url = conf.get("cts_url", None)
if cts_url:
# Create compose directory.
compose_dir = os.path.join(topdir, ci.compose.id)
os.makedirs(compose_dir)
else:
while 1:
ci.compose.id = ci.create_compose_id()
compose_dir = os.path.join(topdir, ci.compose.id)
exists = False
# TODO: callbacks to determine if a composeid was already used
# for callback in already_exists_callbacks:
# if callback(data):
# exists = True
# break
# already_exists_callbacks fallback: does target compose_dir exist?
try:
os.makedirs(compose_dir)
except OSError as ex:
if ex.errno == errno.EEXIST:
exists = True
else:
raise
if exists:
ci = get_compose_info(
conf,
compose_type,
compose_date,
ci.compose.respin + 1,
compose_label,
)
continue
break
write_compose_info(compose_dir, ci)
return compose_dir
@ -221,6 +294,8 @@ class Compose(kobo.log.LoggingBase):
else:
self.cache_region = make_region().configure("dogpile.cache.null")
get_compose_info = staticmethod(get_compose_info)
write_compose_info = staticmethod(write_compose_info)
get_compose_dir = staticmethod(get_compose_dir)
def __getitem__(self, name):

15
pungi/scripts/pungi_koji.py
View File

@ -47,7 +47,8 @@ def main():
group.add_argument(
"--compose-dir",
metavar="PATH",
help="reuse an existing compose directory (DANGEROUS!)",
help="specify compose directory in which the compose will be generated."
"If directory already exists, Pungi will reuse it (DANGEROUS!).",
)
parser.add_argument(
"--label",
@ -199,11 +200,8 @@ def main():
)
else:
opts.compose_dir = os.path.abspath(opts.compose_dir)
if not os.path.isdir(opts.compose_dir):
abort(
"The compose directory does not exist or is not a directory: %s"
% opts.compose_dir
)
if os.path.exists(opts.compose_dir) and not os.path.isdir(opts.compose_dir):
abort("The compose directory is not a directory: %s" % opts.compose_dir)
opts.config = os.path.abspath(opts.config)
@ -275,6 +273,11 @@ def main():
)
else:
compose_dir = opts.compose_dir
if not os.path.exists(compose_dir):
ci = Compose.get_compose_info(
conf, compose_type=compose_type, compose_label=opts.label
)
Compose.write_compose_info(compose_dir, ci)
if opts.print_output_dir:
print("Compose dir: %s" % compose_dir)

77
tests/test_compose.py
View File

@ -11,6 +11,7 @@ import os
import six
import tempfile
import shutil
import json
from pungi.compose import Compose
@ -27,6 +28,27 @@ class ComposeTestCase(unittest.TestCase):
def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
# Basic ComposeInfo metadata used in tests.
self.ci_json = {
"header": {"type": "productmd.composeinfo", "version": mock.ANY},
"payload": {
"compose": {
"date": "20200526",
"id": "test-1.0-20200526.0",
"respin": 0,
"type": "production",
},
"release": {
"internal": False,
"name": "Test",
"short": "test",
"type": "ga",
"version": "1.0",
},
"variants": {},
},
}
def tearDown(self):
shutil.rmtree(self.tmp_dir)
@ -571,6 +593,61 @@ class ComposeTestCase(unittest.TestCase):
d = compose.mkdtemp(prefix="tweak_buildinstall")
self.assertTrue(os.path.isdir(d))
def test_get_compose_info(self):
conf = ConfigWrapper(
release_name="Test",
release_version="1.0",
release_short="test",
release_type="ga",
release_internal=False,
)
ci = Compose.get_compose_info(conf)
ci_json = json.loads(ci.dumps())
self.assertEqual(ci_json, self.ci_json)
def test_get_compose_info_cts(self):
conf = ConfigWrapper(
release_name="Test",
release_version="1.0",
release_short="test",
release_type="ga",
release_internal=False,
cts_url="https://cts.localhost.tld/",
cts_keytab="/tmp/some.keytab",
)
# The `mock.ANY` in ["header"]["version"] cannot be serialized,
# so for this test, we replace it with real version.
ci_copy = dict(self.ci_json)
ci_copy["header"]["version"] = "1.2"
mocked_response = mock.MagicMock()
mocked_response.text = json.dumps(self.ci_json)
mocked_requests = mock.MagicMock()
mocked_requests.post.return_value = mocked_response
mocked_requests_kerberos = mock.MagicMock()
# The `requests` and `requests_kerberos` modules are imported directly
# in the `get_compose_info` function. To patch them, we need to patch
# the `sys.modules` directly so the patched modules are returned by
# `import`.
with mock.patch.dict(
"sys.modules",
requests=mocked_requests,
requests_kerberos=mocked_requests_kerberos,
):
ci = Compose.get_compose_info(conf)
ci_json = json.loads(ci.dumps())
self.assertEqual(ci_json, self.ci_json)
mocked_response.raise_for_status.assert_called_once()
mocked_requests.post.assert_called_once_with(
"https://cts.localhost.tld/api/1/composes/",
auth=mock.ANY,
json={"compose_info": self.ci_json},
)
class StatusTest(unittest.TestCase):
def setUp(self):