almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

pkgset: Query Koji instead of PDC

Browse Source 
PDC is deprecated in upstream. The usecase for getting list of modules
by NS, NSV or NSVC can however be satisfied by querying modules imported
into Koji.

This makes it possible to deprecate PDC configuration.

Merges: https://pagure.io/pungi/pull-request/985
Signed-off-by: Martin Curlej <mcurlej@redhat.com>
This commit is contained in:
Martin Curlej 2018-06-15 09:26:38 +02:00 committed by Lubomír Sedlář
parent 04f68a018f
commit d8c03f6239
4 changed files with 270 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
doc/configuration.rst
View File

@ -1578,26 +1578,6 @@ Media Checksums Settings
``%(release_short)s-%(variant)s-%(version)s-%(date)s%(type_suffix)s.%(respin)s``.
.. _pdc-settings:
PDC Settings
============
Modular compose needs a PDC instance to talk to so that it can query list of
module contents.
**pdc_url**
(*str*) -- URL to the PDC API
**pdc_develop** = ``False``
(*bool*) -- Turning this option on makes the client skip any authentication
assuming the server is open to anyone. This is useful for debugging against
a local instance, but you most likely do not want this in production.
**pdc_insecure** = ``False``
(*bool*) -- Enable this to skip SSL certificate verification. This is a bad
idea in production.
Translate Paths Settings
========================

6
pungi/checks.py
View File

@ -741,9 +741,9 @@ def make_schema():
"global_target": {"type": "string"},
"global_release": {"$ref": "#/definitions/optional_string"},
"pdc_url": {"type": "string"},
"pdc_develop": {"type": "boolean", "default": False},
"pdc_insecure": {"type": "boolean", "default": False},
"pdc_url": {"deprecated": "Koji is queried instead"},
"pdc_develop": {"deprecated": "Koji is queried instead"},
"pdc_insecure": {"deprecated": "Koji is queried instead"},
"koji_profile": {"type": "string"},

133
pungi/phases/pkgset/sources/source_koji.py
View File

@ -39,27 +39,6 @@ from pungi.phases.gather import get_packages_to_gather
import pungi.phases.pkgset.source
try:
from pdc_client import PDCClient
WITH_PDC = True
except:
WITH_PDC = False
def get_pdc_client_session(compose):
if not WITH_PDC:
compose.log_warning("pdc_client module is not installed, "
"support for modules is disabled")
return None
try:
return PDCClient(
server=compose.conf['pdc_url'],
develop=compose.conf['pdc_develop'],
ssl_verify=not compose.conf['pdc_insecure'],
)
except KeyError:
return None
def variant_dict_from_str(compose, module_str):
"""
@ -124,43 +103,73 @@ def variant_dict_from_str(compose, module_str):
@retry(wait_on=IOError)
def get_pdc_modules(compose, session, module_info_str):
def get_koji_modules(compose, koji_wrapper, module_info_str):
"""
:param session : PDCClient instance
:param module_info_str: pdc variant_dict, str, mmd or module dict
:param koji_wrapper : koji wrapper instance
:param module_info_str: str, mmd or module dict
:return final list of module_info which pass repoclosure
"""
koji_proxy = koji_wrapper.koji_proxy
module_info = variant_dict_from_str(compose, module_info_str)
query = dict(
name=module_info['name'],
stream=module_info['stream'],
active=True,
)
if module_info.get('version'):
query['version'] = module_info['version']
if module_info.get('context'):
query['context'] = module_info['context']
retval = session['modules'](page_size=-1, **query)
# we need to format the query string to koji reguirements
query_str = "%s-%s-%s.%s" % (module_info["name"], module_info["stream"],
module_info.get("version", "*"), module_info.get("context", "*"))
query_str = query_str.replace('*.*', '*')
koji_builds = koji_proxy.search(query_str, "build", "glob")
# Error reporting
if not retval:
raise ValueError("Failed to find module in PDC %r" % query)
if not koji_builds:
raise ValueError("Failed to find modules in koji %s" % query_str)
modules = []
for build in koji_builds:
md = koji_proxy.getBuild(build["id"])
# If there is version provided, then all modules with that version will go in.
# In case version is missing, we will find the latest version and include all modules with that version.
if 'version' in query:
modules = retval # all found modules
else:
if not md["extra"]:
continue
try:
version, context = md["release"].split(".")
except ValueError:
version = md["release"]
context = "00000000"
md["stream"] = md["version"]
md["version"] = version
md["context"] = context
try:
md["modulemd"] = md["extra"]["typeinfo"]["module"]["modulemd_str"]
md["tag"] = md["extra"]["typeinfo"]["module"]["content_koji_tag"]
except KeyError:
continue
archives = koji_proxy.listArchives(md["id"])
if not archives:
continue
archive = [a for a in archives
if a["btype"] == "module" and a["filename"] == "modulemd.txt"]
if not archive:
continue
image_id = archive[0]["id"]
rpms = koji_proxy.listRPMs(imageID=image_id)
md["rpms"] = [make_nvra(rpm, add_epoch=True, force_epoch=True, add_rpm=False)
for rpm in rpms]
modules.append(md)
# If there is version provided, then all modules with that version will go
# in. In case version is missing, we will find the latest version and
# include all modules with that version.
if not module_info.get('version'):
# select all found modules with latest version
sorted_retval = sorted(retval, key=lambda item: int(item['version']), reverse=True)
latest_version = int(sorted_retval[0]['version'])
modules = [module for module in sorted_retval if latest_version == int(module['version'])]
sorted_modules = sorted(modules, key=lambda item: int(item['version']), reverse=True)
latest_version = int(sorted_modules[0]['version'])
modules = [module for module in modules if latest_version == int(module['version'])]
return modules
@ -247,42 +256,41 @@ def _log_modulemd(compose, variant, mmd):
% (variant.uid, mmd.dup_nsvc())))
def _get_modules_from_pdc(compose, session, variant, variant_tags):
def _get_modules_from_koji(compose, koji_wrapper, variant, variant_tags):
"""
Loads modules for given `variant` from PDC `session`, adds them to
Loads modules for given `variant` from koji `session`, adds them to
the `variant` and also to `variant_tags` dict.
:param Compose compose: Compose for which the modules are found.
:param PDCClient session: PDC session.
:param koji_wrapper: We will obtain koji session from the wrapper.
:param Variant variant: Variant with modules to find.
:param dict variant_tags: Dict populated by this method. Key is `variant`
and value is list of Koji tags to get the RPMs from.
"""
if not session:
return
# Find out all modules in every variant and add their Koji tags
# to variant and variant_tags list.
for module in variant.get_modules():
pdc_modules = get_pdc_modules(compose, session, module["name"])
for pdc_module in pdc_modules:
mmd = Modulemd.Module.new_from_string(pdc_module["modulemd"])
koji_modules = get_koji_modules(compose, koji_wrapper, module["name"])
for koji_module in koji_modules:
mmd = Modulemd.Module.new_from_string(koji_module["modulemd"])
mmd.upgrade()
_add_module_to_variant(variant, mmd, pdc_module["rpms"])
_add_module_to_variant(variant, mmd, koji_module["rpms"])
_log_modulemd(compose, variant, mmd)
tag = pdc_module["koji_tag"]
uid = ':'.join([pdc_module['name'], pdc_module['stream'],
pdc_module['version'], pdc_module['context']])
tag = koji_module["tag"]
uid = ':'.join([koji_module['name'], koji_module['stream'],
koji_module['version'], koji_module['context']])
variant_tags[variant].append(tag)
# Store mapping module-uid --> koji_tag into variant.
# This is needed in createrepo phase where metadata is exposed by producmd
variant.module_uid_to_koji_tag[uid] = tag
module_msg = "Module '{uid}' in variant '{variant}' will use Koji tag '{tag}' (as a result of querying module '{module}')".format(
uid=uid, variant=variant, tag=tag, module=module["name"])
module_msg = (
"Module '{uid}' in variant '{variant}' will use Koji tag '{tag}' "
"(as a result of querying module '{module}')"
).format(uid=uid, variant=variant, tag=tag, module=module["name"])
compose.log_info("%s" % module_msg)
@ -441,7 +449,6 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
# there are some packages with invalid sigkeys, it raises an exception.
allow_invalid_sigkeys = compose.conf["gather_method"] == "deps"
session = get_pdc_client_session(compose)
for variant in compose.all_variants.values():
# pkgset storing the packages belonging to this particular variant.
variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
@ -467,8 +474,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
elif variant.modules:
included_modules_file = os.path.join(
compose.paths.work.topdir(arch="global"),
"pdc-module-%s.yaml" % variant.uid)
_get_modules_from_pdc(compose, session, variant, variant_tags)
"koji-module-%s.yaml" % variant.uid)
_get_modules_from_koji(compose, koji_wrapper, variant, variant_tags)
# Ensure that every tag added to `variant_tags` is added also to
# `compose_tags`.

205
tests/test_pkgset_source_koji.py
View File

@ -97,7 +97,7 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
self.compose.DEBUG = False
self.koji_wrapper = mock.Mock()
self.pkgset_path = os.path.join(self.topdir, 'work', 'global', 'pkgset_global.pickle')
self.pdc_module_path = os.path.join(self.topdir, 'work', 'global', 'pdc-module-Server.yaml')
self.koji_module_path = os.path.join(self.topdir, 'work', 'global', 'koji-module-Server.yaml')
@mock.patch('six.moves.cPickle.dumps')
@mock.patch('pungi.phases.pkgset.pkgsets.KojiPackageSet')
@ -126,9 +126,8 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
@unittest.skipUnless(Modulemd is not None, 'Modulemd not available') # noqa
@mock.patch('six.moves.cPickle.dumps')
@mock.patch('pungi.phases.pkgset.pkgsets.KojiPackageSet')
@mock.patch('pungi.phases.pkgset.sources.source_koji.get_pdc_modules')
@mock.patch('pungi.phases.pkgset.sources.source_koji.get_pdc_client_session')
def test_pdc_log(self, get_pdc_client_session, get_pdc_modules, KojiPackageSet, pickle_dumps):
@mock.patch('pungi.phases.pkgset.sources.source_koji.get_koji_modules')
def test_pdc_log(self, get_koji_modules, KojiPackageSet, pickle_dumps):
pickle_dumps.return_value = b'DATA'
@ -160,12 +159,12 @@ data:
- MIT
"""
get_pdc_modules.return_value = [
get_koji_modules.return_value = [
{
'abc': 'def',
'modulemd': modulemd1,
'rpms': [],
'koji_tag': 'taggg',
'tag': 'taggg',
'uid': 'modulenamefoo:rhel:1:00000000',
'name': 'modulenamefoo',
'stream': 'rhel',
@ -176,7 +175,7 @@ data:
'abc': 'def',
'modulemd': modulemd2,
'rpms': [],
'koji_tag': 'taggg',
'tag': 'taggg',
'uid': 'modulenamefoo:rhel:4:00000000',
'name': 'modulenamefoo',
'stream': 'rhel',
@ -192,7 +191,7 @@ data:
source_koji.populate_global_pkgset(
self.compose, self.koji_wrapper, '/prefix', 123456)
mmds = Modulemd.Module.new_all_from_file(self.pdc_module_path)
mmds = Modulemd.Module.new_all_from_file(self.koji_module_path)
self.assertEqual(mmds[0].get_name(), "foo")
@mock.patch('six.moves.cPickle.dumps')
@ -311,6 +310,196 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase):
self.assertEqual(pkgsets, expected)
def test_get_koji_modules(self):
mock_build_ids = [{'id': 1065873, 'name': 'testmodule2-master-20180406051653.96c371af'}]
mock_extra = {
'typeinfo': {
'module': {
'content_koji_tag': 'module-b62270b82443edde',
'modulemd_str': mock.Mock()}
}
}
mock_build_md = [
{
'id': 1065873,
'epoch': None,
'extra': mock_extra,
'name': 'testmodule2',
'nvr': 'testmodule2-master-20180406051653.2e6f5e0a',
'release': '20180406051653.2e6f5e0a',
'state': 1,
'version': 'master',
}
]
mock_archives = [
{
"id": 108941,
"btype": "module",
"filename": "modulemd.txt"
}
]
mock_rpms = [
{'arch': 'src',
'epoch': None,
'id': 13640896,
'name': 'perl-List-Compare',
'nvr': 'perl-List-Compare-0.53-9.module_1612+b62270b8',
'release': '9.module_1612+b62270b8',
'version': '0.53'},
{'arch': 'noarch',
'epoch': None,
'id': 13640897,
'name': 'perl-List-Compare',
'nvr': 'perl-List-Compare-0.53-9.module_1612+b62270b8',
'release': '9.module_1612+b62270b8',
'version': '0.53'}
]
self.koji_wrapper.koji_proxy.search.return_value = mock_build_ids
self.koji_wrapper.koji_proxy.getBuild.return_value = mock_build_md[0]
self.koji_wrapper.koji_proxy.listArchives.return_value = mock_archives
self.koji_wrapper.koji_proxy.listRPMs.return_value = mock_rpms
module_info_str = "testmodule2:master:20180406051653:96c371af"
result = source_koji.get_koji_modules(self.compose, self.koji_wrapper, module_info_str)
assert type(result) is list
assert len(result) == 1
module = result[0]
assert type(module) is dict
assert "rpms" in module
assert len(module["rpms"]) == 2
assert "modulemd" in module
assert "stream" in module
assert "context" in module
expected_query = "testmodule2-master-20180406051653.96c371af"
self.koji_wrapper.koji_proxy.search.assert_called_once_with(expected_query, "build",
"glob")
self.koji_wrapper.koji_proxy.getBuild.assert_called_once_with(mock_build_ids[0]["id"])
self.koji_wrapper.koji_proxy.listArchives.assert_called_once_with(mock_build_ids[0]["id"])
self.koji_wrapper.koji_proxy.listRPMs.assert_called_once_with(
imageID=mock_archives[0]["id"])
def test_get_koji_modules_no_version(self):
mock_build_ids = [
{'id': 1065873, 'name': 'testmodule2-master-20180406051653.2e6f5e0a'},
{'id': 1065874, 'name': 'testmodule2-master-20180406051653.96c371af'}
]
mock_extra = [
{
'typeinfo': {
'module': {
'content_koji_tag': 'module-b62270b82443edde',
'modulemd_str': mock.Mock()}
}
},
{
'typeinfo': {
'module': {
'content_koji_tag': 'module-52e40b9cdd3c0f7d',
'modulemd_str': mock.Mock()}
}
}
]
mock_build_md = [
{
'id': 1065873,
'epoch': None,
'extra': mock_extra[0],
'name': 'testmodule2',
'nvr': 'testmodule2-master-20180406051653.2e6f5e0a',
'release': '20180406051653.2e6f5e0a',
'state': 1,
'version': 'master',
},
{
'id': 1065874,
'epoch': None,
'extra': mock_extra[1],
'name': 'testmodule2',
'nvr': 'testmodule2-master-20180406051653.96c371af',
'release': '20180406051653.96c371af',
'state': 1,
'version': 'master',
}
]
mock_archives = [
[{
"id": 108941,
"btype": "module",
"filename": "modulemd.txt"
}],
[{
"id": 108942,
"btype": "module",
"filename": "modulemd.txt"
}],
]
mock_rpms = [
[{'arch': 'src',
'epoch': None,
'id': 13640896,
'name': 'perl-List-Compare',
'nvr': 'perl-List-Compare-0.53-9.module_1612+b62270b8',
'release': '9.module_1612+b62270b8',
'version': '0.53'},
{'arch': 'noarch',
'epoch': None,
'id': 13640897,
'name': 'perl-List-Compare',
'nvr': 'perl-List-Compare-0.53-9.module_1612+b62270b8',
'release': '9.module_1612+b62270b8',
'version': '0.53'}],
[{'arch': 'src',
'epoch': None,
'id': 13640900,
'name': 'perl-List-Compare',
'nvr': 'perl-List-Compare-0.53-9.module_1612+52e40b9c',
'release': '9.module_1612+52e40b9c',
'version': '0.53'},
{'arch': 'noarch',
'epoch': None,
'id': 13640901,
'name': 'perl-List-Compare',
'nvr': 'perl-List-Compare-0.53-9.module_1612+52e40b9c',
'release': '9.module_1612+52e40b9c',
'version': '0.53'}],
]
self.koji_wrapper.koji_proxy.search.return_value = mock_build_ids
self.koji_wrapper.koji_proxy.getBuild.side_effect = mock_build_md
self.koji_wrapper.koji_proxy.listArchives.side_effect = mock_archives
self.koji_wrapper.koji_proxy.listRPMs.side_effect = mock_rpms
module_info_str = "testmodule2:master"
result = source_koji.get_koji_modules(self.compose, self.koji_wrapper, module_info_str)
assert type(result) is list
assert len(result) == 2
module = result[0]
for module in result:
assert type(module) is dict
assert "rpms" in module
assert len(module["rpms"]) == 2
assert "modulemd" in module
assert "stream" in module
assert "context" in module
expected_query = "testmodule2-master-*"
self.koji_wrapper.koji_proxy.search.assert_called_once_with(expected_query, "build",
"glob")
expected_calls = [mock.call(mock_build_ids[0]["id"]), mock.call(mock_build_ids[1]["id"])]
self.koji_wrapper.koji_proxy.getBuild.mock_calls == expected_calls
self.koji_wrapper.koji_proxy.listArchives.mock_calls == expected_calls
expected_rpm_calls = [mock.call(imageID=mock_archives[0][0]["id"]),
mock.call(imageID=mock_archives[1][0]["id"])]
self.koji_wrapper.koji_proxy.listRPMs.mock_calls = expected_rpm_calls
class TestSourceKoji(helpers.PungiTestCase):