pkgset: Query Koji instead of PDC

PDC is deprecated in upstream. The usecase for getting list of modules
by NS, NSV or NSVC can however be satisfied by querying modules imported
into Koji.

This makes it possible to deprecate PDC configuration.

Merges: https://pagure.io/pungi/pull-request/985
Signed-off-by: Martin Curlej <mcurlej@redhat.com>

doc/configuration.rst

@@ -1578,26 +1578,6 @@ Media Checksums Settings
     ``%(release_short)s-%(variant)s-%(version)s-%(date)s%(type_suffix)s.%(respin)s``.
 
 
-.. _pdc-settings:
-
-PDC Settings
-============
-
-Modular compose needs a PDC instance to talk to so that it can query list of
-module contents.
-
-**pdc_url**
-    (*str*) -- URL to the PDC API
-
-**pdc_develop** = ``False``
-    (*bool*) -- Turning this option on makes the client skip any authentication
-    assuming the server is open to anyone. This is useful for debugging against
-    a local instance, but you most likely do not want this in production.
-
-**pdc_insecure** = ``False``
-    (*bool*) -- Enable this to skip SSL certificate verification. This is a bad
-    idea in production.
-
 Translate Paths Settings
 ========================
 

pungi/checks.py

@@ -741,9 +741,9 @@ def make_schema():
             "global_target": {"type": "string"},
             "global_release": {"$ref": "#/definitions/optional_string"},
 
-            "pdc_url": {"type": "string"},
+            "pdc_url": {"deprecated": "Koji is queried instead"},
-            "pdc_develop": {"type": "boolean", "default": False},
+            "pdc_develop": {"deprecated": "Koji is queried instead"},
-            "pdc_insecure": {"type": "boolean", "default": False},
+            "pdc_insecure": {"deprecated": "Koji is queried instead"},
 
             "koji_profile": {"type": "string"},
 

pungi/phases/pkgset/sources/source_koji.py

@@ -39,27 +39,6 @@ from pungi.phases.gather import get_packages_to_gather
 
 import pungi.phases.pkgset.source
 
-try:
-    from pdc_client import PDCClient
-    WITH_PDC = True
-except:
-    WITH_PDC = False
-
-
-def get_pdc_client_session(compose):
-    if not WITH_PDC:
-        compose.log_warning("pdc_client module is not installed, "
-                            "support for modules is disabled")
-        return None
-    try:
-        return PDCClient(
-            server=compose.conf['pdc_url'],
-            develop=compose.conf['pdc_develop'],
-            ssl_verify=not compose.conf['pdc_insecure'],
-        )
-    except KeyError:
-        return None
-
 
 def variant_dict_from_str(compose, module_str):
     """
@@ -124,43 +103,73 @@ def variant_dict_from_str(compose, module_str):
 
 
 @retry(wait_on=IOError)
-def get_pdc_modules(compose, session, module_info_str):
+def get_koji_modules(compose, koji_wrapper, module_info_str):
     """
-    :param session : PDCClient instance
+    :param koji_wrapper : koji wrapper instance
-    :param module_info_str: pdc variant_dict, str, mmd or module dict
+    :param module_info_str: str, mmd or module dict
 
     :return final list of module_info which pass repoclosure
     """
+    koji_proxy = koji_wrapper.koji_proxy
 
     module_info = variant_dict_from_str(compose, module_info_str)
 
-    query = dict(
+    # we need to format the query string to koji reguirements
-        name=module_info['name'],
+    query_str = "%s-%s-%s.%s" % (module_info["name"], module_info["stream"],
-        stream=module_info['stream'],
+                                 module_info.get("version", "*"), module_info.get("context", "*"))
-        active=True,
+    query_str = query_str.replace('*.*', '*')
-    )
-    if module_info.get('version'):
-        query['version'] = module_info['version']
-    if module_info.get('context'):
-        query['context'] = module_info['context']
-
-    retval = session['modules'](page_size=-1, **query)
 
+    koji_builds = koji_proxy.search(query_str, "build", "glob")
     # Error reporting
-    if not retval:
+    if not koji_builds:
-        raise ValueError("Failed to find module in PDC %r" % query)
+        raise ValueError("Failed to find modules in koji %s" % query_str)
 
     modules = []
+    for build in koji_builds:
+        md = koji_proxy.getBuild(build["id"])
 
-    # If there is version provided, then all modules with that version will go in.
+        if not md["extra"]:
-    # In case version is missing, we will find the latest version and include all modules with that version.
+            continue
-    if 'version' in query:
+
-        modules = retval  # all found modules
+        try:
-    else:
+            version, context = md["release"].split(".")
+        except ValueError:
+            version = md["release"]
+            context = "00000000"
+        md["stream"] = md["version"]
+        md["version"] = version
+        md["context"] = context
+
+        try:
+            md["modulemd"] = md["extra"]["typeinfo"]["module"]["modulemd_str"]
+            md["tag"] = md["extra"]["typeinfo"]["module"]["content_koji_tag"]
+        except KeyError:
+            continue
+
+        archives = koji_proxy.listArchives(md["id"])
+        if not archives:
+            continue
+
+        archive = [a for a in archives
+                   if a["btype"] == "module" and a["filename"] == "modulemd.txt"]
+
+        if not archive:
+            continue
+
+        image_id = archive[0]["id"]
+        rpms = koji_proxy.listRPMs(imageID=image_id)
+        md["rpms"] = [make_nvra(rpm, add_epoch=True, force_epoch=True, add_rpm=False)
+                      for rpm in rpms]
+        modules.append(md)
+
+    # If there is version provided, then all modules with that version will go
+    # in. In case version is missing, we will find the latest version and
+    # include all modules with that version.
+    if not module_info.get('version'):
         # select all found modules with latest version
-        sorted_retval = sorted(retval, key=lambda item: int(item['version']), reverse=True)
+        sorted_modules = sorted(modules, key=lambda item: int(item['version']), reverse=True)
-        latest_version = int(sorted_retval[0]['version'])
+        latest_version = int(sorted_modules[0]['version'])
-        modules = [module for module in sorted_retval if latest_version == int(module['version'])]
+        modules = [module for module in modules if latest_version == int(module['version'])]
 
     return modules
 
@@ -247,42 +256,41 @@ def _log_modulemd(compose, variant, mmd):
                                         % (variant.uid, mmd.dup_nsvc())))
 
 
-def _get_modules_from_pdc(compose, session, variant, variant_tags):
+def _get_modules_from_koji(compose, koji_wrapper, variant, variant_tags):
     """
-    Loads modules for given `variant` from PDC `session`, adds them to
+    Loads modules for given `variant` from koji `session`, adds them to
     the `variant` and also to `variant_tags` dict.
 
     :param Compose compose: Compose for which the modules are found.
-    :param PDCClient session: PDC session.
+    :param koji_wrapper: We will obtain koji session from the wrapper.
     :param Variant variant: Variant with modules to find.
     :param dict variant_tags: Dict populated by this method. Key is `variant`
         and value is list of Koji tags to get the RPMs from.
     """
-    if not session:
-        return
 
     # Find out all modules in every variant and add their Koji tags
     # to variant and variant_tags list.
     for module in variant.get_modules():
-        pdc_modules = get_pdc_modules(compose, session, module["name"])
+        koji_modules = get_koji_modules(compose, koji_wrapper, module["name"])
-        for pdc_module in pdc_modules:
+        for koji_module in koji_modules:
-
+            mmd = Modulemd.Module.new_from_string(koji_module["modulemd"])
-            mmd = Modulemd.Module.new_from_string(pdc_module["modulemd"])
             mmd.upgrade()
-            _add_module_to_variant(variant, mmd, pdc_module["rpms"])
+            _add_module_to_variant(variant, mmd, koji_module["rpms"])
             _log_modulemd(compose, variant, mmd)
 
-            tag = pdc_module["koji_tag"]
+            tag = koji_module["tag"]
-            uid = ':'.join([pdc_module['name'], pdc_module['stream'],
+            uid = ':'.join([koji_module['name'], koji_module['stream'],
-                            pdc_module['version'], pdc_module['context']])
+                            koji_module['version'], koji_module['context']])
             variant_tags[variant].append(tag)
 
             # Store mapping module-uid --> koji_tag into variant.
             # This is needed in createrepo phase where metadata is exposed by producmd
             variant.module_uid_to_koji_tag[uid] = tag
 
-            module_msg = "Module '{uid}' in variant '{variant}' will use Koji tag '{tag}' (as a result of querying module '{module}')".format(
+            module_msg = (
-                uid=uid, variant=variant, tag=tag, module=module["name"])
+                "Module '{uid}' in variant '{variant}' will use Koji tag '{tag}' "
+                "(as a result of querying module '{module}')"
+            ).format(uid=uid, variant=variant, tag=tag, module=module["name"])
             compose.log_info("%s" % module_msg)
 
 
@@ -441,7 +449,6 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
     # there are some packages with invalid sigkeys, it raises an exception.
     allow_invalid_sigkeys = compose.conf["gather_method"] == "deps"
 
-    session = get_pdc_client_session(compose)
     for variant in compose.all_variants.values():
         # pkgset storing the packages belonging to this particular variant.
         variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
@@ -467,8 +474,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
         elif variant.modules:
             included_modules_file = os.path.join(
                 compose.paths.work.topdir(arch="global"),
-                "pdc-module-%s.yaml" % variant.uid)
+                "koji-module-%s.yaml" % variant.uid)
-            _get_modules_from_pdc(compose, session, variant, variant_tags)
+            _get_modules_from_koji(compose, koji_wrapper, variant, variant_tags)
 
         # Ensure that every tag added to `variant_tags` is added also to
         # `compose_tags`.

tests/test_pkgset_source_koji.py

@@ -97,7 +97,7 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
         self.compose.DEBUG = False
         self.koji_wrapper = mock.Mock()
         self.pkgset_path = os.path.join(self.topdir, 'work', 'global', 'pkgset_global.pickle')
-        self.pdc_module_path = os.path.join(self.topdir, 'work', 'global', 'pdc-module-Server.yaml')
+        self.koji_module_path = os.path.join(self.topdir, 'work', 'global', 'koji-module-Server.yaml')
 
     @mock.patch('six.moves.cPickle.dumps')
     @mock.patch('pungi.phases.pkgset.pkgsets.KojiPackageSet')
@@ -126,9 +126,8 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
     @unittest.skipUnless(Modulemd is not None, 'Modulemd not available')   # noqa
     @mock.patch('six.moves.cPickle.dumps')
     @mock.patch('pungi.phases.pkgset.pkgsets.KojiPackageSet')
-    @mock.patch('pungi.phases.pkgset.sources.source_koji.get_pdc_modules')
+    @mock.patch('pungi.phases.pkgset.sources.source_koji.get_koji_modules')
-    @mock.patch('pungi.phases.pkgset.sources.source_koji.get_pdc_client_session')
+    def test_pdc_log(self, get_koji_modules, KojiPackageSet, pickle_dumps):
-    def test_pdc_log(self, get_pdc_client_session, get_pdc_modules, KojiPackageSet, pickle_dumps):
 
         pickle_dumps.return_value = b'DATA'
 
@@ -160,12 +159,12 @@ data:
             - MIT
 """
 
-        get_pdc_modules.return_value = [
+        get_koji_modules.return_value = [
             {
                 'abc': 'def',
                 'modulemd': modulemd1,
                 'rpms': [],
-                'koji_tag': 'taggg',
+                'tag': 'taggg',
                 'uid': 'modulenamefoo:rhel:1:00000000',
                 'name': 'modulenamefoo',
                 'stream': 'rhel',
@@ -176,7 +175,7 @@ data:
                 'abc': 'def',
                 'modulemd': modulemd2,
                 'rpms': [],
-                'koji_tag': 'taggg',
+                'tag': 'taggg',
                 'uid': 'modulenamefoo:rhel:4:00000000',
                 'name': 'modulenamefoo',
                 'stream': 'rhel',
@@ -192,7 +191,7 @@ data:
 
         source_koji.populate_global_pkgset(
             self.compose, self.koji_wrapper, '/prefix', 123456)
-        mmds = Modulemd.Module.new_all_from_file(self.pdc_module_path)
+        mmds = Modulemd.Module.new_all_from_file(self.koji_module_path)
         self.assertEqual(mmds[0].get_name(), "foo")
 
     @mock.patch('six.moves.cPickle.dumps')
@@ -311,6 +310,196 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase):
 
         self.assertEqual(pkgsets, expected)
 
+    def test_get_koji_modules(self):
+        mock_build_ids = [{'id': 1065873, 'name': 'testmodule2-master-20180406051653.96c371af'}]
+        mock_extra = {
+            'typeinfo': {
+                'module': {
+                    'content_koji_tag': 'module-b62270b82443edde',
+                    'modulemd_str': mock.Mock()}
+            }
+        }
+        mock_build_md = [
+            {
+                'id': 1065873,
+                'epoch': None,
+                'extra': mock_extra,
+                'name': 'testmodule2',
+                'nvr': 'testmodule2-master-20180406051653.2e6f5e0a',
+                'release': '20180406051653.2e6f5e0a',
+                'state': 1,
+                'version': 'master',
+            }
+        ]
+        mock_archives = [
+            {
+                "id": 108941,
+                "btype": "module",
+                "filename": "modulemd.txt"
+            }
+        ]
+
+        mock_rpms = [
+            {'arch': 'src',
+             'epoch': None,
+             'id': 13640896,
+             'name': 'perl-List-Compare',
+             'nvr': 'perl-List-Compare-0.53-9.module_1612+b62270b8',
+             'release': '9.module_1612+b62270b8',
+             'version': '0.53'},
+            {'arch': 'noarch',
+             'epoch': None,
+             'id': 13640897,
+             'name': 'perl-List-Compare',
+             'nvr': 'perl-List-Compare-0.53-9.module_1612+b62270b8',
+             'release': '9.module_1612+b62270b8',
+             'version': '0.53'}
+
+        ]
+
+        self.koji_wrapper.koji_proxy.search.return_value = mock_build_ids
+        self.koji_wrapper.koji_proxy.getBuild.return_value = mock_build_md[0]
+        self.koji_wrapper.koji_proxy.listArchives.return_value = mock_archives
+        self.koji_wrapper.koji_proxy.listRPMs.return_value = mock_rpms
+
+        module_info_str = "testmodule2:master:20180406051653:96c371af"
+        result = source_koji.get_koji_modules(self.compose, self.koji_wrapper, module_info_str)
+
+        assert type(result) is list
+        assert len(result) == 1
+        module = result[0]
+        assert type(module) is dict
+        assert "rpms" in module
+        assert len(module["rpms"]) == 2
+        assert "modulemd" in module
+        assert "stream" in module
+        assert "context" in module
+
+        expected_query = "testmodule2-master-20180406051653.96c371af"
+        self.koji_wrapper.koji_proxy.search.assert_called_once_with(expected_query, "build",
+                                                                    "glob")
+        self.koji_wrapper.koji_proxy.getBuild.assert_called_once_with(mock_build_ids[0]["id"])
+        self.koji_wrapper.koji_proxy.listArchives.assert_called_once_with(mock_build_ids[0]["id"])
+        self.koji_wrapper.koji_proxy.listRPMs.assert_called_once_with(
+            imageID=mock_archives[0]["id"])
+
+    def test_get_koji_modules_no_version(self):
+        mock_build_ids = [
+            {'id': 1065873, 'name': 'testmodule2-master-20180406051653.2e6f5e0a'},
+            {'id': 1065874, 'name': 'testmodule2-master-20180406051653.96c371af'}
+        ]
+        mock_extra = [
+            {
+                'typeinfo': {
+                    'module': {
+                        'content_koji_tag': 'module-b62270b82443edde',
+                        'modulemd_str': mock.Mock()}
+                }
+            },
+            {
+                'typeinfo': {
+                    'module': {
+                        'content_koji_tag': 'module-52e40b9cdd3c0f7d',
+                        'modulemd_str': mock.Mock()}
+                }
+            }
+        ]
+        mock_build_md = [
+            {
+                'id': 1065873,
+                'epoch': None,
+                'extra': mock_extra[0],
+                'name': 'testmodule2',
+                'nvr': 'testmodule2-master-20180406051653.2e6f5e0a',
+                'release': '20180406051653.2e6f5e0a',
+                'state': 1,
+                'version': 'master',
+            },
+            {
+                'id': 1065874,
+                'epoch': None,
+                'extra': mock_extra[1],
+                'name': 'testmodule2',
+                'nvr': 'testmodule2-master-20180406051653.96c371af',
+                'release': '20180406051653.96c371af',
+                'state': 1,
+                'version': 'master',
+            }
+        ]
+        mock_archives = [
+            [{
+                "id": 108941,
+                "btype": "module",
+                "filename": "modulemd.txt"
+            }],
+            [{
+                "id": 108942,
+                "btype": "module",
+                "filename": "modulemd.txt"
+            }],
+        ]
+
+        mock_rpms = [
+            [{'arch': 'src',
+              'epoch': None,
+              'id': 13640896,
+              'name': 'perl-List-Compare',
+              'nvr': 'perl-List-Compare-0.53-9.module_1612+b62270b8',
+              'release': '9.module_1612+b62270b8',
+              'version': '0.53'},
+             {'arch': 'noarch',
+              'epoch': None,
+              'id': 13640897,
+              'name': 'perl-List-Compare',
+              'nvr': 'perl-List-Compare-0.53-9.module_1612+b62270b8',
+              'release': '9.module_1612+b62270b8',
+              'version': '0.53'}],
+            [{'arch': 'src',
+              'epoch': None,
+              'id': 13640900,
+              'name': 'perl-List-Compare',
+              'nvr': 'perl-List-Compare-0.53-9.module_1612+52e40b9c',
+              'release': '9.module_1612+52e40b9c',
+              'version': '0.53'},
+             {'arch': 'noarch',
+              'epoch': None,
+              'id': 13640901,
+              'name': 'perl-List-Compare',
+              'nvr': 'perl-List-Compare-0.53-9.module_1612+52e40b9c',
+              'release': '9.module_1612+52e40b9c',
+              'version': '0.53'}],
+        ]
+
+        self.koji_wrapper.koji_proxy.search.return_value = mock_build_ids
+        self.koji_wrapper.koji_proxy.getBuild.side_effect = mock_build_md
+        self.koji_wrapper.koji_proxy.listArchives.side_effect = mock_archives
+        self.koji_wrapper.koji_proxy.listRPMs.side_effect = mock_rpms
+
+        module_info_str = "testmodule2:master"
+        result = source_koji.get_koji_modules(self.compose, self.koji_wrapper, module_info_str)
+
+        assert type(result) is list
+        assert len(result) == 2
+        module = result[0]
+        for module in result:
+            assert type(module) is dict
+            assert "rpms" in module
+            assert len(module["rpms"]) == 2
+            assert "modulemd" in module
+            assert "stream" in module
+            assert "context" in module
+
+        expected_query = "testmodule2-master-*"
+        self.koji_wrapper.koji_proxy.search.assert_called_once_with(expected_query, "build",
+                                                                    "glob")
+
+        expected_calls = [mock.call(mock_build_ids[0]["id"]), mock.call(mock_build_ids[1]["id"])]
+        self.koji_wrapper.koji_proxy.getBuild.mock_calls == expected_calls
+        self.koji_wrapper.koji_proxy.listArchives.mock_calls == expected_calls
+        expected_rpm_calls = [mock.call(imageID=mock_archives[0][0]["id"]),
+                              mock.call(imageID=mock_archives[1][0]["id"])]
+        self.koji_wrapper.koji_proxy.listRPMs.mock_calls = expected_rpm_calls
+
 
 class TestSourceKoji(helpers.PungiTestCase):