diff --git a/doc/configuration.rst b/doc/configuration.rst index f4f9a4da..bd26d8bf 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1578,26 +1578,6 @@ Media Checksums Settings ``%(release_short)s-%(variant)s-%(version)s-%(date)s%(type_suffix)s.%(respin)s``. -.. _pdc-settings: - -PDC Settings -============ - -Modular compose needs a PDC instance to talk to so that it can query list of -module contents. - -**pdc_url** - (*str*) -- URL to the PDC API - -**pdc_develop** = ``False`` - (*bool*) -- Turning this option on makes the client skip any authentication - assuming the server is open to anyone. This is useful for debugging against - a local instance, but you most likely do not want this in production. - -**pdc_insecure** = ``False`` - (*bool*) -- Enable this to skip SSL certificate verification. This is a bad - idea in production. - Translate Paths Settings ======================== diff --git a/pungi/checks.py b/pungi/checks.py index a81340a5..c6ac840a 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -741,9 +741,9 @@ def make_schema(): "global_target": {"type": "string"}, "global_release": {"$ref": "#/definitions/optional_string"}, - "pdc_url": {"type": "string"}, - "pdc_develop": {"type": "boolean", "default": False}, - "pdc_insecure": {"type": "boolean", "default": False}, + "pdc_url": {"deprecated": "Koji is queried instead"}, + "pdc_develop": {"deprecated": "Koji is queried instead"}, + "pdc_insecure": {"deprecated": "Koji is queried instead"}, "koji_profile": {"type": "string"}, diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 2dd105e2..32502612 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -39,27 +39,6 @@ from pungi.phases.gather import get_packages_to_gather import pungi.phases.pkgset.source -try: - from pdc_client import PDCClient - WITH_PDC = True -except: - WITH_PDC = False - - -def get_pdc_client_session(compose): - if not WITH_PDC: - compose.log_warning("pdc_client module is not installed, " - "support for modules is disabled") - return None - try: - return PDCClient( - server=compose.conf['pdc_url'], - develop=compose.conf['pdc_develop'], - ssl_verify=not compose.conf['pdc_insecure'], - ) - except KeyError: - return None - def variant_dict_from_str(compose, module_str): """ @@ -124,43 +103,73 @@ def variant_dict_from_str(compose, module_str): @retry(wait_on=IOError) -def get_pdc_modules(compose, session, module_info_str): +def get_koji_modules(compose, koji_wrapper, module_info_str): """ - :param session : PDCClient instance - :param module_info_str: pdc variant_dict, str, mmd or module dict + :param koji_wrapper : koji wrapper instance + :param module_info_str: str, mmd or module dict :return final list of module_info which pass repoclosure """ + koji_proxy = koji_wrapper.koji_proxy module_info = variant_dict_from_str(compose, module_info_str) - query = dict( - name=module_info['name'], - stream=module_info['stream'], - active=True, - ) - if module_info.get('version'): - query['version'] = module_info['version'] - if module_info.get('context'): - query['context'] = module_info['context'] - - retval = session['modules'](page_size=-1, **query) + # we need to format the query string to koji reguirements + query_str = "%s-%s-%s.%s" % (module_info["name"], module_info["stream"], + module_info.get("version", "*"), module_info.get("context", "*")) + query_str = query_str.replace('*.*', '*') + koji_builds = koji_proxy.search(query_str, "build", "glob") # Error reporting - if not retval: - raise ValueError("Failed to find module in PDC %r" % query) + if not koji_builds: + raise ValueError("Failed to find modules in koji %s" % query_str) modules = [] + for build in koji_builds: + md = koji_proxy.getBuild(build["id"]) - # If there is version provided, then all modules with that version will go in. - # In case version is missing, we will find the latest version and include all modules with that version. - if 'version' in query: - modules = retval # all found modules - else: + if not md["extra"]: + continue + + try: + version, context = md["release"].split(".") + except ValueError: + version = md["release"] + context = "00000000" + md["stream"] = md["version"] + md["version"] = version + md["context"] = context + + try: + md["modulemd"] = md["extra"]["typeinfo"]["module"]["modulemd_str"] + md["tag"] = md["extra"]["typeinfo"]["module"]["content_koji_tag"] + except KeyError: + continue + + archives = koji_proxy.listArchives(md["id"]) + if not archives: + continue + + archive = [a for a in archives + if a["btype"] == "module" and a["filename"] == "modulemd.txt"] + + if not archive: + continue + + image_id = archive[0]["id"] + rpms = koji_proxy.listRPMs(imageID=image_id) + md["rpms"] = [make_nvra(rpm, add_epoch=True, force_epoch=True, add_rpm=False) + for rpm in rpms] + modules.append(md) + + # If there is version provided, then all modules with that version will go + # in. In case version is missing, we will find the latest version and + # include all modules with that version. + if not module_info.get('version'): # select all found modules with latest version - sorted_retval = sorted(retval, key=lambda item: int(item['version']), reverse=True) - latest_version = int(sorted_retval[0]['version']) - modules = [module for module in sorted_retval if latest_version == int(module['version'])] + sorted_modules = sorted(modules, key=lambda item: int(item['version']), reverse=True) + latest_version = int(sorted_modules[0]['version']) + modules = [module for module in modules if latest_version == int(module['version'])] return modules @@ -247,42 +256,41 @@ def _log_modulemd(compose, variant, mmd): % (variant.uid, mmd.dup_nsvc()))) -def _get_modules_from_pdc(compose, session, variant, variant_tags): +def _get_modules_from_koji(compose, koji_wrapper, variant, variant_tags): """ - Loads modules for given `variant` from PDC `session`, adds them to + Loads modules for given `variant` from koji `session`, adds them to the `variant` and also to `variant_tags` dict. :param Compose compose: Compose for which the modules are found. - :param PDCClient session: PDC session. + :param koji_wrapper: We will obtain koji session from the wrapper. :param Variant variant: Variant with modules to find. :param dict variant_tags: Dict populated by this method. Key is `variant` and value is list of Koji tags to get the RPMs from. """ - if not session: - return # Find out all modules in every variant and add their Koji tags # to variant and variant_tags list. for module in variant.get_modules(): - pdc_modules = get_pdc_modules(compose, session, module["name"]) - for pdc_module in pdc_modules: - - mmd = Modulemd.Module.new_from_string(pdc_module["modulemd"]) + koji_modules = get_koji_modules(compose, koji_wrapper, module["name"]) + for koji_module in koji_modules: + mmd = Modulemd.Module.new_from_string(koji_module["modulemd"]) mmd.upgrade() - _add_module_to_variant(variant, mmd, pdc_module["rpms"]) + _add_module_to_variant(variant, mmd, koji_module["rpms"]) _log_modulemd(compose, variant, mmd) - tag = pdc_module["koji_tag"] - uid = ':'.join([pdc_module['name'], pdc_module['stream'], - pdc_module['version'], pdc_module['context']]) + tag = koji_module["tag"] + uid = ':'.join([koji_module['name'], koji_module['stream'], + koji_module['version'], koji_module['context']]) variant_tags[variant].append(tag) # Store mapping module-uid --> koji_tag into variant. # This is needed in createrepo phase where metadata is exposed by producmd variant.module_uid_to_koji_tag[uid] = tag - module_msg = "Module '{uid}' in variant '{variant}' will use Koji tag '{tag}' (as a result of querying module '{module}')".format( - uid=uid, variant=variant, tag=tag, module=module["name"]) + module_msg = ( + "Module '{uid}' in variant '{variant}' will use Koji tag '{tag}' " + "(as a result of querying module '{module}')" + ).format(uid=uid, variant=variant, tag=tag, module=module["name"]) compose.log_info("%s" % module_msg) @@ -441,7 +449,6 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): # there are some packages with invalid sigkeys, it raises an exception. allow_invalid_sigkeys = compose.conf["gather_method"] == "deps" - session = get_pdc_client_session(compose) for variant in compose.all_variants.values(): # pkgset storing the packages belonging to this particular variant. variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( @@ -467,8 +474,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): elif variant.modules: included_modules_file = os.path.join( compose.paths.work.topdir(arch="global"), - "pdc-module-%s.yaml" % variant.uid) - _get_modules_from_pdc(compose, session, variant, variant_tags) + "koji-module-%s.yaml" % variant.uid) + _get_modules_from_koji(compose, koji_wrapper, variant, variant_tags) # Ensure that every tag added to `variant_tags` is added also to # `compose_tags`. diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index 63df2156..5ea4c427 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -97,7 +97,7 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase): self.compose.DEBUG = False self.koji_wrapper = mock.Mock() self.pkgset_path = os.path.join(self.topdir, 'work', 'global', 'pkgset_global.pickle') - self.pdc_module_path = os.path.join(self.topdir, 'work', 'global', 'pdc-module-Server.yaml') + self.koji_module_path = os.path.join(self.topdir, 'work', 'global', 'koji-module-Server.yaml') @mock.patch('six.moves.cPickle.dumps') @mock.patch('pungi.phases.pkgset.pkgsets.KojiPackageSet') @@ -126,9 +126,8 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase): @unittest.skipUnless(Modulemd is not None, 'Modulemd not available') # noqa @mock.patch('six.moves.cPickle.dumps') @mock.patch('pungi.phases.pkgset.pkgsets.KojiPackageSet') - @mock.patch('pungi.phases.pkgset.sources.source_koji.get_pdc_modules') - @mock.patch('pungi.phases.pkgset.sources.source_koji.get_pdc_client_session') - def test_pdc_log(self, get_pdc_client_session, get_pdc_modules, KojiPackageSet, pickle_dumps): + @mock.patch('pungi.phases.pkgset.sources.source_koji.get_koji_modules') + def test_pdc_log(self, get_koji_modules, KojiPackageSet, pickle_dumps): pickle_dumps.return_value = b'DATA' @@ -160,12 +159,12 @@ data: - MIT """ - get_pdc_modules.return_value = [ + get_koji_modules.return_value = [ { 'abc': 'def', 'modulemd': modulemd1, 'rpms': [], - 'koji_tag': 'taggg', + 'tag': 'taggg', 'uid': 'modulenamefoo:rhel:1:00000000', 'name': 'modulenamefoo', 'stream': 'rhel', @@ -176,7 +175,7 @@ data: 'abc': 'def', 'modulemd': modulemd2, 'rpms': [], - 'koji_tag': 'taggg', + 'tag': 'taggg', 'uid': 'modulenamefoo:rhel:4:00000000', 'name': 'modulenamefoo', 'stream': 'rhel', @@ -192,7 +191,7 @@ data: source_koji.populate_global_pkgset( self.compose, self.koji_wrapper, '/prefix', 123456) - mmds = Modulemd.Module.new_all_from_file(self.pdc_module_path) + mmds = Modulemd.Module.new_all_from_file(self.koji_module_path) self.assertEqual(mmds[0].get_name(), "foo") @mock.patch('six.moves.cPickle.dumps') @@ -311,6 +310,196 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase): self.assertEqual(pkgsets, expected) + def test_get_koji_modules(self): + mock_build_ids = [{'id': 1065873, 'name': 'testmodule2-master-20180406051653.96c371af'}] + mock_extra = { + 'typeinfo': { + 'module': { + 'content_koji_tag': 'module-b62270b82443edde', + 'modulemd_str': mock.Mock()} + } + } + mock_build_md = [ + { + 'id': 1065873, + 'epoch': None, + 'extra': mock_extra, + 'name': 'testmodule2', + 'nvr': 'testmodule2-master-20180406051653.2e6f5e0a', + 'release': '20180406051653.2e6f5e0a', + 'state': 1, + 'version': 'master', + } + ] + mock_archives = [ + { + "id": 108941, + "btype": "module", + "filename": "modulemd.txt" + } + ] + + mock_rpms = [ + {'arch': 'src', + 'epoch': None, + 'id': 13640896, + 'name': 'perl-List-Compare', + 'nvr': 'perl-List-Compare-0.53-9.module_1612+b62270b8', + 'release': '9.module_1612+b62270b8', + 'version': '0.53'}, + {'arch': 'noarch', + 'epoch': None, + 'id': 13640897, + 'name': 'perl-List-Compare', + 'nvr': 'perl-List-Compare-0.53-9.module_1612+b62270b8', + 'release': '9.module_1612+b62270b8', + 'version': '0.53'} + + ] + + self.koji_wrapper.koji_proxy.search.return_value = mock_build_ids + self.koji_wrapper.koji_proxy.getBuild.return_value = mock_build_md[0] + self.koji_wrapper.koji_proxy.listArchives.return_value = mock_archives + self.koji_wrapper.koji_proxy.listRPMs.return_value = mock_rpms + + module_info_str = "testmodule2:master:20180406051653:96c371af" + result = source_koji.get_koji_modules(self.compose, self.koji_wrapper, module_info_str) + + assert type(result) is list + assert len(result) == 1 + module = result[0] + assert type(module) is dict + assert "rpms" in module + assert len(module["rpms"]) == 2 + assert "modulemd" in module + assert "stream" in module + assert "context" in module + + expected_query = "testmodule2-master-20180406051653.96c371af" + self.koji_wrapper.koji_proxy.search.assert_called_once_with(expected_query, "build", + "glob") + self.koji_wrapper.koji_proxy.getBuild.assert_called_once_with(mock_build_ids[0]["id"]) + self.koji_wrapper.koji_proxy.listArchives.assert_called_once_with(mock_build_ids[0]["id"]) + self.koji_wrapper.koji_proxy.listRPMs.assert_called_once_with( + imageID=mock_archives[0]["id"]) + + def test_get_koji_modules_no_version(self): + mock_build_ids = [ + {'id': 1065873, 'name': 'testmodule2-master-20180406051653.2e6f5e0a'}, + {'id': 1065874, 'name': 'testmodule2-master-20180406051653.96c371af'} + ] + mock_extra = [ + { + 'typeinfo': { + 'module': { + 'content_koji_tag': 'module-b62270b82443edde', + 'modulemd_str': mock.Mock()} + } + }, + { + 'typeinfo': { + 'module': { + 'content_koji_tag': 'module-52e40b9cdd3c0f7d', + 'modulemd_str': mock.Mock()} + } + } + ] + mock_build_md = [ + { + 'id': 1065873, + 'epoch': None, + 'extra': mock_extra[0], + 'name': 'testmodule2', + 'nvr': 'testmodule2-master-20180406051653.2e6f5e0a', + 'release': '20180406051653.2e6f5e0a', + 'state': 1, + 'version': 'master', + }, + { + 'id': 1065874, + 'epoch': None, + 'extra': mock_extra[1], + 'name': 'testmodule2', + 'nvr': 'testmodule2-master-20180406051653.96c371af', + 'release': '20180406051653.96c371af', + 'state': 1, + 'version': 'master', + } + ] + mock_archives = [ + [{ + "id": 108941, + "btype": "module", + "filename": "modulemd.txt" + }], + [{ + "id": 108942, + "btype": "module", + "filename": "modulemd.txt" + }], + ] + + mock_rpms = [ + [{'arch': 'src', + 'epoch': None, + 'id': 13640896, + 'name': 'perl-List-Compare', + 'nvr': 'perl-List-Compare-0.53-9.module_1612+b62270b8', + 'release': '9.module_1612+b62270b8', + 'version': '0.53'}, + {'arch': 'noarch', + 'epoch': None, + 'id': 13640897, + 'name': 'perl-List-Compare', + 'nvr': 'perl-List-Compare-0.53-9.module_1612+b62270b8', + 'release': '9.module_1612+b62270b8', + 'version': '0.53'}], + [{'arch': 'src', + 'epoch': None, + 'id': 13640900, + 'name': 'perl-List-Compare', + 'nvr': 'perl-List-Compare-0.53-9.module_1612+52e40b9c', + 'release': '9.module_1612+52e40b9c', + 'version': '0.53'}, + {'arch': 'noarch', + 'epoch': None, + 'id': 13640901, + 'name': 'perl-List-Compare', + 'nvr': 'perl-List-Compare-0.53-9.module_1612+52e40b9c', + 'release': '9.module_1612+52e40b9c', + 'version': '0.53'}], + ] + + self.koji_wrapper.koji_proxy.search.return_value = mock_build_ids + self.koji_wrapper.koji_proxy.getBuild.side_effect = mock_build_md + self.koji_wrapper.koji_proxy.listArchives.side_effect = mock_archives + self.koji_wrapper.koji_proxy.listRPMs.side_effect = mock_rpms + + module_info_str = "testmodule2:master" + result = source_koji.get_koji_modules(self.compose, self.koji_wrapper, module_info_str) + + assert type(result) is list + assert len(result) == 2 + module = result[0] + for module in result: + assert type(module) is dict + assert "rpms" in module + assert len(module["rpms"]) == 2 + assert "modulemd" in module + assert "stream" in module + assert "context" in module + + expected_query = "testmodule2-master-*" + self.koji_wrapper.koji_proxy.search.assert_called_once_with(expected_query, "build", + "glob") + + expected_calls = [mock.call(mock_build_ids[0]["id"]), mock.call(mock_build_ids[1]["id"])] + self.koji_wrapper.koji_proxy.getBuild.mock_calls == expected_calls + self.koji_wrapper.koji_proxy.listArchives.mock_calls == expected_calls + expected_rpm_calls = [mock.call(imageID=mock_archives[0][0]["id"]), + mock.call(imageID=mock_archives[1][0]["id"])] + self.koji_wrapper.koji_proxy.listRPMs.mock_calls = expected_rpm_calls + class TestSourceKoji(helpers.PungiTestCase):