From 804a0049f6171bd47f8e0cedabcb3cd98b4dd54b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 3 Aug 2017 15:26:34 +0200 Subject: [PATCH] ostree: Simplify configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It makes no sense to repeat the same configuration for multiple architectures. Instead we should just list the architectures as another key in the mapping. There is an option to specify multiple config dicts. This preserves full backwards compatibility, the old config format is still accepted. Fixes: https://pagure.io/pungi/issue/678 Signed-off-by: Lubomír Sedlář --- doc/configuration.rst | 56 +++++++++++++++------------- pungi/checks.py | 76 ++++++++++++++++++++++++++++---------- pungi/phases/ostree.py | 20 +++++++--- tests/test_ostree_phase.py | 57 ++++++++++++++++++++++++++++ 4 files changed, 158 insertions(+), 51 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index c69e0ef6..fc74b067 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1158,15 +1158,16 @@ Example OSTree Settings =============== -The ``ostree`` phase of *Pungi* can create ostree repositories. This is done by -running ``rpm-ostree compose`` in a Koji runroot environment. The ostree -repository itself is not part of the compose and should be located in another -directory. Any new packages in the compose will be added to the repository with -a new commit. +The ``ostree`` phase of *Pungi* can create and update ostree repositories. This +is done by running ``rpm-ostree compose`` in a Koji runroot environment. The +ostree repository itself is not part of the compose and should be located in +another directory. Any new packages in the compose will be added to the +repository with a new commit. **ostree** - (*dict*) -- a variant/arch mapping of configuration. The format should be - ``[(variant_uid_regex, {arch|*: config_dict})]``. + (*dict*) -- a mapping of configuration for each. The format should be + ``{variant_uid_regex: config_dict}``. It is possible to use a list of + configuration dicts as well. The configuration dict for each variant arch pair must have these keys: @@ -1183,6 +1184,9 @@ a new commit. be removed from the tree config file. * ``config_branch`` -- (*str*) Git branch of the repo to use. Defaults to ``master``. + * ``arches`` -- (*[str]*) List of architectures for which to update ostree. + There will be one task per architecture. By default all architectures in + the variant are used. * ``failable`` -- (*[str]*) List of architectures for which this deliverable is not release blocking. * ``update_summary`` -- (*bool*) Update summary metadata after tree composing. @@ -1206,25 +1210,25 @@ Example config -------------- :: - ostree = [ - ("^Atomic$", { - "x86_64": { - "treefile": "fedora-atomic-docker-host.json", - "config_url": "https://git.fedorahosted.org/git/fedora-atomic.git", - "repo": [ - "Server", - "http://example.com/repo/x86_64/os", - {"baseurl": "Everything"}, - {"baseurl": "http://example.com/linux/repo", "exclude": "systemd-container"}, - ], - "keep_original_sources": True, - "ostree_repo": "/mnt/koji/compose/atomic/Rawhide/", - "update_summary": True, - # Automatically generate a reasonable version - "version": "!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN", - } - }) - ] + ostree = { + "^Atomic$": { + "treefile": "fedora-atomic-docker-host.json", + "config_url": "https://git.fedorahosted.org/git/fedora-atomic.git", + "repo": [ + "Server", + "http://example.com/repo/x86_64/os", + {"baseurl": "Everything"}, + {"baseurl": "http://example.com/linux/repo", "exclude": "systemd-container"}, + ], + "keep_original_sources": True, + "ostree_repo": "/mnt/koji/compose/atomic/Rawhide/", + "update_summary": True, + # Automatically generate a reasonable version + "version": "!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN", + # Only run this for x86_64 even if Atomic has more arches + "arches": ["x86_64"], + } + } Ostree Installer Settings diff --git a/pungi/checks.py b/pungi/checks.py index 71c45247..a129cb9b 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -862,27 +862,63 @@ def make_schema(): "additionalProperties": False, }, - "ostree": _variant_arch_mapping({ - "type": "object", - "properties": { - "treefile": {"type": "string"}, - "config_url": {"type": "string"}, - "repo": { - "$ref": "#/definitions/repos", - "alias": "extra_source_repos", - "append": ["repo_from", "source_repo_from"], + "ostree": { + "anyOf": [ + { + "type": "object", + "patternProperties": { + # Warning: this pattern is a variant uid regex, but the + # format does not let us validate it as there is no regular + # expression to describe all regular expressions. + ".+": _one_or_list({ + "type": "object", + "properties": { + "treefile": {"type": "string"}, + "config_url": {"type": "string"}, + "repo": { + "$ref": "#/definitions/repos", + "alias": "extra_source_repos", + "append": ["repo_from", "source_repo_from"], + }, + "keep_original_sources": {"type": "boolean"}, + "ostree_repo": {"type": "string"}, + "arches": {"$ref": "#/definitions/list_of_strings"}, + "failable": {"$ref": "#/definitions/list_of_strings"}, + "update_summary": {"type": "boolean"}, + "version": {"type": "string"}, + "config_branch": {"type": "string"}, + "tag_ref": {"type": "boolean"}, + }, + "required": ["treefile", "config_url", "repo", "ostree_repo"], + "additionalProperties": False, + }), + }, + "additionalProperties": False, }, - "keep_original_sources": {"type": "boolean"}, - "ostree_repo": {"type": "string"}, - "failable": {"$ref": "#/definitions/list_of_strings"}, - "update_summary": {"type": "boolean"}, - "version": {"type": "string"}, - "config_branch": {"type": "string"}, - "tag_ref": {"type": "boolean"}, - }, - "required": ["treefile", "config_url", "repo", "ostree_repo"], - "additionalProperties": False, - }), + # Deprecated in favour of the dict version above. + _variant_arch_mapping({ + "type": "object", + "properties": { + "treefile": {"type": "string"}, + "config_url": {"type": "string"}, + "repo": { + "$ref": "#/definitions/repos", + "alias": "extra_source_repos", + "append": ["repo_from", "source_repo_from"], + }, + "keep_original_sources": {"type": "boolean"}, + "ostree_repo": {"type": "string"}, + "failable": {"$ref": "#/definitions/list_of_strings"}, + "update_summary": {"type": "boolean"}, + "version": {"type": "string"}, + "config_branch": {"type": "string"}, + "tag_ref": {"type": "boolean"}, + }, + "required": ["treefile", "config_url", "repo", "ostree_repo"], + "additionalProperties": False, + }), + ] + }, "ostree_installer": _variant_arch_mapping({ "type": "object", diff --git a/pungi/phases/ostree.py b/pungi/phases/ostree.py index 1d763bfe..1a4783af 100644 --- a/pungi/phases/ostree.py +++ b/pungi/phases/ostree.py @@ -20,12 +20,22 @@ class OSTreePhase(ConfigGuardedPhase): super(OSTreePhase, self).__init__(compose) self.pool = ThreadPool(logger=self.compose._logger) + def _enqueue(self, variant, arch, conf): + self.pool.add(OSTreeThread(self.pool)) + self.pool.queue_put((self.compose, variant, arch, conf)) + def run(self): - for variant in self.compose.get_variants(): - for arch in variant.arches: - for conf in util.get_arch_variant_data(self.compose.conf, self.name, arch, variant): - self.pool.add(OSTreeThread(self.pool)) - self.pool.queue_put((self.compose, variant, arch, conf)) + if isinstance(self.compose.conf.get(self.name), dict): + for variant in self.compose.get_variants(): + for conf in util.get_variant_data(self.compose.conf, self.name, variant): + for arch in conf.get('arches', []) or variant.arches: + self._enqueue(variant, arch, conf) + else: + # Legacy code path to support original configuration. + for variant in self.compose.get_variants(): + for arch in variant.arches: + for conf in util.get_arch_variant_data(self.compose.conf, self.name, arch, variant): + self._enqueue(variant, arch, conf) self.pool.start() diff --git a/tests/test_ostree_phase.py b/tests/test_ostree_phase.py index 9e7e2025..3a9ebf32 100644 --- a/tests/test_ostree_phase.py +++ b/tests/test_ostree_phase.py @@ -43,6 +43,63 @@ class OSTreePhaseTest(helpers.PungiTestCase): phase = ostree.OSTreePhase(compose) self.assertTrue(phase.skip()) + @mock.patch('pungi.phases.ostree.ThreadPool') + def test_run_with_simple_config(self, ThreadPool): + cfg = helpers.IterableMock(get=lambda x, y: None) + compose = helpers.DummyCompose(self.topdir, { + 'ostree': { + '^Everything$': cfg + } + }) + + pool = ThreadPool.return_value + + phase = ostree.OSTreePhase(compose) + phase.run() + + self.assertEqual(len(pool.add.call_args_list), 2) + self.assertEqual(pool.queue_put.call_args_list, + [mock.call((compose, compose.variants['Everything'], 'x86_64', cfg)), + mock.call((compose, compose.variants['Everything'], 'amd64', cfg))]) + + @mock.patch('pungi.phases.ostree.ThreadPool') + def test_run_with_simple_config_limit_arches(self, ThreadPool): + cfg = helpers.IterableMock(get=lambda x, y: ['x86_64']) + compose = helpers.DummyCompose(self.topdir, { + 'ostree': { + '^Everything$': cfg + } + }) + + pool = ThreadPool.return_value + + phase = ostree.OSTreePhase(compose) + phase.run() + + self.assertEqual(len(pool.add.call_args_list), 1) + self.assertEqual(pool.queue_put.call_args_list, + [mock.call((compose, compose.variants['Everything'], 'x86_64', cfg))]) + + @mock.patch('pungi.phases.ostree.ThreadPool') + def test_run_with_simple_config_limit_arches_two_blocks(self, ThreadPool): + cfg1 = helpers.IterableMock(get=lambda x, y: ['x86_64']) + cfg2 = helpers.IterableMock(get=lambda x, y: ['s390x']) + compose = helpers.DummyCompose(self.topdir, { + 'ostree': { + '^Everything$': [cfg1, cfg2], + } + }) + + pool = ThreadPool.return_value + + phase = ostree.OSTreePhase(compose) + phase.run() + + self.assertEqual(len(pool.add.call_args_list), 2) + self.assertEqual(pool.queue_put.call_args_list, + [mock.call((compose, compose.variants['Everything'], 'x86_64', cfg1)), + mock.call((compose, compose.variants['Everything'], 's390x', cfg2))]) + class OSTreeThreadTest(helpers.PungiTestCase):