ostree: Simplify configuration

It makes no sense to repeat the same configuration for multiple
architectures. Instead we should just list the architectures as another
key in the mapping. There is an option to specify multiple config dicts.

This preserves full backwards compatibility, the old config format is
still accepted.

Fixes: https://pagure.io/pungi/issue/678
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

doc/configuration.rst

@@ -1158,15 +1158,16 @@ Example
 OSTree Settings
 ===============
 
-The ``ostree`` phase of *Pungi* can create ostree repositories. This is done by
+The ``ostree`` phase of *Pungi* can create and update ostree repositories. This
-running ``rpm-ostree compose`` in a Koji runroot environment. The ostree
+is done by running ``rpm-ostree compose`` in a Koji runroot environment. The
-repository itself is not part of the compose and should be located in another
+ostree repository itself is not part of the compose and should be located in
-directory. Any new packages in the compose will be added to the repository with
+another directory. Any new packages in the compose will be added to the
-a new commit.
+repository with a new commit.
 
 **ostree**
-    (*dict*) -- a variant/arch mapping of configuration. The format should be
+    (*dict*) -- a mapping of configuration for each. The format should be
-    ``[(variant_uid_regex, {arch|*: config_dict})]``.
+    ``{variant_uid_regex: config_dict}``. It is possible to use a list of
+    configuration dicts as well.
 
     The configuration dict for each variant arch pair must have these keys:
 
@@ -1183,6 +1184,9 @@ a new commit.
       be removed from the tree config file.
     * ``config_branch`` -- (*str*) Git branch of the repo to use. Defaults to
       ``master``.
+    * ``arches`` -- (*[str]*) List of architectures for which to update ostree.
+      There will be one task per architecture. By default all architectures in
+      the variant are used.
     * ``failable`` -- (*[str]*) List of architectures for which this
       deliverable is not release blocking.
     * ``update_summary`` -- (*bool*) Update summary metadata after tree composing.
@@ -1206,9 +1210,8 @@ Example config
 --------------
 ::
 
-    ostree = [
+    ostree = {
-        ("^Atomic$", {
+        "^Atomic$": {
-            "x86_64": {
             "treefile": "fedora-atomic-docker-host.json",
             "config_url": "https://git.fedorahosted.org/git/fedora-atomic.git",
             "repo": [
@@ -1222,9 +1225,10 @@ Example config
             "update_summary": True,
             # Automatically generate a reasonable version
             "version": "!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN",
+            # Only run this for x86_64 even if Atomic has more arches
+            "arches": ["x86_64"],
+        }
     }
-        })
-    ]
 
 
 Ostree Installer Settings

pungi/checks.py

@@ -862,7 +862,41 @@ def make_schema():
                 "additionalProperties": False,
             },
 
-            "ostree": _variant_arch_mapping({
+            "ostree": {
+                "anyOf": [
+                    {
+                        "type": "object",
+                        "patternProperties": {
+                            # Warning: this pattern is a variant uid regex, but the
+                            # format does not let us validate it as there is no regular
+                            # expression to describe all regular expressions.
+                            ".+": _one_or_list({
+                                "type": "object",
+                                "properties": {
+                                    "treefile": {"type": "string"},
+                                    "config_url": {"type": "string"},
+                                    "repo": {
+                                        "$ref": "#/definitions/repos",
+                                        "alias": "extra_source_repos",
+                                        "append": ["repo_from", "source_repo_from"],
+                                    },
+                                    "keep_original_sources": {"type": "boolean"},
+                                    "ostree_repo": {"type": "string"},
+                                    "arches": {"$ref": "#/definitions/list_of_strings"},
+                                    "failable": {"$ref": "#/definitions/list_of_strings"},
+                                    "update_summary": {"type": "boolean"},
+                                    "version": {"type": "string"},
+                                    "config_branch": {"type": "string"},
+                                    "tag_ref": {"type": "boolean"},
+                                },
+                                "required": ["treefile", "config_url", "repo", "ostree_repo"],
+                                "additionalProperties": False,
+                            }),
+                        },
+                        "additionalProperties": False,
+                    },
+                    # Deprecated in favour of the dict version above.
+                    _variant_arch_mapping({
                         "type": "object",
                         "properties": {
                             "treefile": {"type": "string"},
@@ -883,6 +917,8 @@ def make_schema():
                         "required": ["treefile", "config_url", "repo", "ostree_repo"],
                         "additionalProperties": False,
                     }),
+                ]
+            },
 
             "ostree_installer": _variant_arch_mapping({
                 "type": "object",

pungi/phases/ostree.py

@@ -20,12 +20,22 @@ class OSTreePhase(ConfigGuardedPhase):
         super(OSTreePhase, self).__init__(compose)
         self.pool = ThreadPool(logger=self.compose._logger)
 
+    def _enqueue(self, variant, arch, conf):
+        self.pool.add(OSTreeThread(self.pool))
+        self.pool.queue_put((self.compose, variant, arch, conf))
+
     def run(self):
+        if isinstance(self.compose.conf.get(self.name), dict):
+            for variant in self.compose.get_variants():
+                for conf in util.get_variant_data(self.compose.conf, self.name, variant):
+                    for arch in conf.get('arches', []) or variant.arches:
+                        self._enqueue(variant, arch, conf)
+        else:
+            # Legacy code path to support original configuration.
             for variant in self.compose.get_variants():
                 for arch in variant.arches:
                     for conf in util.get_arch_variant_data(self.compose.conf, self.name, arch, variant):
-                    self.pool.add(OSTreeThread(self.pool))
+                        self._enqueue(variant, arch, conf)
-                    self.pool.queue_put((self.compose, variant, arch, conf))
 
         self.pool.start()
 

tests/test_ostree_phase.py

@@ -43,6 +43,63 @@ class OSTreePhaseTest(helpers.PungiTestCase):
         phase = ostree.OSTreePhase(compose)
         self.assertTrue(phase.skip())
 
+    @mock.patch('pungi.phases.ostree.ThreadPool')
+    def test_run_with_simple_config(self, ThreadPool):
+        cfg = helpers.IterableMock(get=lambda x, y: None)
+        compose = helpers.DummyCompose(self.topdir, {
+            'ostree': {
+                '^Everything$': cfg
+            }
+        })
+
+        pool = ThreadPool.return_value
+
+        phase = ostree.OSTreePhase(compose)
+        phase.run()
+
+        self.assertEqual(len(pool.add.call_args_list), 2)
+        self.assertEqual(pool.queue_put.call_args_list,
+                         [mock.call((compose, compose.variants['Everything'], 'x86_64', cfg)),
+                          mock.call((compose, compose.variants['Everything'], 'amd64', cfg))])
+
+    @mock.patch('pungi.phases.ostree.ThreadPool')
+    def test_run_with_simple_config_limit_arches(self, ThreadPool):
+        cfg = helpers.IterableMock(get=lambda x, y: ['x86_64'])
+        compose = helpers.DummyCompose(self.topdir, {
+            'ostree': {
+                '^Everything$': cfg
+            }
+        })
+
+        pool = ThreadPool.return_value
+
+        phase = ostree.OSTreePhase(compose)
+        phase.run()
+
+        self.assertEqual(len(pool.add.call_args_list), 1)
+        self.assertEqual(pool.queue_put.call_args_list,
+                         [mock.call((compose, compose.variants['Everything'], 'x86_64', cfg))])
+
+    @mock.patch('pungi.phases.ostree.ThreadPool')
+    def test_run_with_simple_config_limit_arches_two_blocks(self, ThreadPool):
+        cfg1 = helpers.IterableMock(get=lambda x, y: ['x86_64'])
+        cfg2 = helpers.IterableMock(get=lambda x, y: ['s390x'])
+        compose = helpers.DummyCompose(self.topdir, {
+            'ostree': {
+                '^Everything$': [cfg1, cfg2],
+            }
+        })
+
+        pool = ThreadPool.return_value
+
+        phase = ostree.OSTreePhase(compose)
+        phase.run()
+
+        self.assertEqual(len(pool.add.call_args_list), 2)
+        self.assertEqual(pool.queue_put.call_args_list,
+                         [mock.call((compose, compose.variants['Everything'], 'x86_64', cfg1)),
+                          mock.call((compose, compose.variants['Everything'], 's390x', cfg2))])
+
 
 class OSTreeThreadTest(helpers.PungiTestCase):