Merge #459 Fix building images and live media for addons

bin/pungi-koji

@@ -366,7 +366,7 @@ def run_compose(compose, create_latest_link=True):
             pungi.metadata.write_tree_info(compose, arch, variant)
 
     # write .discinfo and media.repo before ISOs are created
-    for variant in compose.get_variants(recursive=True):
+    for variant in compose.get_variants():
         if variant.type == "addon" or variant.is_empty:
             continue
         for arch in variant.arches + ["src"]:

pungi/compose.py

@@ -97,7 +97,11 @@ class Compose(kobo.log.LoggingBase):
         kobo.log.LoggingBase.__init__(self, logger)
         # TODO: check if minimal conf values are set
         self.conf = conf
+        # This is a dict mapping UID to Variant objects. It only contains top
+        # level variants.
         self.variants = {}
+        # This is a similar mapping, but contains even nested variants.
+        self.all_variants = {}
         self.topdir = os.path.abspath(topdir)
         self.skip_phases = skip_phases or []
         self.just_phases = just_phases or []
@@ -215,19 +219,23 @@ class Compose(kobo.log.LoggingBase):
             parser = VariantsXmlParser(file_obj, tree_arches, tree_variants, logger=self._logger)
             self.variants = parser.parse()
 
+        self.all_variants = {}
+        for variant in self.get_variants():
+            self.all_variants[variant.uid] = variant
+
         # populate ci_base with variants - needed for layered-products (compose_id)
         ####FIXME - compose_to_composeinfo is no longer needed and has been
         ####        removed, but I'm not entirely sure what this is needed for
         ####        or if it is at all
         self.ci_base = compose_to_composeinfo(self)
 
-    def get_variants(self, types=None, arch=None, recursive=False):
+    def get_variants(self, types=None, arch=None):
         result = []
         types = types or ["variant", "optional", "addon", "layered-product"]
         for i in self.variants.itervalues():
             if i.type in types and (not arch or arch in i.arches):
                 result.append(i)
-            result.extend(i.get_variants(types=types, arch=arch, recursive=recursive))
+            result.extend(i.get_variants(types=types, arch=arch))
         return sorted(set(result))
 
     def get_arches(self):

pungi/phases/createiso.py

@@ -65,7 +65,7 @@ class CreateisoPhase(PhaseBase):
         deliverables = []
 
         commands = []
-        for variant in self.compose.get_variants(types=["variant", "layered-product", "optional"], recursive=True):
+        for variant in self.compose.get_variants(types=["variant", "layered-product", "optional"]):
             for arch in variant.arches + ["src"]:
                 skip_iso = get_arch_variant_data(self.compose.conf, "createiso_skip", arch, variant)
                 if skip_iso == [True]:

pungi/phases/gather/__init__.py

@@ -307,7 +307,7 @@ def gather_wrapper(compose, package_sets, path_prefix):
 
     # write packages (package lists) for all variants
     for arch in compose.get_arches():
-        for variant in compose.get_variants(arch=arch, recursive=True):
+        for variant in compose.get_variants(arch=arch):
             pkg_map = result[arch][variant.uid]
             write_packages(compose, arch, variant, pkg_map, path_prefix=path_prefix)
 

pungi/phases/image_build.py

@@ -31,7 +31,7 @@ class ImageBuildPhase(base.ImageConfigMixin, base.ConfigGuardedPhase):
         install_tree_from = image_conf.pop('install_tree_from', variant.uid)
         if '://' in install_tree_from:
             return install_tree_from
-        install_tree_source = self.compose.variants.get(install_tree_from)
+        install_tree_source = self.compose.all_variants.get(install_tree_from)
         if not install_tree_source:
             raise RuntimeError(
                 'There is no variant %s to get install tree from when building image for %s.'
@@ -56,7 +56,7 @@ class ImageBuildPhase(base.ImageConfigMixin, base.ConfigGuardedPhase):
             extras.append(variant.uid)
 
         for extra in extras:
-            v = self.compose.variants.get(extra)
+            v = self.compose.all_variants.get(extra)
             if not v:
                 raise RuntimeError(
                     'There is no variant %s to get repo from when building image for %s.'

pungi/phases/live_images.py

@@ -47,7 +47,7 @@ class LiveImagesPhase(base.ImageConfigMixin, base.ConfigGuardedPhase):
     def _get_extra_repos(self, arch, variant, extras):
         repo = []
         for extra in extras:
-            v = self.compose.variants.get(extra)
+            v = self.compose.all_variants.get(extra)
             if not v:
                 raise RuntimeError(
                     'There is no variant %s to get repo from when building live image for %s.'
@@ -72,7 +72,7 @@ class LiveImagesPhase(base.ImageConfigMixin, base.ConfigGuardedPhase):
         symlink_isos_to = self.compose.conf.get("symlink_isos_to")
         commands = []
 
-        for variant in self.compose.variants.values():
+        for variant in self.compose.all_variants.values():
             for arch in variant.arches + ["src"]:
                 for data in get_arch_variant_data(self.compose.conf, self.name, arch, variant):
                     subvariant = data.get('subvariant', variant.uid)

pungi/phases/livemedia_phase.py

@@ -36,7 +36,7 @@ class LiveMediaPhase(ImageConfigMixin, ConfigGuardedPhase):
             extras.append(variant.uid)
 
         for extra in extras:
-            v = self.compose.variants.get(extra)
+            v = self.compose.all_variants.get(extra)
             if not v:
                 raise RuntimeError(
                     'There is no variant %s to get repo from when building live media for %s.'
@@ -56,7 +56,7 @@ class LiveMediaPhase(ImageConfigMixin, ConfigGuardedPhase):
         if 'install_tree_from' in image_conf:
             variant_uid = image_conf['install_tree_from']
             try:
-                variant = self.compose.variants[variant_uid]
+                variant = self.compose.all_variants[variant_uid]
             except KeyError:
                 raise RuntimeError(
                     'There is no variant %s to get repo from when building live media for %s.'

pungi/phases/ostree.py

@@ -45,7 +45,7 @@ class OSTreeThread(WorkerThread):
                                                (arch, variant.uid, self.num))
         repodir = os.path.join(workdir, 'config_repo')
 
-        source_variant = compose.variants[config['source_repo_from']]
+        source_variant = compose.all_variants[config['source_repo_from']]
         source_repo = translate_path(compose,
                                      compose.paths.compose.repository('$basearch',
                                                                       source_variant,

pungi/phases/ostree_installer.py

@@ -69,7 +69,7 @@ class OstreeInstallerThread(WorkerThread):
         """
         if '://' in source:
             return source.replace('$arch', arch)
-        source_variant = compose.variants[source]
+        source_variant = compose.all_variants[source]
         return translate_path(
             compose, compose.paths.compose.repository(arch, source_variant, create_dir=False))
 

tests/helpers.py

@@ -59,6 +59,7 @@ class DummyCompose(object):
             'Everything': mock.Mock(uid='Everything', arches=['x86_64', 'amd64'],
                                     type='variant', is_empty=False),
         }
+        self.all_variants = self.variants.copy()
         self.log_info = mock.Mock()
         self.log_error = mock.Mock()
         self.log_debug = mock.Mock()
@@ -73,8 +74,14 @@ class DummyCompose(object):
         self.fail_deliverable = mock.Mock()
         self.require_deliverable = mock.Mock()
 
-    def get_variants(self, arch=None, types=None, recursive=None):
+    def setup_optional(self):
-        return [v for v in self.variants.values() if not arch or arch in v.arches]
+        self.all_variants['Server-optional'] = mock.Mock(
+            uid='Server-optional', arches=['x86_64'], type='optional', is_empty=False,
+            parent=self.variants['Server'])
+        self.variants['Server'].variants = {'optional': self.all_variants['Server-optional']}
+
+    def get_variants(self, arch=None, types=None):
+        return [v for v in self.all_variants.values() if not arch or arch in v.arches]
 
     def can_fail(self, variant, arch, deliverable):
         failable = get_arch_variant_data(self.conf, 'failable_deliverables', arch, variant)

tests/test_buildinstall.py

@@ -28,6 +28,7 @@ class BuildInstallCompose(DummyCompose):
                                 type='variant', buildinstallpackages=[],
                                 is_empty=False),
         }
+        self.all_variants = self.variants.copy()
 
 
 class TestBuildinstallPhase(PungiTestCase):

tests/test_imagebuildphase.py

@@ -277,13 +277,14 @@ class TestImageBuildPhase(PungiTestCase):
                             'distro': 'Fedora-20',
                             'disk_size': 3,
                             'arches': ['x86_64'],
-                            'install_tree_from': 'Everything',
+                            'install_tree_from': 'Server-optional',
                         }
                     }
                 ]
             },
             'koji_profile': 'koji',
         })
+        compose.setup_optional()
 
         self.assertEqual(validate(compose.conf), [])
 
@@ -302,7 +303,7 @@ class TestImageBuildPhase(PungiTestCase):
             "format": [('docker', 'tar.xz')],
             "image_conf": {
                 'image-build': {
-                    'install_tree': self.topdir + '/compose/Everything/$arch/os',
+                    'install_tree': self.topdir + '/compose/Server-optional/$arch/os',
                     'kickstart': 'fedora-docker-base.ks',
                     'format': 'docker',
                     'repo': self.topdir + '/compose/Server/$arch/os',
@@ -340,13 +341,14 @@ class TestImageBuildPhase(PungiTestCase):
                             'distro': 'Fedora-20',
                             'disk_size': 3,
                             'arches': ['x86_64'],
-                            'repo_from': 'Everything',
+                            'repo_from': ['Everything', 'Server-optional'],
                         }
                     }
                 ]
             },
             'koji_profile': 'koji',
         })
+        compose.setup_optional()
 
         self.assertEqual(validate(compose.conf), [])
 
@@ -369,6 +371,7 @@ class TestImageBuildPhase(PungiTestCase):
                     'kickstart': 'fedora-docker-base.ks',
                     'format': 'docker',
                     'repo': ','.join([self.topdir + '/compose/Everything/$arch/os',
+                                      self.topdir + '/compose/Server-optional/$arch/os',
                                       self.topdir + '/compose/Server/$arch/os']),
                     'variant': compose.variants['Server'],
                     'target': 'f24',

tests/test_liveimagesphase.py

@@ -25,12 +25,13 @@ class TestLiveImagesPhase(PungiTestCase):
                     'amd64': {
                         'kickstart': 'test.ks',
                         'additional_repos': ['http://example.com/repo/'],
-                        'repo_from': ['Everything'],
+                        'repo_from': ['Everything', 'Server-optional'],
                         'release': None,
                     }
                 })
             ],
         })
+        compose.setup_optional()
 
         self.assertEqual(validate(compose.conf), [])
 
@@ -49,7 +50,8 @@ class TestLiveImagesPhase(PungiTestCase):
                                            'scratch': False,
                                            'repos': [self.topdir + '/compose/Client/amd64/os',
                                                      'http://example.com/repo/',
-                                                     self.topdir + '/compose/Everything/amd64/os'],
+                                                     self.topdir + '/compose/Everything/amd64/os',
+                                                     self.topdir + '/compose/Server-optional/amd64/os'],
                                            'label': '',
                                            'name': None,
                                            'filename': 'image-name',

tests/test_livemediaphase.py

@@ -326,18 +326,19 @@ class TestLiveMediaPhase(PungiTestCase):
                         'scratch': True,
                         'skip_tag': True,
                         'title': 'Custom Title',
-                        'repo_from': ['Everything'],
+                        'repo_from': ['Everything', 'Server-optional'],
                         'repo': ['http://example.com/extra_repo'],
                         'arches': ['x86_64'],
                         'ksversion': '24',
                         'release': None,
-                        'install_tree_from': 'Everything',
+                        'install_tree_from': 'Server-optional',
                         'subvariant': 'Something',
                         'failable': ['*'],
                     }
                 ]
             }
         })
+        compose.setup_optional()
 
         self.assertEqual(validate(compose.conf), [])
 
@@ -359,12 +360,13 @@ class TestLiveMediaPhase(PungiTestCase):
                                          'release': '20151203.t.0',
                                          'repo': ['http://example.com/extra_repo',
                                                   self.topdir + '/compose/Everything/$basearch/os',
+                                                  self.topdir + '/compose/Server-optional/$basearch/os',
                                                   self.topdir + '/compose/Server/$basearch/os'],
                                          'scratch': True,
                                          'skip_tag': True,
                                          'target': 'f24',
                                          'title': 'Custom Title',
-                                         'install_tree': self.topdir + '/compose/Everything/$basearch/os',
+                                         'install_tree': self.topdir + '/compose/Server-optional/$basearch/os',
                                          'version': '25',
                                          'subvariant': 'Something',
                                          'failable_arches': ['*'],