diff --git a/bin/pungi-koji b/bin/pungi-koji index f77808ae..1b149225 100755 --- a/bin/pungi-koji +++ b/bin/pungi-koji @@ -366,7 +366,7 @@ def run_compose(compose, create_latest_link=True): pungi.metadata.write_tree_info(compose, arch, variant) # write .discinfo and media.repo before ISOs are created - for variant in compose.get_variants(recursive=True): + for variant in compose.get_variants(): if variant.type == "addon" or variant.is_empty: continue for arch in variant.arches + ["src"]: diff --git a/pungi/compose.py b/pungi/compose.py index e62b9b9a..b3a967a2 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -97,7 +97,11 @@ class Compose(kobo.log.LoggingBase): kobo.log.LoggingBase.__init__(self, logger) # TODO: check if minimal conf values are set self.conf = conf + # This is a dict mapping UID to Variant objects. It only contains top + # level variants. self.variants = {} + # This is a similar mapping, but contains even nested variants. + self.all_variants = {} self.topdir = os.path.abspath(topdir) self.skip_phases = skip_phases or [] self.just_phases = just_phases or [] @@ -215,19 +219,23 @@ class Compose(kobo.log.LoggingBase): parser = VariantsXmlParser(file_obj, tree_arches, tree_variants, logger=self._logger) self.variants = parser.parse() + self.all_variants = {} + for variant in self.get_variants(): + self.all_variants[variant.uid] = variant + # populate ci_base with variants - needed for layered-products (compose_id) ####FIXME - compose_to_composeinfo is no longer needed and has been #### removed, but I'm not entirely sure what this is needed for #### or if it is at all self.ci_base = compose_to_composeinfo(self) - def get_variants(self, types=None, arch=None, recursive=False): + def get_variants(self, types=None, arch=None): result = [] types = types or ["variant", "optional", "addon", "layered-product"] for i in self.variants.itervalues(): if i.type in types and (not arch or arch in i.arches): result.append(i) - result.extend(i.get_variants(types=types, arch=arch, recursive=recursive)) + result.extend(i.get_variants(types=types, arch=arch)) return sorted(set(result)) def get_arches(self): diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py index d90b45f8..0a7f0ff1 100644 --- a/pungi/phases/createiso.py +++ b/pungi/phases/createiso.py @@ -65,7 +65,7 @@ class CreateisoPhase(PhaseBase): deliverables = [] commands = [] - for variant in self.compose.get_variants(types=["variant", "layered-product", "optional"], recursive=True): + for variant in self.compose.get_variants(types=["variant", "layered-product", "optional"]): for arch in variant.arches + ["src"]: skip_iso = get_arch_variant_data(self.compose.conf, "createiso_skip", arch, variant) if skip_iso == [True]: diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 71a7b243..4eab85db 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -307,7 +307,7 @@ def gather_wrapper(compose, package_sets, path_prefix): # write packages (package lists) for all variants for arch in compose.get_arches(): - for variant in compose.get_variants(arch=arch, recursive=True): + for variant in compose.get_variants(arch=arch): pkg_map = result[arch][variant.uid] write_packages(compose, arch, variant, pkg_map, path_prefix=path_prefix) diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py index d869a5ad..0f8f1604 100644 --- a/pungi/phases/image_build.py +++ b/pungi/phases/image_build.py @@ -31,7 +31,7 @@ class ImageBuildPhase(base.ImageConfigMixin, base.ConfigGuardedPhase): install_tree_from = image_conf.pop('install_tree_from', variant.uid) if '://' in install_tree_from: return install_tree_from - install_tree_source = self.compose.variants.get(install_tree_from) + install_tree_source = self.compose.all_variants.get(install_tree_from) if not install_tree_source: raise RuntimeError( 'There is no variant %s to get install tree from when building image for %s.' @@ -56,7 +56,7 @@ class ImageBuildPhase(base.ImageConfigMixin, base.ConfigGuardedPhase): extras.append(variant.uid) for extra in extras: - v = self.compose.variants.get(extra) + v = self.compose.all_variants.get(extra) if not v: raise RuntimeError( 'There is no variant %s to get repo from when building image for %s.' diff --git a/pungi/phases/live_images.py b/pungi/phases/live_images.py index e2b403df..6ae4fd1f 100644 --- a/pungi/phases/live_images.py +++ b/pungi/phases/live_images.py @@ -47,7 +47,7 @@ class LiveImagesPhase(base.ImageConfigMixin, base.ConfigGuardedPhase): def _get_extra_repos(self, arch, variant, extras): repo = [] for extra in extras: - v = self.compose.variants.get(extra) + v = self.compose.all_variants.get(extra) if not v: raise RuntimeError( 'There is no variant %s to get repo from when building live image for %s.' @@ -72,7 +72,7 @@ class LiveImagesPhase(base.ImageConfigMixin, base.ConfigGuardedPhase): symlink_isos_to = self.compose.conf.get("symlink_isos_to") commands = [] - for variant in self.compose.variants.values(): + for variant in self.compose.all_variants.values(): for arch in variant.arches + ["src"]: for data in get_arch_variant_data(self.compose.conf, self.name, arch, variant): subvariant = data.get('subvariant', variant.uid) diff --git a/pungi/phases/livemedia_phase.py b/pungi/phases/livemedia_phase.py index 35a5894c..0d98dadd 100644 --- a/pungi/phases/livemedia_phase.py +++ b/pungi/phases/livemedia_phase.py @@ -36,7 +36,7 @@ class LiveMediaPhase(ImageConfigMixin, ConfigGuardedPhase): extras.append(variant.uid) for extra in extras: - v = self.compose.variants.get(extra) + v = self.compose.all_variants.get(extra) if not v: raise RuntimeError( 'There is no variant %s to get repo from when building live media for %s.' @@ -56,7 +56,7 @@ class LiveMediaPhase(ImageConfigMixin, ConfigGuardedPhase): if 'install_tree_from' in image_conf: variant_uid = image_conf['install_tree_from'] try: - variant = self.compose.variants[variant_uid] + variant = self.compose.all_variants[variant_uid] except KeyError: raise RuntimeError( 'There is no variant %s to get repo from when building live media for %s.' diff --git a/pungi/phases/ostree.py b/pungi/phases/ostree.py index 4a52f2ff..cac2d4d9 100644 --- a/pungi/phases/ostree.py +++ b/pungi/phases/ostree.py @@ -45,7 +45,7 @@ class OSTreeThread(WorkerThread): (arch, variant.uid, self.num)) repodir = os.path.join(workdir, 'config_repo') - source_variant = compose.variants[config['source_repo_from']] + source_variant = compose.all_variants[config['source_repo_from']] source_repo = translate_path(compose, compose.paths.compose.repository('$basearch', source_variant, diff --git a/pungi/phases/ostree_installer.py b/pungi/phases/ostree_installer.py index 6b0f6708..8cde3d7b 100644 --- a/pungi/phases/ostree_installer.py +++ b/pungi/phases/ostree_installer.py @@ -69,7 +69,7 @@ class OstreeInstallerThread(WorkerThread): """ if '://' in source: return source.replace('$arch', arch) - source_variant = compose.variants[source] + source_variant = compose.all_variants[source] return translate_path( compose, compose.paths.compose.repository(arch, source_variant, create_dir=False)) diff --git a/tests/helpers.py b/tests/helpers.py index 7fa2d6f8..53828a7f 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -59,6 +59,7 @@ class DummyCompose(object): 'Everything': mock.Mock(uid='Everything', arches=['x86_64', 'amd64'], type='variant', is_empty=False), } + self.all_variants = self.variants.copy() self.log_info = mock.Mock() self.log_error = mock.Mock() self.log_debug = mock.Mock() @@ -73,8 +74,14 @@ class DummyCompose(object): self.fail_deliverable = mock.Mock() self.require_deliverable = mock.Mock() - def get_variants(self, arch=None, types=None, recursive=None): - return [v for v in self.variants.values() if not arch or arch in v.arches] + def setup_optional(self): + self.all_variants['Server-optional'] = mock.Mock( + uid='Server-optional', arches=['x86_64'], type='optional', is_empty=False, + parent=self.variants['Server']) + self.variants['Server'].variants = {'optional': self.all_variants['Server-optional']} + + def get_variants(self, arch=None, types=None): + return [v for v in self.all_variants.values() if not arch or arch in v.arches] def can_fail(self, variant, arch, deliverable): failable = get_arch_variant_data(self.conf, 'failable_deliverables', arch, variant) diff --git a/tests/test_buildinstall.py b/tests/test_buildinstall.py index 00f7aa42..ea976067 100644 --- a/tests/test_buildinstall.py +++ b/tests/test_buildinstall.py @@ -28,6 +28,7 @@ class BuildInstallCompose(DummyCompose): type='variant', buildinstallpackages=[], is_empty=False), } + self.all_variants = self.variants.copy() class TestBuildinstallPhase(PungiTestCase): diff --git a/tests/test_imagebuildphase.py b/tests/test_imagebuildphase.py index 2407cf1c..2fad381f 100644 --- a/tests/test_imagebuildphase.py +++ b/tests/test_imagebuildphase.py @@ -277,13 +277,14 @@ class TestImageBuildPhase(PungiTestCase): 'distro': 'Fedora-20', 'disk_size': 3, 'arches': ['x86_64'], - 'install_tree_from': 'Everything', + 'install_tree_from': 'Server-optional', } } ] }, 'koji_profile': 'koji', }) + compose.setup_optional() self.assertEqual(validate(compose.conf), []) @@ -302,7 +303,7 @@ class TestImageBuildPhase(PungiTestCase): "format": [('docker', 'tar.xz')], "image_conf": { 'image-build': { - 'install_tree': self.topdir + '/compose/Everything/$arch/os', + 'install_tree': self.topdir + '/compose/Server-optional/$arch/os', 'kickstart': 'fedora-docker-base.ks', 'format': 'docker', 'repo': self.topdir + '/compose/Server/$arch/os', @@ -340,13 +341,14 @@ class TestImageBuildPhase(PungiTestCase): 'distro': 'Fedora-20', 'disk_size': 3, 'arches': ['x86_64'], - 'repo_from': 'Everything', + 'repo_from': ['Everything', 'Server-optional'], } } ] }, 'koji_profile': 'koji', }) + compose.setup_optional() self.assertEqual(validate(compose.conf), []) @@ -369,6 +371,7 @@ class TestImageBuildPhase(PungiTestCase): 'kickstart': 'fedora-docker-base.ks', 'format': 'docker', 'repo': ','.join([self.topdir + '/compose/Everything/$arch/os', + self.topdir + '/compose/Server-optional/$arch/os', self.topdir + '/compose/Server/$arch/os']), 'variant': compose.variants['Server'], 'target': 'f24', diff --git a/tests/test_liveimagesphase.py b/tests/test_liveimagesphase.py index ac01d13b..6f383164 100644 --- a/tests/test_liveimagesphase.py +++ b/tests/test_liveimagesphase.py @@ -25,12 +25,13 @@ class TestLiveImagesPhase(PungiTestCase): 'amd64': { 'kickstart': 'test.ks', 'additional_repos': ['http://example.com/repo/'], - 'repo_from': ['Everything'], + 'repo_from': ['Everything', 'Server-optional'], 'release': None, } }) ], }) + compose.setup_optional() self.assertEqual(validate(compose.conf), []) @@ -49,7 +50,8 @@ class TestLiveImagesPhase(PungiTestCase): 'scratch': False, 'repos': [self.topdir + '/compose/Client/amd64/os', 'http://example.com/repo/', - self.topdir + '/compose/Everything/amd64/os'], + self.topdir + '/compose/Everything/amd64/os', + self.topdir + '/compose/Server-optional/amd64/os'], 'label': '', 'name': None, 'filename': 'image-name', diff --git a/tests/test_livemediaphase.py b/tests/test_livemediaphase.py index acca0389..6d577de0 100644 --- a/tests/test_livemediaphase.py +++ b/tests/test_livemediaphase.py @@ -326,18 +326,19 @@ class TestLiveMediaPhase(PungiTestCase): 'scratch': True, 'skip_tag': True, 'title': 'Custom Title', - 'repo_from': ['Everything'], + 'repo_from': ['Everything', 'Server-optional'], 'repo': ['http://example.com/extra_repo'], 'arches': ['x86_64'], 'ksversion': '24', 'release': None, - 'install_tree_from': 'Everything', + 'install_tree_from': 'Server-optional', 'subvariant': 'Something', 'failable': ['*'], } ] } }) + compose.setup_optional() self.assertEqual(validate(compose.conf), []) @@ -359,12 +360,13 @@ class TestLiveMediaPhase(PungiTestCase): 'release': '20151203.t.0', 'repo': ['http://example.com/extra_repo', self.topdir + '/compose/Everything/$basearch/os', + self.topdir + '/compose/Server-optional/$basearch/os', self.topdir + '/compose/Server/$basearch/os'], 'scratch': True, 'skip_tag': True, 'target': 'f24', 'title': 'Custom Title', - 'install_tree': self.topdir + '/compose/Everything/$basearch/os', + 'install_tree': self.topdir + '/compose/Server-optional/$basearch/os', 'version': '25', 'subvariant': 'Something', 'failable_arches': ['*'],