extra-isos: Include treeinfo pointing to all variants

This will be used by Anaconda to consume multiple repos for
installation.

JIRA: RCM-36970
JIRA: COMPOSE-2753
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

Fixup

pungi/phases/extra_isos.py

@@ -17,6 +17,7 @@ import os
 
 from kobo.shortcuts import force_list
 from kobo.threads import ThreadPool, WorkerThread
+import productmd.treeinfo
 
 from pungi import createiso
 from pungi.phases.base import ConfigGuardedPhase, PhaseBase, PhaseLoggerMixin
@@ -163,8 +164,19 @@ def get_iso_contents(compose, variant, arch, include_variants, filename, bootabl
         for k, v in iso.get_graft_points([extra_files_dir]).items():
             files[os.path.join(var.uid, k)] = v
 
-    # Add extra files specific for the ISO
     extra_files_dir = compose.paths.work.extra_iso_extra_files_dir(arch, variant)
+
+    original_treeinfo = os.path.join(
+        compose.paths.compose.os_tree(arch=arch, variant=variant), ".treeinfo"
+    )
+    tweak_treeinfo(
+        compose,
+        include_variants,
+        original_treeinfo,
+        os.path.join(extra_files_dir, ".treeinfo"),
+    )
+
+    # Add extra files specific for the ISO
     files.update(iso.get_graft_points([extra_files_dir]))
 
     gp = "%s-graft-points" % iso_dir
@@ -172,6 +184,26 @@ def get_iso_contents(compose, variant, arch, include_variants, filename, bootabl
     return gp
 
 
+def tweak_treeinfo(compose, include_variants, source_file, dest_file):
+    ti = productmd.treeinfo.TreeInfo()
+    ti.load(source_file)
+    for variant_uid in include_variants:
+        variant = compose.all_variants[variant_uid]
+        var = productmd.treeinfo.Variant(ti)
+        var.id = variant.id
+        var.uid = variant.uid
+        var.name = variant.name
+        var.type = variant.type
+        ti.variants.add(var)
+
+    for variant_id in ti.variants:
+        var = ti.variants[variant_id]
+        var.paths.packages = os.path.join(var.uid, "Packages")
+        var.paths.repository = var.uid
+
+    ti.dump(dest_file)
+
+
 def get_filename(compose, variant, arch, format):
     disc_type = compose.conf['disc_types'].get('dvd', 'dvd')
     base_filename = compose.get_image_name(

tests/fixtures/treeinfo

@@ -0,0 +1,48 @@
+[checksums]
+images/boot.iso = sha256:fc8a4be604b6425746f12fa706116eb940f93358f036b8fbbe518b516cb6870c
+
+[general]
+; WARNING.0 = This section provides compatibility with pre-productmd treeinfos.
+; WARNING.1 = Read productmd documentation for details about new format.
+arch = x86_64
+family = Test
+name = Test 1.0
+packagedir = Packages
+platforms = x86_64,xen
+repository = .
+timestamp = 1531881582
+variant = Server
+version = 1.0
+
+[header]
+type = productmd.treeinfo
+version = 1.2
+
+[images-x86_64]
+boot.iso = images/boot.iso
+
+[images-xen]
+initrd = images/pxeboot/initrd.img
+kernel = images/pxeboot/vmlinuz
+
+[release]
+name = Test
+short = T
+version = 1.0
+
+[stage2]
+mainimage = images/install.img
+
+[tree]
+arch = x86_64
+build_timestamp = 1531881582
+platforms = x86_64,xen
+variants = Server
+
+[variant-Server]
+id = Server
+name = Server
+packages = Packages
+repository = .
+type = variant
+uid = Server

tests/fixtures/treeinfo-expected

@@ -0,0 +1,58 @@
+[checksums]
+images/boot.iso = sha256:fc8a4be604b6425746f12fa706116eb940f93358f036b8fbbe518b516cb6870c
+
+[general]
+; WARNING.0 = This section provides compatibility with pre-productmd treeinfos.
+; WARNING.1 = Read productmd documentation for details about new format.
+arch = x86_64
+family = Test
+name = Test 1.0
+packagedir = Client/Packages
+platforms = x86_64,xen
+repository = Client
+timestamp = 1531881582
+variant = Client
+variants = Client,Server
+version = 1.0
+
+[header]
+type = productmd.treeinfo
+version = 1.2
+
+[images-x86_64]
+boot.iso = images/boot.iso
+
+[images-xen]
+initrd = images/pxeboot/initrd.img
+kernel = images/pxeboot/vmlinuz
+
+[release]
+name = Test
+short = T
+version = 1.0
+
+[stage2]
+mainimage = images/install.img
+
+[tree]
+arch = x86_64
+build_timestamp = 1531881582
+platforms = x86_64,xen
+variants = Client,Server
+
+[variant-Client]
+id = Client
+name = Client
+packages = Client/Packages
+repository = Client
+type = variant
+uid = Client
+
+[variant-Server]
+id = Server
+name = Server
+packages = Server/Packages
+repository = Server
+type = variant
+uid = Server
+

tests/helpers.py

@@ -33,7 +33,7 @@ class PungiTestCase(unittest.TestCase):
 
 
 class MockVariant(mock.Mock):
-    def __init__(self, is_empty=False, *args, **kwargs):
+    def __init__(self, is_empty=False, name=None, *args, **kwargs):
         super(MockVariant, self).__init__(*args, is_empty=is_empty, **kwargs)
         self.parent = kwargs.get('parent', None)
         self.mmds = []
@@ -42,6 +42,7 @@ class MockVariant(mock.Mock):
         self.variants = {}
         self.pkgset = mock.Mock(rpms_by_arch={})
         self.modules = None
+        self.name = name
 
     def __str__(self):
         return self.uid
@@ -125,11 +126,11 @@ class DummyCompose(object):
         self.has_comps = True
         self.variants = {
             'Server': MockVariant(uid='Server', arches=['x86_64', 'amd64'],
-                                  type='variant'),
+                                  type='variant', id='Server', name='Server'),
             'Client': MockVariant(uid='Client', arches=['amd64'],
-                                  type='variant'),
+                                  type='variant', id='Client', name='Client'),
             'Everything': MockVariant(uid='Everything', arches=['x86_64', 'amd64'],
-                                      type='variant'),
+                                      type='variant', id='Everything', name='Everything'),
         }
         self.all_variants = self.variants.copy()
 

tests/test_extra_isos_phase.py

@@ -323,6 +323,7 @@ class GetExtraFilesTest(helpers.PungiTestCase):
                                     logger=self.compose._logger)])
 
 
+@mock.patch("pungi.phases.extra_isos.tweak_treeinfo")
 @mock.patch('pungi.wrappers.iso.write_graft_points')
 @mock.patch('pungi.wrappers.iso.get_graft_points')
 class GetIsoContentsTest(helpers.PungiTestCase):
@@ -332,7 +333,7 @@ class GetIsoContentsTest(helpers.PungiTestCase):
         self.compose = helpers.DummyCompose(self.topdir, {})
         self.variant = self.compose.variants['Server']
 
-    def test_non_bootable_binary(self, ggp, wgp):
+    def test_non_bootable_binary(self, ggp, wgp, tt):
         gp = {
             'compose/Client/x86_64/os/Packages': {'f/foo.rpm': '/mnt/f/foo.rpm'},
             'compose/Client/x86_64/os/repodata': {'primary.xml': '/mnt/repodata/primary.xml'},
@@ -371,7 +372,23 @@ class GetIsoContentsTest(helpers.PungiTestCase):
         self.assertDictEqual(dict(wgp.call_args_list[0][0][1]), expected)
         self.assertEqual(wgp.call_args_list[0][1], {'exclude': ["*/lost+found", "*/boot.iso"]})
 
-    def test_source(self, ggp, wgp):
+        # Check correct call to tweak_treeinfo
+        self.assertEqual(
+            tt.call_args_list,
+            [
+                mock.call(
+                    self.compose,
+                    ["Client"],
+                    os.path.join(self.topdir, "compose/Server/x86_64/os/.treeinfo"),
+                    os.path.join(
+                        self.topdir,
+                        "work/x86_64/Server/extra-iso-extra-files/.treeinfo",
+                    )
+                ),
+            ],
+        )
+
+    def test_source(self, ggp, wgp, tt):
         gp = {
             'compose/Client/source/tree/Packages': {'f/foo.rpm': '/mnt/f/foo.rpm'},
             'compose/Client/source/tree/repodata': {'primary.xml': '/mnt/repodata/primary.xml'},
@@ -410,7 +427,23 @@ class GetIsoContentsTest(helpers.PungiTestCase):
         self.assertDictEqual(dict(wgp.call_args_list[0][0][1]), expected)
         self.assertEqual(wgp.call_args_list[0][1], {'exclude': ["*/lost+found", "*/boot.iso"]})
 
-    def test_bootable(self, ggp, wgp):
+        # Check correct call to tweak_treeinfo
+        self.assertEqual(
+            tt.call_args_list,
+            [
+                mock.call(
+                    self.compose,
+                    ["Client"],
+                    os.path.join(self.topdir, "compose/Server/source/tree/.treeinfo"),
+                    os.path.join(
+                        self.topdir,
+                        "work/src/Server/extra-iso-extra-files/.treeinfo",
+                    )
+                ),
+            ],
+        )
+
+    def test_bootable(self, ggp, wgp, tt):
         self.compose.conf['buildinstall_method'] = 'lorax'
 
         bi_dir = os.path.join(self.topdir, 'work/x86_64/buildinstall/Server')
@@ -436,8 +469,13 @@ class GetIsoContentsTest(helpers.PungiTestCase):
         gp_file = os.path.join(self.topdir, 'work/x86_64/iso/my.iso-graft-points')
 
         self.assertEqual(
-            extra_isos.get_iso_contents(self.compose, self.variant, 'x86_64',
+            extra_isos.get_iso_contents(
-                                        ['Client'], 'my.iso', True),
+                self.compose,
+                self.variant,
+                'x86_64',
+                ['Client'],
+                'my.iso',
+                True),
             gp_file
         )
 
@@ -468,6 +506,22 @@ class GetIsoContentsTest(helpers.PungiTestCase):
         self.assertTrue(os.path.exists(os.path.join(iso_dir, 'isolinux/isolinux.bin')))
         self.assertTrue(os.path.exists(os.path.join(iso_dir, 'images/boot.img')))
 
+        # Check correct call to tweak_treeinfo
+        self.assertEqual(
+            tt.call_args_list,
+            [
+                mock.call(
+                    self.compose,
+                    ["Client"],
+                    os.path.join(self.topdir, "compose/Server/x86_64/os/.treeinfo"),
+                    os.path.join(
+                        self.topdir,
+                        "work/x86_64/Server/extra-iso-extra-files/.treeinfo",
+                    )
+                ),
+            ],
+        )
+
 
 class GetFilenameTest(helpers.PungiTestCase):
     def test_use_original_name(self):
@@ -525,5 +579,22 @@ class GetVolumeIDTest(helpers.PungiTestCase):
         self.assertIn('boom', str(ctx.exception))
 
 
+class TweakTreeinfoTest(helpers.PungiTestCase):
+    def test_tweak(self):
+        compose = helpers.DummyCompose(self.topdir, {})
+        input = os.path.join(helpers.FIXTURE_DIR, "treeinfo")
+        output = os.path.join(self.topdir, "actual-treeinfo")
+        expected = os.path.join(helpers.FIXTURE_DIR, "treeinfo-expected")
+        extra_isos.tweak_treeinfo(compose, ["Client"], input, output)
+
+        with open(expected) as f:
+            expected = f.read()
+        with open(output) as f:
+            actual = f.read()
+
+        self.maxDiff = None
+        self.assertEqual(expected, actual)
+
+
 if __name__ == '__main__':
     unittest.main()