extra_iso: Include extra_files.json metadata

This should be on top level of the ISO, and list files added
specifically to the ISO. If there's anything inherited from one some
variant, the files will be listed in metadata in the variant directory.

JIRA: COMPOSE-3069
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

pungi/phases/extra_isos.py

@@ -142,10 +142,23 @@ def get_extra_files(compose, variant, arch, extra_files):
     included in the ISO.
     """
     extra_files_dir = compose.paths.work.extra_iso_extra_files_dir(arch, variant)
+    filelist = []
     for scm_dict in extra_files:
         getter = get_file_from_scm if 'file' in scm_dict else get_dir_from_scm
-        target_path = os.path.join(extra_files_dir, scm_dict.get('target', '').lstrip('/'))
+        target = scm_dict.get("target", "").lstrip("/")
-        getter(scm_dict, target_path, logger=compose._logger)
+        target_path = os.path.join(extra_files_dir, target).rstrip("/")
+        filelist.extend(
+            os.path.join(target, f)
+            for f in getter(scm_dict, target_path, logger=compose._logger)
+        )
+
+    if filelist:
+        metadata.write_extra_files(
+            extra_files_dir,
+            filelist,
+            compose.conf["media_checksums"],
+            logger=compose._logger,
+        )
 
 
 def get_iso_contents(

tests/helpers.py

@@ -149,7 +149,6 @@ class DummyCompose(object):
         self.conf = load_config(PKGSET_REPOS, **config)
         checks.validate(self.conf)
         self.paths = paths.Paths(self)
-        self._logger = mock.Mock()
         self.has_comps = True
         self.variants = {
             'Server': MockVariant(uid='Server', arches=['x86_64', 'amd64'],
@@ -162,7 +161,7 @@ class DummyCompose(object):
         self.all_variants = self.variants.copy()
 
         # for PhaseLoggerMixin
-        self._logger = mock.Mock()
+        self._logger = mock.Mock(name="compose._logger")
         self._logger.handlers = [mock.Mock()]
 
         self.log_info = mock.Mock()

tests/test_extra_isos_phase.py

@@ -368,6 +368,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase):
         self.assertEqual(aitm.call_args_list, [])
 
 
+@mock.patch("pungi.metadata.write_extra_files")
 @mock.patch('pungi.phases.extra_isos.get_file_from_scm')
 @mock.patch('pungi.phases.extra_isos.get_dir_from_scm')
 class GetExtraFilesTest(helpers.PungiTestCase):
@@ -377,14 +378,19 @@ class GetExtraFilesTest(helpers.PungiTestCase):
         self.compose = helpers.DummyCompose(self.topdir, {})
         self.variant = self.compose.variants['Server']
         self.arch = 'x86_64'
+        self.dir = os.path.join(
+            self.topdir, "work", self.arch, self.variant.uid, "extra-iso-extra-files"
+        )
 
-    def test_no_config(self, get_dir, get_file):
+    def test_no_config(self, get_dir, get_file, write_extra):
         extra_isos.get_extra_files(self.compose, self.variant, self.arch, [])
 
         self.assertEqual(get_dir.call_args_list, [])
         self.assertEqual(get_file.call_args_list, [])
+        self.assertEqual(write_extra.call_args_list, [])
 
-    def test_get_file(self, get_dir, get_file):
+    def test_get_file(self, get_dir, get_file, write_extra):
+        get_file.return_value = ["GPL"]
         cfg = {
             'scm': 'git',
             'repo': 'https://pagure.io/pungi.git',
@@ -394,14 +400,28 @@ class GetExtraFilesTest(helpers.PungiTestCase):
         extra_isos.get_extra_files(self.compose, self.variant, self.arch, [cfg])
 
         self.assertEqual(get_dir.call_args_list, [])
-        self.assertEqual(get_file.call_args_list,
+        self.assertEqual(
-                         [mock.call(cfg,
+            get_file.call_args_list,
-                                    os.path.join(self.topdir, 'work',
+            [
-                                                 self.arch, self.variant.uid,
+                mock.call(
-                                                 'extra-iso-extra-files/legalese'),
+                    cfg, os.path.join(self.dir, "legalese"), logger=self.compose._logger
-                                    logger=self.compose._logger)])
+                )
+            ],
+        )
+        self.assertEqual(
+            write_extra.call_args_list,
+            [
+                mock.call(
+                    self.dir,
+                    ["legalese/GPL"],
+                    self.compose.conf["media_checksums"],
+                    logger=self.compose._logger,
+                )
+            ],
+        )
 
-    def test_get_dir(self, get_dir, get_file):
+    def test_get_dir(self, get_dir, get_file, write_extra):
+        get_dir.return_value = ["a", "b"]
         cfg = {
             'scm': 'git',
             'repo': 'https://pagure.io/pungi.git',
@@ -411,12 +431,60 @@ class GetExtraFilesTest(helpers.PungiTestCase):
         extra_isos.get_extra_files(self.compose, self.variant, self.arch, [cfg])
 
         self.assertEqual(get_file.call_args_list, [])
-        self.assertEqual(get_dir.call_args_list,
+        self.assertEqual(
-                         [mock.call(cfg,
+            get_dir.call_args_list,
-                                    os.path.join(self.topdir, 'work',
+            [
-                                                 self.arch, self.variant.uid,
+                mock.call(
-                                                 'extra-iso-extra-files/foo'),
+                    cfg, os.path.join(self.dir, "foo"), logger=self.compose._logger
-                                    logger=self.compose._logger)])
+                )
+            ],
+        )
+        self.assertEqual(
+            write_extra.call_args_list,
+            [
+                mock.call(
+                    self.dir,
+                    ["foo/a", "foo/b"],
+                    self.compose.conf["media_checksums"],
+                    logger=self.compose._logger,
+                )
+            ],
+        )
+
+    def test_get_multiple_files(self, get_dir, get_file, write_extra):
+        get_file.side_effect = [["GPL"], ["setup.py"]]
+        cfg1 = {
+            'scm': 'git',
+            'repo': 'https://pagure.io/pungi.git',
+            'file': 'GPL',
+            'target': 'legalese',
+        }
+        cfg2 = {"scm": "git", "repo": "https://pagure.io/pungi.git", "file": "setup.py"}
+        extra_isos.get_extra_files(self.compose, self.variant, self.arch, [cfg1, cfg2])
+
+        self.assertEqual(get_dir.call_args_list, [])
+        self.assertEqual(
+            get_file.call_args_list,
+            [
+                mock.call(
+                    cfg1,
+                    os.path.join(self.dir, "legalese"),
+                    logger=self.compose._logger,
+                ),
+                mock.call(cfg2, self.dir, logger=self.compose._logger),
+            ],
+        )
+        self.assertEqual(
+            write_extra.call_args_list,
+            [
+                mock.call(
+                    self.dir,
+                    ["legalese/GPL", "setup.py"],
+                    self.compose.conf["media_checksums"],
+                    logger=self.compose._logger,
+                )
+            ],
+        )
 
 
 @mock.patch("pungi.phases.extra_isos.tweak_treeinfo")