Download extra files from container registry

This could be useful for handling flatpak applications in the installer.

All of the specified containers are downloaded into a single oci
layout.

JIRA: RHELCMP-14302
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
(cherry picked from commit 3d5348a6728b4d01cf8770494902e64c99e21a14)

doc/scm_support.rst

@@ -18,6 +18,7 @@ which can contain following keys.
   * ``cvs`` -- copies files from a CVS repository
   * ``rpm`` -- copies files from a package in the compose
   * ``koji`` -- downloads archives from a given build in Koji build system
+  * ``container-image`` -- downloads an artifact from a container registry
 
 * ``repo``
 
@@ -85,6 +86,24 @@ For ``extra_files`` phase either key is valid and should be chosen depending on
 what the actual use case.
 
 
+``container-image`` example
+---------------------------
+
+Example of pulling a container image into the compose. ::
+
+    {
+        # Pull a container into an oci-archive tar file
+        "scm": "container-image",
+        # This is the pull spec including tag. It is passed directly to skopeo
+        # copy with no modification.
+        "repo": "docker://registry.access.redhat.com/ubi9/ubi-minimal:latest",
+        # Key `file` is required, but the value is ignored.
+        "file": "",
+        # Optional subdirectory under Server/<arch>/os
+        "target": "containers",
+    }
+
+
 Caveats
 -------
 

pungi/phases/extra_files.py

@@ -112,7 +112,7 @@ def copy_extra_files(
         target_path = os.path.join(
             extra_files_dir, scm_dict.get("target", "").lstrip("/")
         )
-        getter(scm_dict, target_path, compose=compose)
+        getter(scm_dict, target_path, compose=compose, arch=arch)
 
     if os.listdir(extra_files_dir):
         metadata.populate_extra_files_metadata(

pungi/wrappers/scm.py

@@ -78,7 +78,7 @@ class FileWrapper(ScmBase):
         for i in dirs:
             copy_all(i, target_dir)
 
-    def export_file(self, scm_root, scm_file, target_dir, scm_branch=None):
+    def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, arch=None):
         if scm_root:
             raise ValueError("FileWrapper: 'scm_root' should be empty.")
         self.log_debug(
@@ -117,7 +117,7 @@ class CvsWrapper(ScmBase):
             )
             copy_all(os.path.join(tmp_dir, scm_dir), target_dir)
 
-    def export_file(self, scm_root, scm_file, target_dir, scm_branch=None):
+    def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, arch=None):
         scm_file = scm_file.lstrip("/")
         scm_branch = scm_branch or "HEAD"
         with temp_dir() as tmp_dir:
@@ -243,7 +243,7 @@ class GitWrapper(ScmBase):
 
             copy_all(os.path.join(tmp_dir, scm_dir), target_dir)
 
-    def export_file(self, scm_root, scm_file, target_dir, scm_branch=None):
+    def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, arch=None):
         scm_file = scm_file.lstrip("/")
         scm_branch = scm_branch or "master"
 
@@ -289,7 +289,7 @@ class RpmScmWrapper(ScmBase):
                         )
                     )
 
-    def export_file(self, scm_root, scm_file, target_dir, scm_branch=None):
+    def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, arch=None):
         for rpm in self._list_rpms(scm_root):
             scm_file = scm_file.lstrip("/")
             with temp_dir() as tmp_dir:
@@ -314,7 +314,7 @@ class KojiScmWrapper(ScmBase):
     def export_dir(self, *args, **kwargs):
         raise RuntimeError("Only files can be exported from Koji")
 
-    def export_file(self, scm_root, scm_file, target_dir, scm_branch=None):
+    def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, arch=None):
         if scm_branch:
             self._get_latest_from_tag(scm_branch, scm_root, scm_file, target_dir)
         else:
@@ -351,6 +351,26 @@ class KojiScmWrapper(ScmBase):
             urlretrieve(url, target_file)
 
 
+class ContainerImageScmWrapper(ScmBase):
+
+    def export_dir(self, *args, **kwargs):
+        raise RuntimeError("Containers can only be exported as files")
+
+    def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, arch=None):
+        ARCHES = {"aarch64": "arm64", "x86_64": "amd64"}
+        arch = ARCHES.get(arch, arch)
+        cmd = [
+            "skopeo",
+            "--override-arch=" + arch,
+            "copy",
+            scm_root,
+            "oci:" + target_dir,
+            "--remove-signatures",
+        ]
+        self.log_debug("Exporting container %s to %s: %s", scm_root, target_dir, cmd)
+        run(cmd, can_fail=False)
+
+
 def _get_wrapper(scm_type, *args, **kwargs):
     SCM_WRAPPERS = {
         "file": FileWrapper,
@@ -358,6 +378,7 @@ def _get_wrapper(scm_type, *args, **kwargs):
         "git": GitWrapper,
         "rpm": RpmScmWrapper,
         "koji": KojiScmWrapper,
+        "container-image": ContainerImageScmWrapper,
     }
     try:
         cls = SCM_WRAPPERS[scm_type]
@@ -366,7 +387,7 @@ def _get_wrapper(scm_type, *args, **kwargs):
     return cls(*args, **kwargs)
 
 
-def get_file_from_scm(scm_dict, target_path, compose=None):
+def get_file_from_scm(scm_dict, target_path, compose=None, arch=None):
     """
     Copy one or more files from source control to a target path. A list of files
     created in ``target_path`` is returned.
@@ -420,8 +441,18 @@ def get_file_from_scm(scm_dict, target_path, compose=None):
     files_copied = []
     for i in force_list(scm_file):
         with temp_dir(prefix="scm_checkout_") as tmp_dir:
-            scm.export_file(scm_repo, i, scm_branch=scm_branch, target_dir=tmp_dir)
+            # Most SCM wrappers need a temporary directory: the git repo is
-            files_copied += copy_all(tmp_dir, target_path)
+            # cloned there, and only relevant files are copied out. But this
+            # doesn't work for the container image fetching. That pulls in only
+            # required files, and the final output needs to be done by skopeo
+            # to correctly handle multiple containers landing in the same OCI
+            # archive.
+            dest = target_path if scm_type == "container-image" else tmp_dir
+            scm.export_file(
+                scm_repo, i, scm_branch=scm_branch, target_dir=dest, arch=arch
+            )
+            if dest == tmp_dir:
+                files_copied += copy_all(tmp_dir, target_path)
     return files_copied
 
 
@@ -460,7 +491,7 @@ def get_file(source, destination, compose, overwrite=False):
     return destination
 
 
-def get_dir_from_scm(scm_dict, target_path, compose=None):
+def get_dir_from_scm(scm_dict, target_path, compose=None, arch=None):
     """
     Copy a directory from source control to a target path. A list of files
     created in ``target_path`` is returned.

tests/test_extra_files_phase.py

@@ -223,7 +223,7 @@ class TestCopyFiles(helpers.PungiTestCase):
             )
         )
 
-    def fake_get_file(self, scm_dict, dest, compose):
+    def fake_get_file(self, scm_dict, dest, compose, arch=None):
         self.scm_dict = scm_dict
         helpers.touch(os.path.join(dest, scm_dict["file"]))
         return [scm_dict["file"]]

tests/test_scm.py

@@ -799,3 +799,72 @@ class KojiSCMTestCase(SCMBaseTest):
             dl.call_args_list,
             [mock.call("http://koji.local/koji/images/abc.tar", mock.ANY)],
         )
+
+
+IMAGE_URL = "example.com/image"
+
+
+class ContainerImageScmWrapperTest(SCMBaseTest):
+    def test_get_dir_is_not_implemented(self):
+        with self.assertRaises(RuntimeError):
+            scm.get_dir_from_scm(
+                {"scm": "container-image", "repo": IMAGE_URL, "dir": ""}, self.destdir
+            )
+
+    @parameterized.expand(
+        [
+            ("x86_64", "amd64"),
+            ("aarch64", "arm64"),
+            ("s390x", "s390x"),
+        ]
+    )
+    @mock.patch("pungi.wrappers.scm.run")
+    def test_get_file(self, real_arch, translated_arch, mock_run):
+        scm.get_file_from_scm(
+            {
+                "scm": "container-image",
+                "repo": IMAGE_URL + ":latest",
+                "file": "",
+                "target": "subdir",
+            },
+            self.destdir,
+            arch=real_arch,
+        )
+        scm.get_file_from_scm(
+            {
+                "scm": "container-image",
+                "repo": IMAGE_URL + ":prev",
+                "file": "",
+                "target": "subdir",
+            },
+            self.destdir,
+            arch=real_arch,
+        )
+
+        self.assertCountEqual(
+            mock_run.mock_calls,
+            [
+                mock.call(
+                    [
+                        "skopeo",
+                        f"--override-arch={translated_arch}",
+                        "copy",
+                        IMAGE_URL + ":latest",
+                        f"oci:{self.destdir}",
+                        "--remove-signatures",
+                    ],
+                    can_fail=False,
+                ),
+                mock.call(
+                    [
+                        "skopeo",
+                        f"--override-arch={translated_arch}",
+                        "copy",
+                        IMAGE_URL + ":prev",
+                        f"oci:{self.destdir}",
+                        "--remove-signatures",
+                    ],
+                    can_fail=False,
+                ),
+            ],
+        )