From ed0713c5722caa9fa349827feabdab1a1787007e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 24 Oct 2024 17:10:16 +0200 Subject: [PATCH] Download extra files from container registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This could be useful for handling flatpak applications in the installer. All of the specified containers are downloaded into a single oci layout. JIRA: RHELCMP-14302 Signed-off-by: Lubomír Sedlář (cherry picked from commit 3d5348a6728b4d01cf8770494902e64c99e21a14) --- doc/scm_support.rst | 19 +++++++++ pungi/phases/extra_files.py | 2 +- pungi/wrappers/scm.py | 49 ++++++++++++++++++----- tests/test_extra_files_phase.py | 2 +- tests/test_scm.py | 69 +++++++++++++++++++++++++++++++++ 5 files changed, 130 insertions(+), 11 deletions(-) diff --git a/doc/scm_support.rst b/doc/scm_support.rst index d86d6f67..7496547d 100644 --- a/doc/scm_support.rst +++ b/doc/scm_support.rst @@ -18,6 +18,7 @@ which can contain following keys. * ``cvs`` -- copies files from a CVS repository * ``rpm`` -- copies files from a package in the compose * ``koji`` -- downloads archives from a given build in Koji build system + * ``container-image`` -- downloads an artifact from a container registry * ``repo`` @@ -85,6 +86,24 @@ For ``extra_files`` phase either key is valid and should be chosen depending on what the actual use case. +``container-image`` example +--------------------------- + +Example of pulling a container image into the compose. :: + + { + # Pull a container into an oci-archive tar file + "scm": "container-image", + # This is the pull spec including tag. It is passed directly to skopeo + # copy with no modification. + "repo": "docker://registry.access.redhat.com/ubi9/ubi-minimal:latest", + # Key `file` is required, but the value is ignored. + "file": "", + # Optional subdirectory under Server//os + "target": "containers", + } + + Caveats ------- diff --git a/pungi/phases/extra_files.py b/pungi/phases/extra_files.py index f30880c3..833dc712 100644 --- a/pungi/phases/extra_files.py +++ b/pungi/phases/extra_files.py @@ -112,7 +112,7 @@ def copy_extra_files( target_path = os.path.join( extra_files_dir, scm_dict.get("target", "").lstrip("/") ) - getter(scm_dict, target_path, compose=compose) + getter(scm_dict, target_path, compose=compose, arch=arch) if os.listdir(extra_files_dir): metadata.populate_extra_files_metadata( diff --git a/pungi/wrappers/scm.py b/pungi/wrappers/scm.py index 90bc14d1..9b4bc994 100644 --- a/pungi/wrappers/scm.py +++ b/pungi/wrappers/scm.py @@ -78,7 +78,7 @@ class FileWrapper(ScmBase): for i in dirs: copy_all(i, target_dir) - def export_file(self, scm_root, scm_file, target_dir, scm_branch=None): + def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, arch=None): if scm_root: raise ValueError("FileWrapper: 'scm_root' should be empty.") self.log_debug( @@ -117,7 +117,7 @@ class CvsWrapper(ScmBase): ) copy_all(os.path.join(tmp_dir, scm_dir), target_dir) - def export_file(self, scm_root, scm_file, target_dir, scm_branch=None): + def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, arch=None): scm_file = scm_file.lstrip("/") scm_branch = scm_branch or "HEAD" with temp_dir() as tmp_dir: @@ -243,7 +243,7 @@ class GitWrapper(ScmBase): copy_all(os.path.join(tmp_dir, scm_dir), target_dir) - def export_file(self, scm_root, scm_file, target_dir, scm_branch=None): + def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, arch=None): scm_file = scm_file.lstrip("/") scm_branch = scm_branch or "master" @@ -289,7 +289,7 @@ class RpmScmWrapper(ScmBase): ) ) - def export_file(self, scm_root, scm_file, target_dir, scm_branch=None): + def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, arch=None): for rpm in self._list_rpms(scm_root): scm_file = scm_file.lstrip("/") with temp_dir() as tmp_dir: @@ -314,7 +314,7 @@ class KojiScmWrapper(ScmBase): def export_dir(self, *args, **kwargs): raise RuntimeError("Only files can be exported from Koji") - def export_file(self, scm_root, scm_file, target_dir, scm_branch=None): + def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, arch=None): if scm_branch: self._get_latest_from_tag(scm_branch, scm_root, scm_file, target_dir) else: @@ -351,6 +351,26 @@ class KojiScmWrapper(ScmBase): urlretrieve(url, target_file) +class ContainerImageScmWrapper(ScmBase): + + def export_dir(self, *args, **kwargs): + raise RuntimeError("Containers can only be exported as files") + + def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, arch=None): + ARCHES = {"aarch64": "arm64", "x86_64": "amd64"} + arch = ARCHES.get(arch, arch) + cmd = [ + "skopeo", + "--override-arch=" + arch, + "copy", + scm_root, + "oci:" + target_dir, + "--remove-signatures", + ] + self.log_debug("Exporting container %s to %s: %s", scm_root, target_dir, cmd) + run(cmd, can_fail=False) + + def _get_wrapper(scm_type, *args, **kwargs): SCM_WRAPPERS = { "file": FileWrapper, @@ -358,6 +378,7 @@ def _get_wrapper(scm_type, *args, **kwargs): "git": GitWrapper, "rpm": RpmScmWrapper, "koji": KojiScmWrapper, + "container-image": ContainerImageScmWrapper, } try: cls = SCM_WRAPPERS[scm_type] @@ -366,7 +387,7 @@ def _get_wrapper(scm_type, *args, **kwargs): return cls(*args, **kwargs) -def get_file_from_scm(scm_dict, target_path, compose=None): +def get_file_from_scm(scm_dict, target_path, compose=None, arch=None): """ Copy one or more files from source control to a target path. A list of files created in ``target_path`` is returned. @@ -420,8 +441,18 @@ def get_file_from_scm(scm_dict, target_path, compose=None): files_copied = [] for i in force_list(scm_file): with temp_dir(prefix="scm_checkout_") as tmp_dir: - scm.export_file(scm_repo, i, scm_branch=scm_branch, target_dir=tmp_dir) - files_copied += copy_all(tmp_dir, target_path) + # Most SCM wrappers need a temporary directory: the git repo is + # cloned there, and only relevant files are copied out. But this + # doesn't work for the container image fetching. That pulls in only + # required files, and the final output needs to be done by skopeo + # to correctly handle multiple containers landing in the same OCI + # archive. + dest = target_path if scm_type == "container-image" else tmp_dir + scm.export_file( + scm_repo, i, scm_branch=scm_branch, target_dir=dest, arch=arch + ) + if dest == tmp_dir: + files_copied += copy_all(tmp_dir, target_path) return files_copied @@ -460,7 +491,7 @@ def get_file(source, destination, compose, overwrite=False): return destination -def get_dir_from_scm(scm_dict, target_path, compose=None): +def get_dir_from_scm(scm_dict, target_path, compose=None, arch=None): """ Copy a directory from source control to a target path. A list of files created in ``target_path`` is returned. diff --git a/tests/test_extra_files_phase.py b/tests/test_extra_files_phase.py index 8cb0b6d9..2f35fb8b 100644 --- a/tests/test_extra_files_phase.py +++ b/tests/test_extra_files_phase.py @@ -223,7 +223,7 @@ class TestCopyFiles(helpers.PungiTestCase): ) ) - def fake_get_file(self, scm_dict, dest, compose): + def fake_get_file(self, scm_dict, dest, compose, arch=None): self.scm_dict = scm_dict helpers.touch(os.path.join(dest, scm_dict["file"])) return [scm_dict["file"]] diff --git a/tests/test_scm.py b/tests/test_scm.py index 534492da..a6e8e156 100644 --- a/tests/test_scm.py +++ b/tests/test_scm.py @@ -799,3 +799,72 @@ class KojiSCMTestCase(SCMBaseTest): dl.call_args_list, [mock.call("http://koji.local/koji/images/abc.tar", mock.ANY)], ) + + +IMAGE_URL = "example.com/image" + + +class ContainerImageScmWrapperTest(SCMBaseTest): + def test_get_dir_is_not_implemented(self): + with self.assertRaises(RuntimeError): + scm.get_dir_from_scm( + {"scm": "container-image", "repo": IMAGE_URL, "dir": ""}, self.destdir + ) + + @parameterized.expand( + [ + ("x86_64", "amd64"), + ("aarch64", "arm64"), + ("s390x", "s390x"), + ] + ) + @mock.patch("pungi.wrappers.scm.run") + def test_get_file(self, real_arch, translated_arch, mock_run): + scm.get_file_from_scm( + { + "scm": "container-image", + "repo": IMAGE_URL + ":latest", + "file": "", + "target": "subdir", + }, + self.destdir, + arch=real_arch, + ) + scm.get_file_from_scm( + { + "scm": "container-image", + "repo": IMAGE_URL + ":prev", + "file": "", + "target": "subdir", + }, + self.destdir, + arch=real_arch, + ) + + self.assertCountEqual( + mock_run.mock_calls, + [ + mock.call( + [ + "skopeo", + f"--override-arch={translated_arch}", + "copy", + IMAGE_URL + ":latest", + f"oci:{self.destdir}", + "--remove-signatures", + ], + can_fail=False, + ), + mock.call( + [ + "skopeo", + f"--override-arch={translated_arch}", + "copy", + IMAGE_URL + ":prev", + f"oci:{self.destdir}", + "--remove-signatures", + ], + can_fail=False, + ), + ], + )