almalinux/pungi
15
2
Fork 0
Code Issues Pull Requests Releases Activity

Download extra files from container registry

Browse Source 
This could be useful for handling flatpak applications in the installer.

All of the specified containers are downloaded into a single oci
layout.

JIRA: RHELCMP-14302
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
(cherry picked from commit 3d5348a6728b4d01cf8770494902e64c99e21a14)
This commit is contained in:
Lubomír Sedlář 2024-10-24 17:10:16 +02:00 committed by Stepan Oksanichenko
parent e550458c9f
commit ed0713c572
5 changed files with 130 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
doc/scm_support.rst
View File

@ -18,6 +18,7 @@ which can contain following keys.
* ``cvs`` -- copies files from a CVS repository
* ``rpm`` -- copies files from a package in the compose
* ``koji`` -- downloads archives from a given build in Koji build system
* ``container-image`` -- downloads an artifact from a container registry
* ``repo``
@ -85,6 +86,24 @@ For ``extra_files`` phase either key is valid and should be chosen depending on
what the actual use case.
``container-image`` example
---------------------------
Example of pulling a container image into the compose. ::
{
# Pull a container into an oci-archive tar file
"scm": "container-image",
# This is the pull spec including tag. It is passed directly to skopeo
# copy with no modification.
"repo": "docker://registry.access.redhat.com/ubi9/ubi-minimal:latest",
# Key `file` is required, but the value is ignored.
"file": "",
# Optional subdirectory under Server/<arch>/os
"target": "containers",
}
Caveats
-------

2
pungi/phases/extra_files.py
View File

@ -112,7 +112,7 @@ def copy_extra_files(
target_path = os.path.join(
extra_files_dir, scm_dict.get("target", "").lstrip("/")
)
getter(scm_dict, target_path, compose=compose)
getter(scm_dict, target_path, compose=compose, arch=arch)
if os.listdir(extra_files_dir):
metadata.populate_extra_files_metadata(

49
pungi/wrappers/scm.py
View File

@ -78,7 +78,7 @@ class FileWrapper(ScmBase):
for i in dirs:
copy_all(i, target_dir)
def export_file(self, scm_root, scm_file, target_dir, scm_branch=None):
def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, arch=None):
if scm_root:
raise ValueError("FileWrapper: 'scm_root' should be empty.")
self.log_debug(
@ -117,7 +117,7 @@ class CvsWrapper(ScmBase):
)
copy_all(os.path.join(tmp_dir, scm_dir), target_dir)
def export_file(self, scm_root, scm_file, target_dir, scm_branch=None):
def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, arch=None):
scm_file = scm_file.lstrip("/")
scm_branch = scm_branch or "HEAD"
with temp_dir() as tmp_dir:
@ -243,7 +243,7 @@ class GitWrapper(ScmBase):
copy_all(os.path.join(tmp_dir, scm_dir), target_dir)
def export_file(self, scm_root, scm_file, target_dir, scm_branch=None):
def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, arch=None):
scm_file = scm_file.lstrip("/")
scm_branch = scm_branch or "master"
@ -289,7 +289,7 @@ class RpmScmWrapper(ScmBase):
)
)
def export_file(self, scm_root, scm_file, target_dir, scm_branch=None):
def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, arch=None):
for rpm in self._list_rpms(scm_root):
scm_file = scm_file.lstrip("/")
with temp_dir() as tmp_dir:
@ -314,7 +314,7 @@ class KojiScmWrapper(ScmBase):
def export_dir(self, *args, **kwargs):
raise RuntimeError("Only files can be exported from Koji")
def export_file(self, scm_root, scm_file, target_dir, scm_branch=None):
def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, arch=None):
if scm_branch:
self._get_latest_from_tag(scm_branch, scm_root, scm_file, target_dir)
else:
@ -351,6 +351,26 @@ class KojiScmWrapper(ScmBase):
urlretrieve(url, target_file)
class ContainerImageScmWrapper(ScmBase):
def export_dir(self, *args, **kwargs):
raise RuntimeError("Containers can only be exported as files")
def export_file(self, scm_root, scm_file, target_dir, scm_branch=None, arch=None):
ARCHES = {"aarch64": "arm64", "x86_64": "amd64"}
arch = ARCHES.get(arch, arch)
cmd = [
"skopeo",
"--override-arch=" + arch,
"copy",
scm_root,
"oci:" + target_dir,
"--remove-signatures",
]
self.log_debug("Exporting container %s to %s: %s", scm_root, target_dir, cmd)
run(cmd, can_fail=False)
def _get_wrapper(scm_type, *args, **kwargs):
SCM_WRAPPERS = {
"file": FileWrapper,
@ -358,6 +378,7 @@ def _get_wrapper(scm_type, *args, **kwargs):
"git": GitWrapper,
"rpm": RpmScmWrapper,
"koji": KojiScmWrapper,
"container-image": ContainerImageScmWrapper,
}
try:
cls = SCM_WRAPPERS[scm_type]
@ -366,7 +387,7 @@ def _get_wrapper(scm_type, *args, **kwargs):
return cls(*args, **kwargs)
def get_file_from_scm(scm_dict, target_path, compose=None):
def get_file_from_scm(scm_dict, target_path, compose=None, arch=None):
"""
Copy one or more files from source control to a target path. A list of files
created in ``target_path`` is returned.
@ -420,8 +441,18 @@ def get_file_from_scm(scm_dict, target_path, compose=None):
files_copied = []
for i in force_list(scm_file):
with temp_dir(prefix="scm_checkout_") as tmp_dir:
scm.export_file(scm_repo, i, scm_branch=scm_branch, target_dir=tmp_dir)
files_copied += copy_all(tmp_dir, target_path)
# Most SCM wrappers need a temporary directory: the git repo is
# cloned there, and only relevant files are copied out. But this
# doesn't work for the container image fetching. That pulls in only
# required files, and the final output needs to be done by skopeo
# to correctly handle multiple containers landing in the same OCI
# archive.
dest = target_path if scm_type == "container-image" else tmp_dir
scm.export_file(
scm_repo, i, scm_branch=scm_branch, target_dir=dest, arch=arch
)
if dest == tmp_dir:
files_copied += copy_all(tmp_dir, target_path)
return files_copied
@ -460,7 +491,7 @@ def get_file(source, destination, compose, overwrite=False):
return destination
def get_dir_from_scm(scm_dict, target_path, compose=None):
def get_dir_from_scm(scm_dict, target_path, compose=None, arch=None):
"""
Copy a directory from source control to a target path. A list of files
created in ``target_path`` is returned.

2
tests/test_extra_files_phase.py
View File

@ -223,7 +223,7 @@ class TestCopyFiles(helpers.PungiTestCase):
)
)
def fake_get_file(self, scm_dict, dest, compose):
def fake_get_file(self, scm_dict, dest, compose, arch=None):
self.scm_dict = scm_dict
helpers.touch(os.path.join(dest, scm_dict["file"]))
return [scm_dict["file"]]

69
tests/test_scm.py
View File

@ -799,3 +799,72 @@ class KojiSCMTestCase(SCMBaseTest):
dl.call_args_list,
[mock.call("http://koji.local/koji/images/abc.tar", mock.ANY)],
)
IMAGE_URL = "example.com/image"
class ContainerImageScmWrapperTest(SCMBaseTest):
def test_get_dir_is_not_implemented(self):
with self.assertRaises(RuntimeError):
scm.get_dir_from_scm(
{"scm": "container-image", "repo": IMAGE_URL, "dir": ""}, self.destdir
)
@parameterized.expand(
[
("x86_64", "amd64"),
("aarch64", "arm64"),
("s390x", "s390x"),
]
)
@mock.patch("pungi.wrappers.scm.run")
def test_get_file(self, real_arch, translated_arch, mock_run):
scm.get_file_from_scm(
{
"scm": "container-image",
"repo": IMAGE_URL + ":latest",
"file": "",
"target": "subdir",
},
self.destdir,
arch=real_arch,
)
scm.get_file_from_scm(
{
"scm": "container-image",
"repo": IMAGE_URL + ":prev",
"file": "",
"target": "subdir",
},
self.destdir,
arch=real_arch,
)
self.assertCountEqual(
mock_run.mock_calls,
[
mock.call(
[
"skopeo",
f"--override-arch={translated_arch}",
"copy",
IMAGE_URL + ":latest",
f"oci:{self.destdir}",
"--remove-signatures",
],
can_fail=False,
),
mock.call(
[
"skopeo",
f"--override-arch={translated_arch}",
"copy",
IMAGE_URL + ":prev",
f"oci:{self.destdir}",
"--remove-signatures",
],
can_fail=False,
),
],
)