From 74c8d6ee0bf170fbd4accda2d422afde14712408 Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Tue, 29 Oct 2024 17:16:08 -0400 Subject: [PATCH 01/02] Don't return a VarRefInfoRef from flatpak_remote_state_lookup_ref() The memory management of returning a VarRefInfoRef is tricky - it points to data owned by the summary or the sideload repo. External consumers were always retrieving a copy of the summary metadata, so make the public function do that. --- common/flatpak-dir-private.h | 14 ++++----- common/flatpak-dir.c | 61 +++++++++++++++++++++++++++++------- common/flatpak-transaction.c | 14 +++------ 3 files changed, 61 insertions(+), 28 deletions(-) diff --git a/common/flatpak-dir-private.h b/common/flatpak-dir-private.h index 539476b8..3456660b 100644 --- a/common/flatpak-dir-private.h +++ b/common/flatpak-dir-private.h @@ -126,13 +126,13 @@ gboolean flatpak_remote_state_ensure_subsummary_all_arches (FlatpakRemoteState * GError **error); gboolean flatpak_remote_state_allow_ref (FlatpakRemoteState *self, const char *ref); -gboolean flatpak_remote_state_lookup_ref (FlatpakRemoteState *self, - const char *ref, - char **out_checksum, - guint64 *out_timestamp, - VarRefInfoRef *out_info, - GFile **out_sideload_path, - GError **error); +gboolean flatpak_remote_state_lookup_ref (FlatpakRemoteState *self, + const char *ref, + char **out_checksum, + guint64 *out_timestamp, + GVariant **out_summary_metadata, + GFile **out_sideload_path, + GError **error); GPtrArray *flatpak_remote_state_match_subrefs (FlatpakRemoteState *self, FlatpakDecomposed *ref); GFile *flatpak_remote_state_lookup_sideload_checksum (FlatpakRemoteState *self, diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index 9f398252..99889aa5 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -755,14 +755,14 @@ get_summary_for_ref (FlatpakRemoteState *self, /* Returns TRUE if the ref is found in the summary or cache. * out_checksum and out_variant are only set when the ref is found. */ -gboolean -flatpak_remote_state_lookup_ref (FlatpakRemoteState *self, - const char *ref, - char **out_checksum, - guint64 *out_timestamp, - VarRefInfoRef *out_info, - GFile **out_sideload_path, - GError **error) +static gboolean +flatpak_remote_state_lookup_ref_internal (FlatpakRemoteState *self, + const char *ref, + char **out_checksum, + guint64 *out_timestamp, + VarRefInfoRef *out_info, + GFile **out_sideload_path, + GError **error) { if (!flatpak_remote_state_allow_ref (self, ref)) { @@ -822,12 +822,50 @@ flatpak_remote_state_lookup_ref (FlatpakRemoteState *self, return FALSE; if (out_sideload_path) - *out_sideload_path = g_object_ref (ostree_repo_get_path (ss->repo)); + { + if (ss) + *out_sideload_path = g_object_ref (ostree_repo_get_path (ss->repo)); + else + *out_sideload_path = NULL; + } } return TRUE; } +/* Returns TRUE if the ref is found in the summary or cache. + * out parameters are only set if the ref is found. + */ +gboolean +flatpak_remote_state_lookup_ref (FlatpakRemoteState *self, + const char *ref, + char **out_checksum, + guint64 *out_timestamp, + GVariant **out_summary_metadata, + GFile **out_sideload_path, + GError **error) +{ + + if (out_summary_metadata) + { + VarRefInfoRef info; + + if (flatpak_remote_state_lookup_ref_internal (self, ref, out_checksum, out_timestamp, + &info, out_sideload_path, error)) + { + *out_summary_metadata = var_metadata_dup_to_gvariant (var_ref_info_get_metadata (info)); + return TRUE; + } + else + return FALSE; + } + else + { + return flatpak_remote_state_lookup_ref_internal (self, ref, out_checksum, out_timestamp, + NULL, out_sideload_path, error); + } +} + GPtrArray * flatpak_remote_state_match_subrefs (FlatpakRemoteState *self, FlatpakDecomposed *ref) @@ -995,7 +1033,6 @@ flatpak_remote_state_load_data (FlatpakRemoteState *self, /* Look up from sideload */ g_autofree char *checksum = NULL; guint64 timestamp; - VarRefInfoRef info; FlatpakSideloadState *ss = NULL; g_autoptr(GVariant) commit_data = NULL; g_autoptr(GVariant) commit_metadata = NULL; @@ -1006,7 +1043,7 @@ flatpak_remote_state_load_data (FlatpakRemoteState *self, /* Use sideload refs if any */ if (!flatpak_remote_state_resolve_sideloaded_ref (self, ref, &checksum, ×tamp, - &info, &ss, error)) + NULL, &ss, error)) return FALSE; if (!ostree_repo_load_commit (ss->repo, checksum, &commit_data, NULL, error)) @@ -1095,7 +1132,7 @@ flatpak_remote_state_fetch_image_source (FlatpakRemoteState *self, const char *delta_url = NULL; /* We extract the rev info from the latest, even if we don't use the latest digest, assuming refs don't move */ - if (!flatpak_remote_state_lookup_ref (self, ref, &latest_rev, NULL, &latest_rev_info, NULL, error)) + if (!flatpak_remote_state_lookup_ref_internal (self, ref, &latest_rev, NULL, &latest_rev_info, NULL, error)) return NULL; if (latest_rev == NULL) diff --git a/common/flatpak-transaction.c b/common/flatpak-transaction.c index 1aba1daa..2dd35f53 100644 --- a/common/flatpak-transaction.c +++ b/common/flatpak-transaction.c @@ -3482,7 +3482,6 @@ try_resolve_op_from_metadata (FlatpakTransaction *self, guint64 installed_size = 0; const char *metadata = NULL; VarMetadataRef sparse_cache; - VarRefInfoRef info; g_autofree char *summary_checksum = NULL; /* Ref has to match the actual commit in the summary */ @@ -3499,9 +3498,8 @@ try_resolve_op_from_metadata (FlatpakTransaction *self, metadata_bytes = g_bytes_new (metadata, strlen (metadata)); - if (flatpak_remote_state_lookup_ref (state, flatpak_decomposed_get_ref (op->ref), - NULL, NULL, &info, NULL, NULL)) - op->summary_metadata = var_metadata_dup_to_gvariant (var_ref_info_get_metadata (info)); + flatpak_remote_state_lookup_ref (state, flatpak_decomposed_get_ref (op->ref), + NULL, NULL, &op->summary_metadata, NULL, NULL); op->installed_size = installed_size; op->download_size = download_size; @@ -3704,13 +3702,11 @@ resolve_ops (FlatpakTransaction *self, * Note, we don't have a token here, so this will not work for authenticated apps. * We handle this by catching the 401 http status and retrying. */ g_autoptr(GVariant) commit_data = NULL; - VarRefInfoRef ref_info; /* OCI needs this to get the oci repository for the ref to request the token, so lets always set it here */ - if (op->summary_metadata == NULL && - flatpak_remote_state_lookup_ref (state, flatpak_decomposed_get_ref (op->ref), - NULL, NULL, &ref_info, NULL, NULL)) - op->summary_metadata = var_metadata_dup_to_gvariant (var_ref_info_get_metadata (ref_info)); + if (op->summary_metadata == NULL) + flatpak_remote_state_lookup_ref (state, flatpak_decomposed_get_ref (op->ref), + NULL, NULL, &op->summary_metadata, NULL, NULL); commit_data = flatpak_remote_state_load_ref_commit (state, priv->dir, flatpak_decomposed_get_ref (op->ref), -- 2.47.1 From 751944405e027daf968741a3fbeec188ccf07ab1 Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Tue, 29 Oct 2024 17:19:38 -0400 Subject: [PATCH 02/02] Support sideload repositories for OCI remotes For OCI remotes, the existing sideload repository system doesn't work: identity for OCI remotes is done by manifest digest (disguised as a fake commit ID internally), instead of by ostree commit, so we have no way of knowing whether a sideloaded image matches the summary. Allow specifying a new form of sideload repository with: --sideload-repo=oci: The desired use case for this is preinstalling Flatpaks during OS install, and for this, binding the entire repository to a single collection ID is both inconvenient and not useful, so OCI sideload repostories don't have a defined collection ID - they just apply to all OCI remotes. (And, because of this, they are restricted to the command line.) This is implemented in a straightforward way by adding, throughout the FlatpakTransaction and FlatpakDir code, in parallel, to GFile *sideload_path, FlatpakImageSource *image_source. The new FlatpakImageCollection type represents a set of FlatpakImageSource loaded from the image sideload repository. --- app/flatpak-builtins-install.c | 17 +- app/flatpak-builtins-remote-info.c | 2 +- app/flatpak-builtins-update.c | 4 +- app/flatpak-builtins-utils.c | 33 ++ app/flatpak-builtins-utils.h | 6 + common/flatpak-common-types-private.h | 13 +- common/flatpak-dir-private.h | 12 +- common/flatpak-dir.c | 362 +++++++++++++++++----- common/flatpak-image-collection-private.h | 50 +++ common/flatpak-image-collection.c | 154 +++++++++ common/flatpak-image-source-private.h | 7 + common/flatpak-image-source.c | 25 +- common/flatpak-transaction.c | 74 ++++- common/flatpak-transaction.h | 5 + common/meson.build | 1 + doc/flatpak-install.xml | 9 + doc/flatpak-update.xml | 9 + doc/reference/meson.build | 1 + 18 files changed, 667 insertions(+), 117 deletions(-) create mode 100644 common/flatpak-image-collection-private.h create mode 100644 common/flatpak-image-collection.c diff --git a/app/flatpak-builtins-install.c b/app/flatpak-builtins-install.c index e1f7f312..80a43152 100644 --- a/app/flatpak-builtins-install.c +++ b/app/flatpak-builtins-install.c @@ -132,8 +132,9 @@ read_gpg_data (GCancellable *cancellable, } static FlatpakTransaction * -create_install_transaction (FlatpakDir *dir, - GError **error) +create_install_transaction (FlatpakDir *dir, + GCancellable *cancellable, + GError **error) { g_autoptr(FlatpakTransaction) transaction = NULL; @@ -155,8 +156,8 @@ create_install_transaction (FlatpakDir *dir, flatpak_transaction_set_auto_install_sdk (transaction, opt_include_sdk); flatpak_transaction_set_auto_install_debug (transaction, opt_include_debug); - for (int i = 0; opt_sideload_repos != NULL && opt_sideload_repos[i] != NULL; i++) - flatpak_transaction_add_sideload_repo (transaction, opt_sideload_repos[i]); + if (!setup_sideload_repositories (transaction, opt_sideload_repos, cancellable, error)) + return FALSE; return g_steal_pointer (&transaction); } @@ -194,7 +195,7 @@ install_bundle (FlatpakDir *dir, return FALSE; } - transaction = create_install_transaction (dir, error); + transaction = create_install_transaction (dir, cancellable, error); if (transaction == NULL) return FALSE; @@ -257,7 +258,7 @@ install_from (FlatpakDir *dir, file_data = g_bytes_new_take (g_steal_pointer (&data), data_len); } - transaction = create_install_transaction (dir, error); + transaction = create_install_transaction (dir, cancellable, error); if (transaction == NULL) return FALSE; @@ -303,7 +304,7 @@ install_image (FlatpakDir *dir, return FALSE; } - transaction = create_install_transaction (dir, error); + transaction = create_install_transaction (dir, cancellable, error); if (transaction == NULL) return FALSE; @@ -537,7 +538,7 @@ flatpak_builtin_install (int argc, char **argv, GCancellable *cancellable, GErro default_branch = flatpak_dir_get_remote_default_branch (dir, remote); - transaction = create_install_transaction (dir, error); + transaction = create_install_transaction (dir, cancellable, error); if (transaction == NULL) return FALSE; diff --git a/app/flatpak-builtins-remote-info.c b/app/flatpak-builtins-remote-info.c index 8dee2af7..89fd5c83 100644 --- a/app/flatpak-builtins-remote-info.c +++ b/app/flatpak-builtins-remote-info.c @@ -151,7 +151,7 @@ flatpak_builtin_remote_info (int argc, char **argv, GCancellable *cancellable, G if (opt_commit) commit = g_strdup (opt_commit); else if (!flatpak_remote_state_lookup_ref (state, flatpak_decomposed_get_ref (ref), - &commit, NULL, NULL, NULL, error)) + &commit, NULL, NULL, NULL, NULL, error)) { g_assert (error == NULL || *error != NULL); return FALSE; diff --git a/app/flatpak-builtins-update.c b/app/flatpak-builtins-update.c index b5184a33..5465dc8c 100644 --- a/app/flatpak-builtins-update.c +++ b/app/flatpak-builtins-update.c @@ -150,8 +150,8 @@ flatpak_builtin_update (int argc, if (opt_arch) flatpak_transaction_set_default_arch (transaction, opt_arch); - for (i = 0; opt_sideload_repos != NULL && opt_sideload_repos[i] != NULL; i++) - flatpak_transaction_add_sideload_repo (transaction, opt_sideload_repos[i]); + if (!setup_sideload_repositories (transaction, opt_sideload_repos, cancellable, error)) + return FALSE; g_ptr_array_insert (transactions, 0, transaction); } diff --git a/app/flatpak-builtins-utils.c b/app/flatpak-builtins-utils.c index e539a8c2..01c2cd05 100644 --- a/app/flatpak-builtins-utils.c +++ b/app/flatpak-builtins-utils.c @@ -1450,3 +1450,36 @@ ensure_remote_state_all_arches (FlatpakDir *dir, return TRUE; } + +gboolean +setup_sideload_repositories (FlatpakTransaction *transaction, + char **opt_sideload_repos, + GCancellable *cancellable, + GError **error) +{ + for (int i = 0; opt_sideload_repos != NULL && opt_sideload_repos[i] != NULL; i++) + { + const char *repo = opt_sideload_repos[i]; + if (g_str_has_prefix (repo, "oci:") || g_str_has_prefix (repo, "oci-archive:")) + { + if (!flatpak_transaction_add_sideload_image_collection (transaction, repo, cancellable, error)) + return FALSE; + } + else if (g_str_has_prefix (repo, "file:")) + { + g_autoptr(GFile) file = g_file_new_for_uri (repo); + const char *path = flatpak_file_get_path_cached (file); + flatpak_transaction_add_sideload_repo (transaction, path); + } + else + { + if (g_regex_match_simple ("^[A-Za-z][A-Za-z0-9+.-]*:", repo, + G_REGEX_DEFAULT, G_REGEX_MATCH_DEFAULT)) + return flatpak_fail (error, _("Unknown scheme in sideload location %s"), repo); + + flatpak_transaction_add_sideload_repo (transaction, repo); + } + } + + return TRUE; +} diff --git a/app/flatpak-builtins-utils.h b/app/flatpak-builtins-utils.h index 257a6b95..cd24abf7 100644 --- a/app/flatpak-builtins-utils.h +++ b/app/flatpak-builtins-utils.h @@ -27,6 +27,7 @@ #include "flatpak-utils-private.h" #include "flatpak-dir-private.h" #include "flatpak-permission-dbus-generated.h" +#include "flatpak-transaction.h" /* AS_CHECK_VERSION was introduced in 0.14.0; we still support 0.12.0, so * behave as though versions without this macro are arbitrarily old */ @@ -209,4 +210,9 @@ gboolean ensure_remote_state_all_arches (FlatpakDir *dir, GCancellable *cancellable, GError **error); +gboolean setup_sideload_repositories (FlatpakTransaction *transaction, + char **opt_sideload_repos, + GCancellable *cancellable, + GError **error); + #endif /* __FLATPAK_BUILTINS_UTILS_H__ */ diff --git a/common/flatpak-common-types-private.h b/common/flatpak-common-types-private.h index d7f3913b..6d41f12c 100644 --- a/common/flatpak-common-types-private.h +++ b/common/flatpak-common-types-private.h @@ -51,11 +51,12 @@ typedef enum { FLATPAK_RUN_FLAG_PARENT_SHARE_PIDS = (1 << 21), } FlatpakRunFlags; -typedef struct FlatpakDir FlatpakDir; -typedef struct FlatpakDeploy FlatpakDeploy; -typedef struct _FlatpakImageSource FlatpakImageSource; -typedef struct FlatpakOciRegistry FlatpakOciRegistry; -typedef struct _FlatpakOciManifest FlatpakOciManifest; -typedef struct _FlatpakOciImage FlatpakOciImage; +typedef struct FlatpakDir FlatpakDir; +typedef struct FlatpakDeploy FlatpakDeploy; +typedef struct FlatpakImageCollection FlatpakImageCollection; +typedef struct _FlatpakImageSource FlatpakImageSource; +typedef struct FlatpakOciRegistry FlatpakOciRegistry; +typedef struct _FlatpakOciManifest FlatpakOciManifest; +typedef struct _FlatpakOciImage FlatpakOciImage; #endif /* __FLATPAK_COMMON_TYPES_H__ */ diff --git a/common/flatpak-dir-private.h b/common/flatpak-dir-private.h index 3456660b..ffcff5ff 100644 --- a/common/flatpak-dir-private.h +++ b/common/flatpak-dir-private.h @@ -87,6 +87,7 @@ typedef struct { char *remote_name; gboolean is_file_uri; + gboolean is_oci; char *collection_id; /* New format summary */ @@ -107,6 +108,7 @@ typedef struct int refcount; gint32 default_token_type; GPtrArray *sideload_repos; + GPtrArray *sideload_image_collections; } FlatpakRemoteState; FlatpakRemoteState *flatpak_remote_state_ref (FlatpakRemoteState *remote_state); @@ -132,11 +134,14 @@ gboolean flatpak_remote_state_lookup_ref (FlatpakRemoteState *self, guint64 *out_timestamp, GVariant **out_summary_metadata, GFile **out_sideload_path, + FlatpakImageSource **out_image_Source, GError **error); GPtrArray *flatpak_remote_state_match_subrefs (FlatpakRemoteState *self, FlatpakDecomposed *ref); -GFile *flatpak_remote_state_lookup_sideload_checksum (FlatpakRemoteState *self, - char *checksum); +void flatpak_remote_state_lookup_sideload_checksum (FlatpakRemoteState *self, + char *checksum, + GFile **out_sideload_path, + FlatpakImageSource **out_image_source); gboolean flatpak_remote_state_lookup_cache (FlatpakRemoteState *self, const char *ref, guint64 *download_size, @@ -163,6 +168,8 @@ GVariant *flatpak_remote_state_load_ref_commit (FlatpakRemoteState *self, GError **error); void flatpak_remote_state_add_sideload_dir (FlatpakRemoteState *self, GFile *path); +void flatpak_remote_state_add_sideload_image_collection (FlatpakRemoteState *self, + FlatpakImageCollection *image_collection); G_DEFINE_AUTOPTR_CLEANUP_FUNC (FlatpakDir, g_object_unref) @@ -982,6 +989,7 @@ gboolean flatpak_dir_find_latest_rev (Fla char **out_rev, guint64 *out_timestamp, GFile **out_sideload_path, + FlatpakImageSource **out_image_source, GCancellable *cancellable, GError **error); FlatpakDecomposed * flatpak_dir_get_remote_auto_install_authenticator_ref (FlatpakDir *self, diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index 99889aa5..1c304525 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -52,6 +52,7 @@ #include "flatpak-dir-utils-private.h" #include "flatpak-error.h" #include "flatpak-locale-utils-private.h" +#include "flatpak-image-collection-private.h" #include "flatpak-image-source-private.h" #include "flatpak-oci-registry-private.h" #include "flatpak-ref.h" @@ -375,6 +376,7 @@ flatpak_remote_state_new (void) state->refcount = 1; state->sideload_repos = g_ptr_array_new_with_free_func ((GDestroyNotify)flatpak_sideload_state_free); + state->sideload_image_collections = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); state->subsummaries = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify)variant_maybe_unref); return state; } @@ -408,6 +410,7 @@ flatpak_remote_state_unref (FlatpakRemoteState *remote_state) g_clear_pointer (&remote_state->allow_refs, g_regex_unref); g_clear_pointer (&remote_state->deny_refs, g_regex_unref); g_clear_pointer (&remote_state->sideload_repos, g_ptr_array_unref); + g_clear_pointer (&remote_state->sideload_image_collections, g_ptr_array_unref); g_free (remote_state); } @@ -470,6 +473,13 @@ flatpak_remote_state_add_sideload_repo (FlatpakRemoteState *self, } } +void +flatpak_remote_state_add_sideload_image_collection (FlatpakRemoteState *self, + FlatpakImageCollection *image_collection) +{ + g_ptr_array_add (self->sideload_image_collections, g_object_ref (image_collection)); +} + static void add_sideload_subdirs (GPtrArray *res, GFile *parent, gboolean recurse); @@ -656,31 +666,57 @@ get_timestamp_from_ref_info (VarRefInfoRef info) } -GFile * -flatpak_remote_state_lookup_sideload_checksum (FlatpakRemoteState *self, - char *checksum) +void +flatpak_remote_state_lookup_sideload_checksum (FlatpakRemoteState *self, + char *checksum, + GFile **out_sideload_path, + FlatpakImageSource **out_image_source) { - for (int i = 0; i < self->sideload_repos->len; i++) + if (out_sideload_path) + *out_sideload_path = NULL; + if (out_image_source) + *out_image_source = NULL; + + if (self->is_oci) { - FlatpakSideloadState *ss = g_ptr_array_index (self->sideload_repos, i); - OstreeRepoCommitState commit_state; + if (out_image_source) + { + const char *digest = g_strconcat ("sha256:", checksum, NULL); - if (ostree_repo_load_commit (ss->repo, checksum, NULL, &commit_state, NULL) && - commit_state == OSTREE_REPO_COMMIT_STATE_NORMAL) - return g_object_ref (ostree_repo_get_path (ss->repo)); + for (int i = 0; i < self->sideload_image_collections->len; i++) + { + FlatpakImageCollection *collection = g_ptr_array_index (self->sideload_image_collections, i); + g_autoptr(FlatpakImageSource) image_source = flatpak_image_collection_lookup_digest (collection, digest); + if (image_source) + *out_image_source = g_steal_pointer (&image_source); + } + } } + else if (out_sideload_path) + { + for (int i = 0; i < self->sideload_repos->len; i++) + { + FlatpakSideloadState *ss = g_ptr_array_index (self->sideload_repos, i); + OstreeRepoCommitState commit_state; - return NULL; + if (ostree_repo_load_commit (ss->repo, checksum, NULL, &commit_state, NULL) && + commit_state == OSTREE_REPO_COMMIT_STATE_NORMAL) + { + *out_sideload_path = g_object_ref (ostree_repo_get_path (ss->repo)); + return; + } + } + } } static gboolean -flatpak_remote_state_resolve_sideloaded_ref (FlatpakRemoteState *self, - const char *ref, - char **out_checksum, - guint64 *out_timestamp, - VarRefInfoRef *out_info, - FlatpakSideloadState **out_sideload_state, - GError **error) +flatpak_remote_state_resolve_sideloaded_ref_repos (FlatpakRemoteState *self, + const char *ref, + char **out_checksum, + guint64 *out_timestamp, + VarRefInfoRef *out_info, + FlatpakSideloadState **out_sideload_state, + GError **error) { g_autofree char *latest_checksum = NULL; guint64 latest_timestamp = 0; @@ -725,6 +761,73 @@ flatpak_remote_state_resolve_sideloaded_ref (FlatpakRemoteState *self, return TRUE; } +static gboolean +flatpak_remote_state_resolve_sideloaded_ref_images (FlatpakRemoteState *self, + const char *ref, + char **out_checksum, + guint64 *out_timestamp, + VarRefInfoRef *out_info, + FlatpakImageSource **out_image_source, + GError **error) +{ + g_autoptr(FlatpakImageSource) image_source = NULL; + + for (int i = 0; i < self->sideload_image_collections->len; i++) + { + FlatpakImageCollection *collection = g_ptr_array_index (self->sideload_image_collections, i); + image_source = flatpak_image_collection_lookup_ref (collection, ref); + if (image_source) + break; + } + + if (image_source) + { + if (out_checksum) + { + const char *digest = flatpak_image_source_get_digest (image_source); + g_assert (g_str_has_prefix (digest, "sha256:")); + *out_checksum = g_strdup (digest + 7); + } + + if (out_timestamp) + *out_timestamp = flatpak_image_source_get_commit_timestamp (image_source); + } + + if (image_source == NULL) + return flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, + _("No such ref '%s' in remote %s"), + ref, self->remote_name); + + if (out_image_source) + *out_image_source = g_steal_pointer (&image_source); + + return TRUE; +} + +static gboolean +flatpak_remote_state_resolve_sideloaded_ref (FlatpakRemoteState *self, + const char *ref, + char **out_checksum, + guint64 *out_timestamp, + VarRefInfoRef *out_info, + FlatpakSideloadState **out_sideload_state, + FlatpakImageSource **out_image_source, + GError **error) +{ + if (out_sideload_state) + *out_sideload_state = NULL; + if (out_image_source) + *out_image_source = NULL; + + if (self->is_oci) + return flatpak_remote_state_resolve_sideloaded_ref_images (self, ref, out_checksum, out_timestamp, out_info, + out_image_source, error); + else + return flatpak_remote_state_resolve_sideloaded_ref_repos (self, ref, out_checksum, out_timestamp, out_info, + out_sideload_state, error); + +} + static GVariant * get_summary_for_ref (FlatpakRemoteState *self, const char *ref) @@ -754,6 +857,9 @@ get_summary_for_ref (FlatpakRemoteState *self, /* Returns TRUE if the ref is found in the summary or cache. * out_checksum and out_variant are only set when the ref is found. + * + * NOTE: The _internal() variant has the odd constraint that *out_info is only + * valid if *out_image_source is NULL. */ static gboolean flatpak_remote_state_lookup_ref_internal (FlatpakRemoteState *self, @@ -762,8 +868,11 @@ flatpak_remote_state_lookup_ref_internal (FlatpakRemoteState *self, guint64 *out_timestamp, VarRefInfoRef *out_info, GFile **out_sideload_path, + FlatpakImageSource **out_image_source, GError **error) { + g_assert (out_info == NULL || out_image_source != NULL); + if (!flatpak_remote_state_allow_ref (self, ref)) { return flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, @@ -786,26 +895,7 @@ flatpak_remote_state_lookup_ref_internal (FlatpakRemoteState *self, ref, self->remote_name); /* Even if its available in the summary we want to install it from a sideload repo if available */ - - if (out_sideload_path) - { - g_autoptr(GFile) found_sideload_path = NULL; - - for (int i = 0; i < self->sideload_repos->len; i++) - { - FlatpakSideloadState *ss = g_ptr_array_index (self->sideload_repos, i); - OstreeRepoCommitState commit_state; - - if (ostree_repo_load_commit (ss->repo, checksum, NULL, &commit_state, NULL) && - commit_state == OSTREE_REPO_COMMIT_STATE_NORMAL) - { - found_sideload_path = g_object_ref (ostree_repo_get_path (ss->repo)); - break; - } - } - - *out_sideload_path = g_steal_pointer (&found_sideload_path); - } + flatpak_remote_state_lookup_sideload_checksum (self, checksum, out_sideload_path, out_image_source); if (out_info) *out_info = info; @@ -818,7 +908,7 @@ flatpak_remote_state_lookup_ref_internal (FlatpakRemoteState *self, { FlatpakSideloadState *ss = NULL; - if (!flatpak_remote_state_resolve_sideloaded_ref (self, ref, out_checksum, out_timestamp, out_info, &ss, error)) + if (!flatpak_remote_state_resolve_sideloaded_ref (self, ref, out_checksum, out_timestamp, out_info, &ss, out_image_source, error)) return FALSE; if (out_sideload_path) @@ -843,17 +933,26 @@ flatpak_remote_state_lookup_ref (FlatpakRemoteState *self, guint64 *out_timestamp, GVariant **out_summary_metadata, GFile **out_sideload_path, + FlatpakImageSource **out_image_source, GError **error) { if (out_summary_metadata) { + g_autoptr(FlatpakImageSource) local_image_source; VarRefInfoRef info; if (flatpak_remote_state_lookup_ref_internal (self, ref, out_checksum, out_timestamp, - &info, out_sideload_path, error)) + &info, out_sideload_path, &local_image_source, error)) { - *out_summary_metadata = var_metadata_dup_to_gvariant (var_ref_info_get_metadata (info)); + if (local_image_source) + *out_summary_metadata = flatpak_image_source_make_summary_metadata (local_image_source); + else + *out_summary_metadata = var_metadata_dup_to_gvariant (var_ref_info_get_metadata (info)); + + if (out_image_source) + *out_image_source = g_steal_pointer (&local_image_source); + return TRUE; } else @@ -862,7 +961,7 @@ flatpak_remote_state_lookup_ref (FlatpakRemoteState *self, else { return flatpak_remote_state_lookup_ref_internal (self, ref, out_checksum, out_timestamp, - NULL, out_sideload_path, error); + NULL, out_sideload_path, out_image_source, error); } } @@ -1034,6 +1133,7 @@ flatpak_remote_state_load_data (FlatpakRemoteState *self, g_autofree char *checksum = NULL; guint64 timestamp; FlatpakSideloadState *ss = NULL; + g_autoptr(FlatpakImageSource) image_source = NULL; g_autoptr(GVariant) commit_data = NULL; g_autoptr(GVariant) commit_metadata = NULL; const char *xa_metadata = NULL; @@ -1043,13 +1143,23 @@ flatpak_remote_state_load_data (FlatpakRemoteState *self, /* Use sideload refs if any */ if (!flatpak_remote_state_resolve_sideloaded_ref (self, ref, &checksum, ×tamp, - NULL, &ss, error)) + NULL, &ss, &image_source, error)) return FALSE; - if (!ostree_repo_load_commit (ss->repo, checksum, &commit_data, NULL, error)) - return FALSE; + if (image_source) + { + g_autoptr(GVariantBuilder) metadata_builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); + flatpak_image_source_build_commit_metadata (image_source, metadata_builder); + commit_metadata = g_variant_builder_end (metadata_builder); + } + else + { + if (!ostree_repo_load_commit (ss->repo, checksum, &commit_data, NULL, error)) + return FALSE; + + commit_metadata = g_variant_get_child_value (commit_data, 0); + } - commit_metadata = g_variant_get_child_value (commit_data, 0); g_variant_lookup (commit_metadata, "xa.metadata", "&s", &xa_metadata); if (xa_metadata == NULL) return flatpak_fail (error, "No xa.metadata in sideload commit %s ref %s", checksum, ref); @@ -1124,15 +1234,11 @@ flatpak_remote_state_fetch_image_source (FlatpakRemoteState *self, GError **error) { g_autoptr(FlatpakImageSource) image_source = NULL; - g_autofree char *oci_digest = NULL; g_autofree char *latest_rev = NULL; VarRefInfoRef latest_rev_info; - VarMetadataRef metadata; - const char *oci_repository = NULL; - const char *delta_url = NULL; /* We extract the rev info from the latest, even if we don't use the latest digest, assuming refs don't move */ - if (!flatpak_remote_state_lookup_ref_internal (self, ref, &latest_rev, NULL, &latest_rev_info, NULL, error)) + if (!flatpak_remote_state_lookup_ref_internal (self, ref, &latest_rev, NULL, &latest_rev_info, NULL, &image_source, error)) return NULL; if (latest_rev == NULL) @@ -1143,23 +1249,31 @@ flatpak_remote_state_fetch_image_source (FlatpakRemoteState *self, return NULL; } - metadata = var_ref_info_get_metadata (latest_rev_info); - oci_repository = var_metadata_lookup_string (metadata, "xa.oci-repository", NULL); - delta_url = var_metadata_lookup_string (metadata, "xa.delta-url", NULL); + if (image_source == NULL) + { + VarMetadataRef metadata; + g_autofree char *oci_digest = NULL; + const char *oci_repository = NULL; + const char *delta_url = NULL; - oci_digest = g_strconcat ("sha256:", opt_rev ? opt_rev : latest_rev, NULL); + metadata = var_ref_info_get_metadata (latest_rev_info); + oci_repository = var_metadata_lookup_string (metadata, "xa.oci-repository", NULL); + delta_url = var_metadata_lookup_string (metadata, "xa.delta-url", NULL); - image_source = flatpak_remote_state_new_image_source (self, oci_repository, oci_digest, token, cancellable, error); - if (image_source == NULL) - return NULL; + oci_digest = g_strconcat ("sha256:", opt_rev ? opt_rev : latest_rev, NULL); - if (g_strcmp0 (flatpak_image_source_get_ref (image_source), ref) != 0) - { - flatpak_fail_error (error, FLATPAK_ERROR_INVALID_DATA, _("Commit has no requested ref ‘%s’ in ref binding metadata"), ref); - return NULL; - } + image_source = flatpak_remote_state_new_image_source (self, oci_repository, oci_digest, token, cancellable, error); + if (image_source == NULL) + return NULL; + + if (g_strcmp0 (flatpak_image_source_get_ref (image_source), ref) != 0) + { + flatpak_fail_error (error, FLATPAK_ERROR_INVALID_DATA, _("Commit has no requested ref ‘%s’ in ref binding metadata"), ref); + return NULL; + } - flatpak_image_source_set_delta_url (image_source, delta_url); + flatpak_image_source_set_delta_url (image_source, delta_url); + } return g_steal_pointer (&image_source); } @@ -1299,7 +1413,7 @@ flatpak_remote_state_load_ref_commit (FlatpakRemoteState *self, if (opt_commit == NULL) { - if (!flatpak_remote_state_lookup_ref (self, ref, &commit, NULL, NULL, NULL, error)) + if (!flatpak_remote_state_lookup_ref (self, ref, &commit, NULL, NULL, NULL, NULL, error)) return NULL; if (commit == NULL) @@ -1317,12 +1431,26 @@ flatpak_remote_state_load_ref_commit (FlatpakRemoteState *self, if (ostree_repo_load_commit (dir->repo, commit, &commit_data, NULL, NULL)) goto out; - for (int i = 0; i < self->sideload_repos->len; i++) + if (self->is_oci) { - FlatpakSideloadState *ss = g_ptr_array_index (self->sideload_repos, i); + g_autoptr(FlatpakImageSource) image_source; - if (ostree_repo_load_commit (ss->repo, commit, &commit_data, NULL, NULL)) - goto out; + flatpak_remote_state_lookup_sideload_checksum (self, commit, NULL, &image_source); + if (image_source) + { + commit_data = flatpak_image_source_make_fake_commit (image_source); + goto out; + } + } + else + { + for (int i = 0; i < self->sideload_repos->len; i++) + { + FlatpakSideloadState *ss = g_ptr_array_index (self->sideload_repos, i); + + if (ostree_repo_load_commit (ss->repo, commit, &commit_data, NULL, NULL)) + goto out; + } } if (flatpak_dir_get_remote_oci (dir, self->remote_name)) @@ -5034,6 +5162,7 @@ flatpak_dir_find_latest_rev (FlatpakDir *self, char **out_rev, guint64 *out_timestamp, GFile **out_sideload_path, + FlatpakImageSource **out_image_source, GCancellable *cancellable, GError **error) { @@ -5041,7 +5170,7 @@ flatpak_dir_find_latest_rev (FlatpakDir *self, g_return_val_if_fail (out_rev != NULL, FALSE); - if (!flatpak_remote_state_lookup_ref (state, ref, &latest_rev, out_timestamp, NULL, out_sideload_path, error)) + if (!flatpak_remote_state_lookup_ref (state, ref, &latest_rev, out_timestamp, NULL, out_sideload_path, out_image_source, error)) return FALSE; if (latest_rev == NULL) return flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, @@ -5286,10 +5415,10 @@ flatpak_dir_update_appstream (FlatpakDir *self, used_branch = new_branch; if (!is_oci) { - if (!flatpak_dir_find_latest_rev (self, state, used_branch, NULL, &appstream_commit, NULL, &appstream_sideload_path, cancellable, &first_error)) + if (!flatpak_dir_find_latest_rev (self, state, used_branch, NULL, &appstream_commit, NULL, &appstream_sideload_path, NULL, cancellable, &first_error)) { used_branch = old_branch; - if (!flatpak_dir_find_latest_rev (self, state, used_branch, NULL, &appstream_commit, NULL, &appstream_sideload_path, cancellable, &second_error)) + if (!flatpak_dir_find_latest_rev (self, state, used_branch, NULL, &appstream_commit, NULL, &appstream_sideload_path, NULL, cancellable, &second_error)) { g_prefix_error (&first_error, "Error updating appstream2: "); g_prefix_error (&second_error, "Error updating appstream: "); @@ -6095,7 +6224,7 @@ flatpak_dir_pull (FlatpakDir *self, { rev = g_strdup (opt_rev); } - else if (!flatpak_remote_state_lookup_ref (state, ref, &rev, NULL, NULL, NULL, error)) + else if (!flatpak_remote_state_lookup_ref (state, ref, &rev, NULL, NULL, NULL, NULL, error)) { g_assert (error == NULL || *error != NULL); return FALSE; @@ -10489,7 +10618,7 @@ flatpak_dir_check_for_update (FlatpakDir *self, else { if (!flatpak_dir_find_latest_rev (self, state, flatpak_decomposed_get_ref (ref), checksum_or_latest, &latest_rev, - NULL, NULL, cancellable, error)) + NULL, NULL, NULL, cancellable, error)) return NULL; } @@ -12771,6 +12900,7 @@ _flatpak_dir_get_remote_state (FlatpakDir *self, if (!ostree_repo_remote_get_url (self->repo, remote_or_uri, &url, error)) return NULL; + state->is_oci = flatpak_dir_get_remote_oci (self, remote_or_uri); state->default_token_type = flatpak_dir_get_remote_default_token_type (self, remote_or_uri); } @@ -13104,6 +13234,56 @@ populate_hash_table_from_refs_map (GHashTable *ret_all_refs, } +static void +populate_hash_table_from_image_collection (GHashTable *ret_all_refs, + GHashTable *ref_timestamps, + FlatpakImageCollection *image_collection, + const char *opt_collection_id, + FlatpakRemoteState *state) +{ + g_autoptr(GPtrArray) sources = flatpak_image_collection_get_sources (image_collection); + + for (guint i = 0; i < sources->len; i++) + { + FlatpakImageSource *image_source = g_ptr_array_index (sources, i); + const char *ref_name = flatpak_image_source_get_ref (image_source); + const char *digest = flatpak_image_source_get_digest (image_source); + const char *checksum; + guint64 *new_timestamp = NULL; + g_autoptr(FlatpakDecomposed) decomposed = NULL; + + if (!flatpak_remote_state_allow_ref (state, ref_name)) + continue; + + g_assert (g_str_has_prefix (digest, "sha256:")); + checksum = digest + 7; + + decomposed = flatpak_decomposed_new_from_col_ref (ref_name, opt_collection_id, NULL); + if (decomposed == NULL) + continue; + + if (ref_timestamps) + { + guint64 timestamp = flatpak_image_source_get_commit_timestamp (image_source); + gpointer value; + + if (g_hash_table_lookup_extended (ref_timestamps, ref_name, NULL, &value)) + { + guint64 *old_timestamp = value; + if (*old_timestamp >= timestamp) + continue; /* New timestamp is older, skip this commit */ + } + + new_timestamp = g_memdup2 (×tamp, sizeof (guint64)); + } + + g_hash_table_replace (ret_all_refs, g_steal_pointer (&decomposed), g_strdup (checksum)); + if (new_timestamp) + g_hash_table_replace (ref_timestamps, g_strdup (ref_name), new_timestamp); + } +} + + /* This tries to list all available remote refs but also tries to keep * working when offline, so it looks in sideloaded repos. Also it uses * in-memory cached summaries which ostree doesn't. */ @@ -13175,25 +13355,37 @@ flatpak_dir_list_all_remote_refs (FlatpakDir *self, ref_map = var_summary_get_ref_map (summary); populate_hash_table_from_refs_map (ret_all_refs, NULL, ref_map, main_collection_id, state); } - else if (state->collection_id) + else { g_autoptr(GHashTable) ref_mtimes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); /* No main summary, add just all sideloded refs, with the latest version of each checksum */ - for (int i = 0; i < state->sideload_repos->len; i++) + if (state->collection_id) { - FlatpakSideloadState *ss = g_ptr_array_index (state->sideload_repos, i); + for (int i = 0; i < state->sideload_repos->len; i++) + { + FlatpakSideloadState *ss = g_ptr_array_index (state->sideload_repos, i); - summary = var_summary_from_gvariant (ss->summary); - exts = var_summary_get_metadata (summary); + summary = var_summary_from_gvariant (ss->summary); + exts = var_summary_get_metadata (summary); - if (var_metadata_lookup (exts, "ostree.summary.collection-map", NULL, &v)) - { - VarCollectionMapRef map = var_collection_map_from_variant (v); + if (var_metadata_lookup (exts, "ostree.summary.collection-map", NULL, &v)) + { + VarCollectionMapRef map = var_collection_map_from_variant (v); - if (var_collection_map_lookup (map, state->collection_id, NULL, &ref_map)) - populate_hash_table_from_refs_map (ret_all_refs, ref_mtimes, ref_map, NULL, state); + if (var_collection_map_lookup (map, state->collection_id, NULL, &ref_map)) + populate_hash_table_from_refs_map (ret_all_refs, ref_mtimes, ref_map, NULL, state); + } + } + } + + if (state->is_oci) + { + for (int i = 0; i < state->sideload_image_collections->len; i++) + { + FlatpakImageCollection *collection = g_ptr_array_index (state->sideload_image_collections, i); + populate_hash_table_from_image_collection (ret_all_refs, ref_mtimes, collection, NULL, state); } } } @@ -15773,7 +15965,7 @@ flatpak_dir_find_remote_related_for_metadata (FlatpakDir *self, if (extension_ref == NULL) continue; - if (flatpak_remote_state_lookup_ref (state, flatpak_decomposed_get_ref (extension_ref), &checksum, NULL, NULL, NULL, NULL)) + if (flatpak_remote_state_lookup_ref (state, flatpak_decomposed_get_ref (extension_ref), &checksum, NULL, NULL, NULL, NULL, NULL)) { if (flatpak_filters_allow_ref (NULL, masked, flatpak_decomposed_get_ref (extension_ref))) add_related (self, related, state->remote_name, extension, extension_ref, checksum, @@ -15788,7 +15980,7 @@ flatpak_dir_find_remote_related_for_metadata (FlatpakDir *self, g_autofree char *subref_checksum = NULL; if (flatpak_remote_state_lookup_ref (state, flatpak_decomposed_get_ref (subref_ref), - &subref_checksum, NULL, NULL, NULL, NULL) && + &subref_checksum, NULL, NULL, NULL, NULL, NULL) && flatpak_filters_allow_ref (NULL, masked, flatpak_decomposed_get_ref (subref_ref))) add_related (self, related, state->remote_name, extension, subref_ref, subref_checksum, no_autodownload, download_if, autoprune_unless, autodelete, locale_subset); diff --git a/common/flatpak-image-collection-private.h b/common/flatpak-image-collection-private.h new file mode 100644 index 00000000..2848f95f --- /dev/null +++ b/common/flatpak-image-collection-private.h @@ -0,0 +1,50 @@ +/* + * Copyright © 2024 Red Hat, Inc + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see . + * + * Authors: + * Owen Taylor + */ + +#ifndef __FLATPAK_IMAGE_COLLECTION_H__ +#define __FLATPAK_IMAGE_COLLECTION_H__ + +#include +#include + +#include +#include + +#define FLATPAK_TYPE_IMAGE_COLLECTION flatpak_image_collection_get_type () +#define FLATPAK_IMAGE_COLLECTION(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), FLATPAK_TYPE_IMAGE_COLLECTION, FlatpakImageCollection)) +#define FLATPAK_IS_IMAGE_COLLECTION(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), FLATPAK_TYPE_IMAGE_COLLECTION)) + +GType flatpak_image_collection_get_type (void); + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (FlatpakImageCollection, g_object_unref) + + +FlatpakImageCollection *flatpak_image_collection_new (const char *location, + GCancellable *cancellable, + GError **error); + +FlatpakImageSource *flatpak_image_collection_lookup_ref (FlatpakImageCollection *self, + const char *ref); +FlatpakImageSource *flatpak_image_collection_lookup_digest (FlatpakImageCollection *self, + const char *digest); + +GPtrArray *flatpak_image_collection_get_sources (FlatpakImageCollection *self); + +#endif /* __FLATPAK_IMAGE_COLLECTION_H__ */ diff --git a/common/flatpak-image-collection.c b/common/flatpak-image-collection.c new file mode 100644 index 00000000..bdb41838 --- /dev/null +++ b/common/flatpak-image-collection.c @@ -0,0 +1,154 @@ +/* vi:set et sw=2 sts=2 cin cino=t0,f0,(0,{s,>2s,n-s,^-s,e-s: + * Copyright © 2024 Red Hat, Inc + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see . + * + * Authors: + * Owen Taylor + */ + +#include + +#include "flatpak-image-collection-private.h" +#include "flatpak-oci-registry-private.h" + +struct FlatpakImageCollection +{ + GObject parent; + + GPtrArray *sources; +}; + +typedef struct +{ + GObjectClass parent_class; +} FlatpakImageCollectionClass; + +G_DEFINE_TYPE (FlatpakImageCollection, flatpak_image_collection, G_TYPE_OBJECT) + + +static void +flatpak_image_collection_finalize (GObject *object) +{ + FlatpakImageCollection *self = FLATPAK_IMAGE_COLLECTION (object); + + g_ptr_array_free (self->sources, TRUE); + + G_OBJECT_CLASS (flatpak_image_collection_parent_class)->finalize (object); +} + +static void +flatpak_image_collection_class_init (FlatpakImageCollectionClass *klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + + object_class->finalize = flatpak_image_collection_finalize; +} + +static void +flatpak_image_collection_init (FlatpakImageCollection *self) +{ + self->sources = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); +} + +FlatpakImageCollection * +flatpak_image_collection_new (const char *location, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(FlatpakImageCollection) self = NULL; + g_autoptr(FlatpakOciRegistry) registry = NULL; + g_autoptr(FlatpakOciIndex) index = NULL; + gsize i; + + self = g_object_new (FLATPAK_TYPE_IMAGE_COLLECTION, NULL); + + if (g_str_has_prefix (location, "oci:")) + { + g_autoptr(GFile) dir = g_file_new_for_path (location + 4); + g_autofree char *uri = g_file_get_uri (dir); + + registry = flatpak_oci_registry_new (uri, FALSE, -1, cancellable, error); + if (registry == NULL) + return NULL; + } + else if (g_str_has_prefix (location, "oci-archive:")) + { + g_autoptr(GFile) file = g_file_new_for_path (location + 12); + registry = flatpak_oci_registry_new_for_archive (file, cancellable, error); + if (registry == NULL) + return NULL; + } + else + { + flatpak_fail (error, "Can't parse image collection location %s", location); + return NULL; + } + + index = flatpak_oci_registry_load_index (registry, cancellable, error); + if (index == NULL) + return NULL; + + for (i = 0; index->manifests[i] != NULL; i++) + { + g_autoptr(GError) local_error = NULL; + FlatpakOciManifestDescriptor *descriptor = index->manifests[i]; + g_autoptr(FlatpakImageSource) image_source = flatpak_image_source_new (registry, NULL, + descriptor->parent.digest, + cancellable, &local_error); + if (image_source == NULL) + { + g_info ("Can't load manifest in image collection: %s", local_error->message); + continue; + } + + g_ptr_array_add (self->sources, g_steal_pointer (&image_source)); + } + + return g_steal_pointer (&self); +} + +FlatpakImageSource * +flatpak_image_collection_lookup_ref (FlatpakImageCollection *self, + const char *ref) +{ + for (guint i = 0; i < self->sources->len; i++) + { + FlatpakImageSource *source = g_ptr_array_index (self->sources, i); + if (strcmp (flatpak_image_source_get_ref (source), ref) == 0) + return g_object_ref (source); + } + + return NULL; +} + +FlatpakImageSource * +flatpak_image_collection_lookup_digest (FlatpakImageCollection *self, + const char *digest) +{ + for (guint i = 0; i < self->sources->len; i++) + { + FlatpakImageSource *source = g_ptr_array_index (self->sources, i); + if (strcmp (flatpak_image_source_get_digest (source), digest) == 0) + return g_object_ref (source); + } + + return NULL; +} + +GPtrArray * +flatpak_image_collection_get_sources (FlatpakImageCollection *self) +{ + return g_ptr_array_ref (self->sources); +} diff --git a/common/flatpak-image-source-private.h b/common/flatpak-image-source-private.h index 94e4cc1a..597a8174 100644 --- a/common/flatpak-image-source-private.h +++ b/common/flatpak-image-source-private.h @@ -32,6 +32,12 @@ G_DECLARE_FINAL_TYPE (FlatpakImageSource, FLATPAK, IMAGE_SOURCE, GObject) +FlatpakImageSource *flatpak_image_source_new (FlatpakOciRegistry *registry, + const char *repository, + const char *digest, + GCancellable *cancellable, + GError **error); + FlatpakImageSource *flatpak_image_source_new_local (GFile *file, const char *reference, GCancellable *cancellable, @@ -70,4 +76,5 @@ void flatpak_image_source_build_commit_metadata (FlatpakImageSource *self, GVariantBuilder *metadata_builder); GVariant *flatpak_image_source_make_fake_commit (FlatpakImageSource *image_source); +GVariant *flatpak_image_source_make_summary_metadata (FlatpakImageSource *self); #endif /* __FLATPAK_IMAGE_SOURCE_H__ */ diff --git a/common/flatpak-image-source.c b/common/flatpak-image-source.c index 9d13cf0d..a31f1084 100644 --- a/common/flatpak-image-source.c +++ b/common/flatpak-image-source.c @@ -68,7 +68,7 @@ flatpak_image_source_init (FlatpakImageSource *self) { } -static FlatpakImageSource * +FlatpakImageSource * flatpak_image_source_new (FlatpakOciRegistry *registry, const char *repository, const char *digest, @@ -78,6 +78,12 @@ flatpak_image_source_new (FlatpakOciRegistry *registry, g_autoptr(FlatpakImageSource) self = NULL; g_autoptr(FlatpakOciVersioned) versioned = NULL; + if (!g_str_has_prefix (digest, "sha256:")) + { + flatpak_fail_error (error, FLATPAK_ERROR_INVALID_DATA, _("Only sha256 image checksums are supported")); + return NULL; + } + self = g_object_new (FLATPAK_TYPE_IMAGE_SOURCE, NULL); self->registry = g_object_ref (registry); self->repository = g_strdup (repository); @@ -494,3 +500,20 @@ flatpak_image_source_make_fake_commit (FlatpakImageSource *self) ostree_checksum_to_bytes_v ("0000000000000000000000000000000000000000000000000000000000000000"), ostree_checksum_to_bytes_v ("0000000000000000000000000000000000000000000000000000000000000000"))); } + +GVariant * +flatpak_image_source_make_summary_metadata (FlatpakImageSource *self) +{ + g_autoptr(GVariantBuilder) ref_metadata_builder = NULL; + + ref_metadata_builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); + + if (self->repository) + g_variant_builder_add (ref_metadata_builder, "{sv}", "xa.oci-repository", + g_variant_new_string (self->repository)); + if (self->delta_url) + g_variant_builder_add (ref_metadata_builder, "{sv}", "xa.delta-url", + g_variant_new_string (self->delta_url)); + + return g_variant_ref_sink (g_variant_builder_end (ref_metadata_builder)); +} diff --git a/common/flatpak-transaction.c b/common/flatpak-transaction.c index 2dd35f53..b498e4d4 100644 --- a/common/flatpak-transaction.c +++ b/common/flatpak-transaction.c @@ -26,6 +26,8 @@ #include "flatpak-auth-private.h" #include "flatpak-dir-private.h" #include "flatpak-error.h" +#include "flatpak-image-collection-private.h" +#include "flatpak-image-collection-private.h" #include "flatpak-image-source-private.h" #include "flatpak-installation-private.h" #include "flatpak-oci-registry-private.h" @@ -172,6 +174,7 @@ typedef struct _FlatpakTransactionPrivate GHashTable *remote_states; /* (element-type utf8 FlatpakRemoteState) */ GPtrArray *extra_dependency_dirs; GPtrArray *extra_sideload_repos; + GPtrArray *sideload_image_collections; GList *ops; GPtrArray *added_origin_remotes; @@ -524,6 +527,38 @@ flatpak_transaction_add_sideload_repo (FlatpakTransaction *self, g_strdup (path)); } +/** + * flatpak_transaction_add_sideload_image_collection: + * @self: a #FlatpakTransaction + * @location: source of images for installation + * + * Adds a set of images to be used as source for installation. This is similar + * to flatpak_transaction_add_sideload_repo(), but the Flatpaks are stored + * as OCI images rather than ostree commits, and the images are used for + * all OCI remotes without regard to collection ID. + * + * Currently @location should be either 'oci:' or 'oci-archive:'. + * Additional schemes may be added in the future. + * + * Since: 1.7.1 + */ +gboolean +flatpak_transaction_add_sideload_image_collection (FlatpakTransaction *self, + const char *location, + GCancellable *cancellable, + GError **error) +{ + FlatpakTransactionPrivate *priv = flatpak_transaction_get_instance_private (self); + g_autoptr(FlatpakImageCollection) collection = NULL; + + collection = flatpak_image_collection_new (location, cancellable, error); + if (collection == NULL) + return FALSE; + + g_ptr_array_add (priv->sideload_image_collections, g_steal_pointer (&collection)); + return TRUE; +} + /** * flatpak_transaction_add_default_dependency_sources: * @self: a #FlatpakTransaction @@ -1033,6 +1068,7 @@ flatpak_transaction_finalize (GObject *object) g_ptr_array_free (priv->extra_dependency_dirs, TRUE); g_ptr_array_free (priv->extra_sideload_repos, TRUE); + g_ptr_array_free (priv->sideload_image_collections, TRUE); G_OBJECT_CLASS (flatpak_transaction_parent_class)->finalize (object); } @@ -1508,6 +1544,7 @@ flatpak_transaction_init (FlatpakTransaction *self) priv->added_origin_remotes = g_ptr_array_new_with_free_func (g_free); priv->extra_dependency_dirs = g_ptr_array_new_with_free_func (g_object_unref); priv->extra_sideload_repos = g_ptr_array_new_with_free_func (g_free); + priv->sideload_image_collections = g_ptr_array_new_with_free_func (g_object_unref); priv->can_run = TRUE; } @@ -2085,6 +2122,13 @@ flatpak_transaction_ensure_remote_state (FlatpakTransaction *self, g_autoptr(GFile) f = g_file_new_for_path (path); flatpak_remote_state_add_sideload_dir (state, f); } + + for (int i = 0; i < priv->sideload_image_collections->len; i++) + { + FlatpakImageCollection *collection = g_ptr_array_index (priv->sideload_image_collections, i); + flatpak_remote_state_add_sideload_image_collection (state, collection); + } + } if (opt_arch != NULL && @@ -2368,7 +2412,7 @@ search_for_dependency (FlatpakTransaction *self, continue; } - if (flatpak_remote_state_lookup_ref (state, flatpak_decomposed_get_ref (runtime_ref), NULL, NULL, NULL, NULL, NULL)) + if (flatpak_remote_state_lookup_ref (state, flatpak_decomposed_get_ref (runtime_ref), NULL, NULL, NULL, NULL, NULL, NULL)) g_ptr_array_add (found, g_strdup (remote)); } @@ -3243,7 +3287,7 @@ flatpak_transaction_add_auto_install (FlatpakTransaction *self, g_autoptr(FlatpakRemoteState) state = flatpak_transaction_ensure_remote_state (self, FLATPAK_TRANSACTION_OPERATION_UPDATE, remote, NULL, NULL); if (state != NULL && - flatpak_remote_state_lookup_ref (state, flatpak_decomposed_get_ref (auto_install_ref), NULL, NULL, NULL, NULL, NULL)) + flatpak_remote_state_lookup_ref (state, flatpak_decomposed_get_ref (auto_install_ref), NULL, NULL, NULL, NULL, NULL, NULL)) { g_info ("Auto adding install of %s from remote %s", flatpak_decomposed_get_ref (auto_install_ref), remote); @@ -3424,6 +3468,7 @@ resolve_op_from_commit (FlatpakTransaction *self, FlatpakTransactionOperation *op, const char *checksum, GFile *sideload_path, + FlatpakImageSource *image_source, GVariant *commit_data, GError **error) { @@ -3463,7 +3508,7 @@ resolve_op_from_commit (FlatpakTransaction *self, flatpak_decomposed_get_ref (eolr_decomposed)); } - return resolve_op_end (self, op, checksum, sideload_path, NULL, metadata_bytes, error); + return resolve_op_end (self, op, checksum, sideload_path, image_source, metadata_bytes, error); } /* NOTE: In case of non-available summary this returns FALSE with a @@ -3474,6 +3519,7 @@ try_resolve_op_from_metadata (FlatpakTransaction *self, FlatpakTransactionOperation *op, const char *checksum, GFile *sideload_path, + FlatpakImageSource *image_source, FlatpakRemoteState *state, GError **error) { @@ -3487,7 +3533,7 @@ try_resolve_op_from_metadata (FlatpakTransaction *self, /* Ref has to match the actual commit in the summary */ if ((state->summary == NULL && state->index == NULL) || !flatpak_remote_state_lookup_ref (state, flatpak_decomposed_get_ref (op->ref), - &summary_checksum, NULL, NULL, NULL, NULL) || + &summary_checksum, NULL, NULL, NULL, NULL, NULL) || strcmp (summary_checksum, checksum) != 0) return FALSE; @@ -3499,7 +3545,7 @@ try_resolve_op_from_metadata (FlatpakTransaction *self, metadata_bytes = g_bytes_new (metadata, strlen (metadata)); flatpak_remote_state_lookup_ref (state, flatpak_decomposed_get_ref (op->ref), - NULL, NULL, &op->summary_metadata, NULL, NULL); + NULL, NULL, &op->summary_metadata, NULL, NULL, NULL); op->installed_size = installed_size; op->download_size = download_size; @@ -3629,36 +3675,39 @@ resolve_ops (FlatpakTransaction *self, if (commit_data == NULL) return FALSE; - if (!resolve_op_from_commit (self, op, checksum, NULL, commit_data, error)) + if (!resolve_op_from_commit (self, op, checksum, NULL, NULL, commit_data, error)) return FALSE; } else { g_autoptr(GError) local_error = NULL; g_autoptr(GFile) sideload_path = NULL; + g_autoptr(FlatpakImageSource) image_source = NULL; if (op->commit != NULL) { checksum = g_strdup (op->commit); /* Check if this is available offline and if so, use that */ - sideload_path = flatpak_remote_state_lookup_sideload_checksum (state, op->commit); + flatpak_remote_state_lookup_sideload_checksum (state, op->commit, &sideload_path, &image_source); } else { g_autofree char *latest_checksum = NULL; g_autoptr(GFile) latest_sideload_path = NULL; + g_autoptr(FlatpakImageSource) latest_image_source = NULL; g_autofree char *local_checksum = NULL; guint64 latest_timestamp; g_autoptr(GVariant) local_commit_data = flatpak_dir_read_latest_commit (priv->dir, op->remote, op->ref, &local_checksum, NULL, NULL); if (flatpak_dir_find_latest_rev (priv->dir, state, flatpak_decomposed_get_ref (op->ref), op->commit, - &latest_checksum, &latest_timestamp, &latest_sideload_path, + &latest_checksum, &latest_timestamp, &latest_sideload_path, &latest_image_source, cancellable, &local_error)) { /* If we found the latest in a sideload repo, it may be older that what is locally available, check timestamps. * Note: If the timestamps are equal (timestamp granularity issue), assume we want to update */ - if (latest_sideload_path != NULL && local_commit_data && latest_timestamp != 0 && + if ((latest_sideload_path != NULL || latest_image_source != NULL) && + local_commit_data && latest_timestamp != 0 && ostree_commit_get_timestamp (local_commit_data) > latest_timestamp) { g_info ("Installed commit %s newer than sideloaded %s, ignoring", local_checksum, latest_checksum); @@ -3669,6 +3718,7 @@ resolve_ops (FlatpakTransaction *self, /* Otherwise, use whatever we found */ checksum = g_steal_pointer (&latest_checksum); sideload_path = g_steal_pointer (&latest_sideload_path); + image_source = g_steal_pointer (&latest_image_source); } } else @@ -3689,7 +3739,7 @@ resolve_ops (FlatpakTransaction *self, } /* First try to resolve via metadata (if remote is available and its metadata matches the commit version) */ - if (!try_resolve_op_from_metadata (self, op, checksum, sideload_path, state, &local_error)) + if (!try_resolve_op_from_metadata (self, op, checksum, sideload_path, image_source, state, &local_error)) { if (local_error) { @@ -3706,7 +3756,7 @@ resolve_ops (FlatpakTransaction *self, /* OCI needs this to get the oci repository for the ref to request the token, so lets always set it here */ if (op->summary_metadata == NULL) flatpak_remote_state_lookup_ref (state, flatpak_decomposed_get_ref (op->ref), - NULL, NULL, &op->summary_metadata, NULL, NULL); + NULL, NULL, &op->summary_metadata, NULL, NULL, NULL); commit_data = flatpak_remote_state_load_ref_commit (state, priv->dir, flatpak_decomposed_get_ref (op->ref), @@ -3732,7 +3782,7 @@ resolve_ops (FlatpakTransaction *self, return FALSE; } - if (!resolve_op_from_commit (self, op, checksum, sideload_path, commit_data, error)) + if (!resolve_op_from_commit (self, op, checksum, sideload_path, image_source, commit_data, error)) return FALSE; } } diff --git a/common/flatpak-transaction.h b/common/flatpak-transaction.h index 3d3f4ce8..dfa3d039 100644 --- a/common/flatpak-transaction.h +++ b/common/flatpak-transaction.h @@ -272,6 +272,11 @@ FLATPAK_EXTERN void flatpak_transaction_add_sideload_repo (FlatpakTransaction *self, const char *path); FLATPAK_EXTERN +gboolean flatpak_transaction_add_sideload_image_collection (FlatpakTransaction *self, + const char *location, + GCancellable *cancellable, + GError **error); +FLATPAK_EXTERN void flatpak_transaction_add_default_dependency_sources (FlatpakTransaction *self); FLATPAK_EXTERN gboolean flatpak_transaction_run (FlatpakTransaction *transaction, diff --git a/common/meson.build b/common/meson.build index c14337be..37911c4c 100644 --- a/common/meson.build +++ b/common/meson.build @@ -173,6 +173,7 @@ sources = [ 'flatpak-error.c', 'flatpak-exports.c', 'flatpak-glib-backports.c', + 'flatpak-image-collection.c', 'flatpak-image-source.c', 'flatpak-installation.c', 'flatpak-installed-ref.c', diff --git a/doc/flatpak-install.xml b/doc/flatpak-install.xml index 99aa5469..985b94b2 100644 --- a/doc/flatpak-install.xml +++ b/doc/flatpak-install.xml @@ -84,6 +84,15 @@ system-wide basis (see flatpak1). + + For OCI remotes, sideload repositories can also be specified as + or , + where path points to an OCI image layout or archive of an OCI image layout. + (See containers-transports5.) + When specified in this fashion, the images found in the image layout are available for all OCI remotes, + without regard to collection ID. For this reason, there is no directory equivalent - + it can only be specified on the command line. + The alternative form of the command (with , , or ) allows to install directly from a source. The source can be a diff --git a/doc/flatpak-update.xml b/doc/flatpak-update.xml index feaaf278..096ee310 100644 --- a/doc/flatpak-update.xml +++ b/doc/flatpak-update.xml @@ -75,6 +75,15 @@ system-wide basis (see flatpak1). + + For OCI remotes, sideload repositories can also be specified as + or , + where path points to an OCI image layout or archive of an OCI image layout. + (See containers-transports5.) + When specified in this fashion, the images found in the image layout are available for all OCI remotes, + without regard to collection ID. For this reason, there is no directory equivalent - + it can only be specified on the command line. + Note that updating a runtime is different from installing a different branch, and runtime updates are expected to keep diff --git a/doc/reference/meson.build b/doc/reference/meson.build index 92f1482e..0854030e 100644 --- a/doc/reference/meson.build +++ b/doc/reference/meson.build @@ -46,6 +46,7 @@ gnome.gtkdoc( 'flatpak-document-dbus-generated.h', 'flatpak-enum-types.h', 'flatpak-exports-private.h', + 'flatpak-image-collection-private.h', 'flatpak-image-source-private.h', 'flatpak-installed-ref-private.h', 'flatpak-json-oci-private.h', -- 2.47.1