418 lines
16 KiB
Diff
418 lines
16 KiB
Diff
From 465a6a59279bd7fa2680c626ca0f10c059276668 Mon Sep 17 00:00:00 2001
|
|
From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= <amatej@redhat.com>
|
|
Date: Wed, 9 Feb 2022 13:18:41 +0100
|
|
Subject: [PATCH 29/34] Use dnf solv userdata to check versions and checksum
|
|
(RhBug:2027445)
|
|
|
|
Remove unused functions for checksums
|
|
|
|
= changelog =
|
|
msg: Write and check versions and checksums for solvfile cache by using new dnf solvfile userdata (RhBug:2027445)
|
|
It is not possible to use old cache files, therefore cache regeneration is triggered automatically.
|
|
type: bugfix
|
|
resolves: https://bugzilla.redhat.com/show_bug.cgi?id=2027445
|
|
---
|
|
libdnf/dnf-sack.cpp | 254 ++++++++++++++++++++++--------------
|
|
libdnf/hy-iutil-private.hpp | 2 -
|
|
libdnf/hy-iutil.cpp | 20 ---
|
|
3 files changed, 156 insertions(+), 120 deletions(-)
|
|
|
|
diff --git a/libdnf/dnf-sack.cpp b/libdnf/dnf-sack.cpp
|
|
index b9baeaef..61f4807c 100644
|
|
--- a/libdnf/dnf-sack.cpp
|
|
+++ b/libdnf/dnf-sack.cpp
|
|
@@ -225,17 +225,39 @@ dnf_sack_new(void)
|
|
return DNF_SACK(g_object_new(DNF_TYPE_SACK, NULL));
|
|
}
|
|
|
|
-static int
|
|
-can_use_repomd_cache(FILE *fp_solv, unsigned char cs_repomd[CHKSUM_BYTES])
|
|
-{
|
|
- unsigned char cs_cache[CHKSUM_BYTES];
|
|
-
|
|
- if (fp_solv &&
|
|
- !checksum_read(cs_cache, fp_solv) &&
|
|
- !checksum_cmp(cs_cache, cs_repomd))
|
|
- return 1;
|
|
+// Try to load cached solv file into repo otherwise return FALSE
|
|
+static gboolean
|
|
+try_to_use_cached_solvfile(const char *path, Repo *repo, int flags, const unsigned char *checksum, GError **err){
|
|
+ FILE *fp_cache = fopen(path, "r");
|
|
+ if (!fp_cache) {
|
|
+ // Missing cache files (ENOENT) are not an error and can even be expected in some cases
|
|
+ // (such as when repo doesn't have updateinfo/prestodelta metadata).
|
|
+ // Use g_debug in order not to pollute the log by default with such entries.
|
|
+ if (errno == ENOENT) {
|
|
+ g_debug("Failed to open solvfile cache: %s: %s", path, strerror(errno));
|
|
+ } else {
|
|
+ g_warning("Failed to open solvfile cache: %s: %s", path, strerror(errno));
|
|
+ }
|
|
+ return FALSE;
|
|
+ }
|
|
+ std::unique_ptr<SolvUserdata> solv_userdata = solv_userdata_read(fp_cache);
|
|
+ gboolean ret = TRUE;
|
|
+ if (solv_userdata && solv_userdata_verify(solv_userdata.get(), checksum)) {
|
|
+ // after reading the header rewind to the begining
|
|
+ fseek(fp_cache, 0, SEEK_SET);
|
|
+ if (repo_add_solv(repo, fp_cache, flags)) {
|
|
+ g_set_error (err,
|
|
+ DNF_ERROR,
|
|
+ DNF_ERROR_INTERNAL_ERROR,
|
|
+ _("repo_add_solv() has failed."));
|
|
+ ret = FALSE;
|
|
+ }
|
|
+ } else {
|
|
+ ret = FALSE;
|
|
+ }
|
|
|
|
- return 0;
|
|
+ fclose(fp_cache);
|
|
+ return ret;
|
|
}
|
|
|
|
void
|
|
@@ -375,33 +397,27 @@ load_ext(DnfSack *sack, HyRepo hrepo, _hy_repo_repodata which_repodata,
|
|
gboolean done = FALSE;
|
|
|
|
char *fn_cache = dnf_sack_give_cache_fn(sack, name, suffix);
|
|
- fp = fopen(fn_cache, "r");
|
|
assert(libdnf::repoGetImpl(hrepo)->checksum);
|
|
- if (can_use_repomd_cache(fp, libdnf::repoGetImpl(hrepo)->checksum)) {
|
|
- int flags = 0;
|
|
- /* the updateinfo is not a real extension */
|
|
- if (which_repodata != _HY_REPODATA_UPDATEINFO)
|
|
- flags |= REPO_EXTEND_SOLVABLES;
|
|
- /* do not pollute the main pool with directory component ids */
|
|
- if (which_repodata == _HY_REPODATA_FILENAMES || which_repodata == _HY_REPODATA_OTHER)
|
|
- flags |= REPO_LOCALPOOL;
|
|
- done = TRUE;
|
|
+
|
|
+ int flags = 0;
|
|
+ /* the updateinfo is not a real extension */
|
|
+ if (which_repodata != _HY_REPODATA_UPDATEINFO)
|
|
+ flags |= REPO_EXTEND_SOLVABLES;
|
|
+ /* do not pollute the main pool with directory component ids */
|
|
+ if (which_repodata == _HY_REPODATA_FILENAMES || which_repodata == _HY_REPODATA_OTHER)
|
|
+ flags |= REPO_LOCALPOOL;
|
|
+ if (try_to_use_cached_solvfile(fn_cache, repo, flags, libdnf::repoGetImpl(hrepo)->checksum, error)) {
|
|
g_debug("%s: using cache file: %s", __func__, fn_cache);
|
|
- ret = repo_add_solv(repo, fp, flags);
|
|
- if (ret) {
|
|
- g_set_error_literal (error,
|
|
- DNF_ERROR,
|
|
- DNF_ERROR_INTERNAL_ERROR,
|
|
- _("failed to add solv"));
|
|
- return FALSE;
|
|
- } else {
|
|
- repo_update_state(hrepo, which_repodata, _HY_LOADED_CACHE);
|
|
- repo_set_repodata(hrepo, which_repodata, repo->nrepodata - 1);
|
|
- }
|
|
+ done = TRUE;
|
|
+ repo_update_state(hrepo, which_repodata, _HY_LOADED_CACHE);
|
|
+ repo_set_repodata(hrepo, which_repodata, repo->nrepodata - 1);
|
|
}
|
|
+ if (error && *error) {
|
|
+ g_prefix_error(error, _("Loading extension cache %s (%d) failed: "), fn_cache, which_repodata);
|
|
+ return FALSE;
|
|
+ }
|
|
+
|
|
g_free(fn_cache);
|
|
- if (fp)
|
|
- fclose(fp);
|
|
if (done)
|
|
return TRUE;
|
|
|
|
@@ -514,35 +530,53 @@ write_main(DnfSack *sack, HyRepo hrepo, int switchtosolv, GError **error)
|
|
strerror(errno));
|
|
goto done;
|
|
}
|
|
- rc = repo_write(repo, fp);
|
|
- rc |= checksum_write(repoImpl->checksum, fp);
|
|
- rc |= fclose(fp);
|
|
+
|
|
+ SolvUserdata solv_userdata;
|
|
+ if (solv_userdata_fill(&solv_userdata, repoImpl->checksum, error)) {
|
|
+ ret = FALSE;
|
|
+ fclose(fp);
|
|
+ goto done;
|
|
+ }
|
|
+
|
|
+ Repowriter *writer = repowriter_create(repo);
|
|
+ repowriter_set_userdata(writer, &solv_userdata, solv_userdata_size);
|
|
+ rc = repowriter_write(writer, fp);
|
|
+ repowriter_free(writer);
|
|
if (rc) {
|
|
+ ret = FALSE;
|
|
+ fclose(fp);
|
|
+ g_set_error(error,
|
|
+ DNF_ERROR,
|
|
+ DNF_ERROR_INTERNAL_ERROR,
|
|
+ _("While writing primary cache %s repowriter write failed: %i, error: %s"),
|
|
+ tmp_fn_templ, rc, pool_errstr(repo->pool));
|
|
+ goto done;
|
|
+ }
|
|
+
|
|
+ if (fclose(fp)) {
|
|
ret = FALSE;
|
|
g_set_error (error,
|
|
DNF_ERROR,
|
|
DNF_ERROR_FILE_INVALID,
|
|
- _("write_main() failed writing data: %i"), rc);
|
|
+ _("Failed closing tmp file %s: %s"),
|
|
+ tmp_fn_templ, strerror(errno));
|
|
goto done;
|
|
}
|
|
}
|
|
if (switchtosolv && repo_is_one_piece(repo)) {
|
|
+ repo_empty(repo, 1);
|
|
/* switch over to written solv file activate paging */
|
|
- FILE *fp = fopen(tmp_fn_templ, "r");
|
|
- if (fp) {
|
|
- repo_empty(repo, 1);
|
|
- rc = repo_add_solv(repo, fp, 0);
|
|
- fclose(fp);
|
|
- if (rc) {
|
|
- /* this is pretty fatal */
|
|
- ret = FALSE;
|
|
- g_set_error_literal (error,
|
|
- DNF_ERROR,
|
|
- DNF_ERROR_FILE_INVALID,
|
|
- _("write_main() failed to re-load "
|
|
- "written solv file"));
|
|
- goto done;
|
|
- }
|
|
+ gboolean loaded = try_to_use_cached_solvfile(tmp_fn_templ, repo, 0, repoImpl->checksum, error);
|
|
+ if (error && *error) {
|
|
+ g_prefix_error(error, _("Failed to use newly written primary cache: %s: "), tmp_fn_templ);
|
|
+ ret = FALSE;
|
|
+ goto done;
|
|
+ }
|
|
+ if (!loaded) {
|
|
+ g_set_error(error, DNF_ERROR, DNF_ERROR_INTERNAL_ERROR,
|
|
+ _("Failed to use newly written primary cache: %s"), tmp_fn_templ);
|
|
+ ret = FALSE;
|
|
+ goto done;
|
|
}
|
|
}
|
|
|
|
@@ -569,20 +603,6 @@ write_ext_updateinfo_filter(Repo *repo, Repokey *key, void *kfdata)
|
|
return repo_write_stdkeyfilter(repo, key, 0);
|
|
}
|
|
|
|
-static int
|
|
-write_ext_updateinfo(HyRepo hrepo, Repodata *data, FILE *fp)
|
|
-{
|
|
- auto repoImpl = libdnf::repoGetImpl(hrepo);
|
|
- Repo *repo = repoImpl->libsolvRepo;
|
|
- int oldstart = repo->start;
|
|
- repo->start = repoImpl->main_end;
|
|
- repo->nsolvables -= repoImpl->main_nsolvables;
|
|
- int res = repo_write_filtered(repo, fp, write_ext_updateinfo_filter, data, 0);
|
|
- repo->start = oldstart;
|
|
- repo->nsolvables += repoImpl->main_nsolvables;
|
|
- return res;
|
|
-}
|
|
-
|
|
static gboolean
|
|
write_ext(DnfSack *sack, HyRepo hrepo, _hy_repo_repodata which_repodata,
|
|
const char *suffix, GError **error)
|
|
@@ -611,37 +631,78 @@ write_ext(DnfSack *sack, HyRepo hrepo, _hy_repo_repodata which_repodata,
|
|
FILE *fp = fdopen(tmp_fd, "w+");
|
|
|
|
g_debug("%s: storing %s to: %s", __func__, repo->name, tmp_fn_templ);
|
|
- if (which_repodata != _HY_REPODATA_UPDATEINFO)
|
|
- ret |= repodata_write(data, fp);
|
|
- else
|
|
- ret |= write_ext_updateinfo(hrepo, data, fp);
|
|
- ret |= checksum_write(repoImpl->checksum, fp);
|
|
- ret |= fclose(fp);
|
|
+
|
|
+ SolvUserdata solv_userdata;
|
|
+ if (solv_userdata_fill(&solv_userdata, repoImpl->checksum, error)) {
|
|
+ fclose(fp);
|
|
+ success = FALSE;
|
|
+ goto done;
|
|
+ }
|
|
+
|
|
+ Repowriter *writer = repowriter_create(repo);
|
|
+ repowriter_set_userdata(writer, &solv_userdata, solv_userdata_size);
|
|
+ if (which_repodata != _HY_REPODATA_UPDATEINFO) {
|
|
+ repowriter_set_repodatarange(writer, data->repodataid, data->repodataid + 1);
|
|
+ repowriter_set_flags(writer, REPOWRITER_NO_STORAGE_SOLVABLE);
|
|
+ ret = repowriter_write(writer, fp);
|
|
+ } else {
|
|
+ // write only updateinfo repodata
|
|
+ int oldstart = repo->start;
|
|
+ repo->start = repoImpl->main_end;
|
|
+ repo->nsolvables -= repoImpl->main_nsolvables;
|
|
+ repowriter_set_flags(writer, REPOWRITER_LEGACY);
|
|
+ repowriter_set_keyfilter(writer, write_ext_updateinfo_filter, data);
|
|
+ repowriter_set_keyqueue(writer, 0);
|
|
+ ret = repowriter_write(writer, fp);
|
|
+ repo->start = oldstart;
|
|
+ repo->nsolvables += repoImpl->main_nsolvables;
|
|
+ }
|
|
+ repowriter_free(writer);
|
|
if (ret) {
|
|
+ success = FALSE;
|
|
+ fclose(fp);
|
|
+ g_set_error (error,
|
|
+ DNF_ERROR,
|
|
+ DNF_ERROR_INTERNAL_ERROR,
|
|
+ _("While writing extension cache %s (%d): repowriter write failed: %i, error: %s"),
|
|
+ tmp_fn_templ, which_repodata, ret, pool_errstr(repo->pool));
|
|
+ goto done;
|
|
+ }
|
|
+
|
|
+ if (fclose(fp)) {
|
|
success = FALSE;
|
|
g_set_error (error,
|
|
DNF_ERROR,
|
|
- DNF_ERROR_FAILED,
|
|
- _("write_ext(%1$d) has failed: %2$d"),
|
|
- which_repodata, ret);
|
|
+ DNF_ERROR_FILE_INVALID,
|
|
+ _("While writing extension cache (%d): cannot close temporary file: %s"),
|
|
+ which_repodata, tmp_fn_templ);
|
|
goto done;
|
|
}
|
|
}
|
|
|
|
if (repo_is_one_piece(repo) && which_repodata != _HY_REPODATA_UPDATEINFO) {
|
|
/* switch over to written solv file activate paging */
|
|
- FILE *fp = fopen(tmp_fn_templ, "r");
|
|
- if (fp) {
|
|
- int flags = REPO_USE_LOADING | REPO_EXTEND_SOLVABLES;
|
|
- /* do not pollute the main pool with directory component ids */
|
|
- if (which_repodata == _HY_REPODATA_FILENAMES || which_repodata == _HY_REPODATA_OTHER)
|
|
- flags |= REPO_LOCALPOOL;
|
|
- repodata_extend_block(data, repo->start, repo->end - repo->start);
|
|
- data->state = REPODATA_LOADING;
|
|
- repo_add_solv(repo, fp, flags);
|
|
- data->state = REPODATA_AVAILABLE;
|
|
- fclose(fp);
|
|
+ int flags = REPO_USE_LOADING | REPO_EXTEND_SOLVABLES;
|
|
+ /* do not pollute the main pool with directory component ids */
|
|
+ if (which_repodata == _HY_REPODATA_FILENAMES || which_repodata == _HY_REPODATA_OTHER)
|
|
+ flags |= REPO_LOCALPOOL;
|
|
+ repodata_extend_block(data, repo->start, repo->end - repo->start);
|
|
+ data->state = REPODATA_LOADING;
|
|
+ int loaded = try_to_use_cached_solvfile(tmp_fn_templ, repo, flags, repoImpl->checksum, error);
|
|
+ if (error && *error) {
|
|
+ g_prefix_error(error, _("Failed to use newly written extension cache: %s (%d): "),
|
|
+ tmp_fn_templ, which_repodata);
|
|
+ success = FALSE;
|
|
+ goto done;
|
|
+ }
|
|
+ if (!loaded) {
|
|
+ g_set_error(error, DNF_ERROR, DNF_ERROR_INTERNAL_ERROR,
|
|
+ _("Failed to use newly written extension cache: %s (%d)"), tmp_fn_templ, which_repodata);
|
|
+ success = FALSE;
|
|
+ goto done;
|
|
}
|
|
+
|
|
+ data->state = REPODATA_AVAILABLE;
|
|
}
|
|
|
|
if (!mv(tmp_fn_templ, fn, error)) {
|
|
@@ -672,7 +733,7 @@ load_yum_repo(DnfSack *sack, HyRepo hrepo, GError **error)
|
|
|
|
FILE *fp_primary = NULL;
|
|
FILE *fp_repomd = NULL;
|
|
- FILE *fp_cache = fopen(fn_cache, "r");
|
|
+
|
|
if (!fn_repomd) {
|
|
g_set_error (error,
|
|
DNF_ERROR,
|
|
@@ -693,18 +754,17 @@ load_yum_repo(DnfSack *sack, HyRepo hrepo, GError **error)
|
|
}
|
|
checksum_fp(repoImpl->checksum, fp_repomd);
|
|
|
|
- if (can_use_repomd_cache(fp_cache, repoImpl->checksum)) {
|
|
+ if (try_to_use_cached_solvfile(fn_cache, repo, 0, repoImpl->checksum, error)) {
|
|
const char *chksum = pool_checksum_str(pool, repoImpl->checksum);
|
|
g_debug("using cached %s (0x%s)", name, chksum);
|
|
- if (repo_add_solv(repo, fp_cache, 0)) {
|
|
- g_set_error (error,
|
|
- DNF_ERROR,
|
|
- DNF_ERROR_INTERNAL_ERROR,
|
|
- _("repo_add_solv() has failed."));
|
|
- retval = FALSE;
|
|
- goto out;
|
|
- }
|
|
repoImpl->state_main = _HY_LOADED_CACHE;
|
|
+ goto out;
|
|
+ }
|
|
+
|
|
+ if (error && *error) {
|
|
+ g_prefix_error(error, _("While loading repository failed to use %s: "), fn_cache);
|
|
+ retval = FALSE;
|
|
+ goto out;
|
|
} else {
|
|
auto primary = hrepo->getMetadataPath(MD_TYPE_PRIMARY);
|
|
if (primary.empty()) {
|
|
@@ -733,8 +793,6 @@ load_yum_repo(DnfSack *sack, HyRepo hrepo, GError **error)
|
|
repoImpl->state_main = _HY_LOADED_FETCH;
|
|
}
|
|
out:
|
|
- if (fp_cache)
|
|
- fclose(fp_cache);
|
|
if (fp_repomd)
|
|
fclose(fp_repomd);
|
|
if (fp_primary)
|
|
diff --git a/libdnf/hy-iutil-private.hpp b/libdnf/hy-iutil-private.hpp
|
|
index d498c032..efc91c63 100644
|
|
--- a/libdnf/hy-iutil-private.hpp
|
|
+++ b/libdnf/hy-iutil-private.hpp
|
|
@@ -52,9 +52,7 @@ int solv_userdata_verify(const SolvUserdata *solv_userdata, const unsigned char
|
|
/* crypto utils */
|
|
int checksum_cmp(const unsigned char *cs1, const unsigned char *cs2);
|
|
int checksum_fp(unsigned char *out, FILE *fp);
|
|
-int checksum_read(unsigned char *csout, FILE *fp);
|
|
int checksum_stat(unsigned char *out, FILE *fp);
|
|
-int checksum_write(const unsigned char *cs, FILE *fp);
|
|
int checksumt_l2h(int type);
|
|
const char *pool_checksum_str(Pool *pool, const unsigned char *chksum);
|
|
|
|
diff --git a/libdnf/hy-iutil.cpp b/libdnf/hy-iutil.cpp
|
|
index f81ca52f..c409a10a 100644
|
|
--- a/libdnf/hy-iutil.cpp
|
|
+++ b/libdnf/hy-iutil.cpp
|
|
@@ -142,17 +142,6 @@ checksum_fp(unsigned char *out, FILE *fp)
|
|
return 0;
|
|
}
|
|
|
|
-/* calls rewind(fp) before returning */
|
|
-int
|
|
-checksum_read(unsigned char *csout, FILE *fp)
|
|
-{
|
|
- if (fseek(fp, -32, SEEK_END) ||
|
|
- fread(csout, CHKSUM_BYTES, 1, fp) != 1)
|
|
- return 1;
|
|
- rewind(fp);
|
|
- return 0;
|
|
-}
|
|
-
|
|
/* does not move the fp position */
|
|
int
|
|
checksum_stat(unsigned char *out, FILE *fp)
|
|
@@ -174,15 +163,6 @@ checksum_stat(unsigned char *out, FILE *fp)
|
|
return 0;
|
|
}
|
|
|
|
-/* moves fp to the end of file */
|
|
-int checksum_write(const unsigned char *cs, FILE *fp)
|
|
-{
|
|
- if (fseek(fp, 0, SEEK_END) ||
|
|
- fwrite(cs, CHKSUM_BYTES, 1, fp) != 1)
|
|
- return 1;
|
|
- return 0;
|
|
-}
|
|
-
|
|
static std::array<char, solv_userdata_solv_toolversion_size>
|
|
get_padded_solv_toolversion()
|
|
{
|
|
--
|
|
2.31.1
|
|
|