From 465a6a59279bd7fa2680c626ca0f10c059276668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= Date: Wed, 9 Feb 2022 13:18:41 +0100 Subject: [PATCH 29/34] Use dnf solv userdata to check versions and checksum (RhBug:2027445) Remove unused functions for checksums = changelog = msg: Write and check versions and checksums for solvfile cache by using new dnf solvfile userdata (RhBug:2027445) It is not possible to use old cache files, therefore cache regeneration is triggered automatically. type: bugfix resolves: https://bugzilla.redhat.com/show_bug.cgi?id=2027445 --- libdnf/dnf-sack.cpp | 254 ++++++++++++++++++++++-------------- libdnf/hy-iutil-private.hpp | 2 - libdnf/hy-iutil.cpp | 20 --- 3 files changed, 156 insertions(+), 120 deletions(-) diff --git a/libdnf/dnf-sack.cpp b/libdnf/dnf-sack.cpp index b9baeaef..61f4807c 100644 --- a/libdnf/dnf-sack.cpp +++ b/libdnf/dnf-sack.cpp @@ -225,17 +225,39 @@ dnf_sack_new(void) return DNF_SACK(g_object_new(DNF_TYPE_SACK, NULL)); } -static int -can_use_repomd_cache(FILE *fp_solv, unsigned char cs_repomd[CHKSUM_BYTES]) -{ - unsigned char cs_cache[CHKSUM_BYTES]; - - if (fp_solv && - !checksum_read(cs_cache, fp_solv) && - !checksum_cmp(cs_cache, cs_repomd)) - return 1; +// Try to load cached solv file into repo otherwise return FALSE +static gboolean +try_to_use_cached_solvfile(const char *path, Repo *repo, int flags, const unsigned char *checksum, GError **err){ + FILE *fp_cache = fopen(path, "r"); + if (!fp_cache) { + // Missing cache files (ENOENT) are not an error and can even be expected in some cases + // (such as when repo doesn't have updateinfo/prestodelta metadata). + // Use g_debug in order not to pollute the log by default with such entries. + if (errno == ENOENT) { + g_debug("Failed to open solvfile cache: %s: %s", path, strerror(errno)); + } else { + g_warning("Failed to open solvfile cache: %s: %s", path, strerror(errno)); + } + return FALSE; + } + std::unique_ptr solv_userdata = solv_userdata_read(fp_cache); + gboolean ret = TRUE; + if (solv_userdata && solv_userdata_verify(solv_userdata.get(), checksum)) { + // after reading the header rewind to the begining + fseek(fp_cache, 0, SEEK_SET); + if (repo_add_solv(repo, fp_cache, flags)) { + g_set_error (err, + DNF_ERROR, + DNF_ERROR_INTERNAL_ERROR, + _("repo_add_solv() has failed.")); + ret = FALSE; + } + } else { + ret = FALSE; + } - return 0; + fclose(fp_cache); + return ret; } void @@ -375,33 +397,27 @@ load_ext(DnfSack *sack, HyRepo hrepo, _hy_repo_repodata which_repodata, gboolean done = FALSE; char *fn_cache = dnf_sack_give_cache_fn(sack, name, suffix); - fp = fopen(fn_cache, "r"); assert(libdnf::repoGetImpl(hrepo)->checksum); - if (can_use_repomd_cache(fp, libdnf::repoGetImpl(hrepo)->checksum)) { - int flags = 0; - /* the updateinfo is not a real extension */ - if (which_repodata != _HY_REPODATA_UPDATEINFO) - flags |= REPO_EXTEND_SOLVABLES; - /* do not pollute the main pool with directory component ids */ - if (which_repodata == _HY_REPODATA_FILENAMES || which_repodata == _HY_REPODATA_OTHER) - flags |= REPO_LOCALPOOL; - done = TRUE; + + int flags = 0; + /* the updateinfo is not a real extension */ + if (which_repodata != _HY_REPODATA_UPDATEINFO) + flags |= REPO_EXTEND_SOLVABLES; + /* do not pollute the main pool with directory component ids */ + if (which_repodata == _HY_REPODATA_FILENAMES || which_repodata == _HY_REPODATA_OTHER) + flags |= REPO_LOCALPOOL; + if (try_to_use_cached_solvfile(fn_cache, repo, flags, libdnf::repoGetImpl(hrepo)->checksum, error)) { g_debug("%s: using cache file: %s", __func__, fn_cache); - ret = repo_add_solv(repo, fp, flags); - if (ret) { - g_set_error_literal (error, - DNF_ERROR, - DNF_ERROR_INTERNAL_ERROR, - _("failed to add solv")); - return FALSE; - } else { - repo_update_state(hrepo, which_repodata, _HY_LOADED_CACHE); - repo_set_repodata(hrepo, which_repodata, repo->nrepodata - 1); - } + done = TRUE; + repo_update_state(hrepo, which_repodata, _HY_LOADED_CACHE); + repo_set_repodata(hrepo, which_repodata, repo->nrepodata - 1); } + if (error && *error) { + g_prefix_error(error, _("Loading extension cache %s (%d) failed: "), fn_cache, which_repodata); + return FALSE; + } + g_free(fn_cache); - if (fp) - fclose(fp); if (done) return TRUE; @@ -514,35 +530,53 @@ write_main(DnfSack *sack, HyRepo hrepo, int switchtosolv, GError **error) strerror(errno)); goto done; } - rc = repo_write(repo, fp); - rc |= checksum_write(repoImpl->checksum, fp); - rc |= fclose(fp); + + SolvUserdata solv_userdata; + if (solv_userdata_fill(&solv_userdata, repoImpl->checksum, error)) { + ret = FALSE; + fclose(fp); + goto done; + } + + Repowriter *writer = repowriter_create(repo); + repowriter_set_userdata(writer, &solv_userdata, solv_userdata_size); + rc = repowriter_write(writer, fp); + repowriter_free(writer); if (rc) { + ret = FALSE; + fclose(fp); + g_set_error(error, + DNF_ERROR, + DNF_ERROR_INTERNAL_ERROR, + _("While writing primary cache %s repowriter write failed: %i, error: %s"), + tmp_fn_templ, rc, pool_errstr(repo->pool)); + goto done; + } + + if (fclose(fp)) { ret = FALSE; g_set_error (error, DNF_ERROR, DNF_ERROR_FILE_INVALID, - _("write_main() failed writing data: %i"), rc); + _("Failed closing tmp file %s: %s"), + tmp_fn_templ, strerror(errno)); goto done; } } if (switchtosolv && repo_is_one_piece(repo)) { + repo_empty(repo, 1); /* switch over to written solv file activate paging */ - FILE *fp = fopen(tmp_fn_templ, "r"); - if (fp) { - repo_empty(repo, 1); - rc = repo_add_solv(repo, fp, 0); - fclose(fp); - if (rc) { - /* this is pretty fatal */ - ret = FALSE; - g_set_error_literal (error, - DNF_ERROR, - DNF_ERROR_FILE_INVALID, - _("write_main() failed to re-load " - "written solv file")); - goto done; - } + gboolean loaded = try_to_use_cached_solvfile(tmp_fn_templ, repo, 0, repoImpl->checksum, error); + if (error && *error) { + g_prefix_error(error, _("Failed to use newly written primary cache: %s: "), tmp_fn_templ); + ret = FALSE; + goto done; + } + if (!loaded) { + g_set_error(error, DNF_ERROR, DNF_ERROR_INTERNAL_ERROR, + _("Failed to use newly written primary cache: %s"), tmp_fn_templ); + ret = FALSE; + goto done; } } @@ -569,20 +603,6 @@ write_ext_updateinfo_filter(Repo *repo, Repokey *key, void *kfdata) return repo_write_stdkeyfilter(repo, key, 0); } -static int -write_ext_updateinfo(HyRepo hrepo, Repodata *data, FILE *fp) -{ - auto repoImpl = libdnf::repoGetImpl(hrepo); - Repo *repo = repoImpl->libsolvRepo; - int oldstart = repo->start; - repo->start = repoImpl->main_end; - repo->nsolvables -= repoImpl->main_nsolvables; - int res = repo_write_filtered(repo, fp, write_ext_updateinfo_filter, data, 0); - repo->start = oldstart; - repo->nsolvables += repoImpl->main_nsolvables; - return res; -} - static gboolean write_ext(DnfSack *sack, HyRepo hrepo, _hy_repo_repodata which_repodata, const char *suffix, GError **error) @@ -611,37 +631,78 @@ write_ext(DnfSack *sack, HyRepo hrepo, _hy_repo_repodata which_repodata, FILE *fp = fdopen(tmp_fd, "w+"); g_debug("%s: storing %s to: %s", __func__, repo->name, tmp_fn_templ); - if (which_repodata != _HY_REPODATA_UPDATEINFO) - ret |= repodata_write(data, fp); - else - ret |= write_ext_updateinfo(hrepo, data, fp); - ret |= checksum_write(repoImpl->checksum, fp); - ret |= fclose(fp); + + SolvUserdata solv_userdata; + if (solv_userdata_fill(&solv_userdata, repoImpl->checksum, error)) { + fclose(fp); + success = FALSE; + goto done; + } + + Repowriter *writer = repowriter_create(repo); + repowriter_set_userdata(writer, &solv_userdata, solv_userdata_size); + if (which_repodata != _HY_REPODATA_UPDATEINFO) { + repowriter_set_repodatarange(writer, data->repodataid, data->repodataid + 1); + repowriter_set_flags(writer, REPOWRITER_NO_STORAGE_SOLVABLE); + ret = repowriter_write(writer, fp); + } else { + // write only updateinfo repodata + int oldstart = repo->start; + repo->start = repoImpl->main_end; + repo->nsolvables -= repoImpl->main_nsolvables; + repowriter_set_flags(writer, REPOWRITER_LEGACY); + repowriter_set_keyfilter(writer, write_ext_updateinfo_filter, data); + repowriter_set_keyqueue(writer, 0); + ret = repowriter_write(writer, fp); + repo->start = oldstart; + repo->nsolvables += repoImpl->main_nsolvables; + } + repowriter_free(writer); if (ret) { + success = FALSE; + fclose(fp); + g_set_error (error, + DNF_ERROR, + DNF_ERROR_INTERNAL_ERROR, + _("While writing extension cache %s (%d): repowriter write failed: %i, error: %s"), + tmp_fn_templ, which_repodata, ret, pool_errstr(repo->pool)); + goto done; + } + + if (fclose(fp)) { success = FALSE; g_set_error (error, DNF_ERROR, - DNF_ERROR_FAILED, - _("write_ext(%1$d) has failed: %2$d"), - which_repodata, ret); + DNF_ERROR_FILE_INVALID, + _("While writing extension cache (%d): cannot close temporary file: %s"), + which_repodata, tmp_fn_templ); goto done; } } if (repo_is_one_piece(repo) && which_repodata != _HY_REPODATA_UPDATEINFO) { /* switch over to written solv file activate paging */ - FILE *fp = fopen(tmp_fn_templ, "r"); - if (fp) { - int flags = REPO_USE_LOADING | REPO_EXTEND_SOLVABLES; - /* do not pollute the main pool with directory component ids */ - if (which_repodata == _HY_REPODATA_FILENAMES || which_repodata == _HY_REPODATA_OTHER) - flags |= REPO_LOCALPOOL; - repodata_extend_block(data, repo->start, repo->end - repo->start); - data->state = REPODATA_LOADING; - repo_add_solv(repo, fp, flags); - data->state = REPODATA_AVAILABLE; - fclose(fp); + int flags = REPO_USE_LOADING | REPO_EXTEND_SOLVABLES; + /* do not pollute the main pool with directory component ids */ + if (which_repodata == _HY_REPODATA_FILENAMES || which_repodata == _HY_REPODATA_OTHER) + flags |= REPO_LOCALPOOL; + repodata_extend_block(data, repo->start, repo->end - repo->start); + data->state = REPODATA_LOADING; + int loaded = try_to_use_cached_solvfile(tmp_fn_templ, repo, flags, repoImpl->checksum, error); + if (error && *error) { + g_prefix_error(error, _("Failed to use newly written extension cache: %s (%d): "), + tmp_fn_templ, which_repodata); + success = FALSE; + goto done; + } + if (!loaded) { + g_set_error(error, DNF_ERROR, DNF_ERROR_INTERNAL_ERROR, + _("Failed to use newly written extension cache: %s (%d)"), tmp_fn_templ, which_repodata); + success = FALSE; + goto done; } + + data->state = REPODATA_AVAILABLE; } if (!mv(tmp_fn_templ, fn, error)) { @@ -672,7 +733,7 @@ load_yum_repo(DnfSack *sack, HyRepo hrepo, GError **error) FILE *fp_primary = NULL; FILE *fp_repomd = NULL; - FILE *fp_cache = fopen(fn_cache, "r"); + if (!fn_repomd) { g_set_error (error, DNF_ERROR, @@ -693,18 +754,17 @@ load_yum_repo(DnfSack *sack, HyRepo hrepo, GError **error) } checksum_fp(repoImpl->checksum, fp_repomd); - if (can_use_repomd_cache(fp_cache, repoImpl->checksum)) { + if (try_to_use_cached_solvfile(fn_cache, repo, 0, repoImpl->checksum, error)) { const char *chksum = pool_checksum_str(pool, repoImpl->checksum); g_debug("using cached %s (0x%s)", name, chksum); - if (repo_add_solv(repo, fp_cache, 0)) { - g_set_error (error, - DNF_ERROR, - DNF_ERROR_INTERNAL_ERROR, - _("repo_add_solv() has failed.")); - retval = FALSE; - goto out; - } repoImpl->state_main = _HY_LOADED_CACHE; + goto out; + } + + if (error && *error) { + g_prefix_error(error, _("While loading repository failed to use %s: "), fn_cache); + retval = FALSE; + goto out; } else { auto primary = hrepo->getMetadataPath(MD_TYPE_PRIMARY); if (primary.empty()) { @@ -733,8 +793,6 @@ load_yum_repo(DnfSack *sack, HyRepo hrepo, GError **error) repoImpl->state_main = _HY_LOADED_FETCH; } out: - if (fp_cache) - fclose(fp_cache); if (fp_repomd) fclose(fp_repomd); if (fp_primary) diff --git a/libdnf/hy-iutil-private.hpp b/libdnf/hy-iutil-private.hpp index d498c032..efc91c63 100644 --- a/libdnf/hy-iutil-private.hpp +++ b/libdnf/hy-iutil-private.hpp @@ -52,9 +52,7 @@ int solv_userdata_verify(const SolvUserdata *solv_userdata, const unsigned char /* crypto utils */ int checksum_cmp(const unsigned char *cs1, const unsigned char *cs2); int checksum_fp(unsigned char *out, FILE *fp); -int checksum_read(unsigned char *csout, FILE *fp); int checksum_stat(unsigned char *out, FILE *fp); -int checksum_write(const unsigned char *cs, FILE *fp); int checksumt_l2h(int type); const char *pool_checksum_str(Pool *pool, const unsigned char *chksum); diff --git a/libdnf/hy-iutil.cpp b/libdnf/hy-iutil.cpp index f81ca52f..c409a10a 100644 --- a/libdnf/hy-iutil.cpp +++ b/libdnf/hy-iutil.cpp @@ -142,17 +142,6 @@ checksum_fp(unsigned char *out, FILE *fp) return 0; } -/* calls rewind(fp) before returning */ -int -checksum_read(unsigned char *csout, FILE *fp) -{ - if (fseek(fp, -32, SEEK_END) || - fread(csout, CHKSUM_BYTES, 1, fp) != 1) - return 1; - rewind(fp); - return 0; -} - /* does not move the fp position */ int checksum_stat(unsigned char *out, FILE *fp) @@ -174,15 +163,6 @@ checksum_stat(unsigned char *out, FILE *fp) return 0; } -/* moves fp to the end of file */ -int checksum_write(const unsigned char *cs, FILE *fp) -{ - if (fseek(fp, 0, SEEK_END) || - fwrite(cs, CHKSUM_BYTES, 1, fp) != 1) - return 1; - return 0; -} - static std::array get_padded_solv_toolversion() { -- 2.31.1