Add zstd compression support
Resolves: RHEL-67689 Specifies git VCS for `%autosetup` because zstd tests contain binary data and their patch cannot be applied by the `patch` tool which is used by default.
This commit is contained in:
parent
92c1111ef8
commit
cc4d2c3ef6
492
0002-Add-zstd-compression-support.patch
Normal file
492
0002-Add-zstd-compression-support.patch
Normal file
@ -0,0 +1,492 @@
|
||||
From 126a79f7e313090c0bb09993f7bace43a7d05e7b Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= <amatej@redhat.com>
|
||||
Date: Tue, 17 Aug 2021 14:27:47 +0200
|
||||
Subject: [PATCH 1/4] Add zstd compression support
|
||||
|
||||
---
|
||||
CMakeLists.txt | 9 ++
|
||||
README.md | 1 +
|
||||
createrepo_c.spec | 9 +-
|
||||
doc/createrepo_c.8 | 2 +-
|
||||
src/CMakeLists.txt | 1 +
|
||||
src/cmd_parser.c | 6 +-
|
||||
src/compression_wrapper.c | 192 ++++++++++++++++++++++++++++
|
||||
src/compression_wrapper.h | 1 +
|
||||
src/error.h | 2 +
|
||||
src/python/createrepo_c/__init__.py | 3 +
|
||||
src/python/createrepo_cmodule.c | 1 +
|
||||
11 files changed, 223 insertions(+), 4 deletions(-)
|
||||
|
||||
diff --git a/CMakeLists.txt b/CMakeLists.txt
|
||||
index b016960..40d43b6 100644
|
||||
--- a/CMakeLists.txt
|
||||
+++ b/CMakeLists.txt
|
||||
@@ -21,6 +21,8 @@ if(NOT BUILD_LIBCREATEREPO_C_SHARED)
|
||||
set(CMAKE_POSITION_INDEPENDENT_CODE 1)
|
||||
endif()
|
||||
|
||||
+option(WITH_ZSTD "Build with zstd support" ON)
|
||||
+
|
||||
option(CREATEREPO_C_INSTALL_DEVELOPMENT "Install createrepo_c development files." ON)
|
||||
option(CREATEREPO_C_INSTALL_MANPAGES "Install createrepo_c man-pages." ON)
|
||||
|
||||
@@ -100,6 +102,13 @@ IF (WITH_LIBMODULEMD)
|
||||
SET (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DWITH_LIBMODULEMD")
|
||||
ENDIF (WITH_LIBMODULEMD)
|
||||
|
||||
+if (WITH_ZSTD)
|
||||
+ pkg_check_modules(ZSTD REQUIRED libzstd)
|
||||
+ include_directories(${ZSTD_INCLUDE_DIRS})
|
||||
+ SET (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DWITH_ZSTD")
|
||||
+ SET (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DWITH_ZSTD")
|
||||
+endif()
|
||||
+
|
||||
# Threaded XZ Compression
|
||||
# Note: This option is disabled by default, because Createrepo_c
|
||||
# parallelize a lot of tasks (including compression) by default, this
|
||||
diff --git a/README.md b/README.md
|
||||
index de2cd01..95b03ce 100644
|
||||
--- a/README.md
|
||||
+++ b/README.md
|
||||
@@ -25,6 +25,7 @@ Package build requires - Pkg name in Fedora/Ubuntu:
|
||||
* xz (http://tukaani.org/xz/) - xz-devel/liblzma-dev
|
||||
* zchunk (https://github.com/zchunk/zchunk) - zchunk-devel/
|
||||
* zlib (http://www.zlib.net/) - zlib-devel/zlib1g-dev
|
||||
+* libzstd (http://facebook.github.io/zstd/) - libzstd-devel/libzstd-dev
|
||||
* *Documentation:* doxygen (http://doxygen.org/) - doxygen/doxygen
|
||||
* *Documentation:* sphinx (http://sphinx-doc.org/) - python3-sphinx/python3-sphinx
|
||||
* **Test requires:** check (http://check.sourceforge.net/) - check-devel/check
|
||||
diff --git a/createrepo_c.spec b/createrepo_c.spec
|
||||
index 0105a71..afa45f6 100644
|
||||
--- a/createrepo_c.spec
|
||||
+++ b/createrepo_c.spec
|
||||
@@ -18,8 +18,11 @@
|
||||
|
||||
%if 0%{?rhel} && 0%{?rhel} < 8
|
||||
%bcond_with libmodulemd
|
||||
+# dnf supports zstd since 8.4: https://bugzilla.redhat.com/show_bug.cgi?id=1914876
|
||||
+%bcond_with zstd
|
||||
%else
|
||||
%bcond_without libmodulemd
|
||||
+%bcond_without zstd
|
||||
%endif
|
||||
|
||||
%if 0%{?rhel} && 0%{?rhel} <= 8
|
||||
@@ -65,6 +68,9 @@ Requires: rpm >= 4.9.0
|
||||
%if %{with drpm}
|
||||
BuildRequires: drpm-devel >= 0.4.0
|
||||
%endif
|
||||
+%if %{with zstd}
|
||||
+BuildRequires: pkgconfig(libzstd)
|
||||
+%endif
|
||||
|
||||
%if 0%{?fedora} || 0%{?rhel} > 7
|
||||
Obsoletes: createrepo < 0.11.0
|
||||
@@ -114,7 +120,8 @@ pushd build-py3
|
||||
-DWITH_ZCHUNK=%{?with_zchunk:ON}%{!?with_zchunk:OFF} \
|
||||
-DWITH_LIBMODULEMD=%{?with_libmodulemd:ON}%{!?with_libmodulemd:OFF} \
|
||||
-DWITH_LEGACY_HASHES=%{?with_legacy_hashes:ON}%{!?with_legacy_hashes:OFF} \
|
||||
- -DENABLE_DRPM=%{?with_drpm:ON}%{!?with_drpm:OFF}
|
||||
+ -DENABLE_DRPM=%{?with_drpm:ON}%{!?with_drpm:OFF} \
|
||||
+ -DWITH_ZSTD=%{?with_zstd:ON}%{!?with_zstd:OFF}
|
||||
make %{?_smp_mflags} RPM_OPT_FLAGS="%{optflags}"
|
||||
# Build C documentation
|
||||
make doc-c
|
||||
diff --git a/doc/createrepo_c.8 b/doc/createrepo_c.8
|
||||
index 86d0dc7..bf9862b 100644
|
||||
--- a/doc/createrepo_c.8
|
||||
+++ b/doc/createrepo_c.8
|
||||
@@ -161,7 +161,7 @@ Number of workers to spawn to read rpms.
|
||||
Use xz for repodata compression.
|
||||
.SS \-\-compress\-type COMPRESSION_TYPE
|
||||
.sp
|
||||
-Which compression type to use.
|
||||
+Which compression type to use. Supported compressions are: bzip2, gzip, zck, zstd, xz.
|
||||
.SS \-\-general\-compress\-type COMPRESSION_TYPE
|
||||
.sp
|
||||
Which compression type to use (even for primary, filelists and other xml).
|
||||
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
|
||||
index d118750..1689d5d 100644
|
||||
--- a/src/CMakeLists.txt
|
||||
+++ b/src/CMakeLists.txt
|
||||
@@ -80,6 +80,7 @@ TARGET_LINK_LIBRARIES(libcreaterepo_c ${SQLITE3_LIBRARIES})
|
||||
TARGET_LINK_LIBRARIES(libcreaterepo_c ${ZLIB_LIBRARY})
|
||||
TARGET_LINK_LIBRARIES(libcreaterepo_c ${ZCK_LIBRARIES})
|
||||
TARGET_LINK_LIBRARIES(libcreaterepo_c ${DRPM_LIBRARIES})
|
||||
+TARGET_LINK_LIBRARIES(libcreaterepo_c ${ZSTD_LIBRARIES})
|
||||
|
||||
SET_TARGET_PROPERTIES(libcreaterepo_c PROPERTIES
|
||||
OUTPUT_NAME "createrepo_c"
|
||||
diff --git a/src/cmd_parser.c b/src/cmd_parser.c
|
||||
index 0e79b40..f8b027e 100644
|
||||
--- a/src/cmd_parser.c
|
||||
+++ b/src/cmd_parser.c
|
||||
@@ -158,9 +158,9 @@ static GOptionEntry cmd_entries[] =
|
||||
{ "xz", 0, 0, G_OPTION_ARG_NONE, &(_cmd_options.xz_compression),
|
||||
"Use xz for repodata compression.", NULL },
|
||||
{ "compress-type", 0, 0, G_OPTION_ARG_STRING, &(_cmd_options.compress_type),
|
||||
- "Which compression type to use.", "COMPRESSION_TYPE" },
|
||||
+ "Which compression type to use for additional metadata files (comps, updateinfo, etc). Supported compressions are: bzip2, gzip, zck, zstd, xz.", "COMPRESSION_TYPE" },
|
||||
{ "general-compress-type", 0, 0, G_OPTION_ARG_STRING, &(_cmd_options.general_compress_type),
|
||||
- "Which compression type to use (even for primary, filelists and other xml).",
|
||||
+ "Which compression type to use (even for primary, filelists and other xml). Supported compressions are: bzip2, gzip, zck, zstd, xz.",
|
||||
"COMPRESSION_TYPE" },
|
||||
#ifdef WITH_ZCHUNK
|
||||
{ "zck", 0, 0, G_OPTION_ARG_NONE, &(_cmd_options.zck_compression),
|
||||
@@ -284,6 +284,8 @@ check_and_set_compression_type(const char *type_str,
|
||||
*type = CR_CW_BZ2_COMPRESSION;
|
||||
} else if (!strcmp(compress_str->str, "xz")) {
|
||||
*type = CR_CW_XZ_COMPRESSION;
|
||||
+ } else if (!strcmp(compress_str->str, "zstd")) {
|
||||
+ *type = CR_CW_ZSTD_COMPRESSION;
|
||||
} else {
|
||||
g_set_error(err, ERR_DOMAIN, CRE_BADARG,
|
||||
"Unknown/Unsupported compression type \"%s\"", type_str);
|
||||
diff --git a/src/compression_wrapper.c b/src/compression_wrapper.c
|
||||
index b23c345..9100222 100644
|
||||
--- a/src/compression_wrapper.c
|
||||
+++ b/src/compression_wrapper.c
|
||||
@@ -35,6 +35,9 @@
|
||||
#endif // WITH_ZCHUNK
|
||||
#include "error.h"
|
||||
#include "compression_wrapper.h"
|
||||
+#ifdef WITH_ZSTD
|
||||
+#include <zstd.h>
|
||||
+#endif
|
||||
|
||||
|
||||
#define ERR_DOMAIN CREATEREPO_C_ERROR
|
||||
@@ -118,6 +121,17 @@ typedef struct {
|
||||
unsigned char buffer[XZ_BUFFER_SIZE];
|
||||
} XzFile;
|
||||
|
||||
+#ifdef WITH_ZSTD
|
||||
+#define CR_CW_ZSTD_COMPRESSION_LEVEL 9
|
||||
+typedef struct {
|
||||
+ void *buffer;
|
||||
+ size_t buffer_size;
|
||||
+ ZSTD_inBuffer zib;
|
||||
+ ZSTD_outBuffer zob;
|
||||
+ void * context; //ZSTD_{C,D}Ctx
|
||||
+} ZstdFile;
|
||||
+#endif
|
||||
+
|
||||
cr_CompressionType
|
||||
cr_detect_compression(const char *filename, GError **err)
|
||||
{
|
||||
@@ -151,6 +165,9 @@ cr_detect_compression(const char *filename, GError **err)
|
||||
} else if (g_str_has_suffix(filename, ".zck"))
|
||||
{
|
||||
return CR_CW_ZCK_COMPRESSION;
|
||||
+ } else if (g_str_has_suffix(filename, ".zst"))
|
||||
+ {
|
||||
+ return CR_CW_ZSTD_COMPRESSION;
|
||||
} else if (g_str_has_suffix(filename, ".xml") ||
|
||||
g_str_has_suffix(filename, ".tar") ||
|
||||
g_str_has_suffix(filename, ".yaml") ||
|
||||
@@ -192,6 +209,11 @@ cr_detect_compression(const char *filename, GError **err)
|
||||
type = CR_CW_GZ_COMPRESSION;
|
||||
}
|
||||
|
||||
+ else if (g_str_has_prefix(mime_type, "application/zstd"))
|
||||
+ {
|
||||
+ type = CR_CW_ZSTD_COMPRESSION;
|
||||
+ }
|
||||
+
|
||||
else if (g_str_has_prefix(mime_type, "application/x-bzip2") ||
|
||||
g_str_has_prefix(mime_type, "application/x-bz2") ||
|
||||
g_str_has_prefix(mime_type, "application/bzip2") ||
|
||||
@@ -255,6 +277,8 @@ cr_compression_type(const char *name)
|
||||
type = CR_CW_XZ_COMPRESSION;
|
||||
if (!g_strcmp0(name_lower, "zck"))
|
||||
type = CR_CW_ZCK_COMPRESSION;
|
||||
+ if (!g_strcmp0(name_lower, "zstd"))
|
||||
+ type = CR_CW_ZSTD_COMPRESSION;
|
||||
g_free(name_lower);
|
||||
|
||||
return type;
|
||||
@@ -272,6 +296,8 @@ cr_compression_suffix(cr_CompressionType comtype)
|
||||
return ".xz";
|
||||
case CR_CW_ZCK_COMPRESSION:
|
||||
return ".zck";
|
||||
+ case CR_CW_ZSTD_COMPRESSION:
|
||||
+ return ".zst";
|
||||
default:
|
||||
return NULL;
|
||||
}
|
||||
@@ -413,6 +439,56 @@ cr_sopen(const char *filename,
|
||||
}
|
||||
break;
|
||||
|
||||
+ case (CR_CW_ZSTD_COMPRESSION): { // ------------------------------------
|
||||
+#ifdef WITH_ZSTD
|
||||
+ FILE *f = fopen(filename, mode_str);
|
||||
+
|
||||
+ if (!f) {
|
||||
+ g_set_error(err, ERR_DOMAIN, CRE_IO, "fopen(): %s", g_strerror(errno));
|
||||
+ break;
|
||||
+ }
|
||||
+
|
||||
+ file->INNERFILE = f;
|
||||
+
|
||||
+ ZstdFile *zstd_file = g_malloc0(sizeof(ZstdFile));
|
||||
+
|
||||
+ if (mode == CR_CW_MODE_WRITE) {
|
||||
+ if ((zstd_file->context = (void *) ZSTD_createCCtx()) == NULL) {
|
||||
+ g_set_error(err, ERR_DOMAIN, CRE_ZSTD, "%s",
|
||||
+ "Failed to create ZSTD context.");
|
||||
+ g_free(zstd_file);
|
||||
+ fclose(f);
|
||||
+ break;
|
||||
+ }
|
||||
+ size_t ret = ZSTD_CCtx_setParameter(zstd_file->context, ZSTD_c_compressionLevel, CR_CW_ZSTD_COMPRESSION_LEVEL);
|
||||
+ if (ZSTD_isError(ret)) {
|
||||
+ g_set_error(err, ERR_DOMAIN, CRE_ZSTD, "%s",
|
||||
+ ZSTD_getErrorName(ret));
|
||||
+ g_free(zstd_file);
|
||||
+ fclose(f);
|
||||
+ break;
|
||||
+ }
|
||||
+ zstd_file->buffer_size = ZSTD_CStreamOutSize();
|
||||
+ } else {
|
||||
+ if ((zstd_file->context = (void *) ZSTD_createDCtx()) == NULL) {
|
||||
+ g_free(zstd_file);
|
||||
+ fclose(f);
|
||||
+ g_set_error(err, ERR_DOMAIN, CRE_IO, "%s",
|
||||
+ "Failed to create ZSTD context.");
|
||||
+ break;
|
||||
+ }
|
||||
+ zstd_file->buffer_size = ZSTD_DStreamInSize();
|
||||
+ }
|
||||
+ zstd_file->buffer = g_malloc(zstd_file->buffer_size);
|
||||
+ file->FILE = (void *) zstd_file;
|
||||
+
|
||||
+ break;
|
||||
+#else
|
||||
+ g_set_error(err, ERR_DOMAIN, CRE_IO, "createrepo_c wasn't compiled with zstd support");
|
||||
+ break;
|
||||
+#endif // WITH_ZSTD
|
||||
+ }
|
||||
+
|
||||
case (CR_CW_BZ2_COMPRESSION): { // ------------------------------------
|
||||
FILE *f = fopen(filename, mode_str);
|
||||
file->INNERFILE = f;
|
||||
@@ -769,6 +845,43 @@ cr_close(CR_FILE *cr_file, GError **err)
|
||||
}
|
||||
break;
|
||||
|
||||
+ case (CR_CW_ZSTD_COMPRESSION): // --------------------------------------
|
||||
+#ifdef WITH_ZSTD
|
||||
+ ZstdFile * zstd = (ZstdFile *) cr_file->FILE;
|
||||
+ if (cr_file->mode == CR_CW_MODE_READ) {
|
||||
+ ZSTD_freeDCtx(zstd->context);
|
||||
+ } else {
|
||||
+ size_t remaining;
|
||||
+ // No more new input just finish flushing compression data
|
||||
+ ZSTD_inBuffer zip = { NULL, 0, 0 };
|
||||
+ do {
|
||||
+ zstd->zob.dst = zstd->buffer;
|
||||
+ zstd->zob.size = zstd->buffer_size;
|
||||
+ zstd->zob.pos = 0;
|
||||
+
|
||||
+ remaining = ZSTD_compressStream2(zstd->context, &zstd->zob , &zip, ZSTD_e_end);
|
||||
+ if (ZSTD_isError(remaining)) {
|
||||
+ g_set_error(err, ERR_DOMAIN, CRE_ZSTD, "%s", ZSTD_getErrorName(remaining));
|
||||
+ break;
|
||||
+ } else if (zstd->zob.pos != fwrite(zstd->buffer, 1, zstd->zob.pos, cr_file->INNERFILE)) {
|
||||
+ g_set_error(err, ERR_DOMAIN, CRE_IO, "cr_close ZSTD fwrite failed");
|
||||
+ break;
|
||||
+ }
|
||||
+ } while(remaining != 0);
|
||||
+ ZSTD_freeCCtx(zstd->context);
|
||||
+ }
|
||||
+
|
||||
+ fclose(cr_file->INNERFILE);
|
||||
+ g_free(zstd->buffer);
|
||||
+ g_free(cr_file->FILE);
|
||||
+
|
||||
+ ret = CRE_OK;
|
||||
+ break;
|
||||
+#else
|
||||
+ g_set_error(err, ERR_DOMAIN, CRE_IO, "createrepo_c wasn't compiled with zstd support");
|
||||
+ break;
|
||||
+#endif // WITH_ZSTD
|
||||
+
|
||||
case (CR_CW_BZ2_COMPRESSION): // --------------------------------------
|
||||
if (cr_file->mode == CR_CW_MODE_READ)
|
||||
BZ2_bzReadClose(&rc, (BZFILE *) cr_file->FILE);
|
||||
@@ -980,6 +1093,43 @@ cr_read(CR_FILE *cr_file, void *buffer, unsigned int len, GError **err)
|
||||
}
|
||||
break;
|
||||
|
||||
+ case (CR_CW_ZSTD_COMPRESSION): // ---------------------------------------
|
||||
+#ifdef WITH_ZSTD
|
||||
+ ZstdFile * zstd = (ZstdFile *) cr_file->FILE;
|
||||
+
|
||||
+ ZSTD_outBuffer zob = {buffer, len, 0};
|
||||
+
|
||||
+ while (zob.pos < zob.size) {
|
||||
+ // Re-fill compressed data buffer
|
||||
+ if (zstd->zib.pos >= zstd->zib.size) {
|
||||
+ zstd->zib.size = fread(zstd->buffer, 1, zstd->buffer_size, cr_file->INNERFILE);
|
||||
+ if (zstd->zib.size == 0) {
|
||||
+ break; //EOF
|
||||
+ }
|
||||
+ zstd->zib.src = zstd->buffer;
|
||||
+ zstd->zib.pos = 0;
|
||||
+ }
|
||||
+
|
||||
+ // Decompress chunk
|
||||
+ int decomp_ret = ZSTD_decompressStream(zstd->context, &zob, &zstd->zib);
|
||||
+ if (ZSTD_isError(decomp_ret)) {
|
||||
+ ret = CR_CW_ERR;
|
||||
+ g_set_error(err, ERR_DOMAIN, CRE_ZSTD, "%s", ZSTD_getErrorName(decomp_ret));
|
||||
+ break;
|
||||
+ }
|
||||
+
|
||||
+ }
|
||||
+
|
||||
+ if (!(err && *err)) {
|
||||
+ ret = zob.pos;
|
||||
+ }
|
||||
+
|
||||
+ break;
|
||||
+#else
|
||||
+ g_set_error(err, ERR_DOMAIN, CRE_IO, "createrepo_c wasn't compiled with zstd support");
|
||||
+ break;
|
||||
+#endif // WITH_ZSTD
|
||||
+
|
||||
case (CR_CW_BZ2_COMPRESSION): // --------------------------------------
|
||||
ret = BZ2_bzRead(&bzerror, (BZFILE *) cr_file->FILE, buffer, len);
|
||||
if (!ret && bzerror == BZ_SEQUENCE_ERROR)
|
||||
@@ -1214,6 +1364,44 @@ cr_write(CR_FILE *cr_file, const void *buffer, unsigned int len, GError **err)
|
||||
}
|
||||
break;
|
||||
|
||||
+ case (CR_CW_ZSTD_COMPRESSION): // ---------------------------------------
|
||||
+#ifdef WITH_ZSTD
|
||||
+ ZstdFile * zstd = (ZstdFile *) cr_file->FILE;
|
||||
+ ZSTD_inBuffer zib = {buffer, len, 0};
|
||||
+
|
||||
+ while (zib.pos < zib.size) {
|
||||
+ zstd->zob.dst = zstd->buffer;
|
||||
+ zstd->zob.size = zstd->buffer_size;
|
||||
+ zstd->zob.pos = 0;
|
||||
+
|
||||
+ // Compress chunk into buffer
|
||||
+ size_t remaining = ZSTD_compressStream2(zstd->context, &zstd->zob , &zib, ZSTD_e_continue);
|
||||
+ if (ZSTD_isError(remaining)) {
|
||||
+ g_set_error(err, ERR_DOMAIN, CRE_ZSTD, "%s", ZSTD_getErrorName(remaining));
|
||||
+ break;
|
||||
+ }
|
||||
+
|
||||
+ // Write compressed buffer
|
||||
+ if (zstd->zob.pos > 0) {
|
||||
+ size_t nw = fwrite(zstd->buffer, 1, zstd->zob.pos, cr_file->INNERFILE);
|
||||
+ if (nw != zstd->zob.pos) {
|
||||
+ g_set_error(err, ERR_DOMAIN, CRE_IO, "cr_write zstd write failed");
|
||||
+ break;
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ }
|
||||
+
|
||||
+ if (!(err && *err)) {
|
||||
+ ret = zib.pos;
|
||||
+ }
|
||||
+
|
||||
+ break;
|
||||
+#else
|
||||
+ g_set_error(err, ERR_DOMAIN, CRE_IO, "createrepo_c wasn't compiled with zstd support");
|
||||
+ break;
|
||||
+#endif // WITH_ZSTD
|
||||
+
|
||||
case (CR_CW_BZ2_COMPRESSION): // --------------------------------------
|
||||
BZ2_bzWrite(&bzerror, (BZFILE *) cr_file->FILE, (void *) buffer, len);
|
||||
if (bzerror == BZ_OK) {
|
||||
@@ -1361,6 +1549,7 @@ cr_puts(CR_FILE *cr_file, const char *str, GError **err)
|
||||
case (CR_CW_BZ2_COMPRESSION): // --------------------------------------
|
||||
case (CR_CW_XZ_COMPRESSION): // ---------------------------------------
|
||||
case (CR_CW_ZCK_COMPRESSION): // --------------------------------------
|
||||
+ case (CR_CW_ZSTD_COMPRESSION): // --------------------------------------
|
||||
len = strlen(str);
|
||||
ret = cr_write(cr_file, str, len, err);
|
||||
if (ret != (int) len)
|
||||
@@ -1398,6 +1587,7 @@ cr_end_chunk(CR_FILE *cr_file, GError **err)
|
||||
case (CR_CW_GZ_COMPRESSION): // ---------------------------------------
|
||||
case (CR_CW_BZ2_COMPRESSION): // --------------------------------------
|
||||
case (CR_CW_XZ_COMPRESSION): // ---------------------------------------
|
||||
+ case (CR_CW_ZSTD_COMPRESSION): // ---------------------------------------
|
||||
break;
|
||||
case (CR_CW_ZCK_COMPRESSION): { // ------------------------------------
|
||||
#ifdef WITH_ZCHUNK
|
||||
@@ -1450,6 +1640,7 @@ cr_set_autochunk(CR_FILE *cr_file, gboolean auto_chunk, GError **err)
|
||||
case (CR_CW_GZ_COMPRESSION): // ---------------------------------------
|
||||
case (CR_CW_BZ2_COMPRESSION): // --------------------------------------
|
||||
case (CR_CW_XZ_COMPRESSION): // ---------------------------------------
|
||||
+ case (CR_CW_ZSTD_COMPRESSION): // ---------------------------------------
|
||||
break;
|
||||
case (CR_CW_ZCK_COMPRESSION): { // ------------------------------------
|
||||
#ifdef WITH_ZCHUNK
|
||||
@@ -1524,6 +1715,7 @@ cr_printf(GError **err, CR_FILE *cr_file, const char *format, ...)
|
||||
case (CR_CW_BZ2_COMPRESSION): // --------------------------------------
|
||||
case (CR_CW_XZ_COMPRESSION): // ---------------------------------------
|
||||
case (CR_CW_ZCK_COMPRESSION): // --------------------------------------
|
||||
+ case (CR_CW_ZSTD_COMPRESSION): // --------------------------------------
|
||||
tmp_ret = cr_write(cr_file, buf, ret, err);
|
||||
if (tmp_ret != (int) ret)
|
||||
ret = CR_CW_ERR;
|
||||
diff --git a/src/compression_wrapper.h b/src/compression_wrapper.h
|
||||
index 72e0078..32936d0 100644
|
||||
--- a/src/compression_wrapper.h
|
||||
+++ b/src/compression_wrapper.h
|
||||
@@ -42,6 +42,7 @@ typedef enum {
|
||||
CR_CW_BZ2_COMPRESSION, /*!< BZip2 compression */
|
||||
CR_CW_XZ_COMPRESSION, /*!< XZ compression */
|
||||
CR_CW_ZCK_COMPRESSION, /*!< ZCK compression */
|
||||
+ CR_CW_ZSTD_COMPRESSION, /*!< ZSTD compression */
|
||||
CR_CW_COMPRESSION_SENTINEL, /*!< Sentinel of the list */
|
||||
} cr_CompressionType;
|
||||
|
||||
diff --git a/src/error.h b/src/error.h
|
||||
index b925bc7..8738032 100644
|
||||
--- a/src/error.h
|
||||
+++ b/src/error.h
|
||||
@@ -98,6 +98,8 @@ typedef enum {
|
||||
(34) ZCK library related error */
|
||||
CRE_MODULEMD, /*!<
|
||||
(35) modulemd related error */
|
||||
+ CRE_ZSTD, /*!<
|
||||
+ (36) Zstd library related error */
|
||||
CRE_SENTINEL, /*!<
|
||||
(XX) Sentinel */
|
||||
} cr_Error;
|
||||
diff --git a/src/python/createrepo_c/__init__.py b/src/python/createrepo_c/__init__.py
|
||||
index 440e559..21f6f74 100644
|
||||
--- a/src/python/createrepo_c/__init__.py
|
||||
+++ b/src/python/createrepo_c/__init__.py
|
||||
@@ -59,6 +59,9 @@ XZ = _createrepo_c.XZ_COMPRESSION
|
||||
#: Zchunk compression alias
|
||||
ZCK = _createrepo_c.ZCK_COMPRESSION
|
||||
|
||||
+#: Zstd compression alias
|
||||
+ZSTD = _createrepo_c.ZSTD_COMPRESSION
|
||||
+
|
||||
HT_KEY_DEFAULT = _createrepo_c.HT_KEY_DEFAULT #: Default key (hash)
|
||||
HT_KEY_HASH = _createrepo_c.HT_KEY_HASH #: Package hash as a key
|
||||
HT_KEY_NAME = _createrepo_c.HT_KEY_NAME #: Package name as a key
|
||||
diff --git a/src/python/createrepo_cmodule.c b/src/python/createrepo_cmodule.c
|
||||
index ba6cad6..64ac4ec 100644
|
||||
--- a/src/python/createrepo_cmodule.c
|
||||
+++ b/src/python/createrepo_cmodule.c
|
||||
@@ -264,6 +264,7 @@ PyInit__createrepo_c(void)
|
||||
PyModule_AddIntConstant(m, "BZ2_COMPRESSION", CR_CW_BZ2_COMPRESSION);
|
||||
PyModule_AddIntConstant(m, "XZ_COMPRESSION", CR_CW_XZ_COMPRESSION);
|
||||
PyModule_AddIntConstant(m, "ZCK_COMPRESSION", CR_CW_ZCK_COMPRESSION);
|
||||
+ PyModule_AddIntConstant(m, "ZSTD_COMPRESSION", CR_CW_ZSTD_COMPRESSION);
|
||||
|
||||
/* Zchunk support */
|
||||
#ifdef WITH_ZCHUNK
|
||||
--
|
||||
2.48.1
|
||||
|
358
0003-Add-unittests-for-zstd-compression.patch
Normal file
358
0003-Add-unittests-for-zstd-compression.patch
Normal file
@ -0,0 +1,358 @@
|
||||
From 06ff7fc8adbdbbe7d4d7782e278b1a2e20d9ab8b Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= <amatej@redhat.com>
|
||||
Date: Tue, 17 Aug 2021 14:28:05 +0200
|
||||
Subject: [PATCH 2/4] Add unittests for zstd compression
|
||||
|
||||
---
|
||||
.../python/tests/test_compression_wrapper.py | 8 +-
|
||||
tests/test_compression_wrapper.c | 161 ++++++++++++++++++
|
||||
tests/testdata/compressed_files/00_plain.foo5 | Bin 0 -> 13 bytes
|
||||
.../compressed_files/00_plain.txt.zst | Bin 0 -> 13 bytes
|
||||
tests/testdata/compressed_files/01_plain.foo5 | Bin 0 -> 51 bytes
|
||||
.../compressed_files/01_plain.txt.zst | Bin 0 -> 51 bytes
|
||||
6 files changed, 168 insertions(+), 1 deletion(-)
|
||||
create mode 100644 tests/testdata/compressed_files/00_plain.foo5
|
||||
create mode 100644 tests/testdata/compressed_files/00_plain.txt.zst
|
||||
create mode 100644 tests/testdata/compressed_files/01_plain.foo5
|
||||
create mode 100644 tests/testdata/compressed_files/01_plain.txt.zst
|
||||
|
||||
diff --git a/tests/python/tests/test_compression_wrapper.py b/tests/python/tests/test_compression_wrapper.py
|
||||
index c8957a2..7719fdd 100644
|
||||
--- a/tests/python/tests/test_compression_wrapper.py
|
||||
+++ b/tests/python/tests/test_compression_wrapper.py
|
||||
@@ -15,6 +15,7 @@ class TestCaseCompressionWrapper(unittest.TestCase):
|
||||
self.assertEqual(cr.compression_suffix(cr.BZ2), ".bz2")
|
||||
self.assertEqual(cr.compression_suffix(cr.XZ), ".xz")
|
||||
self.assertEqual(cr.compression_suffix(cr.ZCK), ".zck")
|
||||
+ self.assertEqual(cr.compression_suffix(cr.ZSTD), ".zst")
|
||||
|
||||
def test_detect_compression(self):
|
||||
|
||||
@@ -69,6 +70,11 @@ class TestCaseCompressionWrapper(unittest.TestCase):
|
||||
#comtype = cr.detect_compression(path)
|
||||
#self.assertEqual(comtype, cr.ZCK)
|
||||
|
||||
+ # Bad suffix - zstd compression
|
||||
+ path = os.path.join(COMPRESSED_FILES_PATH, "01_plain.foo5")
|
||||
+ comtype = cr.detect_compression(path)
|
||||
+ self.assertEqual(comtype, cr.ZSTD)
|
||||
+
|
||||
def test_compression_type(self):
|
||||
self.assertEqual(cr.compression_type(None), cr.UNKNOWN_COMPRESSION)
|
||||
self.assertEqual(cr.compression_type(""), cr.UNKNOWN_COMPRESSION)
|
||||
@@ -77,4 +83,4 @@ class TestCaseCompressionWrapper(unittest.TestCase):
|
||||
self.assertEqual(cr.compression_type("xz"), cr.XZ)
|
||||
self.assertEqual(cr.compression_type("XZ"), cr.XZ)
|
||||
self.assertEqual(cr.compression_type("zck"), cr.ZCK)
|
||||
-
|
||||
+ self.assertEqual(cr.compression_type("zstd"), cr.ZSTD)
|
||||
diff --git a/tests/test_compression_wrapper.c b/tests/test_compression_wrapper.c
|
||||
index edd4fe5..6326156 100644
|
||||
--- a/tests/test_compression_wrapper.c
|
||||
+++ b/tests/test_compression_wrapper.c
|
||||
@@ -37,10 +37,12 @@
|
||||
#define FILE_COMPRESSED_0_GZ TEST_COMPRESSED_FILES_PATH"/00_plain.txt.gz"
|
||||
#define FILE_COMPRESSED_0_BZ2 TEST_COMPRESSED_FILES_PATH"/00_plain.txt.bz2"
|
||||
#define FILE_COMPRESSED_0_XZ TEST_COMPRESSED_FILES_PATH"/00_plain.txt.xz"
|
||||
+#define FILE_COMPRESSED_0_ZSTD TEST_COMPRESSED_FILES_PATH"/00_plain.txt.zst"
|
||||
#define FILE_COMPRESSED_0_PLAIN_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/00_plain.foo0"
|
||||
#define FILE_COMPRESSED_0_GZ_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/00_plain.foo1"
|
||||
#define FILE_COMPRESSED_0_BZ2_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/00_plain.foo2"
|
||||
#define FILE_COMPRESSED_0_XZ_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/00_plain.foo3"
|
||||
+#define FILE_COMPRESSED_0_ZSTD_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/00_plain.foo5"
|
||||
|
||||
#define FILE_COMPRESSED_1_CONTENT "foobar foobar foobar foobar test test\nfolkjsaflkjsadokf\n"
|
||||
#define FILE_COMPRESSED_1_CONTENT_LEN 56
|
||||
@@ -48,11 +50,13 @@
|
||||
#define FILE_COMPRESSED_1_GZ TEST_COMPRESSED_FILES_PATH"/01_plain.txt.gz"
|
||||
#define FILE_COMPRESSED_1_BZ2 TEST_COMPRESSED_FILES_PATH"/01_plain.txt.bz2"
|
||||
#define FILE_COMPRESSED_1_XZ TEST_COMPRESSED_FILES_PATH"/01_plain.txt.xz"
|
||||
+#define FILE_COMPRESSED_1_ZSTD TEST_COMPRESSED_FILES_PATH"/01_plain.txt.zst"
|
||||
#define FILE_COMPRESSED_1_ZCK TEST_COMPRESSED_FILES_PATH"/01_plain.txt.zck"
|
||||
#define FILE_COMPRESSED_1_PLAIN_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/01_plain.foo0"
|
||||
#define FILE_COMPRESSED_1_GZ_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/01_plain.foo1"
|
||||
#define FILE_COMPRESSED_1_BZ2_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/01_plain.foo2"
|
||||
#define FILE_COMPRESSED_1_XZ_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/01_plain.foo3"
|
||||
+#define FILE_COMPRESSED_1_ZSTD_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/01_plain.foo5"
|
||||
|
||||
|
||||
static void
|
||||
@@ -98,6 +102,11 @@ test_cr_compression_suffix(void)
|
||||
|
||||
suffix = cr_compression_suffix(CR_CW_XZ_COMPRESSION);
|
||||
g_assert_cmpstr(suffix, ==, ".xz");
|
||||
+
|
||||
+#ifdef WITH_ZSTD
|
||||
+ suffix = cr_compression_suffix(CR_CW_ZSTD_COMPRESSION);
|
||||
+ g_assert_cmpstr(suffix, ==, ".zst");
|
||||
+#endif // WITH_ZSTD
|
||||
}
|
||||
|
||||
static void
|
||||
@@ -134,6 +143,11 @@ test_cr_compression_type(void)
|
||||
|
||||
type = cr_compression_type("xz");
|
||||
g_assert_cmpint(type, ==, CR_CW_XZ_COMPRESSION);
|
||||
+
|
||||
+#ifdef WITH_ZSTD
|
||||
+ type = cr_compression_type("zstd");
|
||||
+ g_assert_cmpint(type, ==, CR_CW_ZSTD_COMPRESSION);
|
||||
+#endif // WITH_ZSTD
|
||||
}
|
||||
|
||||
static void
|
||||
@@ -177,6 +191,17 @@ test_cr_detect_compression(void)
|
||||
ret = cr_detect_compression(FILE_COMPRESSED_1_XZ, &tmp_err);
|
||||
g_assert_cmpint(ret, ==, CR_CW_XZ_COMPRESSION);
|
||||
g_assert(!tmp_err);
|
||||
+
|
||||
+#ifdef WITH_ZSTD
|
||||
+ // Zstd
|
||||
+
|
||||
+ ret = cr_detect_compression(FILE_COMPRESSED_0_ZSTD, &tmp_err);
|
||||
+ g_assert_cmpint(ret, ==, CR_CW_ZSTD_COMPRESSION);
|
||||
+ g_assert(!tmp_err);
|
||||
+ ret = cr_detect_compression(FILE_COMPRESSED_1_ZSTD, &tmp_err);
|
||||
+ g_assert_cmpint(ret, ==, CR_CW_ZSTD_COMPRESSION);
|
||||
+ g_assert(!tmp_err);
|
||||
+#endif // WITH_ZSTD
|
||||
}
|
||||
|
||||
|
||||
@@ -221,6 +246,17 @@ test_cr_detect_compression_bad_suffix(void)
|
||||
ret = cr_detect_compression(FILE_COMPRESSED_1_XZ_BAD_SUFFIX, &tmp_err);
|
||||
g_assert_cmpint(ret, ==, CR_CW_XZ_COMPRESSION);
|
||||
g_assert(!tmp_err);
|
||||
+
|
||||
+#ifdef WITH_ZSTD
|
||||
+ // Zstd
|
||||
+
|
||||
+ ret = cr_detect_compression(FILE_COMPRESSED_0_ZSTD_BAD_SUFFIX, &tmp_err);
|
||||
+ g_assert_cmpint(ret, ==, CR_CW_ZSTD_COMPRESSION);
|
||||
+ g_assert(!tmp_err);
|
||||
+ ret = cr_detect_compression(FILE_COMPRESSED_1_ZSTD_BAD_SUFFIX, &tmp_err);
|
||||
+ g_assert_cmpint(ret, ==, CR_CW_ZSTD_COMPRESSION);
|
||||
+ g_assert(!tmp_err);
|
||||
+#endif // WITH_ZSTD
|
||||
}
|
||||
|
||||
|
||||
@@ -235,6 +271,16 @@ test_helper_cw_input(const char *filename,
|
||||
char buffer[COMPRESSED_BUFFER_LEN+1];
|
||||
GError *tmp_err = NULL;
|
||||
|
||||
+ if (ctype != CR_CW_AUTO_DETECT_COMPRESSION) {
|
||||
+ cr_CompressionType detected_type = cr_detect_compression(filename, &tmp_err);
|
||||
+ g_assert(!tmp_err);
|
||||
+ if (ctype != detected_type) {
|
||||
+ printf("detected_type: %i does not match passed type: %i for filename: %s\n",
|
||||
+ detected_type, ctype, filename);
|
||||
+ g_assert(0);
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
file = cr_open(filename, CR_CW_MODE_READ, ctype, &tmp_err);
|
||||
g_assert(file);
|
||||
g_assert(!tmp_err);
|
||||
@@ -281,6 +327,15 @@ test_cr_read_with_autodetection(void)
|
||||
FILE_COMPRESSED_0_CONTENT, FILE_COMPRESSED_0_CONTENT_LEN);
|
||||
test_helper_cw_input(FILE_COMPRESSED_1_XZ, CR_CW_AUTO_DETECT_COMPRESSION,
|
||||
FILE_COMPRESSED_1_CONTENT, FILE_COMPRESSED_1_CONTENT_LEN);
|
||||
+
|
||||
+#ifdef WITH_ZSTD
|
||||
+ // Zstd
|
||||
+
|
||||
+ test_helper_cw_input(FILE_COMPRESSED_0_ZSTD, CR_CW_AUTO_DETECT_COMPRESSION,
|
||||
+ FILE_COMPRESSED_0_CONTENT, FILE_COMPRESSED_0_CONTENT_LEN);
|
||||
+ test_helper_cw_input(FILE_COMPRESSED_1_ZSTD, CR_CW_AUTO_DETECT_COMPRESSION,
|
||||
+ FILE_COMPRESSED_1_CONTENT, FILE_COMPRESSED_1_CONTENT_LEN);
|
||||
+#endif // WITH_ZSTD
|
||||
}
|
||||
|
||||
|
||||
@@ -447,6 +502,29 @@ outputtest_cw_output(Outputtest *outputtest,
|
||||
test_helper_cw_output(OUTPUT_TYPE_PRINTF, outputtest->tmp_filename,
|
||||
CR_CW_XZ_COMPRESSION, FILE_COMPRESSED_1_CONTENT,
|
||||
FILE_COMPRESSED_1_CONTENT_LEN);
|
||||
+
|
||||
+#ifdef WITH_ZSTD
|
||||
+ // Zstd
|
||||
+
|
||||
+ test_helper_cw_output(OUTPUT_TYPE_WRITE, outputtest->tmp_filename,
|
||||
+ CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_0_CONTENT,
|
||||
+ FILE_COMPRESSED_0_CONTENT_LEN);
|
||||
+ test_helper_cw_output(OUTPUT_TYPE_WRITE, outputtest->tmp_filename,
|
||||
+ CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_1_CONTENT,
|
||||
+ FILE_COMPRESSED_1_CONTENT_LEN);
|
||||
+ test_helper_cw_output(OUTPUT_TYPE_PUTS, outputtest->tmp_filename,
|
||||
+ CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_0_CONTENT,
|
||||
+ FILE_COMPRESSED_0_CONTENT_LEN);
|
||||
+ test_helper_cw_output(OUTPUT_TYPE_PUTS, outputtest->tmp_filename,
|
||||
+ CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_1_CONTENT,
|
||||
+ FILE_COMPRESSED_1_CONTENT_LEN);
|
||||
+ test_helper_cw_output(OUTPUT_TYPE_PRINTF, outputtest->tmp_filename,
|
||||
+ CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_0_CONTENT,
|
||||
+ FILE_COMPRESSED_0_CONTENT_LEN);
|
||||
+ test_helper_cw_output(OUTPUT_TYPE_PRINTF, outputtest->tmp_filename,
|
||||
+ CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_1_CONTENT,
|
||||
+ FILE_COMPRESSED_1_CONTENT_LEN);
|
||||
+#endif // WITH_ZSTD
|
||||
}
|
||||
|
||||
|
||||
@@ -508,6 +586,15 @@ test_cr_error_handling(void)
|
||||
g_error_free(tmp_err);
|
||||
tmp_err = NULL;
|
||||
|
||||
+#ifdef WITH_ZSTD
|
||||
+ f = cr_open("/", CR_CW_MODE_WRITE, CR_CW_ZSTD_COMPRESSION, &tmp_err);
|
||||
+ g_assert(!f);
|
||||
+ g_assert(tmp_err);
|
||||
+ g_assert_cmpint(tmp_err->code, ==, CRE_IO);
|
||||
+ g_error_free(tmp_err);
|
||||
+ tmp_err = NULL;
|
||||
+#endif // WITH_ZSTD
|
||||
+
|
||||
// Opening plain text file as compressed
|
||||
|
||||
char buf[256];
|
||||
@@ -550,6 +637,21 @@ test_cr_error_handling(void)
|
||||
ret = cr_close(f, &tmp_err);
|
||||
g_assert_cmpint(ret, ==, CRE_OK);
|
||||
g_assert(!tmp_err);
|
||||
+
|
||||
+#ifdef WITH_ZSTD
|
||||
+ f = cr_open(FILE_COMPRESSED_1_PLAIN, CR_CW_MODE_READ,
|
||||
+ CR_CW_ZSTD_COMPRESSION, &tmp_err);
|
||||
+ g_assert(f);
|
||||
+ ret = cr_read(f, buf, 256, &tmp_err);
|
||||
+ g_assert_cmpint(ret, ==, -1);
|
||||
+ g_assert(tmp_err);
|
||||
+ g_assert_cmpint(tmp_err->code, ==, CRE_ZSTD);
|
||||
+ g_error_free(tmp_err);
|
||||
+ tmp_err = NULL;
|
||||
+ ret = cr_close(f, &tmp_err);
|
||||
+ g_assert_cmpint(ret, ==, CRE_OK);
|
||||
+ g_assert(!tmp_err);
|
||||
+#endif // WITH_ZSTD
|
||||
}
|
||||
|
||||
|
||||
@@ -670,6 +772,33 @@ test_contentstating_singlewrite(Outputtest *outputtest,
|
||||
g_assert_cmpstr(stat->checksum, ==, content_sha256);
|
||||
cr_contentstat_free(stat, &tmp_err);
|
||||
g_assert(!tmp_err);
|
||||
+
|
||||
+#ifdef WITH_ZSTD
|
||||
+ // zstd compression
|
||||
+ stat = cr_contentstat_new(CR_CHECKSUM_SHA256, &tmp_err);
|
||||
+ g_assert(stat);
|
||||
+ g_assert(!tmp_err);
|
||||
+
|
||||
+ f = cr_sopen(outputtest->tmp_filename,
|
||||
+ CR_CW_MODE_WRITE,
|
||||
+ CR_CW_ZSTD_COMPRESSION,
|
||||
+ stat,
|
||||
+ &tmp_err);
|
||||
+ g_assert(f);
|
||||
+ g_assert(!tmp_err);
|
||||
+
|
||||
+ ret = cr_write(f, content, content_len, &tmp_err);
|
||||
+ g_assert_cmpint(ret, ==, content_len);
|
||||
+ g_assert(!tmp_err);
|
||||
+
|
||||
+ cr_close(f, &tmp_err);
|
||||
+ g_assert(!tmp_err);
|
||||
+
|
||||
+ g_assert_cmpint(stat->size, ==, content_len);
|
||||
+ g_assert_cmpstr(stat->checksum, ==, content_sha256);
|
||||
+ cr_contentstat_free(stat, &tmp_err);
|
||||
+ g_assert(!tmp_err);
|
||||
+#endif // WITH_ZSTD
|
||||
}
|
||||
|
||||
static void
|
||||
@@ -715,6 +844,38 @@ test_contentstating_multiwrite(Outputtest *outputtest,
|
||||
g_assert_cmpstr(stat->checksum, ==, content_sha256);
|
||||
cr_contentstat_free(stat, &tmp_err);
|
||||
g_assert(!tmp_err);
|
||||
+
|
||||
+#ifdef WITH_ZSTD
|
||||
+ // Zstd compression
|
||||
+
|
||||
+ stat = cr_contentstat_new(CR_CHECKSUM_SHA256, &tmp_err);
|
||||
+ g_assert(stat);
|
||||
+ g_assert(!tmp_err);
|
||||
+
|
||||
+ f = cr_sopen(outputtest->tmp_filename,
|
||||
+ CR_CW_MODE_WRITE,
|
||||
+ CR_CW_ZSTD_COMPRESSION,
|
||||
+ stat,
|
||||
+ &tmp_err);
|
||||
+ g_assert(f);
|
||||
+ g_assert(!tmp_err);
|
||||
+
|
||||
+ ret = cr_write(f, content, 10, &tmp_err);
|
||||
+ g_assert_cmpint(ret, ==, 10);
|
||||
+ g_assert(!tmp_err);
|
||||
+
|
||||
+ ret = cr_write(f, content+10, 29, &tmp_err);
|
||||
+ g_assert_cmpint(ret, ==, 29);
|
||||
+ g_assert(!tmp_err);
|
||||
+
|
||||
+ cr_close(f, &tmp_err);
|
||||
+ g_assert(!tmp_err);
|
||||
+
|
||||
+ g_assert_cmpint(stat->size, ==, content_len);
|
||||
+ g_assert_cmpstr(stat->checksum, ==, content_sha256);
|
||||
+ cr_contentstat_free(stat, &tmp_err);
|
||||
+ g_assert(!tmp_err);
|
||||
+#endif // WITH_ZSTD
|
||||
}
|
||||
|
||||
static void
|
||||
diff --git a/tests/testdata/compressed_files/00_plain.foo5 b/tests/testdata/compressed_files/00_plain.foo5
|
||||
new file mode 100644
|
||||
index 0000000000000000000000000000000000000000..e58c09d56ab1845ad8c524844d4acd0515de2df5
|
||||
GIT binary patch
|
||||
literal 13
|
||||
UcmdPcs{dDofsuh>=F1y_03PE6+5i9m
|
||||
|
||||
literal 0
|
||||
HcmV?d00001
|
||||
|
||||
diff --git a/tests/testdata/compressed_files/00_plain.txt.zst b/tests/testdata/compressed_files/00_plain.txt.zst
|
||||
new file mode 100644
|
||||
index 0000000000000000000000000000000000000000..e58c09d56ab1845ad8c524844d4acd0515de2df5
|
||||
GIT binary patch
|
||||
literal 13
|
||||
UcmdPcs{dDofsuh>=F1y_03PE6+5i9m
|
||||
|
||||
literal 0
|
||||
HcmV?d00001
|
||||
|
||||
diff --git a/tests/testdata/compressed_files/01_plain.foo5 b/tests/testdata/compressed_files/01_plain.foo5
|
||||
new file mode 100644
|
||||
index 0000000000000000000000000000000000000000..9d9c470fc36aae29007d667145a3dedf23a825c2
|
||||
GIT binary patch
|
||||
literal 51
|
||||
zcmdPcs{dET!jzHWLt1`*Qeu%pNosKkS6Y5fc2;p>8kkJU&raiFV$fi^7~aE^zvB)7
|
||||
D=VTJ`
|
||||
|
||||
literal 0
|
||||
HcmV?d00001
|
||||
|
||||
diff --git a/tests/testdata/compressed_files/01_plain.txt.zst b/tests/testdata/compressed_files/01_plain.txt.zst
|
||||
new file mode 100644
|
||||
index 0000000000000000000000000000000000000000..9d9c470fc36aae29007d667145a3dedf23a825c2
|
||||
GIT binary patch
|
||||
literal 51
|
||||
zcmdPcs{dET!jzHWLt1`*Qeu%pNosKkS6Y5fc2;p>8kkJU&raiFV$fi^7~aE^zvB)7
|
||||
D=VTJ`
|
||||
|
||||
literal 0
|
||||
HcmV?d00001
|
||||
|
||||
--
|
||||
2.48.1
|
||||
|
72
0004-Fix-error-a-label-can-only-be-part-of-a-statement.patch
Normal file
72
0004-Fix-error-a-label-can-only-be-part-of-a-statement.patch
Normal file
@ -0,0 +1,72 @@
|
||||
From 37efd285ba34f53451d11ad2390604e1d46536b0 Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= <amatej@redhat.com>
|
||||
Date: Wed, 17 May 2023 09:34:05 +0200
|
||||
Subject: [PATCH 3/4] Fix: error: a label can only be part of a statement
|
||||
|
||||
Declaration is not a statement.
|
||||
This seems to work with newer versions of GCC but the C language
|
||||
standard only allows statements to follow a label.
|
||||
---
|
||||
src/compression_wrapper.c | 11 ++++++-----
|
||||
1 file changed, 6 insertions(+), 5 deletions(-)
|
||||
|
||||
diff --git a/src/compression_wrapper.c b/src/compression_wrapper.c
|
||||
index 9100222..af06543 100644
|
||||
--- a/src/compression_wrapper.c
|
||||
+++ b/src/compression_wrapper.c
|
||||
@@ -845,7 +845,7 @@ cr_close(CR_FILE *cr_file, GError **err)
|
||||
}
|
||||
break;
|
||||
|
||||
- case (CR_CW_ZSTD_COMPRESSION): // --------------------------------------
|
||||
+ case (CR_CW_ZSTD_COMPRESSION): { // --------------------------------------
|
||||
#ifdef WITH_ZSTD
|
||||
ZstdFile * zstd = (ZstdFile *) cr_file->FILE;
|
||||
if (cr_file->mode == CR_CW_MODE_READ) {
|
||||
@@ -881,7 +881,7 @@ cr_close(CR_FILE *cr_file, GError **err)
|
||||
g_set_error(err, ERR_DOMAIN, CRE_IO, "createrepo_c wasn't compiled with zstd support");
|
||||
break;
|
||||
#endif // WITH_ZSTD
|
||||
-
|
||||
+ }
|
||||
case (CR_CW_BZ2_COMPRESSION): // --------------------------------------
|
||||
if (cr_file->mode == CR_CW_MODE_READ)
|
||||
BZ2_bzReadClose(&rc, (BZFILE *) cr_file->FILE);
|
||||
@@ -1093,7 +1093,7 @@ cr_read(CR_FILE *cr_file, void *buffer, unsigned int len, GError **err)
|
||||
}
|
||||
break;
|
||||
|
||||
- case (CR_CW_ZSTD_COMPRESSION): // ---------------------------------------
|
||||
+ case (CR_CW_ZSTD_COMPRESSION): { // ---------------------------------------
|
||||
#ifdef WITH_ZSTD
|
||||
ZstdFile * zstd = (ZstdFile *) cr_file->FILE;
|
||||
|
||||
@@ -1129,7 +1129,7 @@ cr_read(CR_FILE *cr_file, void *buffer, unsigned int len, GError **err)
|
||||
g_set_error(err, ERR_DOMAIN, CRE_IO, "createrepo_c wasn't compiled with zstd support");
|
||||
break;
|
||||
#endif // WITH_ZSTD
|
||||
-
|
||||
+ }
|
||||
case (CR_CW_BZ2_COMPRESSION): // --------------------------------------
|
||||
ret = BZ2_bzRead(&bzerror, (BZFILE *) cr_file->FILE, buffer, len);
|
||||
if (!ret && bzerror == BZ_SEQUENCE_ERROR)
|
||||
@@ -1364,7 +1364,7 @@ cr_write(CR_FILE *cr_file, const void *buffer, unsigned int len, GError **err)
|
||||
}
|
||||
break;
|
||||
|
||||
- case (CR_CW_ZSTD_COMPRESSION): // ---------------------------------------
|
||||
+ case (CR_CW_ZSTD_COMPRESSION): { // ---------------------------------------
|
||||
#ifdef WITH_ZSTD
|
||||
ZstdFile * zstd = (ZstdFile *) cr_file->FILE;
|
||||
ZSTD_inBuffer zib = {buffer, len, 0};
|
||||
@@ -1401,6 +1401,7 @@ cr_write(CR_FILE *cr_file, const void *buffer, unsigned int len, GError **err)
|
||||
g_set_error(err, ERR_DOMAIN, CRE_IO, "createrepo_c wasn't compiled with zstd support");
|
||||
break;
|
||||
#endif // WITH_ZSTD
|
||||
+ }
|
||||
|
||||
case (CR_CW_BZ2_COMPRESSION): // --------------------------------------
|
||||
BZ2_bzWrite(&bzerror, (BZFILE *) cr_file->FILE, (void *) buffer, len);
|
||||
--
|
||||
2.48.1
|
||||
|
37
0005-Set-compression-level-for-zstd-to-level-10.patch
Normal file
37
0005-Set-compression-level-for-zstd-to-level-10.patch
Normal file
@ -0,0 +1,37 @@
|
||||
From 84cdfec03f8adfd891227cd2e44b744d0b55b2ff Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Dirk=20M=C3=BCller?= <dirk@dmllr.de>
|
||||
Date: Wed, 8 Nov 2023 23:31:44 +0100
|
||||
Subject: [PATCH 4/4] Set compression level for zstd to level 10
|
||||
|
||||
This requires a good amount of saving due to larger dictionary
|
||||
size with only marginal more time effort for compression. decompression
|
||||
time is unaffected.
|
||||
---
|
||||
src/compression_wrapper.c | 11 ++++++++++-
|
||||
1 file changed, 10 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/src/compression_wrapper.c b/src/compression_wrapper.c
|
||||
index af06543..175806a 100644
|
||||
--- a/src/compression_wrapper.c
|
||||
+++ b/src/compression_wrapper.c
|
||||
@@ -122,7 +122,16 @@ typedef struct {
|
||||
} XzFile;
|
||||
|
||||
#ifdef WITH_ZSTD
|
||||
-#define CR_CW_ZSTD_COMPRESSION_LEVEL 9
|
||||
+
|
||||
+/** level 10 or 11 are good choices for the XML files that we generate.
|
||||
+ * level 10 requires ~ 18% more time with 1% saving over level 9
|
||||
+ * level 11 requires ~ 50% more time with 1.5% saving over level 9
|
||||
+ * level 12 requires ~ 65% more time with 1.5% saving over level 9
|
||||
+ * level 13 requires ~ 260% more time with 1.55% saving over level 9
|
||||
+*/
|
||||
+
|
||||
+#define CR_CW_ZSTD_COMPRESSION_LEVEL 10
|
||||
+
|
||||
typedef struct {
|
||||
void *buffer;
|
||||
size_t buffer_size;
|
||||
--
|
||||
2.48.1
|
||||
|
@ -25,11 +25,15 @@
|
||||
Summary: Creates a common metadata repository
|
||||
Name: createrepo_c
|
||||
Version: 0.20.1
|
||||
Release: 2%{?dist}
|
||||
Release: 3%{?dist}
|
||||
License: GPLv2+
|
||||
URL: https://github.com/rpm-software-management/createrepo_c
|
||||
Source0: %{url}/archive/%{version}/%{name}-%{version}.tar.gz
|
||||
Patch1: 0001-Test_compare_contents_instead_of_checksum-cleanup.patch
|
||||
Patch2: 0002-Add-zstd-compression-support.patch
|
||||
Patch3: 0003-Add-unittests-for-zstd-compression.patch
|
||||
Patch4: 0004-Fix-error-a-label-can-only-be-part-of-a-statement.patch
|
||||
Patch5: 0005-Set-compression-level-for-zstd-to-level-10.patch
|
||||
|
||||
BuildRequires: cmake
|
||||
BuildRequires: gcc
|
||||
@ -98,7 +102,7 @@ Requires: %{name}-libs = %{version}-%{release}
|
||||
Python 3 bindings for the createrepo_c library.
|
||||
|
||||
%prep
|
||||
%autosetup -p1
|
||||
%autosetup -p1 -S git
|
||||
|
||||
mkdir build-py3
|
||||
|
||||
@ -176,6 +180,9 @@ ln -sr %{buildroot}%{_bindir}/modifyrepo_c %{buildroot}%{_bindir}/modifyrepo
|
||||
%{python3_sitearch}/%{name}-%{version}-py%{python3_version}.egg-info
|
||||
|
||||
%changelog
|
||||
* Mon Mar 10 2025 Ales Matej <amatej@redhat.com> - 0.20.1-3
|
||||
- Add zstd compression support (RHEL-67689)
|
||||
|
||||
* Mon Jun 26 2023 Jaroslav Rohel <jrohel@redhat.com> - 0.20.1-2
|
||||
- Change test to compare contents instead of checksum, cleanup (RhBug:2130179)
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user