From 06ff7fc8adbdbbe7d4d7782e278b1a2e20d9ab8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= Date: Tue, 17 Aug 2021 14:28:05 +0200 Subject: [PATCH 2/4] Add unittests for zstd compression --- .../python/tests/test_compression_wrapper.py | 8 +- tests/test_compression_wrapper.c | 161 ++++++++++++++++++ tests/testdata/compressed_files/00_plain.foo5 | Bin 0 -> 13 bytes .../compressed_files/00_plain.txt.zst | Bin 0 -> 13 bytes tests/testdata/compressed_files/01_plain.foo5 | Bin 0 -> 51 bytes .../compressed_files/01_plain.txt.zst | Bin 0 -> 51 bytes 6 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 tests/testdata/compressed_files/00_plain.foo5 create mode 100644 tests/testdata/compressed_files/00_plain.txt.zst create mode 100644 tests/testdata/compressed_files/01_plain.foo5 create mode 100644 tests/testdata/compressed_files/01_plain.txt.zst diff --git a/tests/python/tests/test_compression_wrapper.py b/tests/python/tests/test_compression_wrapper.py index c8957a2..7719fdd 100644 --- a/tests/python/tests/test_compression_wrapper.py +++ b/tests/python/tests/test_compression_wrapper.py @@ -15,6 +15,7 @@ class TestCaseCompressionWrapper(unittest.TestCase): self.assertEqual(cr.compression_suffix(cr.BZ2), ".bz2") self.assertEqual(cr.compression_suffix(cr.XZ), ".xz") self.assertEqual(cr.compression_suffix(cr.ZCK), ".zck") + self.assertEqual(cr.compression_suffix(cr.ZSTD), ".zst") def test_detect_compression(self): @@ -69,6 +70,11 @@ class TestCaseCompressionWrapper(unittest.TestCase): #comtype = cr.detect_compression(path) #self.assertEqual(comtype, cr.ZCK) + # Bad suffix - zstd compression + path = os.path.join(COMPRESSED_FILES_PATH, "01_plain.foo5") + comtype = cr.detect_compression(path) + self.assertEqual(comtype, cr.ZSTD) + def test_compression_type(self): self.assertEqual(cr.compression_type(None), cr.UNKNOWN_COMPRESSION) self.assertEqual(cr.compression_type(""), cr.UNKNOWN_COMPRESSION) @@ -77,4 +83,4 @@ class TestCaseCompressionWrapper(unittest.TestCase): self.assertEqual(cr.compression_type("xz"), cr.XZ) self.assertEqual(cr.compression_type("XZ"), cr.XZ) self.assertEqual(cr.compression_type("zck"), cr.ZCK) - + self.assertEqual(cr.compression_type("zstd"), cr.ZSTD) diff --git a/tests/test_compression_wrapper.c b/tests/test_compression_wrapper.c index edd4fe5..6326156 100644 --- a/tests/test_compression_wrapper.c +++ b/tests/test_compression_wrapper.c @@ -37,10 +37,12 @@ #define FILE_COMPRESSED_0_GZ TEST_COMPRESSED_FILES_PATH"/00_plain.txt.gz" #define FILE_COMPRESSED_0_BZ2 TEST_COMPRESSED_FILES_PATH"/00_plain.txt.bz2" #define FILE_COMPRESSED_0_XZ TEST_COMPRESSED_FILES_PATH"/00_plain.txt.xz" +#define FILE_COMPRESSED_0_ZSTD TEST_COMPRESSED_FILES_PATH"/00_plain.txt.zst" #define FILE_COMPRESSED_0_PLAIN_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/00_plain.foo0" #define FILE_COMPRESSED_0_GZ_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/00_plain.foo1" #define FILE_COMPRESSED_0_BZ2_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/00_plain.foo2" #define FILE_COMPRESSED_0_XZ_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/00_plain.foo3" +#define FILE_COMPRESSED_0_ZSTD_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/00_plain.foo5" #define FILE_COMPRESSED_1_CONTENT "foobar foobar foobar foobar test test\nfolkjsaflkjsadokf\n" #define FILE_COMPRESSED_1_CONTENT_LEN 56 @@ -48,11 +50,13 @@ #define FILE_COMPRESSED_1_GZ TEST_COMPRESSED_FILES_PATH"/01_plain.txt.gz" #define FILE_COMPRESSED_1_BZ2 TEST_COMPRESSED_FILES_PATH"/01_plain.txt.bz2" #define FILE_COMPRESSED_1_XZ TEST_COMPRESSED_FILES_PATH"/01_plain.txt.xz" +#define FILE_COMPRESSED_1_ZSTD TEST_COMPRESSED_FILES_PATH"/01_plain.txt.zst" #define FILE_COMPRESSED_1_ZCK TEST_COMPRESSED_FILES_PATH"/01_plain.txt.zck" #define FILE_COMPRESSED_1_PLAIN_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/01_plain.foo0" #define FILE_COMPRESSED_1_GZ_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/01_plain.foo1" #define FILE_COMPRESSED_1_BZ2_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/01_plain.foo2" #define FILE_COMPRESSED_1_XZ_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/01_plain.foo3" +#define FILE_COMPRESSED_1_ZSTD_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/01_plain.foo5" static void @@ -98,6 +102,11 @@ test_cr_compression_suffix(void) suffix = cr_compression_suffix(CR_CW_XZ_COMPRESSION); g_assert_cmpstr(suffix, ==, ".xz"); + +#ifdef WITH_ZSTD + suffix = cr_compression_suffix(CR_CW_ZSTD_COMPRESSION); + g_assert_cmpstr(suffix, ==, ".zst"); +#endif // WITH_ZSTD } static void @@ -134,6 +143,11 @@ test_cr_compression_type(void) type = cr_compression_type("xz"); g_assert_cmpint(type, ==, CR_CW_XZ_COMPRESSION); + +#ifdef WITH_ZSTD + type = cr_compression_type("zstd"); + g_assert_cmpint(type, ==, CR_CW_ZSTD_COMPRESSION); +#endif // WITH_ZSTD } static void @@ -177,6 +191,17 @@ test_cr_detect_compression(void) ret = cr_detect_compression(FILE_COMPRESSED_1_XZ, &tmp_err); g_assert_cmpint(ret, ==, CR_CW_XZ_COMPRESSION); g_assert(!tmp_err); + +#ifdef WITH_ZSTD + // Zstd + + ret = cr_detect_compression(FILE_COMPRESSED_0_ZSTD, &tmp_err); + g_assert_cmpint(ret, ==, CR_CW_ZSTD_COMPRESSION); + g_assert(!tmp_err); + ret = cr_detect_compression(FILE_COMPRESSED_1_ZSTD, &tmp_err); + g_assert_cmpint(ret, ==, CR_CW_ZSTD_COMPRESSION); + g_assert(!tmp_err); +#endif // WITH_ZSTD } @@ -221,6 +246,17 @@ test_cr_detect_compression_bad_suffix(void) ret = cr_detect_compression(FILE_COMPRESSED_1_XZ_BAD_SUFFIX, &tmp_err); g_assert_cmpint(ret, ==, CR_CW_XZ_COMPRESSION); g_assert(!tmp_err); + +#ifdef WITH_ZSTD + // Zstd + + ret = cr_detect_compression(FILE_COMPRESSED_0_ZSTD_BAD_SUFFIX, &tmp_err); + g_assert_cmpint(ret, ==, CR_CW_ZSTD_COMPRESSION); + g_assert(!tmp_err); + ret = cr_detect_compression(FILE_COMPRESSED_1_ZSTD_BAD_SUFFIX, &tmp_err); + g_assert_cmpint(ret, ==, CR_CW_ZSTD_COMPRESSION); + g_assert(!tmp_err); +#endif // WITH_ZSTD } @@ -235,6 +271,16 @@ test_helper_cw_input(const char *filename, char buffer[COMPRESSED_BUFFER_LEN+1]; GError *tmp_err = NULL; + if (ctype != CR_CW_AUTO_DETECT_COMPRESSION) { + cr_CompressionType detected_type = cr_detect_compression(filename, &tmp_err); + g_assert(!tmp_err); + if (ctype != detected_type) { + printf("detected_type: %i does not match passed type: %i for filename: %s\n", + detected_type, ctype, filename); + g_assert(0); + } + } + file = cr_open(filename, CR_CW_MODE_READ, ctype, &tmp_err); g_assert(file); g_assert(!tmp_err); @@ -281,6 +327,15 @@ test_cr_read_with_autodetection(void) FILE_COMPRESSED_0_CONTENT, FILE_COMPRESSED_0_CONTENT_LEN); test_helper_cw_input(FILE_COMPRESSED_1_XZ, CR_CW_AUTO_DETECT_COMPRESSION, FILE_COMPRESSED_1_CONTENT, FILE_COMPRESSED_1_CONTENT_LEN); + +#ifdef WITH_ZSTD + // Zstd + + test_helper_cw_input(FILE_COMPRESSED_0_ZSTD, CR_CW_AUTO_DETECT_COMPRESSION, + FILE_COMPRESSED_0_CONTENT, FILE_COMPRESSED_0_CONTENT_LEN); + test_helper_cw_input(FILE_COMPRESSED_1_ZSTD, CR_CW_AUTO_DETECT_COMPRESSION, + FILE_COMPRESSED_1_CONTENT, FILE_COMPRESSED_1_CONTENT_LEN); +#endif // WITH_ZSTD } @@ -447,6 +502,29 @@ outputtest_cw_output(Outputtest *outputtest, test_helper_cw_output(OUTPUT_TYPE_PRINTF, outputtest->tmp_filename, CR_CW_XZ_COMPRESSION, FILE_COMPRESSED_1_CONTENT, FILE_COMPRESSED_1_CONTENT_LEN); + +#ifdef WITH_ZSTD + // Zstd + + test_helper_cw_output(OUTPUT_TYPE_WRITE, outputtest->tmp_filename, + CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_0_CONTENT, + FILE_COMPRESSED_0_CONTENT_LEN); + test_helper_cw_output(OUTPUT_TYPE_WRITE, outputtest->tmp_filename, + CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_1_CONTENT, + FILE_COMPRESSED_1_CONTENT_LEN); + test_helper_cw_output(OUTPUT_TYPE_PUTS, outputtest->tmp_filename, + CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_0_CONTENT, + FILE_COMPRESSED_0_CONTENT_LEN); + test_helper_cw_output(OUTPUT_TYPE_PUTS, outputtest->tmp_filename, + CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_1_CONTENT, + FILE_COMPRESSED_1_CONTENT_LEN); + test_helper_cw_output(OUTPUT_TYPE_PRINTF, outputtest->tmp_filename, + CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_0_CONTENT, + FILE_COMPRESSED_0_CONTENT_LEN); + test_helper_cw_output(OUTPUT_TYPE_PRINTF, outputtest->tmp_filename, + CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_1_CONTENT, + FILE_COMPRESSED_1_CONTENT_LEN); +#endif // WITH_ZSTD } @@ -508,6 +586,15 @@ test_cr_error_handling(void) g_error_free(tmp_err); tmp_err = NULL; +#ifdef WITH_ZSTD + f = cr_open("/", CR_CW_MODE_WRITE, CR_CW_ZSTD_COMPRESSION, &tmp_err); + g_assert(!f); + g_assert(tmp_err); + g_assert_cmpint(tmp_err->code, ==, CRE_IO); + g_error_free(tmp_err); + tmp_err = NULL; +#endif // WITH_ZSTD + // Opening plain text file as compressed char buf[256]; @@ -550,6 +637,21 @@ test_cr_error_handling(void) ret = cr_close(f, &tmp_err); g_assert_cmpint(ret, ==, CRE_OK); g_assert(!tmp_err); + +#ifdef WITH_ZSTD + f = cr_open(FILE_COMPRESSED_1_PLAIN, CR_CW_MODE_READ, + CR_CW_ZSTD_COMPRESSION, &tmp_err); + g_assert(f); + ret = cr_read(f, buf, 256, &tmp_err); + g_assert_cmpint(ret, ==, -1); + g_assert(tmp_err); + g_assert_cmpint(tmp_err->code, ==, CRE_ZSTD); + g_error_free(tmp_err); + tmp_err = NULL; + ret = cr_close(f, &tmp_err); + g_assert_cmpint(ret, ==, CRE_OK); + g_assert(!tmp_err); +#endif // WITH_ZSTD } @@ -670,6 +772,33 @@ test_contentstating_singlewrite(Outputtest *outputtest, g_assert_cmpstr(stat->checksum, ==, content_sha256); cr_contentstat_free(stat, &tmp_err); g_assert(!tmp_err); + +#ifdef WITH_ZSTD + // zstd compression + stat = cr_contentstat_new(CR_CHECKSUM_SHA256, &tmp_err); + g_assert(stat); + g_assert(!tmp_err); + + f = cr_sopen(outputtest->tmp_filename, + CR_CW_MODE_WRITE, + CR_CW_ZSTD_COMPRESSION, + stat, + &tmp_err); + g_assert(f); + g_assert(!tmp_err); + + ret = cr_write(f, content, content_len, &tmp_err); + g_assert_cmpint(ret, ==, content_len); + g_assert(!tmp_err); + + cr_close(f, &tmp_err); + g_assert(!tmp_err); + + g_assert_cmpint(stat->size, ==, content_len); + g_assert_cmpstr(stat->checksum, ==, content_sha256); + cr_contentstat_free(stat, &tmp_err); + g_assert(!tmp_err); +#endif // WITH_ZSTD } static void @@ -715,6 +844,38 @@ test_contentstating_multiwrite(Outputtest *outputtest, g_assert_cmpstr(stat->checksum, ==, content_sha256); cr_contentstat_free(stat, &tmp_err); g_assert(!tmp_err); + +#ifdef WITH_ZSTD + // Zstd compression + + stat = cr_contentstat_new(CR_CHECKSUM_SHA256, &tmp_err); + g_assert(stat); + g_assert(!tmp_err); + + f = cr_sopen(outputtest->tmp_filename, + CR_CW_MODE_WRITE, + CR_CW_ZSTD_COMPRESSION, + stat, + &tmp_err); + g_assert(f); + g_assert(!tmp_err); + + ret = cr_write(f, content, 10, &tmp_err); + g_assert_cmpint(ret, ==, 10); + g_assert(!tmp_err); + + ret = cr_write(f, content+10, 29, &tmp_err); + g_assert_cmpint(ret, ==, 29); + g_assert(!tmp_err); + + cr_close(f, &tmp_err); + g_assert(!tmp_err); + + g_assert_cmpint(stat->size, ==, content_len); + g_assert_cmpstr(stat->checksum, ==, content_sha256); + cr_contentstat_free(stat, &tmp_err); + g_assert(!tmp_err); +#endif // WITH_ZSTD } static void diff --git a/tests/testdata/compressed_files/00_plain.foo5 b/tests/testdata/compressed_files/00_plain.foo5 new file mode 100644 index 0000000000000000000000000000000000000000..e58c09d56ab1845ad8c524844d4acd0515de2df5 GIT binary patch literal 13 UcmdPcs{dDofsuh>=F1y_03PE6+5i9m literal 0 HcmV?d00001 diff --git a/tests/testdata/compressed_files/00_plain.txt.zst b/tests/testdata/compressed_files/00_plain.txt.zst new file mode 100644 index 0000000000000000000000000000000000000000..e58c09d56ab1845ad8c524844d4acd0515de2df5 GIT binary patch literal 13 UcmdPcs{dDofsuh>=F1y_03PE6+5i9m literal 0 HcmV?d00001 diff --git a/tests/testdata/compressed_files/01_plain.foo5 b/tests/testdata/compressed_files/01_plain.foo5 new file mode 100644 index 0000000000000000000000000000000000000000..9d9c470fc36aae29007d667145a3dedf23a825c2 GIT binary patch literal 51 zcmdPcs{dET!jzHWLt1`*Qeu%pNosKkS6Y5fc2;p>8kkJU&raiFV$fi^7~aE^zvB)7 D=VTJ` literal 0 HcmV?d00001 diff --git a/tests/testdata/compressed_files/01_plain.txt.zst b/tests/testdata/compressed_files/01_plain.txt.zst new file mode 100644 index 0000000000000000000000000000000000000000..9d9c470fc36aae29007d667145a3dedf23a825c2 GIT binary patch literal 51 zcmdPcs{dET!jzHWLt1`*Qeu%pNosKkS6Y5fc2;p>8kkJU&raiFV$fi^7~aE^zvB)7 D=VTJ` literal 0 HcmV?d00001 -- 2.48.1