diff -up dovecot-2.3.16/configure.ac.keeplzma dovecot-2.3.16/configure.ac --- dovecot-2.3.16/configure.ac.keeplzma 2021-08-06 11:25:51.000000000 +0200 +++ dovecot-2.3.16/configure.ac 2022-02-28 13:58:02.337149927 +0100 @@ -173,7 +173,7 @@ AS_HELP_STRING([--with-bzlib], [Build wi want_bzlib=auto) AC_ARG_WITH(lzma, -AS_HELP_STRING([--with-lzma], [Build with LZMA decompression support (auto)]), +AS_HELP_STRING([--with-lzma], [Build with LZMA compression support (auto)]), TEST_WITH(lzma, $withval), want_lzma=auto) diff -up dovecot-2.3.16/run-test-valgrind.supp.keeplzma dovecot-2.3.16/run-test-valgrind.supp --- dovecot-2.3.16/run-test-valgrind.supp.keeplzma 2021-08-06 11:25:51.000000000 +0200 +++ dovecot-2.3.16/run-test-valgrind.supp 2022-02-28 13:58:02.337149927 +0100 @@ -5,6 +5,17 @@ obj:*/bash } { + + Memcheck:Cond + obj:/lib/x86_64-linux-gnu/liblzma.so.5.* + obj:/lib/x86_64-linux-gnu/liblzma.so.5.* + obj:/lib/x86_64-linux-gnu/liblzma.so.5.* + obj:/lib/x86_64-linux-gnu/liblzma.so.5.* + obj:/lib/x86_64-linux-gnu/liblzma.so.5.* + fun:lzma_stream_encoder + fun:lzma_easy_encoder +} +{ Memcheck:Leak fun:malloc diff -up dovecot-2.3.16/src/lib-compression/compression.c.keeplzma dovecot-2.3.16/src/lib-compression/compression.c --- dovecot-2.3.16/src/lib-compression/compression.c.keeplzma 2021-08-06 11:25:51.000000000 +0200 +++ dovecot-2.3.16/src/lib-compression/compression.c 2022-02-28 14:22:32.467944396 +0100 @@ -25,6 +25,7 @@ #endif #ifndef HAVE_LZMA # define i_stream_create_lzma NULL +# define o_stream_create_lzma NULL #endif #ifndef HAVE_LZ4 # define i_stream_create_lz4 NULL @@ -216,7 +217,7 @@ const struct compression_handler compres .ext = ".xz", .is_compressed = is_compressed_xz, .create_istream = i_stream_create_lzma, - .create_ostream = NULL, + .create_ostream = o_stream_create_lzma, .get_min_level = compression_get_min_level_unsupported, .get_default_level = compression_get_default_level_unsupported, .get_max_level = compression_get_max_level_unsupported, diff -up dovecot-2.3.16/src/lib-compression/Makefile.am.keeplzma dovecot-2.3.16/src/lib-compression/Makefile.am --- dovecot-2.3.16/src/lib-compression/Makefile.am.keeplzma 2021-08-06 11:25:51.000000000 +0200 +++ dovecot-2.3.16/src/lib-compression/Makefile.am 2022-02-28 13:58:02.337149927 +0100 @@ -13,6 +13,7 @@ libcompression_la_SOURCES = \ istream-zlib.c \ istream-bzlib.c \ istream-zstd.c \ + ostream-lzma.c \ ostream-lz4.c \ ostream-zlib.c \ ostream-bzlib.c \ diff -up dovecot-2.3.16/src/lib-compression/ostream-lzma.c.keeplzma dovecot-2.3.16/src/lib-compression/ostream-lzma.c --- dovecot-2.3.16/src/lib-compression/ostream-lzma.c.keeplzma 2022-02-28 13:58:02.338149934 +0100 +++ dovecot-2.3.16/src/lib-compression/ostream-lzma.c 2022-02-28 13:58:02.338149934 +0100 @@ -0,0 +1,263 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" + +#ifdef HAVE_LZMA + +#include "ostream-private.h" +#include "ostream-zlib.h" +#include + +#define CHUNK_SIZE (1024*64) + +struct lzma_ostream { + struct ostream_private ostream; + lzma_stream strm; + + unsigned char outbuf[CHUNK_SIZE]; + unsigned int outbuf_offset, outbuf_used; + + bool flushed:1; +}; + +static void o_stream_lzma_close(struct iostream_private *stream, + bool close_parent) +{ + struct lzma_ostream *zstream = (struct lzma_ostream *)stream; + i_assert(zstream->ostream.finished || + zstream->ostream.ostream.stream_errno != 0 || + zstream->ostream.error_handling_disabled); + lzma_end(&zstream->strm); + if (close_parent) + o_stream_close(zstream->ostream.parent); +} + +static int o_stream_zlib_send_outbuf(struct lzma_ostream *zstream) +{ + ssize_t ret; + size_t size; + + if (zstream->outbuf_used == 0) + return 1; + + size = zstream->outbuf_used - zstream->outbuf_offset; + i_assert(size > 0); + ret = o_stream_send(zstream->ostream.parent, + zstream->outbuf + zstream->outbuf_offset, size); + if (ret < 0) { + o_stream_copy_error_from_parent(&zstream->ostream); + return -1; + } + if ((size_t)ret != size) { + zstream->outbuf_offset += ret; + return 0; + } + zstream->outbuf_offset = 0; + zstream->outbuf_used = 0; + return 1; +} + +static ssize_t +o_stream_lzma_send_chunk(struct lzma_ostream *zstream, + const void *data, size_t size) +{ + lzma_stream *zs = &zstream->strm; + int ret; + + i_assert(zstream->outbuf_used == 0); + + zs->next_in = (void *)data; + zs->avail_in = size; + while (zs->avail_in > 0) { + if (zs->avail_out == 0) { + /* previous block was compressed. send it and start + compression for a new block. */ + zs->next_out = zstream->outbuf; + zs->avail_out = sizeof(zstream->outbuf); + + zstream->outbuf_used = sizeof(zstream->outbuf); + if ((ret = o_stream_zlib_send_outbuf(zstream)) < 0) + return -1; + if (ret == 0) { + /* parent stream's buffer full */ + break; + } + } + + ret = lzma_code(zs, LZMA_RUN); + switch (ret) { + case LZMA_OK: + break; + case LZMA_MEM_ERROR: + i_fatal_status(FATAL_OUTOFMEM, + "lzma.write(%s): Out of memory", + o_stream_get_name(&zstream->ostream.ostream)); + default: + i_panic("lzma.write(%s) failed with unexpected code %d", + o_stream_get_name(&zstream->ostream.ostream), ret); + } + } + size -= zs->avail_in; + + return size; +} + +static int o_stream_lzma_send_flush(struct lzma_ostream *zstream, bool final) +{ + lzma_stream *zs = &zstream->strm; + size_t len; + bool done = FALSE; + int ret; + + i_assert(zs->avail_in == 0); + + if (zstream->flushed) { + i_assert(zstream->outbuf_used == 0); + return 1; + } + + if ((ret = o_stream_flush_parent_if_needed(&zstream->ostream)) <= 0) + return ret; + if ((ret = o_stream_zlib_send_outbuf(zstream)) <= 0) + return ret; + + if (!final) + return 1; + + i_assert(zstream->outbuf_used == 0); + do { + len = sizeof(zstream->outbuf) - zs->avail_out; + if (len != 0) { + zs->next_out = zstream->outbuf; + zs->avail_out = sizeof(zstream->outbuf); + + zstream->outbuf_used = len; + if ((ret = o_stream_zlib_send_outbuf(zstream)) <= 0) + return ret; + if (done) + break; + } + ret = lzma_code(zs, LZMA_FINISH); + switch (ret) { + case LZMA_OK: + /* still unfinished - need to call lzma_code() again */ + break; + case LZMA_STREAM_END: + /* output is fully finished */ + done = TRUE; + break; + case LZMA_MEM_ERROR: + i_fatal_status(FATAL_OUTOFMEM, + "lzma.write(%s): Out of memory", + o_stream_get_name(&zstream->ostream.ostream)); + default: + i_panic("lzma.write(%s) flush failed with unexpected code %d", + o_stream_get_name(&zstream->ostream.ostream), ret); + } + } while (zs->avail_out != sizeof(zstream->outbuf)); + + if (final) + zstream->flushed = TRUE; + i_assert(zstream->outbuf_used == 0); + return 1; +} + +static int o_stream_lzma_flush(struct ostream_private *stream) +{ + struct lzma_ostream *zstream = (struct lzma_ostream *)stream; + int ret; + + if ((ret = o_stream_lzma_send_flush(zstream, stream->finished)) < 0) + return -1; + else if (ret > 0) + return o_stream_flush_parent(stream); + return ret; +} + +static size_t +o_stream_lzma_get_buffer_used_size(const struct ostream_private *stream) +{ + const struct lzma_ostream *zstream = + (const struct lzma_ostream *)stream; + + /* outbuf has already compressed data that we're trying to send to the + parent stream. We're not including lzma's internal compression + buffer size. */ + return (zstream->outbuf_used - zstream->outbuf_offset) + + o_stream_get_buffer_used_size(stream->parent); +} + +static size_t +o_stream_lzma_get_buffer_avail_size(const struct ostream_private *stream) +{ + /* FIXME: not correct - this is counting compressed size, which may be + too larger than uncompressed size in some situations. Fixing would + require some kind of additional buffering. */ + return o_stream_get_buffer_avail_size(stream->parent); +} + +static ssize_t +o_stream_lzma_sendv(struct ostream_private *stream, + const struct const_iovec *iov, unsigned int iov_count) +{ + struct lzma_ostream *zstream = (struct lzma_ostream *)stream; + ssize_t ret, bytes = 0; + unsigned int i; + + if ((ret = o_stream_zlib_send_outbuf(zstream)) <= 0) { + /* error / we still couldn't flush existing data to + parent stream. */ + return ret; + } + + for (i = 0; i < iov_count; i++) { + ret = o_stream_lzma_send_chunk(zstream, iov[i].iov_base, + iov[i].iov_len); + if (ret < 0) + return -1; + bytes += ret; + if ((size_t)ret != iov[i].iov_len) + break; + } + stream->ostream.offset += bytes; + + /* avail_in!=0 check is used to detect errors. if it's non-zero here + it simply means we didn't send all the data */ + zstream->strm.avail_in = 0; + return bytes; +} + +struct ostream *o_stream_create_lzma(struct ostream *output, int level) +{ + struct lzma_ostream *zstream; + lzma_ret ret; + + i_assert(level >= 1 && level <= 9); + + zstream = i_new(struct lzma_ostream, 1); + zstream->ostream.sendv = o_stream_lzma_sendv; + zstream->ostream.flush = o_stream_lzma_flush; + zstream->ostream.get_buffer_used_size = + o_stream_lzma_get_buffer_used_size; + zstream->ostream.get_buffer_avail_size = + o_stream_lzma_get_buffer_avail_size; + zstream->ostream.iostream.close = o_stream_lzma_close; + + ret = lzma_easy_encoder(&zstream->strm, level, LZMA_CHECK_CRC64); + switch (ret) { + case LZMA_OK: + break; + case LZMA_MEM_ERROR: + i_fatal_status(FATAL_OUTOFMEM, "lzma: Out of memory"); + case LZMA_OPTIONS_ERROR: + i_fatal("lzma: Invalid level"); + default: + i_fatal("lzma_easy_encoder() failed with %d", ret); + } + + zstream->strm.next_out = zstream->outbuf; + zstream->strm.avail_out = sizeof(zstream->outbuf); + return o_stream_create(&zstream->ostream, output, + o_stream_get_fd(output)); +} +#endif diff -up dovecot-2.3.16/src/lib-compression/ostream-zlib.h.keeplzma dovecot-2.3.16/src/lib-compression/ostream-zlib.h --- dovecot-2.3.16/src/lib-compression/ostream-zlib.h.keeplzma 2021-08-06 11:25:51.000000000 +0200 +++ dovecot-2.3.16/src/lib-compression/ostream-zlib.h 2022-02-28 13:58:02.338149934 +0100 @@ -4,6 +4,7 @@ struct ostream *o_stream_create_gz(struct ostream *output, int level); struct ostream *o_stream_create_deflate(struct ostream *output, int level); struct ostream *o_stream_create_bz2(struct ostream *output, int level); +struct ostream *o_stream_create_lzma(struct ostream *output, int level); struct ostream *o_stream_create_lz4(struct ostream *output, int level); struct ostream *o_stream_create_zstd(struct ostream *output, int level); diff -up dovecot-2.3.16/src/lib-compression/test-compression.c.keeplzma dovecot-2.3.16/src/lib-compression/test-compression.c --- dovecot-2.3.16/src/lib-compression/test-compression.c.keeplzma 2021-08-06 11:25:51.000000000 +0200 +++ dovecot-2.3.16/src/lib-compression/test-compression.c 2022-02-28 13:58:02.338149934 +0100 @@ -730,7 +730,6 @@ static void test_compression_int(bool au for (i = 0; compression_handlers[i].name != NULL; i++) { if (compression_handlers[i].create_istream != NULL && - compression_handlers[i].create_ostream != NULL && (!autodetect || compression_handlers[i].is_compressed != NULL)) T_BEGIN { if (compression_handlers[i].is_compressed != NULL &&