dovecot/dovecot-2.3.16-keeplzma.patch

diff -up dovecot-2.3.16/configure.ac.keeplzma dovecot-2.3.16/configure.ac
--- dovecot-2.3.16/configure.ac.keeplzma	2021-08-06 11:25:51.000000000 +0200
+++ dovecot-2.3.16/configure.ac	2022-02-28 13:58:02.337149927 +0100
@@ -173,7 +173,7 @@ AS_HELP_STRING([--with-bzlib], [Build wi
   want_bzlib=auto)
 
 AC_ARG_WITH(lzma,
-AS_HELP_STRING([--with-lzma], [Build with LZMA decompression support (auto)]),
+AS_HELP_STRING([--with-lzma], [Build with LZMA compression support (auto)]),
   TEST_WITH(lzma, $withval),
   want_lzma=auto)
 
diff -up dovecot-2.3.16/run-test-valgrind.supp.keeplzma dovecot-2.3.16/run-test-valgrind.supp
--- dovecot-2.3.16/run-test-valgrind.supp.keeplzma	2021-08-06 11:25:51.000000000 +0200
+++ dovecot-2.3.16/run-test-valgrind.supp	2022-02-28 13:58:02.337149927 +0100
@@ -5,6 +5,17 @@
    obj:*/bash
 }
 {
+   <liblzma>
+   Memcheck:Cond
+   obj:/lib/x86_64-linux-gnu/liblzma.so.5.*
+   obj:/lib/x86_64-linux-gnu/liblzma.so.5.*
+   obj:/lib/x86_64-linux-gnu/liblzma.so.5.*
+   obj:/lib/x86_64-linux-gnu/liblzma.so.5.*
+   obj:/lib/x86_64-linux-gnu/liblzma.so.5.*
+   fun:lzma_stream_encoder
+   fun:lzma_easy_encoder
+}
+{
    <openssl_centos6_i386_v1_0_1_compression_methods>
    Memcheck:Leak
    fun:malloc
diff -up dovecot-2.3.16/src/lib-compression/compression.c.keeplzma dovecot-2.3.16/src/lib-compression/compression.c
--- dovecot-2.3.16/src/lib-compression/compression.c.keeplzma	2021-08-06 11:25:51.000000000 +0200
+++ dovecot-2.3.16/src/lib-compression/compression.c	2022-02-28 14:22:32.467944396 +0100
@@ -25,6 +25,7 @@
 #endif
 #ifndef HAVE_LZMA
 #  define i_stream_create_lzma NULL
+#  define o_stream_create_lzma NULL
 #endif
 #ifndef HAVE_LZ4
 #  define i_stream_create_lz4 NULL
@@ -216,7 +217,7 @@ const struct compression_handler compres
 		.ext = ".xz",
 		.is_compressed = is_compressed_xz,
 		.create_istream = i_stream_create_lzma,
-		.create_ostream = NULL,
+		.create_ostream = o_stream_create_lzma,
 		.get_min_level = compression_get_min_level_unsupported,
 		.get_default_level = compression_get_default_level_unsupported,
 		.get_max_level = compression_get_max_level_unsupported,
diff -up dovecot-2.3.16/src/lib-compression/Makefile.am.keeplzma dovecot-2.3.16/src/lib-compression/Makefile.am
--- dovecot-2.3.16/src/lib-compression/Makefile.am.keeplzma	2021-08-06 11:25:51.000000000 +0200
+++ dovecot-2.3.16/src/lib-compression/Makefile.am	2022-02-28 13:58:02.337149927 +0100
@@ -13,6 +13,7 @@ libcompression_la_SOURCES = \
 	istream-zlib.c \
 	istream-bzlib.c \
 	istream-zstd.c \
+	ostream-lzma.c \
 	ostream-lz4.c \
 	ostream-zlib.c \
 	ostream-bzlib.c \
diff -up dovecot-2.3.16/src/lib-compression/ostream-lzma.c.keeplzma dovecot-2.3.16/src/lib-compression/ostream-lzma.c
--- dovecot-2.3.16/src/lib-compression/ostream-lzma.c.keeplzma	2022-02-28 13:58:02.338149934 +0100
+++ dovecot-2.3.16/src/lib-compression/ostream-lzma.c	2022-02-28 13:58:02.338149934 +0100
@@ -0,0 +1,263 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+
+#ifdef HAVE_LZMA
+
+#include "ostream-private.h"
+#include "ostream-zlib.h"
+#include <lzma.h>
+
+#define CHUNK_SIZE (1024*64)
+
+struct lzma_ostream {
+	struct ostream_private ostream;
+	lzma_stream strm;
+
+	unsigned char outbuf[CHUNK_SIZE];
+	unsigned int outbuf_offset, outbuf_used;
+
+	bool flushed:1;
+};
+
+static void o_stream_lzma_close(struct iostream_private *stream,
+				bool close_parent)
+{
+	struct lzma_ostream *zstream = (struct lzma_ostream *)stream;
+	i_assert(zstream->ostream.finished ||
+		 zstream->ostream.ostream.stream_errno != 0 ||
+		 zstream->ostream.error_handling_disabled);
+	lzma_end(&zstream->strm);
+	if (close_parent)
+		o_stream_close(zstream->ostream.parent);
+}
+
+static int o_stream_zlib_send_outbuf(struct lzma_ostream *zstream)
+{
+	ssize_t ret;
+	size_t size;
+
+	if (zstream->outbuf_used == 0)
+		return 1;
+
+	size = zstream->outbuf_used - zstream->outbuf_offset;
+	i_assert(size > 0);
+	ret = o_stream_send(zstream->ostream.parent,
+			    zstream->outbuf + zstream->outbuf_offset, size);
+	if (ret < 0) {
+		o_stream_copy_error_from_parent(&zstream->ostream);
+		return -1;
+	}
+	if ((size_t)ret != size) {
+		zstream->outbuf_offset += ret;
+		return 0;
+	}
+	zstream->outbuf_offset = 0;
+	zstream->outbuf_used = 0;
+	return 1;
+}
+
+static ssize_t
+o_stream_lzma_send_chunk(struct lzma_ostream *zstream,
+			  const void *data, size_t size)
+{
+	lzma_stream *zs = &zstream->strm;
+	int ret;
+
+	i_assert(zstream->outbuf_used == 0);
+
+	zs->next_in = (void *)data;
+	zs->avail_in = size;
+	while (zs->avail_in > 0) {
+		if (zs->avail_out == 0) {
+			/* previous block was compressed. send it and start
+			   compression for a new block. */
+			zs->next_out = zstream->outbuf;
+			zs->avail_out = sizeof(zstream->outbuf);
+
+			zstream->outbuf_used = sizeof(zstream->outbuf);
+			if ((ret = o_stream_zlib_send_outbuf(zstream)) < 0)
+				return -1;
+			if (ret == 0) {
+				/* parent stream's buffer full */
+				break;
+			}
+		}
+
+		ret = lzma_code(zs, LZMA_RUN);
+		switch (ret) {
+		case LZMA_OK:
+			break;
+		case LZMA_MEM_ERROR:
+			i_fatal_status(FATAL_OUTOFMEM,
+				       "lzma.write(%s): Out of memory",
+				       o_stream_get_name(&zstream->ostream.ostream));
+		default:
+			i_panic("lzma.write(%s) failed with unexpected code %d",
+				o_stream_get_name(&zstream->ostream.ostream), ret);
+		}
+	}
+	size -= zs->avail_in;
+
+	return size;
+}
+
+static int o_stream_lzma_send_flush(struct lzma_ostream *zstream, bool final)
+{
+	lzma_stream *zs = &zstream->strm;
+	size_t len;
+	bool done = FALSE;
+	int ret;
+
+	i_assert(zs->avail_in == 0);
+
+	if (zstream->flushed) {
+		i_assert(zstream->outbuf_used == 0);
+		return 1;
+	}
+
+	if ((ret = o_stream_flush_parent_if_needed(&zstream->ostream)) <= 0)
+		return ret;
+	if ((ret = o_stream_zlib_send_outbuf(zstream)) <= 0)
+		return ret;
+
+	if (!final)
+		return 1;
+
+	i_assert(zstream->outbuf_used == 0);
+	do {
+		len = sizeof(zstream->outbuf) - zs->avail_out;
+		if (len != 0) {
+			zs->next_out = zstream->outbuf;
+			zs->avail_out = sizeof(zstream->outbuf);
+
+			zstream->outbuf_used = len;
+			if ((ret = o_stream_zlib_send_outbuf(zstream)) <= 0)
+				return ret;
+			if (done)
+				break;
+		}
+		ret = lzma_code(zs, LZMA_FINISH);
+		switch (ret) {
+		case LZMA_OK:
+			/* still unfinished - need to call lzma_code() again */
+			break;
+		case LZMA_STREAM_END:
+			/* output is fully finished */
+			done = TRUE;
+			break;
+		case LZMA_MEM_ERROR:
+			i_fatal_status(FATAL_OUTOFMEM,
+				       "lzma.write(%s): Out of memory",
+				       o_stream_get_name(&zstream->ostream.ostream));
+		default:
+			i_panic("lzma.write(%s) flush failed with unexpected code %d",
+				o_stream_get_name(&zstream->ostream.ostream), ret);
+		}
+	} while (zs->avail_out != sizeof(zstream->outbuf));
+
+	if (final)
+		zstream->flushed = TRUE;
+	i_assert(zstream->outbuf_used == 0);
+	return 1;
+}
+
+static int o_stream_lzma_flush(struct ostream_private *stream)
+{
+	struct lzma_ostream *zstream = (struct lzma_ostream *)stream;
+	int ret;
+
+	if ((ret = o_stream_lzma_send_flush(zstream, stream->finished)) < 0)
+		return -1;
+	else if (ret > 0)
+		return o_stream_flush_parent(stream);
+	return ret;
+}
+
+static size_t
+o_stream_lzma_get_buffer_used_size(const struct ostream_private *stream)
+{
+	const struct lzma_ostream *zstream =
+		(const struct lzma_ostream *)stream;
+
+	/* outbuf has already compressed data that we're trying to send to the
+	   parent stream. We're not including lzma's internal compression
+	   buffer size. */
+	return (zstream->outbuf_used - zstream->outbuf_offset) +
+		o_stream_get_buffer_used_size(stream->parent);
+}
+
+static size_t
+o_stream_lzma_get_buffer_avail_size(const struct ostream_private *stream)
+{
+	/* FIXME: not correct - this is counting compressed size, which may be
+	   too larger than uncompressed size in some situations. Fixing would
+	   require some kind of additional buffering. */
+	return o_stream_get_buffer_avail_size(stream->parent);
+}
+
+static ssize_t
+o_stream_lzma_sendv(struct ostream_private *stream,
+		    const struct const_iovec *iov, unsigned int iov_count)
+{
+	struct lzma_ostream *zstream = (struct lzma_ostream *)stream;
+	ssize_t ret, bytes = 0;
+	unsigned int i;
+
+	if ((ret = o_stream_zlib_send_outbuf(zstream)) <= 0) {
+		/* error / we still couldn't flush existing data to
+		   parent stream. */
+		return ret;
+	}
+
+	for (i = 0; i < iov_count; i++) {
+		ret = o_stream_lzma_send_chunk(zstream, iov[i].iov_base,
+						iov[i].iov_len);
+		if (ret < 0)
+			return -1;
+		bytes += ret;
+		if ((size_t)ret != iov[i].iov_len)
+			break;
+	}
+	stream->ostream.offset += bytes;
+
+	/* avail_in!=0 check is used to detect errors. if it's non-zero here
+	   it simply means we didn't send all the data */
+	zstream->strm.avail_in = 0;
+	return bytes;
+}
+
+struct ostream *o_stream_create_lzma(struct ostream *output, int level)
+{
+	struct lzma_ostream *zstream;
+	lzma_ret ret;
+
+	i_assert(level >= 1 && level <= 9);
+
+	zstream = i_new(struct lzma_ostream, 1);
+	zstream->ostream.sendv = o_stream_lzma_sendv;
+	zstream->ostream.flush = o_stream_lzma_flush;
+	zstream->ostream.get_buffer_used_size =
+		o_stream_lzma_get_buffer_used_size;
+	zstream->ostream.get_buffer_avail_size =
+		o_stream_lzma_get_buffer_avail_size;
+	zstream->ostream.iostream.close = o_stream_lzma_close;
+
+	ret = lzma_easy_encoder(&zstream->strm, level, LZMA_CHECK_CRC64);
+	switch (ret) {
+	case LZMA_OK:
+		break;
+	case LZMA_MEM_ERROR:
+		i_fatal_status(FATAL_OUTOFMEM, "lzma: Out of memory");
+	case LZMA_OPTIONS_ERROR:
+		i_fatal("lzma: Invalid level");
+	default:
+		i_fatal("lzma_easy_encoder() failed with %d", ret);
+	}
+
+	zstream->strm.next_out = zstream->outbuf;
+	zstream->strm.avail_out = sizeof(zstream->outbuf);
+	return o_stream_create(&zstream->ostream, output,
+			       o_stream_get_fd(output));
+}
+#endif
diff -up dovecot-2.3.16/src/lib-compression/ostream-zlib.h.keeplzma dovecot-2.3.16/src/lib-compression/ostream-zlib.h
--- dovecot-2.3.16/src/lib-compression/ostream-zlib.h.keeplzma	2021-08-06 11:25:51.000000000 +0200
+++ dovecot-2.3.16/src/lib-compression/ostream-zlib.h	2022-02-28 13:58:02.338149934 +0100
@@ -4,6 +4,7 @@
 struct ostream *o_stream_create_gz(struct ostream *output, int level);
 struct ostream *o_stream_create_deflate(struct ostream *output, int level);
 struct ostream *o_stream_create_bz2(struct ostream *output, int level);
+struct ostream *o_stream_create_lzma(struct ostream *output, int level);
 struct ostream *o_stream_create_lz4(struct ostream *output, int level);
 struct ostream *o_stream_create_zstd(struct ostream *output, int level);
 
diff -up dovecot-2.3.16/src/lib-compression/test-compression.c.keeplzma dovecot-2.3.16/src/lib-compression/test-compression.c
--- dovecot-2.3.16/src/lib-compression/test-compression.c.keeplzma	2021-08-06 11:25:51.000000000 +0200
+++ dovecot-2.3.16/src/lib-compression/test-compression.c	2022-02-28 13:58:02.338149934 +0100
@@ -730,7 +730,6 @@ static void test_compression_int(bool au
 
 	for (i = 0; compression_handlers[i].name != NULL; i++) {
 		if (compression_handlers[i].create_istream != NULL &&
-		    compression_handlers[i].create_ostream != NULL &&
 		    (!autodetect ||
 		     compression_handlers[i].is_compressed != NULL)) T_BEGIN {
 			if (compression_handlers[i].is_compressed != NULL &&