Rebase to alpha2, prevent uncontrolled sqlite WAL growth

.gitignore

@@ -51,3 +51,4 @@
 /rpm-4.17.0-rc1.tar.bz2
 /rpm-4.17.0.tar.bz2
 /rpm-4.18.0-alpha1.tar.bz2
+/rpm-4.18.0-alpha2.tar.bz2

0001-Fix-OpenPGP-key-ID-parsing-regression.patch

@@ -1,31 +0,0 @@
-From 329e2ec47f6cd3bc7ecf85abd9f2ab179c59914f Mon Sep 17 00:00:00 2001
-Message-Id: <329e2ec47f6cd3bc7ecf85abd9f2ab179c59914f.1650871821.git.pmatilai@redhat.com>
-From: Demi Marie Obenour <demi@invisiblethingslab.com>
-Date: Thu, 14 Apr 2022 15:38:11 -0400
-Subject: [PATCH] Fix OpenPGP key ID parsing regression
-
-This fixes a regression in 598a771d8b4f4f480d4990ccf59b978d537201dd,
-which caused RPM to parse key flags from a hashed key ID subpacket.  As
-a result, RPM would wrongly reject a signature that had both key ID and
-key usage flags subpackets in the hashed section.
-
-(backported from commit 7f830132fe717d4b31c035bb3d08379451e3cd81)
----
- rpmio/rpmpgp.c | 1 +
- 1 file changed, 1 insertion(+)
-
-diff --git a/rpmio/rpmpgp.c b/rpmio/rpmpgp.c
-index deea65eae..242b34e19 100644
---- a/rpmio/rpmpgp.c
-+++ b/rpmio/rpmpgp.c
-@@ -528,6 +528,7 @@ static int pgpPrtSubType(const uint8_t *h, size_t hlen, pgpSigType sigtype,
- 		_digp->saved |= PGPDIG_SAVED_ID;
- 		memcpy(_digp->signid, p+1, sizeof(_digp->signid));
- 	    }
-+	    break;
- 	case PGPSUBTYPE_KEY_FLAGS: /* Key usage flags */
- 	    /* Subpackets in the unhashed section cannot be trusted */
- 	    if (!hashed)
--- 
-2.35.1
-

0001-Fix-a-typo-in-doUntar-that-breaks-verbose-uncompress.patch

@@ -1,27 +0,0 @@
-From bb36aac905cf8eb470f085574641d92c8d8ca20f Mon Sep 17 00:00:00 2001
-Message-Id: <bb36aac905cf8eb470f085574641d92c8d8ca20f.1651042591.git.pmatilai@redhat.com>
-From: Mikolaj Izdebski <mizdebsk@redhat.com>
-Date: Wed, 27 Apr 2022 07:54:58 +0200
-Subject: [PATCH] Fix a typo in doUntar that breaks verbose uncompression
-
-See https://bugzilla.redhat.com/show_bug.cgi?id=2079127
----
- build/parsePrep.c | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/build/parsePrep.c b/build/parsePrep.c
-index 8cd2b84e6..bee820d11 100644
---- a/build/parsePrep.c
-+++ b/build/parsePrep.c
-@@ -130,7 +130,7 @@ static char *doUntar(rpmSpec spec, uint32_t c, int quietly)
-     }
- 
-     buf = rpmExpand("%{__rpmuncompress} -x ",
--		    quietly ? "" : "-v", sp->path, NULL);
-+		    quietly ? "" : "-v ", sp->path, NULL);
-     rstrcat(&buf,
- 	"\nSTATUS=$?\n"
- 	"if [ $STATUS -ne 0 ]; then\n"
--- 
-2.35.1
-

0001-Fix-regression-in-rubygem-unpacking-2040.patch

@@ -1,39 +0,0 @@
-From cbcd9dd38dd1d0379e5d25ed3c1b1e96353e4014 Mon Sep 17 00:00:00 2001
-Message-Id: <cbcd9dd38dd1d0379e5d25ed3c1b1e96353e4014.1651130985.git.pmatilai@redhat.com>
-From: Panu Matilainen <pmatilai@redhat.com>
-Date: Thu, 28 Apr 2022 10:17:16 +0300
-Subject: [PATCH] Fix regression in rubygem unpacking (#2040)
-
-Externalizing the source unpack in commit
-cd5d667e99f931504a512b591fcde7ed92cee344 required changing the way
-rubygems are unpacked, and now the .gemspec file gets unpacked into
-different place. Oops. Reported and initial patch by Vit Ondruch.
-
-Fixes: #2040
----
- tools/rpmuncompress.c | 5 +++--
- 1 file changed, 3 insertions(+), 2 deletions(-)
-
-diff --git a/tools/rpmuncompress.c b/tools/rpmuncompress.c
-index 609fb0f53..55aac4c44 100644
---- a/tools/rpmuncompress.c
-+++ b/tools/rpmuncompress.c
-@@ -97,12 +97,13 @@ static char *doUntar(const char *fn)
- 	if (needtar) {
- 	    rasprintf(&buf, "%s '%s' | %s %s -", zipper, fn, tar, taropts);
- 	} else if (at->compressed == COMPRESSED_GEM) {
--	    size_t nvlen = strlen(fn) - 3;
-+	    const char *bn = basename(fn);
-+	    size_t nvlen = strlen(bn) - 3;
- 	    char *gem = rpmGetPath("%{__gem}", NULL);
- 	    char *gemspec = NULL;
- 	    char gemnameversion[nvlen];
- 
--	    rstrlcpy(gemnameversion, fn, nvlen);
-+	    rstrlcpy(gemnameversion, bn, nvlen);
- 	    gemspec = rpmGetPath("", gemnameversion, ".gemspec", NULL);
- 
- 	    rasprintf(&buf, "%s '%s' && %s spec '%s' --ruby > '%s'",
--- 
-2.35.1
-

0001-Fix-regressions-on-dynamic-buildrequires-cli-switche.patch

@@ -1,108 +0,0 @@
-From 0aedb84b3c6535b0befb2cc64912645d927b9dfd Mon Sep 17 00:00:00 2001
-Message-Id: <0aedb84b3c6535b0befb2cc64912645d927b9dfd.1650986658.git.pmatilai@redhat.com>
-From: Panu Matilainen <pmatilai@redhat.com>
-Date: Tue, 26 Apr 2022 18:08:14 +0300
-Subject: [PATCH] Fix regressions on dynamic buildrequires cli switches
- (RhBug:2078744)
-
-This effectively reverts commits b8935fb23518e26aa7b9316d27f2565813d20291
-and ad8b9bd2ca93cf4319680f056bb40bfc24661991 and separating -br and -bd
-from the normal build flow: these are so different from regular builds
-and in particular, have special semantics wrt --nodeps that trying to
-cram them into the normal flow only makes it far harder to read and
-understand.
----
- rpmbuild.c        | 22 +++++++++++++++-------
- tests/rpmbuild.at | 32 ++++++++++++++++++++++++++++++++
- 2 files changed, 47 insertions(+), 7 deletions(-)
-
-diff --git a/rpmbuild.c b/rpmbuild.c
-index de78976ff..abe164e32 100644
---- a/rpmbuild.c
-+++ b/rpmbuild.c
-@@ -663,24 +663,32 @@ int main(int argc, char *argv[])
- 	/* fallthrough */
-     case 'f':
- 	ba->buildAmount |= RPMBUILD_CONF;
-+	ba->buildAmount |= RPMBUILD_BUILDREQUIRES;
-+	if (!noDeps) {
-+	    ba->buildAmount |= RPMBUILD_DUMPBUILDREQUIRES;
-+	    ba->buildAmount |= RPMBUILD_CHECKBUILDREQUIRES;
-+	}
- 	if ((buildChar == 'f') && shortCircuit)
- 	    break;
- 	/* fallthrough */
-+    case 'p':
-+	ba->buildAmount |= RPMBUILD_PREP;
-+	break;
-+    case 'l':
-+	ba->buildAmount |= RPMBUILD_FILECHECK;
-+	break;
-     case 'r':
-+	/* fallthrough */
-     case 'd':
-+	if (!shortCircuit)
-+	    ba->buildAmount |= RPMBUILD_PREP;
- 	ba->buildAmount |= RPMBUILD_BUILDREQUIRES;
- 	ba->buildAmount |= RPMBUILD_DUMPBUILDREQUIRES;
- 	if (!noDeps)
- 	    ba->buildAmount |= RPMBUILD_CHECKBUILDREQUIRES;
--	if ((buildChar == 'r' || buildChar == 'd') && shortCircuit)
-+	if (buildChar == 'd')
- 	    break;
- 	/* fallthrough */
--    case 'p':
--	ba->buildAmount |= RPMBUILD_PREP;
--	break;
--    case 'l':
--	ba->buildAmount |= RPMBUILD_FILECHECK;
--	break;
-     case 's':
- 	ba->buildAmount |= RPMBUILD_PACKAGESOURCE;
- 	break;
-diff --git a/tests/rpmbuild.at b/tests/rpmbuild.at
-index b7bd0d7f0..6142bb1ae 100644
---- a/tests/rpmbuild.at
-+++ b/tests/rpmbuild.at
-@@ -1707,6 +1707,38 @@ runroot rpmbuild \
- )
- AT_CLEANUP
- 
-+# Test that -br creates an src.rpm on success
-+AT_SETUP([rpmbuild -br success])
-+AT_KEYWORDS([build])
-+AT_CHECK([
-+RPMDB_INIT
-+
-+runroot rpmbuild \
-+  -br  /data/SPECS/mini.spec
-+],
-+[0],
-+[Wrote: /build/SRPMS/mini-1-1.src.rpm
-+],
-+[],
-+)
-+AT_CLEANUP
-+
-+# Test that -br creates an src.rpm on success
-+AT_SETUP([rpmbuild -br success])
-+AT_KEYWORDS([build])
-+AT_CHECK([
-+RPMDB_INIT
-+
-+runroot rpmbuild \
-+  -br  /data/SPECS/mini.spec
-+],
-+[0],
-+[Wrote: /build/SRPMS/mini-1-1.src.rpm
-+],
-+[],
-+)
-+AT_CLEANUP
-+
- # ------------------------------
- # Check dynamic build requires
- AT_SETUP([rpmbuild -bd with errors])
--- 
-2.35.1
-

0001-Prevent-uncontrolled-sqlite-WAL-growth-during-large-.patch

@@ -0,0 +1,39 @@
+From 33c8b214425c2860fdfe485d597e210b9723a792 Mon Sep 17 00:00:00 2001
+Message-Id: <33c8b214425c2860fdfe485d597e210b9723a792.1653292401.git.pmatilai@redhat.com>
+From: Panu Matilainen <pmatilai@redhat.com>
+Date: Mon, 23 May 2022 10:42:13 +0300
+Subject: [PATCH] Prevent uncontrolled sqlite WAL growth during large
+ transactions
+
+Sqlite WAL threshold of 1000 pages is way too low for rpmdb as a single
+header often exceeds that, but disabling the checkpointing entirely can
+cause the WAL to grow to multiple gigabytes during large distro update
+transactions, which isn't healty either. Bump the threshold to 10000
+pages which caps the WAL size to tens of megabytes, which hopefully
+is a reasonable balance between performance on rotational disks (anybody
+remember those?) and diskpace consumption.
+
+Also drop no longer meaningful link to %_flush_io configuration.
+---
+ lib/backend/sqlite.c | 5 ++---
+ 1 file changed, 2 insertions(+), 3 deletions(-)
+
+diff --git a/lib/backend/sqlite.c b/lib/backend/sqlite.c
+index 31f77e845..d5389736b 100644
+--- a/lib/backend/sqlite.c
++++ b/lib/backend/sqlite.c
+@@ -180,9 +180,8 @@ static int sqlite_init(rpmdb rdb, const char * dbhome)
+ 		int one = 1;
+ 		/* Annoying but necessary to support non-privileged readers */
+ 		sqlite3_file_control(sdb, NULL, SQLITE_FCNTL_PERSIST_WAL, &one);
+-
+-		if (!rpmExpandNumeric("%{?_flush_io}"))
+-		    sqlexec(sdb, "PRAGMA wal_autocheckpoint = 0");
++		/* Sqlite default threshold is way too low for rpmdb */
++		sqlexec(sdb, "PRAGMA wal_autocheckpoint = 10000");
+ 	    }
+ 	}
+ 
+-- 
+2.35.3
+

0001-Revert-Fix-shared-colored-files-not-removed-on-erasu.patch

@@ -1,98 +0,0 @@
-From 474ba1c63e3b790221b21fbd835427146431f874 Mon Sep 17 00:00:00 2001
-Message-Id: <474ba1c63e3b790221b21fbd835427146431f874.1650878284.git.pmatilai@redhat.com>
-From: Panu Matilainen <pmatilai@redhat.com>
-Date: Mon, 25 Apr 2022 12:17:19 +0300
-Subject: [PATCH] Revert "Fix shared colored files not removed on erasure
- regression in >= 4.14.0"
-
-This causes a worse regression by removing files on update, revert
-for further investigation.
-
-This reverts commit 9e4caf0fc536d1244b298abd9dc4c535b6560d69.
----
- lib/transaction.c | 34 +++-------------------------------
- tests/rpme.at     |  2 ++
- 2 files changed, 5 insertions(+), 31 deletions(-)
-
-diff --git a/lib/transaction.c b/lib/transaction.c
-index ee3552c48..456c0c6de 100644
---- a/lib/transaction.c
-+++ b/lib/transaction.c
-@@ -516,28 +516,6 @@ static void handleInstInstalledFile(const rpmts ts, rpmte p, rpmfiles fi, int fx
-     }
- }
- 
--static rpmFileAction getSkipAction(rpmfiles files, int i)
--{
--    rpmFileAction action = FA_UNKNOWN;
--
--    switch (rpmfilesFState(files, i)) {
--    case RPMFILE_STATE_REPLACED:
--    case RPMFILE_STATE_NOTINSTALLED:
--	action = FA_SKIPNSTATE;
--	break;
--    case RPMFILE_STATE_NETSHARED:
--	action = FA_SKIPNETSHARED;
--	break;
--    case RPMFILE_STATE_WRONGCOLOR:
--	action = FA_SKIPCOLOR;
--	break;
--    default:
--	break;
--    }
--
--    return action;
--}
--
- /**
-  * Update disk space needs on each partition for this package's files.
-  */
-@@ -706,22 +684,16 @@ assert(otherFi != NULL);
-                 /* Here is an overlapped added file we don't want to nuke. */
- 		if (rpmfsGetAction(otherFs, otherFileNum) != FA_ERASE) {
- 		    /* On updates, don't remove files. */
--		    if (!getSkipAction(otherFi, otherFileNum)) {
--			rpmfsSetAction(fs, i, FA_SKIP);
--			break;
--		    }
-+		    rpmfsSetAction(fs, i, FA_SKIP);
-+		    break;
- 		}
- 		/* Here is an overlapped removed file: skip in previous. */
- 		rpmfsSetAction(otherFs, otherFileNum, FA_SKIP);
- 	    }
- 	    if (XFA_SKIPPING(rpmfsGetAction(fs, i)))
- 		break;
--
- 	    if (rpmfilesFState(fi, i) != RPMFILE_STATE_NORMAL) {
--		rpmFileAction skip = getSkipAction(fi, i);
--		if (skip) {
--		    rpmfsSetAction(fs, i, skip);
--		}
-+		rpmfsSetAction(fs, i, FA_SKIP);
- 		break;
- 	    }
- 		
-diff --git a/tests/rpme.at b/tests/rpme.at
-index ec73700c3..f46d64a6b 100644
---- a/tests/rpme.at
-+++ b/tests/rpme.at
-@@ -167,6 +167,7 @@ AT_CLEANUP
- 
- # Test that shared colored files actually get removed regardless of order 1
- AT_SETUP([rpm -e and verify colored files removed 2.1])
-+AT_XFAIL_IF([test $RPM_XFAIL -ne 0])
- AT_KEYWORDS([install erase rpmdb])
- RPMDB_INIT
- AT_CHECK([
-@@ -191,6 +192,7 @@ AT_CLEANUP
- 
- # Test that shared colored files actually get removed regardless of order 2
- AT_SETUP([rpm -e and verify colored files removed 2.2])
-+AT_XFAIL_IF([test $RPM_XFAIL -ne 0])
- AT_KEYWORDS([install erase rpmdb])
- RPMDB_INIT
- AT_CHECK([
--- 
-2.35.1
-

rpm.spec

@@ -29,8 +29,8 @@
 %define rpmhome /usr/lib/rpm
 
 %global rpmver 4.18.0
-%global snapver alpha1
+%global snapver alpha2
-%global baserelease 6
+%global baserelease 1
 %global sover 9
 
 %global srcver %{rpmver}%{?snapver:-%{snapver}}
@@ -56,11 +56,7 @@ Patch1: rpm-4.17.x-siteconfig.patch
 Patch3: rpm-4.9.90-no-man-dirs.patch
 
 # Patches already upstream:
-Patch100: 0001-Fix-OpenPGP-key-ID-parsing-regression.patch
+Patch100: 0001-Prevent-uncontrolled-sqlite-WAL-growth-during-large-.patch
-Patch101: 0001-Revert-Fix-shared-colored-files-not-removed-on-erasu.patch
-Patch102: 0001-Fix-regressions-on-dynamic-buildrequires-cli-switche.patch
-Patch103: 0001-Fix-a-typo-in-doUntar-that-breaks-verbose-uncompress.patch
-Patch104: 0001-Fix-regression-in-rubygem-unpacking-2040.patch
 
 # These are not yet upstream
 Patch906: rpm-4.7.1-geode-i686.patch
@@ -615,6 +611,10 @@ fi
 %doc docs/librpm/html/*
 
 %changelog
+* Mon May 23 2022 Panu Matilainen <pmatilai@redhat.com> - 4.18.0-0.alpha2.1
+- Rebase to 4.18.0-0.alpha2
+- Prevent uncontrolled sqlite WAL growth during large transactions
+
 * Thu Apr 28 2022 Panu Matilainen <pmatilai@redhat.com> - 4.18.0-0.alpha1.6
 - Fix rubygem unpack regression, causing rubygem builds to fail
 

sources

@@ -1 +1 @@
-SHA512 (rpm-4.18.0-alpha1.tar.bz2) = e9d88edc7755543d760cf9273cc015cd1266422da7b57916c8e8abaa8e284c351121da9991a37b88bf335381e02bd85daca94836b57327e6a88bb2a8118eef3e
+SHA512 (rpm-4.18.0-alpha2.tar.bz2) = 76282bbcdf7cfb3b6240ef2e2650cfbee784bddbfa85a09a92c02038e44b19cf8f20bf94b6d5464773d04ee838e136b788506ca28bdf36ed3a017c8e7172990e