Backport patches to enhance synchronization of rpm transaction to swdb

0002-transaction-Set-an-error-for-first-tsi-in-unknown-state.patch

@@ -0,0 +1,27 @@
+From 41cbb0ff39c68a7eb3eaa8573a4b4be97c080c46 Mon Sep 17 00:00:00 2001
+From: Jaroslav Mracek <jmracek@redhat.com>
+Date: Mon, 3 Jun 2019 19:26:55 +0200
+Subject: [PATCH] [transaction] Set an error for first tsi in unknown state
+
+---
+ dnf/yum/rpmtrans.py | 4 ++++
+ 1 file changed, 4 insertions(+)
+
+diff --git a/dnf/yum/rpmtrans.py b/dnf/yum/rpmtrans.py
+index 6a60b5a..50d8b81 100644
+--- a/dnf/yum/rpmtrans.py
++++ b/dnf/yum/rpmtrans.py
+@@ -386,6 +386,10 @@ class RPMTransaction(object):
+         msg = "Error unpacking rpm package %s" % tsi.pkg
+         for display in self.displays:
+             display.error(msg)
++        for tsi1 in transaction_list:
++            if tsi1.state == libdnf.transaction.TransactionItemState_UNKNOWN:
++                tsi1.state = libdnf.transaction.TransactionItemState_ERROR
++                return
+         tsi.state = libdnf.transaction.TransactionItemState_ERROR
+ 
+     def _scriptError(self, amount, total, key):
+--
+libgit2 0.27.8
+

0003-history-Dont-store-failed-transactions-as-succeeded.patch

@@ -0,0 +1,90 @@
+From 94a074b9747a437c478e388dc76ed90d06d8a666 Mon Sep 17 00:00:00 2001
+From: Daniel Mach <dmach@redhat.com>
+Date: Wed, 5 Jun 2019 08:29:03 +0200
+Subject: [PATCH] [history] Don't store failed transactions as succeeded.
+
+---
+ dnf/base.py       |  7 ++++---
+ dnf/db/history.py | 16 ++++++++++------
+ 2 files changed, 14 insertions(+), 9 deletions(-)
+
+diff --git a/dnf/base.py b/dnf/base.py
+index 7a388ec..add6403 100644
+--- a/dnf/base.py
++++ b/dnf/base.py
+@@ -987,15 +987,16 @@ class Base(object):
+             # particular element failed and if not, decide that is the
+             # case.
+             failed = [el for el in self._ts if el.Failed()]
+-            if len(failed) > 0:
++            if failed:
+                 for te in failed:
+                     te_nevra = dnf.util._te_nevra(te)
+                     for tsi in self._transaction:
+                         if str(tsi) == te_nevra:
+                             tsi.state = libdnf.transaction.TransactionItemState_ERROR
+ 
+                 errstring = _('Errors occurred during transaction.')
+                 logger.debug(errstring)
++                self.history.end(rpmdbv)
+             else:
+                 login = dnf.util.get_effective_login()
+                 msg = _("Failed to obtain the transaction lock "
+@@ -1006,7 +1007,7 @@ class Base(object):
+         else:
+             if self._record_history():
+                 herrors = [ucd(x) for x in errors]
+-                self.history.end(rpmdbv, 2, errors=herrors)
++                self.history.end(rpmdbv)
+ 
+             logger.critical(_("Transaction couldn't start:"))
+             for e in errors:
+@@ -1069,7 +1070,7 @@ class Base(object):
+             count = display_banner(tsi.pkg, count)
+ 
+         rpmdbv = rpmdb_sack._rpmdb_version()
+-        self.history.end(rpmdbv, 0)
++        self.history.end(rpmdbv)
+ 
+         timer()
+         self._trans_success = True
+diff --git a/dnf/db/history.py b/dnf/db/history.py
+index dc9c53b..7f22f97 100644
+--- a/dnf/db/history.py
++++ b/dnf/db/history.py
+@@ -486,22 +486,26 @@ class SwdbInterface(object):
+             self.swdb.log_error(self._tid, error)
+     '''
+ 
+-    # TODO: rename to end_transaction?
+-    def end(self, end_rpmdb_version="", return_code=0, errors=None):
+-        assert return_code or not errors
+-        # TODO: fix return_code
+-        return_code = not bool(return_code)
++    def end(self, end_rpmdb_version="", return_code=None, errors=None):
+         if not hasattr(self, '_tid'):
+             return  # Failed at beg() time
+ 
++        if return_code is None:
++            # return_code/state auto-detection
++            return_code = libdnf.transaction.TransactionState_DONE
++            for tsi in self.rpm:
++                if tsi.state == libdnf.transaction.TransactionItemState_ERROR:
++                    return_code = libdnf.transaction.TransactionState_ERROR
++                    break
++
+         for file_descriptor, line in self._output:
+             self.swdb.addConsoleOutputLine(file_descriptor, line)
+         self._output = []
+ 
+         self.swdb.endTransaction(
+             int(time.time()),
+             str(end_rpmdb_version),
+-            bool(return_code)
++            return_code,
+         )
+ 
+         # Closing and cleanup is done in the close() method.
+--
+libgit2 0.27.8
+

0004-transaction-Add-RPMCALLBACK_INST_STARTSTOP-callback-handlers.patch

@@ -0,0 +1,73 @@
+From f7f30977df77032c1b97d86d5df9e9c907f3a7e3 Mon Sep 17 00:00:00 2001
+From: Daniel Mach <dmach@redhat.com>
+Date: Wed, 5 Jun 2019 21:10:05 +0200
+Subject: [PATCH] [transaction] Add RPMCALLBACK_INST_{START,STOP} callback handlers.
+
+---
+ dnf/yum/rpmtrans.py | 23 ++++++++++++++++++-----
+ 1 file changed, 18 insertions(+), 5 deletions(-)
+
+diff --git a/dnf/yum/rpmtrans.py b/dnf/yum/rpmtrans.py
+index 50d8b81..0834a8c 100644
+--- a/dnf/yum/rpmtrans.py
++++ b/dnf/yum/rpmtrans.py
+@@ -262,6 +262,10 @@ class RPMTransaction(object):
+                 return self._instOpenFile(key)
+             elif what == rpm.RPMCALLBACK_INST_CLOSE_FILE:
+                 self._instCloseFile(key)
++            elif what == rpm.RPMCALLBACK_INST_START:
++                self._inst_start(key)
++            elif what == rpm.RPMCALLBACK_INST_STOP:
++                self._inst_stop(key)
+             elif what == rpm.RPMCALLBACK_INST_PROGRESS:
+                 self._instProgress(amount, total, key)
+             elif what == rpm.RPMCALLBACK_UNINST_START:
+@@ -321,12 +325,17 @@ class RPMTransaction(object):
+             return self.fd.fileno()
+ 
+     def _instCloseFile(self, key):
+-        transaction_list = self._extract_cbkey(key)
+         self.fd.close()
+         self.fd = None
+ 
++    def _inst_start(self, key):
++        pass
++
++    def _inst_stop(self, key):
+         if self.test or not self.trans_running:
+             return
++
++        transaction_list = self._extract_cbkey(key)
+         for tsi in transaction_list:
+             if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN:
+                 tsi.state = libdnf.transaction.TransactionItemState_DONE
+@@ -376,6 +385,10 @@ class RPMTransaction(object):
+ 
+     def _cpioError(self, key):
+         transaction_list = self._extract_cbkey(key)
++        for tsi in transaction_list:
++            if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN:
++                tsi.state = libdnf.transaction.TransactionItemState_ERROR
++                break
+         msg = "Error in cpio payload of rpm package %s" % transaction_list[0].pkg
+         for display in self.displays:
+             display.error(msg)
+@@ -386,11 +399,11 @@ class RPMTransaction(object):
+         msg = "Error unpacking rpm package %s" % tsi.pkg
+         for display in self.displays:
+             display.error(msg)
+-        for tsi1 in transaction_list:
+-            if tsi1.state == libdnf.transaction.TransactionItemState_UNKNOWN:
+-                tsi1.state = libdnf.transaction.TransactionItemState_ERROR
++
++        for tsi in transaction_list:
++           if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN:
++                tsi.state = libdnf.transaction.TransactionItemState_ERROR
+                 return
+-        tsi.state = libdnf.transaction.TransactionItemState_ERROR
+ 
+     def _scriptError(self, amount, total, key):
+         # "amount" carries the failed scriptlet tag,
+--
+libgit2 0.27.8
+

0005-Change-synchronization-of-rpm-transaction-to-swdb.patch

@@ -0,0 +1,167 @@
+From 62c1214fba0fee703fef092bd5ceff0dd215a2f6 Mon Sep 17 00:00:00 2001
+From: Jaroslav Mracek <jmracek@redhat.com>
+Date: Thu, 13 Jun 2019 12:02:07 +0200
+Subject: [PATCH] Change synchronization of rpm transaction to swdb
+
+---
+ dnf/base.py         | 21 +++++----------------
+ dnf/util.py         | 36 ++++++++++++++++++++++++++++++++++++
+ dnf/yum/rpmtrans.py | 22 +++-------------------
+ 3 files changed, 44 insertions(+), 35 deletions(-)
+
+diff --git a/dnf/base.py b/dnf/base.py
+index add6403..237ad39 100644
+--- a/dnf/base.py
++++ b/dnf/base.py
+@@ -978,40 +978,29 @@ class Base(object):
+                 os.nice(onice)
+             except:
+                 pass
++        dnf.util._sync_rpm_trans_with_swdb(self._ts, self._transaction)
+ 
+         if errors is None:
+             pass
+         elif len(errors) == 0:
+             # this is a particularly tricky case happening also when rpm failed
+             # to obtain the transaction lock. We can only try to see if a
+             # particular element failed and if not, decide that is the
+             # case.
+             failed = [el for el in self._ts if el.Failed()]
+-            if failed:
+-                for te in failed:
+-                    te_nevra = dnf.util._te_nevra(te)
+-                    for tsi in self._transaction:
+-                        if str(tsi) == te_nevra:
+-                            tsi.state = libdnf.transaction.TransactionItemState_ERROR
+-
+-                errstring = _('Errors occurred during transaction.')
+-                logger.debug(errstring)
+-                self.history.end(rpmdbv)
+-            else:
++            if not failed:
+                 login = dnf.util.get_effective_login()
+                 msg = _("Failed to obtain the transaction lock "
+                         "(logged in as: %s).")
+                 logger.critical(msg, login)
+                 msg = _('Could not run transaction.')
+                 raise dnf.exceptions.Error(msg)
+         else:
+-            if self._record_history():
+-                herrors = [ucd(x) for x in errors]
+-                self.history.end(rpmdbv)
+-
+             logger.critical(_("Transaction couldn't start:"))
+             for e in errors:
+-                logger.critical(e[0])  # should this be 'to_unicoded'?
++                logger.critical(ucd(e[0]))
++            if self._record_history() and not self._ts.isTsFlagSet(rpm.RPMTRANS_FLAG_TEST):
++                self.history.end(rpmdbv)
+             msg = _("Could not run transaction.")
+             raise dnf.exceptions.Error(msg)
+ 
+diff --git a/dnf/util.py b/dnf/util.py
+index 2f1abe6..9d0288e 100644
+--- a/dnf/util.py
++++ b/dnf/util.py
+@@ -396,6 +396,42 @@ def _te_nevra(te):
+     return nevra + te.V() + '-' + te.R() + '.' + te.A()
+ 
+ 
++def _sync_rpm_trans_with_swdb(rpm_transaction, swdb_transaction):
++    revert_actions = {libdnf.transaction.TransactionItemAction_DOWNGRADED,
++                      libdnf.transaction.TransactionItemAction_OBSOLETED,
++                      libdnf.transaction.TransactionItemAction_REMOVE,
++                      libdnf.transaction.TransactionItemAction_UPGRADED,
++                      libdnf.transaction.TransactionItemAction_REINSTALLED}
++    cached_tsi = [tsi for tsi in swdb_transaction]
++    el_not_found = False
++    error = False
++    for rpm_el in rpm_transaction:
++        te_nevra = _te_nevra(rpm_el)
++        tsi = rpm_el.Key()
++        if tsi is None or not hasattr(tsi, "pkg"):
++            for tsi_candidate in cached_tsi:
++                if tsi_candidate.state != libdnf.transaction.TransactionItemState_UNKNOWN:
++                    continue
++                if tsi_candidate.action not in revert_actions:
++                    continue
++                if str(tsi_candidate) == te_nevra:
++                    tsi = tsi_candidate
++                    break
++        if tsi is None or not hasattr(tsi, "pkg"):
++            logger.critical(_("TransactionItem not found for key: {}").format(te_nevra))
++            el_not_found = True
++            continue
++        if rpm_el.Failed():
++            tsi.state = libdnf.transaction.TransactionItemState_ERROR
++            error = True
++        else:
++            tsi.state = libdnf.transaction.TransactionItemState_DONE
++    if error:
++        logger.debug(_('Errors occurred during transaction.'))
++
++    return el_not_found
++
++
+ class tmpdir(object):
+     def __init__(self):
+         prefix = '%s-' % dnf.const.PREFIX
+diff --git a/dnf/yum/rpmtrans.py b/dnf/yum/rpmtrans.py
+index 0834a8c..8a5bd47 100644
+--- a/dnf/yum/rpmtrans.py
++++ b/dnf/yum/rpmtrans.py
+@@ -336,10 +336,7 @@ class RPMTransaction(object):
+             return
+ 
+         transaction_list = self._extract_cbkey(key)
+-        for tsi in transaction_list:
+-            if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN:
+-                tsi.state = libdnf.transaction.TransactionItemState_DONE
+-                break
++        tsi = transaction_list[0]
+ 
+         for display in self.displays:
+             display.filelog(tsi.pkg, tsi.action)
+@@ -370,10 +367,7 @@ class RPMTransaction(object):
+ 
+     def _unInstStop(self, key):
+         transaction_list = self._extract_cbkey(key)
+-        for tsi in transaction_list:
+-            if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN:
+-                tsi.state = libdnf.transaction.TransactionItemState_DONE
+-                break
++        tsi = transaction_list[0]
+ 
+         for display in self.displays:
+             display.filelog(tsi.pkg, tsi.action)
+@@ -385,26 +379,16 @@ class RPMTransaction(object):
+ 
+     def _cpioError(self, key):
+         transaction_list = self._extract_cbkey(key)
+-        for tsi in transaction_list:
+-            if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN:
+-                tsi.state = libdnf.transaction.TransactionItemState_ERROR
+-                break
+         msg = "Error in cpio payload of rpm package %s" % transaction_list[0].pkg
+         for display in self.displays:
+             display.error(msg)
+ 
+     def _unpackError(self, key):
+         transaction_list = self._extract_cbkey(key)
+-        tsi = transaction_list[0]
+-        msg = "Error unpacking rpm package %s" % tsi.pkg
++        msg = "Error unpacking rpm package %s" % transaction_list[0].pkg
+         for display in self.displays:
+             display.error(msg)
+ 
+-        for tsi in transaction_list:
+-           if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN:
+-                tsi.state = libdnf.transaction.TransactionItemState_ERROR
+-                return
+-
+     def _scriptError(self, amount, total, key):
+         # "amount" carries the failed scriptlet tag,
+         # "total" carries fatal/non-fatal status
+--
+libgit2 0.27.8
+

0006-Add-detailed-debug-login-for-swdbrpm-transaction.patch

@@ -0,0 +1,57 @@
+From 9c34a5d3af3ac21b208c90f424e404b4bf68837b Mon Sep 17 00:00:00 2001
+From: Jaroslav Mracek <jmracek@redhat.com>
+Date: Tue, 25 Jun 2019 11:44:37 +0200
+Subject: [PATCH] Add detailed debug login for swdb/rpm transaction
+
+---
+ dnf/util.py | 25 +++++++++++++++++++++++--
+ 1 file changed, 23 insertions(+), 2 deletions(-)
+
+diff --git a/dnf/util.py b/dnf/util.py
+index 9d0288e..71f61ab 100644
+--- a/dnf/util.py
++++ b/dnf/util.py
+@@ -396,6 +396,23 @@ def _te_nevra(te):
+     return nevra + te.V() + '-' + te.R() + '.' + te.A()
+ 
+ 
++def _log_rpm_trans_with_swdb(rpm_transaction, swdb_transaction):
++    logger.debug("Logging transaction elements")
++    for rpm_el in rpm_transaction:
++        tsi = rpm_el.Key()
++        tsi_state = None
++        if tsi is not None:
++            tsi_state = tsi.state
++        msg = "RPM element: '{}', Key(): '{}', Key state: '{}', Failed() '{}': ".format(
++            _te_nevra(rpm_el), tsi, tsi_state, rpm_el.Failed())
++        logger.debug(msg)
++    for tsi in swdb_transaction:
++        msg = "SWDB element: '{}', State: '{}', Action: '{}', From repo: '{}', Reason: '{}', " \
++              "Get reason: '{}'".format(str(tsi), tsi.state, tsi.action, tsi.from_repo, tsi.reason,
++                                        tsi.get_reason())
++        logger.debug(msg)
++
++
+ def _sync_rpm_trans_with_swdb(rpm_transaction, swdb_transaction):
+     revert_actions = {libdnf.transaction.TransactionItemAction_DOWNGRADED,
+                       libdnf.transaction.TransactionItemAction_OBSOLETED,
+@@ -426,10 +443,14 @@ def _sync_rpm_trans_with_swdb(rpm_transaction, swdb_transaction):
+             error = True
+         else:
+             tsi.state = libdnf.transaction.TransactionItemState_DONE
++    for tsi in cached_tsi:
++        if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN:
++            logger.critical(_("TransactionSWDBItem not found for key: {}").format(str(tsi)))
++            el_not_found = True
+     if error:
+         logger.debug(_('Errors occurred during transaction.'))
+-
+-    return el_not_found
++    if el_not_found:
++        _log_rpm_trans_with_swdb(rpm_transaction, cached_tsi)
+ 
+ 
+ class tmpdir(object):
+--
+libgit2 0.27.8
+

dnf.spec

@@ -80,13 +80,19 @@ It supports RPMs, modules and comps groups & environments.
 
 Name:           dnf
 Version:        4.2.5
-Release:        1%{?dist}
+Release:        2%{?dist}
 Summary:        %{pkg_summary}
 # For a breakdown of the licensing, see PACKAGE-LICENSING
 License:        GPLv2+ and GPLv2 and GPL
 URL:            https://github.com/rpm-software-management/dnf
 Source0:        %{url}/archive/%{version}/%{name}-%{version}.tar.gz
 Patch0001:      0001-Revert-Add-best-as-default-behavior-RhBug16707761671683.patch
+Patch0002:      0002-transaction-Set-an-error-for-first-tsi-in-unknown-state.patch
+Patch0003:      0003-history-Dont-store-failed-transactions-as-succeeded.patch
+Patch0004:      0004-transaction-Add-RPMCALLBACK_INST_STARTSTOP-callback-handlers.patch
+Patch0005:      0005-Change-synchronization-of-rpm-transaction-to-swdb.patch
+Patch0006:      0006-Add-detailed-debug-login-for-swdbrpm-transaction.patch
+
 BuildArch:      noarch
 BuildRequires:  cmake
 BuildRequires:  gettext
@@ -500,6 +506,9 @@ ln -sr  %{buildroot}%{confdir}/vars %{buildroot}%{_sysconfdir}/yum/vars
 %endif
 
 %changelog
+* Thu Jun 27 2019 Pavla Kratochvilova <pkratoch@redhat.com> - 4.2.5-2
+- Backport patches to enhance synchronization of rpm transaction to swdb
+
 * Thu Apr 25 2019 Pavla Kratochvilova <pkratoch@redhat.com> - 4.2.5-1
 - Update to 4.2.5
 - Fix multilib obsoletes (RhBug:1672947)