From acedef08fbdd49fae44b5bc348a9ffb36a74a9ad Mon Sep 17 00:00:00 2001 From: Pavla Kratochvilova Date: Wed, 26 Jun 2019 16:24:53 +0200 Subject: [PATCH] Backport patches to enhance synchronization of rpm transaction to swdb --- ...error-for-first-tsi-in-unknown-state.patch | 27 +++ ...ore-failed-transactions-as-succeeded.patch | 90 ++++++++++ ...ACK_INST_STARTSTOP-callback-handlers.patch | 73 ++++++++ ...onization-of-rpm-transaction-to-swdb.patch | 167 ++++++++++++++++++ ...-debug-login-for-swdbrpm-transaction.patch | 57 ++++++ dnf.spec | 11 +- 6 files changed, 424 insertions(+), 1 deletion(-) create mode 100644 0002-transaction-Set-an-error-for-first-tsi-in-unknown-state.patch create mode 100644 0003-history-Dont-store-failed-transactions-as-succeeded.patch create mode 100644 0004-transaction-Add-RPMCALLBACK_INST_STARTSTOP-callback-handlers.patch create mode 100644 0005-Change-synchronization-of-rpm-transaction-to-swdb.patch create mode 100644 0006-Add-detailed-debug-login-for-swdbrpm-transaction.patch diff --git a/0002-transaction-Set-an-error-for-first-tsi-in-unknown-state.patch b/0002-transaction-Set-an-error-for-first-tsi-in-unknown-state.patch new file mode 100644 index 0000000..f23b64f --- /dev/null +++ b/0002-transaction-Set-an-error-for-first-tsi-in-unknown-state.patch @@ -0,0 +1,27 @@ +From 41cbb0ff39c68a7eb3eaa8573a4b4be97c080c46 Mon Sep 17 00:00:00 2001 +From: Jaroslav Mracek +Date: Mon, 3 Jun 2019 19:26:55 +0200 +Subject: [PATCH] [transaction] Set an error for first tsi in unknown state + +--- + dnf/yum/rpmtrans.py | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/dnf/yum/rpmtrans.py b/dnf/yum/rpmtrans.py +index 6a60b5a..50d8b81 100644 +--- a/dnf/yum/rpmtrans.py ++++ b/dnf/yum/rpmtrans.py +@@ -386,6 +386,10 @@ class RPMTransaction(object): + msg = "Error unpacking rpm package %s" % tsi.pkg + for display in self.displays: + display.error(msg) ++ for tsi1 in transaction_list: ++ if tsi1.state == libdnf.transaction.TransactionItemState_UNKNOWN: ++ tsi1.state = libdnf.transaction.TransactionItemState_ERROR ++ return + tsi.state = libdnf.transaction.TransactionItemState_ERROR + + def _scriptError(self, amount, total, key): +-- +libgit2 0.27.8 + diff --git a/0003-history-Dont-store-failed-transactions-as-succeeded.patch b/0003-history-Dont-store-failed-transactions-as-succeeded.patch new file mode 100644 index 0000000..fa84b16 --- /dev/null +++ b/0003-history-Dont-store-failed-transactions-as-succeeded.patch @@ -0,0 +1,90 @@ +From 94a074b9747a437c478e388dc76ed90d06d8a666 Mon Sep 17 00:00:00 2001 +From: Daniel Mach +Date: Wed, 5 Jun 2019 08:29:03 +0200 +Subject: [PATCH] [history] Don't store failed transactions as succeeded. + +--- + dnf/base.py | 7 ++++--- + dnf/db/history.py | 16 ++++++++++------ + 2 files changed, 14 insertions(+), 9 deletions(-) + +diff --git a/dnf/base.py b/dnf/base.py +index 7a388ec..add6403 100644 +--- a/dnf/base.py ++++ b/dnf/base.py +@@ -987,15 +987,16 @@ class Base(object): + # particular element failed and if not, decide that is the + # case. + failed = [el for el in self._ts if el.Failed()] +- if len(failed) > 0: ++ if failed: + for te in failed: + te_nevra = dnf.util._te_nevra(te) + for tsi in self._transaction: + if str(tsi) == te_nevra: + tsi.state = libdnf.transaction.TransactionItemState_ERROR + + errstring = _('Errors occurred during transaction.') + logger.debug(errstring) ++ self.history.end(rpmdbv) + else: + login = dnf.util.get_effective_login() + msg = _("Failed to obtain the transaction lock " +@@ -1006,7 +1007,7 @@ class Base(object): + else: + if self._record_history(): + herrors = [ucd(x) for x in errors] +- self.history.end(rpmdbv, 2, errors=herrors) ++ self.history.end(rpmdbv) + + logger.critical(_("Transaction couldn't start:")) + for e in errors: +@@ -1069,7 +1070,7 @@ class Base(object): + count = display_banner(tsi.pkg, count) + + rpmdbv = rpmdb_sack._rpmdb_version() +- self.history.end(rpmdbv, 0) ++ self.history.end(rpmdbv) + + timer() + self._trans_success = True +diff --git a/dnf/db/history.py b/dnf/db/history.py +index dc9c53b..7f22f97 100644 +--- a/dnf/db/history.py ++++ b/dnf/db/history.py +@@ -486,22 +486,26 @@ class SwdbInterface(object): + self.swdb.log_error(self._tid, error) + ''' + +- # TODO: rename to end_transaction? +- def end(self, end_rpmdb_version="", return_code=0, errors=None): +- assert return_code or not errors +- # TODO: fix return_code +- return_code = not bool(return_code) ++ def end(self, end_rpmdb_version="", return_code=None, errors=None): + if not hasattr(self, '_tid'): + return # Failed at beg() time + ++ if return_code is None: ++ # return_code/state auto-detection ++ return_code = libdnf.transaction.TransactionState_DONE ++ for tsi in self.rpm: ++ if tsi.state == libdnf.transaction.TransactionItemState_ERROR: ++ return_code = libdnf.transaction.TransactionState_ERROR ++ break ++ + for file_descriptor, line in self._output: + self.swdb.addConsoleOutputLine(file_descriptor, line) + self._output = [] + + self.swdb.endTransaction( + int(time.time()), + str(end_rpmdb_version), +- bool(return_code) ++ return_code, + ) + + # Closing and cleanup is done in the close() method. +-- +libgit2 0.27.8 + diff --git a/0004-transaction-Add-RPMCALLBACK_INST_STARTSTOP-callback-handlers.patch b/0004-transaction-Add-RPMCALLBACK_INST_STARTSTOP-callback-handlers.patch new file mode 100644 index 0000000..102216b --- /dev/null +++ b/0004-transaction-Add-RPMCALLBACK_INST_STARTSTOP-callback-handlers.patch @@ -0,0 +1,73 @@ +From f7f30977df77032c1b97d86d5df9e9c907f3a7e3 Mon Sep 17 00:00:00 2001 +From: Daniel Mach +Date: Wed, 5 Jun 2019 21:10:05 +0200 +Subject: [PATCH] [transaction] Add RPMCALLBACK_INST_{START,STOP} callback handlers. + +--- + dnf/yum/rpmtrans.py | 23 ++++++++++++++++++----- + 1 file changed, 18 insertions(+), 5 deletions(-) + +diff --git a/dnf/yum/rpmtrans.py b/dnf/yum/rpmtrans.py +index 50d8b81..0834a8c 100644 +--- a/dnf/yum/rpmtrans.py ++++ b/dnf/yum/rpmtrans.py +@@ -262,6 +262,10 @@ class RPMTransaction(object): + return self._instOpenFile(key) + elif what == rpm.RPMCALLBACK_INST_CLOSE_FILE: + self._instCloseFile(key) ++ elif what == rpm.RPMCALLBACK_INST_START: ++ self._inst_start(key) ++ elif what == rpm.RPMCALLBACK_INST_STOP: ++ self._inst_stop(key) + elif what == rpm.RPMCALLBACK_INST_PROGRESS: + self._instProgress(amount, total, key) + elif what == rpm.RPMCALLBACK_UNINST_START: +@@ -321,12 +325,17 @@ class RPMTransaction(object): + return self.fd.fileno() + + def _instCloseFile(self, key): +- transaction_list = self._extract_cbkey(key) + self.fd.close() + self.fd = None + ++ def _inst_start(self, key): ++ pass ++ ++ def _inst_stop(self, key): + if self.test or not self.trans_running: + return ++ ++ transaction_list = self._extract_cbkey(key) + for tsi in transaction_list: + if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN: + tsi.state = libdnf.transaction.TransactionItemState_DONE +@@ -376,6 +385,10 @@ class RPMTransaction(object): + + def _cpioError(self, key): + transaction_list = self._extract_cbkey(key) ++ for tsi in transaction_list: ++ if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN: ++ tsi.state = libdnf.transaction.TransactionItemState_ERROR ++ break + msg = "Error in cpio payload of rpm package %s" % transaction_list[0].pkg + for display in self.displays: + display.error(msg) +@@ -386,11 +399,11 @@ class RPMTransaction(object): + msg = "Error unpacking rpm package %s" % tsi.pkg + for display in self.displays: + display.error(msg) +- for tsi1 in transaction_list: +- if tsi1.state == libdnf.transaction.TransactionItemState_UNKNOWN: +- tsi1.state = libdnf.transaction.TransactionItemState_ERROR ++ ++ for tsi in transaction_list: ++ if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN: ++ tsi.state = libdnf.transaction.TransactionItemState_ERROR + return +- tsi.state = libdnf.transaction.TransactionItemState_ERROR + + def _scriptError(self, amount, total, key): + # "amount" carries the failed scriptlet tag, +-- +libgit2 0.27.8 + diff --git a/0005-Change-synchronization-of-rpm-transaction-to-swdb.patch b/0005-Change-synchronization-of-rpm-transaction-to-swdb.patch new file mode 100644 index 0000000..ebf6ec6 --- /dev/null +++ b/0005-Change-synchronization-of-rpm-transaction-to-swdb.patch @@ -0,0 +1,167 @@ +From 62c1214fba0fee703fef092bd5ceff0dd215a2f6 Mon Sep 17 00:00:00 2001 +From: Jaroslav Mracek +Date: Thu, 13 Jun 2019 12:02:07 +0200 +Subject: [PATCH] Change synchronization of rpm transaction to swdb + +--- + dnf/base.py | 21 +++++---------------- + dnf/util.py | 36 ++++++++++++++++++++++++++++++++++++ + dnf/yum/rpmtrans.py | 22 +++------------------- + 3 files changed, 44 insertions(+), 35 deletions(-) + +diff --git a/dnf/base.py b/dnf/base.py +index add6403..237ad39 100644 +--- a/dnf/base.py ++++ b/dnf/base.py +@@ -978,40 +978,29 @@ class Base(object): + os.nice(onice) + except: + pass ++ dnf.util._sync_rpm_trans_with_swdb(self._ts, self._transaction) + + if errors is None: + pass + elif len(errors) == 0: + # this is a particularly tricky case happening also when rpm failed + # to obtain the transaction lock. We can only try to see if a + # particular element failed and if not, decide that is the + # case. + failed = [el for el in self._ts if el.Failed()] +- if failed: +- for te in failed: +- te_nevra = dnf.util._te_nevra(te) +- for tsi in self._transaction: +- if str(tsi) == te_nevra: +- tsi.state = libdnf.transaction.TransactionItemState_ERROR +- +- errstring = _('Errors occurred during transaction.') +- logger.debug(errstring) +- self.history.end(rpmdbv) +- else: ++ if not failed: + login = dnf.util.get_effective_login() + msg = _("Failed to obtain the transaction lock " + "(logged in as: %s).") + logger.critical(msg, login) + msg = _('Could not run transaction.') + raise dnf.exceptions.Error(msg) + else: +- if self._record_history(): +- herrors = [ucd(x) for x in errors] +- self.history.end(rpmdbv) +- + logger.critical(_("Transaction couldn't start:")) + for e in errors: +- logger.critical(e[0]) # should this be 'to_unicoded'? ++ logger.critical(ucd(e[0])) ++ if self._record_history() and not self._ts.isTsFlagSet(rpm.RPMTRANS_FLAG_TEST): ++ self.history.end(rpmdbv) + msg = _("Could not run transaction.") + raise dnf.exceptions.Error(msg) + +diff --git a/dnf/util.py b/dnf/util.py +index 2f1abe6..9d0288e 100644 +--- a/dnf/util.py ++++ b/dnf/util.py +@@ -396,6 +396,42 @@ def _te_nevra(te): + return nevra + te.V() + '-' + te.R() + '.' + te.A() + + ++def _sync_rpm_trans_with_swdb(rpm_transaction, swdb_transaction): ++ revert_actions = {libdnf.transaction.TransactionItemAction_DOWNGRADED, ++ libdnf.transaction.TransactionItemAction_OBSOLETED, ++ libdnf.transaction.TransactionItemAction_REMOVE, ++ libdnf.transaction.TransactionItemAction_UPGRADED, ++ libdnf.transaction.TransactionItemAction_REINSTALLED} ++ cached_tsi = [tsi for tsi in swdb_transaction] ++ el_not_found = False ++ error = False ++ for rpm_el in rpm_transaction: ++ te_nevra = _te_nevra(rpm_el) ++ tsi = rpm_el.Key() ++ if tsi is None or not hasattr(tsi, "pkg"): ++ for tsi_candidate in cached_tsi: ++ if tsi_candidate.state != libdnf.transaction.TransactionItemState_UNKNOWN: ++ continue ++ if tsi_candidate.action not in revert_actions: ++ continue ++ if str(tsi_candidate) == te_nevra: ++ tsi = tsi_candidate ++ break ++ if tsi is None or not hasattr(tsi, "pkg"): ++ logger.critical(_("TransactionItem not found for key: {}").format(te_nevra)) ++ el_not_found = True ++ continue ++ if rpm_el.Failed(): ++ tsi.state = libdnf.transaction.TransactionItemState_ERROR ++ error = True ++ else: ++ tsi.state = libdnf.transaction.TransactionItemState_DONE ++ if error: ++ logger.debug(_('Errors occurred during transaction.')) ++ ++ return el_not_found ++ ++ + class tmpdir(object): + def __init__(self): + prefix = '%s-' % dnf.const.PREFIX +diff --git a/dnf/yum/rpmtrans.py b/dnf/yum/rpmtrans.py +index 0834a8c..8a5bd47 100644 +--- a/dnf/yum/rpmtrans.py ++++ b/dnf/yum/rpmtrans.py +@@ -336,10 +336,7 @@ class RPMTransaction(object): + return + + transaction_list = self._extract_cbkey(key) +- for tsi in transaction_list: +- if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN: +- tsi.state = libdnf.transaction.TransactionItemState_DONE +- break ++ tsi = transaction_list[0] + + for display in self.displays: + display.filelog(tsi.pkg, tsi.action) +@@ -370,10 +367,7 @@ class RPMTransaction(object): + + def _unInstStop(self, key): + transaction_list = self._extract_cbkey(key) +- for tsi in transaction_list: +- if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN: +- tsi.state = libdnf.transaction.TransactionItemState_DONE +- break ++ tsi = transaction_list[0] + + for display in self.displays: + display.filelog(tsi.pkg, tsi.action) +@@ -385,26 +379,16 @@ class RPMTransaction(object): + + def _cpioError(self, key): + transaction_list = self._extract_cbkey(key) +- for tsi in transaction_list: +- if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN: +- tsi.state = libdnf.transaction.TransactionItemState_ERROR +- break + msg = "Error in cpio payload of rpm package %s" % transaction_list[0].pkg + for display in self.displays: + display.error(msg) + + def _unpackError(self, key): + transaction_list = self._extract_cbkey(key) +- tsi = transaction_list[0] +- msg = "Error unpacking rpm package %s" % tsi.pkg ++ msg = "Error unpacking rpm package %s" % transaction_list[0].pkg + for display in self.displays: + display.error(msg) + +- for tsi in transaction_list: +- if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN: +- tsi.state = libdnf.transaction.TransactionItemState_ERROR +- return +- + def _scriptError(self, amount, total, key): + # "amount" carries the failed scriptlet tag, + # "total" carries fatal/non-fatal status +-- +libgit2 0.27.8 + diff --git a/0006-Add-detailed-debug-login-for-swdbrpm-transaction.patch b/0006-Add-detailed-debug-login-for-swdbrpm-transaction.patch new file mode 100644 index 0000000..7fcffdb --- /dev/null +++ b/0006-Add-detailed-debug-login-for-swdbrpm-transaction.patch @@ -0,0 +1,57 @@ +From 9c34a5d3af3ac21b208c90f424e404b4bf68837b Mon Sep 17 00:00:00 2001 +From: Jaroslav Mracek +Date: Tue, 25 Jun 2019 11:44:37 +0200 +Subject: [PATCH] Add detailed debug login for swdb/rpm transaction + +--- + dnf/util.py | 25 +++++++++++++++++++++++-- + 1 file changed, 23 insertions(+), 2 deletions(-) + +diff --git a/dnf/util.py b/dnf/util.py +index 9d0288e..71f61ab 100644 +--- a/dnf/util.py ++++ b/dnf/util.py +@@ -396,6 +396,23 @@ def _te_nevra(te): + return nevra + te.V() + '-' + te.R() + '.' + te.A() + + ++def _log_rpm_trans_with_swdb(rpm_transaction, swdb_transaction): ++ logger.debug("Logging transaction elements") ++ for rpm_el in rpm_transaction: ++ tsi = rpm_el.Key() ++ tsi_state = None ++ if tsi is not None: ++ tsi_state = tsi.state ++ msg = "RPM element: '{}', Key(): '{}', Key state: '{}', Failed() '{}': ".format( ++ _te_nevra(rpm_el), tsi, tsi_state, rpm_el.Failed()) ++ logger.debug(msg) ++ for tsi in swdb_transaction: ++ msg = "SWDB element: '{}', State: '{}', Action: '{}', From repo: '{}', Reason: '{}', " \ ++ "Get reason: '{}'".format(str(tsi), tsi.state, tsi.action, tsi.from_repo, tsi.reason, ++ tsi.get_reason()) ++ logger.debug(msg) ++ ++ + def _sync_rpm_trans_with_swdb(rpm_transaction, swdb_transaction): + revert_actions = {libdnf.transaction.TransactionItemAction_DOWNGRADED, + libdnf.transaction.TransactionItemAction_OBSOLETED, +@@ -426,10 +443,14 @@ def _sync_rpm_trans_with_swdb(rpm_transaction, swdb_transaction): + error = True + else: + tsi.state = libdnf.transaction.TransactionItemState_DONE ++ for tsi in cached_tsi: ++ if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN: ++ logger.critical(_("TransactionSWDBItem not found for key: {}").format(str(tsi))) ++ el_not_found = True + if error: + logger.debug(_('Errors occurred during transaction.')) +- +- return el_not_found ++ if el_not_found: ++ _log_rpm_trans_with_swdb(rpm_transaction, cached_tsi) + + + class tmpdir(object): +-- +libgit2 0.27.8 + diff --git a/dnf.spec b/dnf.spec index eae8cee..cfe70e6 100644 --- a/dnf.spec +++ b/dnf.spec @@ -80,13 +80,19 @@ It supports RPMs, modules and comps groups & environments. Name: dnf Version: 4.2.5 -Release: 1%{?dist} +Release: 2%{?dist} Summary: %{pkg_summary} # For a breakdown of the licensing, see PACKAGE-LICENSING License: GPLv2+ and GPLv2 and GPL URL: https://github.com/rpm-software-management/dnf Source0: %{url}/archive/%{version}/%{name}-%{version}.tar.gz Patch0001: 0001-Revert-Add-best-as-default-behavior-RhBug16707761671683.patch +Patch0002: 0002-transaction-Set-an-error-for-first-tsi-in-unknown-state.patch +Patch0003: 0003-history-Dont-store-failed-transactions-as-succeeded.patch +Patch0004: 0004-transaction-Add-RPMCALLBACK_INST_STARTSTOP-callback-handlers.patch +Patch0005: 0005-Change-synchronization-of-rpm-transaction-to-swdb.patch +Patch0006: 0006-Add-detailed-debug-login-for-swdbrpm-transaction.patch + BuildArch: noarch BuildRequires: cmake BuildRequires: gettext @@ -500,6 +506,9 @@ ln -sr %{buildroot}%{confdir}/vars %{buildroot}%{_sysconfdir}/yum/vars %endif %changelog +* Thu Jun 27 2019 Pavla Kratochvilova - 4.2.5-2 +- Backport patches to enhance synchronization of rpm transaction to swdb + * Thu Apr 25 2019 Pavla Kratochvilova - 4.2.5-1 - Update to 4.2.5 - Fix multilib obsoletes (RhBug:1672947)