From a9d30657b1865fa6efbced51e744b0450298e48a Mon Sep 17 00:00:00 2001 From: Cole Robinson Date: Wed, 16 Apr 2014 10:32:52 -0400 Subject: [PATCH] engine: More work to fix cascading error dialogs If a remote network connection stalls, the tick queue becomes backed up while we wait for the hung connection to continue. While this happens, the queue is filled up with other requests to poll the hung connection. When the connection finally times out, the tick thread closes the connection via an idle callback. However before that callback gets a chance to run, all the other poll requests for the dead connection are processed, all launching their own error dialog. Mark the connection as 'closing' before conn.close is scheduled, and use it to short circuit the tick() routine. (cherry picked from commit 1f7604b2413a7a4431703f300fd854489e864c71) --- virtManager/connection.py | 46 ++++++++++++++++++++++++++++++++++++++++++-- virtManager/engine.py | 49 +++++++++++------------------------------------ 2 files changed, 55 insertions(+), 40 deletions(-) diff --git a/virtManager/connection.py b/virtManager/connection.py index 27d5bd9..ffc2f27 100644 --- a/virtManager/connection.py +++ b/virtManager/connection.py @@ -89,6 +89,7 @@ class vmmConnection(vmmGObject): self.connectThread = None self.connectError = None self._backend = virtinst.VirtualConnection(self._uri) + self._closing = False self._caps = None self._caps_xml = None @@ -959,6 +960,10 @@ class vmmConnection(vmmGObject): self.config.set_conn_autoconnect(self.get_uri(), val) def close(self): + if self.state != self.STATE_DISCONNECTED: + logging.debug("conn.close() uri=%s", self.get_uri()) + self._closing = True + def cleanup(devs): for dev in devs.values(): try: @@ -1004,6 +1009,7 @@ class vmmConnection(vmmGObject): self.vms = {} self._change_state(self.STATE_DISCONNECTED) + self._closing = False def _cleanup(self): self.close() @@ -1166,7 +1172,43 @@ class vmmConnection(vmmGObject): kwargs["stats_update"] = False self.idle_emit("priority-tick", kwargs) - def tick(self, stats_update, + def tick(self, *args, **kwargs): + e = None + try: + self._tick(*args, **kwargs) + except KeyboardInterrupt: + raise + except Exception, e: + pass + + if e is None: + return + + from_remote = getattr(libvirt, "VIR_FROM_REMOTE", None) + from_rpc = getattr(libvirt, "VIR_FROM_RPC", None) + sys_error = getattr(libvirt, "VIR_ERR_SYSTEM_ERROR", None) + + dom = -1 + code = -1 + if isinstance(e, libvirt.libvirtError): + dom = e.get_error_domain() + code = e.get_error_code() + + logging.debug("Error polling connection %s", + self.get_uri(), exc_info=True) + + if (dom in [from_remote, from_rpc] and + code in [sys_error]): + e = None + logging.debug("Not showing user error since libvirtd " + "appears to have stopped.") + + self._closing = True + self.idle_add(self.close) + if e: + raise e # pylint: disable=raising-bad-type + + def _tick(self, stats_update, pollvm=False, pollnet=False, pollpool=False, polliface=False, pollnodedev=False, pollmedia=False, @@ -1175,7 +1217,7 @@ class vmmConnection(vmmGObject): main update function: polls for new objects, updates stats, ... @force: Perform the requested polling even if async events are in use """ - if self.state != self.STATE_ACTIVE: + if self.state != self.STATE_ACTIVE or self._closing: return if not pollvm: diff --git a/virtManager/engine.py b/virtManager/engine.py index 0e5e15d..e41c829 100644 --- a/virtManager/engine.py +++ b/virtManager/engine.py @@ -28,8 +28,8 @@ import logging import re import Queue import threading +import traceback -import libvirt from virtinst import util from virtManager import packageutils @@ -338,46 +338,19 @@ class vmmEngine(vmmGObject): def _handle_tick_queue(self): while True: - ignore1, ignore2, obj, kwargs = self._tick_queue.get() - self._tick_single_conn(obj, kwargs) + ignore1, ignore2, conn, kwargs = self._tick_queue.get() + try: + conn.tick(**kwargs) + except Exception, e: + tb = "".join(traceback.format_exc()) + error_msg = (_("Error polling connection '%s': %s") + % (conn.get_uri(), e)) + self.idle_add(lambda: self.err.show_err(error_msg, + details=tb)) + self._tick_queue.task_done() return 1 - def _tick_single_conn(self, conn, kwargs): - e = None - try: - conn.tick(**kwargs) - except KeyboardInterrupt: - raise - except Exception, e: - pass - - if e is None: - return - - from_remote = getattr(libvirt, "VIR_FROM_REMOTE", None) - from_rpc = getattr(libvirt, "VIR_FROM_RPC", None) - sys_error = getattr(libvirt, "VIR_ERR_SYSTEM_ERROR", None) - - dom = -1 - code = -1 - if isinstance(e, libvirt.libvirtError): - dom = e.get_error_domain() - code = e.get_error_code() - - if (dom in [from_remote, from_rpc] and - code in [sys_error]): - logging.exception("Could not refresh connection %s", - conn.get_uri()) - logging.debug("Closing connection since libvirtd " - "appears to have stopped") - else: - error_msg = _("Error polling connection '%s': %s") \ - % (conn.get_uri(), e) - self.idle_add(lambda: self.err.show_err(error_msg)) - - self.idle_add(conn.close) - def increment_window_counter(self, src): ignore = src