qemu-kvm/kvm-s390x-pci-drive-ISM-reset-from-subsystem-reset.patch

From dda71c431be22772f3241af45b62737c988e85d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?C=C3=A9dric=20Le=20Goater?= <clg@redhat.com>
Date: Tue, 23 Jan 2024 13:59:24 +0100
Subject: [PATCH 3/3] s390x/pci: drive ISM reset from subsystem reset
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

RH-Author: Cédric Le Goater <clg@redhat.com>
RH-MergeRequest: 349: s390x: Fix reset ordering of passthrough ISM devices
RH-Jira: RHEL-22411
RH-Acked-by: Thomas Huth <thuth@redhat.com>
RH-Acked-by: Cornelia Huck <cohuck@redhat.com>
RH-Commit: [3/3] 42e89595dd5e24538a2d3f075391b4534497eece

JIRA: https://issues.redhat.com/browse/RHEL-22411

commit 68c691ca99a2538d6a53a70ce8a9ce06ee307ff1
Author: Matthew Rosato <mjrosato@linux.ibm.com>
Date:   Thu Jan 18 13:51:51 2024 -0500

    s390x/pci: drive ISM reset from subsystem reset

    ISM devices are sensitive to manipulation of the IOMMU, so the ISM device
    needs to be reset before the vfio-pci device is reset (triggering a full
    UNMAP).  In order to ensure this occurs, trigger ISM device resets from
    subsystem_reset before triggering the PCI bus reset (which will also
    trigger vfio-pci reset).  This only needs to be done for ISM devices
    which were enabled for use by the guest.
    Further, ensure that AIF is disabled as part of the reset event.

    Fixes: ef1535901a ("s390x: do a subsystem reset before the unprotect on reboot")
    Fixes: 03451953c7 ("s390x/pci: reset ISM passthrough devices on shutdown and system reset")
    Reported-by: Cédric Le Goater <clg@redhat.com>
    Signed-off-by: Matthew Rosato <mjrosato@linux.ibm.com>
    Message-ID: <20240118185151.265329-4-mjrosato@linux.ibm.com>
    Reviewed-by: Eric Farman <farman@linux.ibm.com>
    Reviewed-by: Cédric Le Goater <clg@redhat.com>
    Signed-off-by: Thomas Huth <thuth@redhat.com>

Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
 hw/s390x/s390-pci-bus.c         | 26 +++++++++++++++++---------
 hw/s390x/s390-virtio-ccw.c      |  8 ++++++++
 include/hw/s390x/s390-pci-bus.h |  1 +
 3 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index 2d92848b0f..a8953693b9 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -160,20 +160,12 @@ static void s390_pci_shutdown_notifier(Notifier *n, void *opaque)
     pci_device_reset(pbdev->pdev);
 }
 
-static void s390_pci_reset_cb(void *opaque)
-{
-    S390PCIBusDevice *pbdev = opaque;
-
-    pci_device_reset(pbdev->pdev);
-}
-
 static void s390_pci_perform_unplug(S390PCIBusDevice *pbdev)
 {
     HotplugHandler *hotplug_ctrl;
 
     if (pbdev->pft == ZPCI_PFT_ISM) {
         notifier_remove(&pbdev->shutdown_notifier);
-        qemu_unregister_reset(s390_pci_reset_cb, pbdev);
     }
 
     /* Unplug the PCI device */
@@ -1137,7 +1129,6 @@ static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
             if (pbdev->pft == ZPCI_PFT_ISM) {
                 pbdev->shutdown_notifier.notify = s390_pci_shutdown_notifier;
                 qemu_register_shutdown_notifier(&pbdev->shutdown_notifier);
-                qemu_register_reset(s390_pci_reset_cb, pbdev);
             }
         } else {
             pbdev->fh |= FH_SHM_EMUL;
@@ -1284,6 +1275,23 @@ static void s390_pci_enumerate_bridge(PCIBus *bus, PCIDevice *pdev,
     pci_default_write_config(pdev, PCI_SUBORDINATE_BUS, s->bus_no, 1);
 }
 
+void s390_pci_ism_reset(void)
+{
+    S390pciState *s = s390_get_phb();
+
+    S390PCIBusDevice *pbdev, *next;
+
+    /* Trigger reset event for each passthrough ISM device currently in-use */
+    QTAILQ_FOREACH_SAFE(pbdev, &s->zpci_devs, link, next) {
+        if (pbdev->interp && pbdev->pft == ZPCI_PFT_ISM &&
+            pbdev->fh & FH_MASK_ENABLE) {
+            s390_pci_kvm_aif_disable(pbdev);
+
+            pci_device_reset(pbdev->pdev);
+        }
+    }
+}
+
 static void s390_pcihost_reset(DeviceState *dev)
 {
     S390pciState *s = S390_PCI_HOST_BRIDGE(dev);
diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index 94434c3bb1..51e5b39888 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -108,6 +108,14 @@ static void subsystem_reset(void)
     DeviceState *dev;
     int i;
 
+    /*
+     * ISM firmware is sensitive to unexpected changes to the IOMMU, which can
+     * occur during reset of the vfio-pci device (unmap of entire aperture).
+     * Ensure any passthrough ISM devices are reset now, while CPUs are paused
+     * but before vfio-pci cleanup occurs.
+     */
+    s390_pci_ism_reset();
+
     for (i = 0; i < ARRAY_SIZE(reset_dev_types); i++) {
         dev = DEVICE(object_resolve_path_type("", reset_dev_types[i], NULL));
         if (dev) {
diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h
index 7a658f5e30..2bfad5563a 100644
--- a/include/hw/s390x/s390-pci-bus.h
+++ b/include/hw/s390x/s390-pci-bus.h
@@ -401,5 +401,6 @@ S390PCIBusDevice *s390_pci_find_dev_by_target(S390pciState *s,
                                               const char *target);
 S390PCIBusDevice *s390_pci_find_next_avail_dev(S390pciState *s,
                                                S390PCIBusDevice *pbdev);
+void s390_pci_ism_reset(void);
 
 #endif
-- 
2.41.0