From c885b17e09ab19a3e8d3b2e1765963811af6f764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Le=20Goater?= Date: Mon, 22 Jan 2024 09:25:53 +0100 Subject: [PATCH 098/101] s390x/pci: drive ISM reset from subsystem reset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RH-Author: Cédric Le Goater RH-MergeRequest: 215: s390x: Fix reset ordering of passthrough ISM devices RH-Jira: RHEL-21169 RH-Acked-by: Thomas Huth RH-Acked-by: Cornelia Huck RH-Commit: [3/3] 426cf156a2c67e6dcd7483a769fa3741e2700504 (clegoate/qemu-kvm-c9s) JIRA: https://issues.redhat.com/browse/RHEL-21169 commit 68c691ca99a2538d6a53a70ce8a9ce06ee307ff1 Author: Matthew Rosato Date: Thu Jan 18 13:51:51 2024 -0500 s390x/pci: drive ISM reset from subsystem reset ISM devices are sensitive to manipulation of the IOMMU, so the ISM device needs to be reset before the vfio-pci device is reset (triggering a full UNMAP). In order to ensure this occurs, trigger ISM device resets from subsystem_reset before triggering the PCI bus reset (which will also trigger vfio-pci reset). This only needs to be done for ISM devices which were enabled for use by the guest. Further, ensure that AIF is disabled as part of the reset event. Fixes: ef1535901a ("s390x: do a subsystem reset before the unprotect on reboot") Fixes: 03451953c7 ("s390x/pci: reset ISM passthrough devices on shutdown and system reset") Reported-by: Cédric Le Goater Signed-off-by: Matthew Rosato Message-ID: <20240118185151.265329-4-mjrosato@linux.ibm.com> Reviewed-by: Eric Farman Reviewed-by: Cédric Le Goater Signed-off-by: Thomas Huth Signed-off-by: Cédric Le Goater --- hw/s390x/s390-pci-bus.c | 26 +++++++++++++++++--------- hw/s390x/s390-virtio-ccw.c | 8 ++++++++ include/hw/s390x/s390-pci-bus.h | 1 + 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c index 347580ebac..3e57d5faca 100644 --- a/hw/s390x/s390-pci-bus.c +++ b/hw/s390x/s390-pci-bus.c @@ -151,20 +151,12 @@ static void s390_pci_shutdown_notifier(Notifier *n, void *opaque) pci_device_reset(pbdev->pdev); } -static void s390_pci_reset_cb(void *opaque) -{ - S390PCIBusDevice *pbdev = opaque; - - pci_device_reset(pbdev->pdev); -} - static void s390_pci_perform_unplug(S390PCIBusDevice *pbdev) { HotplugHandler *hotplug_ctrl; if (pbdev->pft == ZPCI_PFT_ISM) { notifier_remove(&pbdev->shutdown_notifier); - qemu_unregister_reset(s390_pci_reset_cb, pbdev); } /* Unplug the PCI device */ @@ -1132,7 +1124,6 @@ static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev, if (pbdev->pft == ZPCI_PFT_ISM) { pbdev->shutdown_notifier.notify = s390_pci_shutdown_notifier; qemu_register_shutdown_notifier(&pbdev->shutdown_notifier); - qemu_register_reset(s390_pci_reset_cb, pbdev); } } else { pbdev->fh |= FH_SHM_EMUL; @@ -1279,6 +1270,23 @@ static void s390_pci_enumerate_bridge(PCIBus *bus, PCIDevice *pdev, pci_default_write_config(pdev, PCI_SUBORDINATE_BUS, s->bus_no, 1); } +void s390_pci_ism_reset(void) +{ + S390pciState *s = s390_get_phb(); + + S390PCIBusDevice *pbdev, *next; + + /* Trigger reset event for each passthrough ISM device currently in-use */ + QTAILQ_FOREACH_SAFE(pbdev, &s->zpci_devs, link, next) { + if (pbdev->interp && pbdev->pft == ZPCI_PFT_ISM && + pbdev->fh & FH_MASK_ENABLE) { + s390_pci_kvm_aif_disable(pbdev); + + pci_device_reset(pbdev->pdev); + } + } +} + static void s390_pcihost_reset(DeviceState *dev) { S390pciState *s = S390_PCI_HOST_BRIDGE(dev); diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c index e26ce26f5a..24f4773179 100644 --- a/hw/s390x/s390-virtio-ccw.c +++ b/hw/s390x/s390-virtio-ccw.c @@ -118,6 +118,14 @@ static void subsystem_reset(void) DeviceState *dev; int i; + /* + * ISM firmware is sensitive to unexpected changes to the IOMMU, which can + * occur during reset of the vfio-pci device (unmap of entire aperture). + * Ensure any passthrough ISM devices are reset now, while CPUs are paused + * but before vfio-pci cleanup occurs. + */ + s390_pci_ism_reset(); + for (i = 0; i < ARRAY_SIZE(reset_dev_types); i++) { dev = DEVICE(object_resolve_path_type("", reset_dev_types[i], NULL)); if (dev) { diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h index 435e788867..2c43ea123f 100644 --- a/include/hw/s390x/s390-pci-bus.h +++ b/include/hw/s390x/s390-pci-bus.h @@ -401,5 +401,6 @@ S390PCIBusDevice *s390_pci_find_dev_by_target(S390pciState *s, const char *target); S390PCIBusDevice *s390_pci_find_next_avail_dev(S390pciState *s, S390PCIBusDevice *pbdev); +void s390_pci_ism_reset(void); #endif -- 2.39.3