Fix wayland on virt efi setups

Resolves: #2172291

0001-udev-Stick-with-wayland-on-hybrid-nvidia-with-vendor.patch

@@ -1,7 +1,8 @@
-From 5d363136eccb0c00368112e1856bb37ed26c6b16 Mon Sep 17 00:00:00 2001
+From 500404b383c3d6a06e6bebd5f060f480c445df9e Mon Sep 17 00:00:00 2001
 From: rpm-build <rpm-build>
 Date: Tue, 27 Sep 2022 10:00:11 -0400
-Subject: [PATCH] udev: Stick with wayland on hybrid nvidia with vendor driver
+Subject: [PATCH 1/2] udev: Stick with wayland on hybrid nvidia with vendor
+ driver
 
 F36 defaulted to wayland for users with hybrid graphics that
 use the vendor nvidia driver.
@@ -13,7 +14,7 @@ commit brings that behavior back.
  1 file changed, 4 insertions(+), 2 deletions(-)
 
 diff --git a/data/61-gdm.rules.in b/data/61-gdm.rules.in
-index 5bc9941..d13f0b8 100644
+index 5bc9941c..d13f0b8b 100644
 --- a/data/61-gdm.rules.in
 +++ b/data/61-gdm.rules.in
 @@ -57,66 +57,68 @@ TEST{0711}!="/usr/bin/nvidia-sleep.sh", GOTO="gdm_disable_wayland"
@@ -88,5 +89,5 @@ index 5bc9941..d13f0b8 100644
  # For nvidia drivers versions Above 510, keep Wayland by default
  ATTR{version}=="[5-9][1-9][0-9].*", GOTO="gdm_end"
 -- 
-2.35.1
+2.39.2
 

0002-udev-Try-to-detect-virtual-graphics-and-EFI-better.patch

@@ -0,0 +1,149 @@
+From f89343fc35aaa2418550f02955bfdd971791db1e Mon Sep 17 00:00:00 2001
+From: Ray Strode <rstrode@redhat.com>
+Date: Wed, 1 Mar 2023 22:50:49 -0500
+Subject: [PATCH 2/2] udev: Try to detect virtual graphics and EFI better
+
+With EFI and simpledrm we may end up booting without a card0 but
+with a card1 drm device. This defeats GDM's simplistic hybrid
+graphics check.
+
+Also, our detection for virtio graphics wasn't working so well, because
+it wrote the results into an environment variable of a non-parent
+device.
+
+This commit gets the detection logic working better for virt/efi setups.
+---
+ data/61-gdm.rules.in | 19 +++++++++----------
+ 1 file changed, 9 insertions(+), 10 deletions(-)
+
+diff --git a/data/61-gdm.rules.in b/data/61-gdm.rules.in
+index d13f0b8b..423f89cb 100644
+--- a/data/61-gdm.rules.in
++++ b/data/61-gdm.rules.in
+@@ -1,114 +1,113 @@
+-# identify virtio graphics cards to find passthrough setups
+-SUBSYSTEM!="virtio", GOTO="gdm_virtio_device_end"
+-ACTION!="add", GOTO="gdm_virtio_device_end"
+-ATTR{vendor}=="0x1af4", ATTR{device}=="0x0010", RUN+="/usr/bin/touch /run/udev/gdm-machine-has-virtual-gpu", ENV{GDM_MACHINE_HAS_VIRTUAL_GPU}="1", GOTO="gdm_virtio_device_end"
+-LABEL="gdm_virtio_device_end"
+-
+ SUBSYSTEM!="pci", GOTO="gdm_pci_device_end"
+-ACTION!="bind", GOTO="gdm_pci_device_end"
++ACTION!="bind", ACTION!="add", GOTO="gdm_pci_device_end"
+ 
+ # identify virtio graphics cards to find passthrough setups
+ # cirrus
+ ATTR{vendor}=="0x1013", ATTR{device}=="0x00b8", ATTR{subsystem_vendor}=="0x1af4", ATTR{subsystem_device}=="0x1100", RUN+="/usr/bin/touch /run/udev/gdm-machine-has-virtual-gpu", ENV{GDM_MACHINE_HAS_VIRTUAL_GPU}="1", GOTO="gdm_pci_device_end"
++ATTR{vendor}=="0x1af4", ATTR{device}=="0x1050", ATTR{subsystem_vendor}=="0x1af4", ATTR{subsystem_device}=="0x1100", RUN+="/usr/bin/touch /run/udev/gdm-machine-has-virtual-gpu", ENV{GDM_MACHINE_HAS_VIRTUAL_GPU}="1", GOTO="gdm_pci_device_end"
+ # vga
+ ATTR{vendor}=="0x1b36", ATTR{device}=="0x0100", RUN+="/usr/bin/touch /run/udev/gdm-machine-has-virtual-gpu", ENV{GDM_MACHINE_HAS_VIRTUAL_GPU}="1", GOTO="gdm_pci_device_end"
+ # qxl
+ ATTR{vendor}=="0x1234", ATTR{device}=="0x1111", RUN+="/usr/bin/touch /run/udev/gdm-machine-has-virtual-gpu", ENV{GDM_MACHINE_HAS_VIRTUAL_GPU}="1", GOTO="gdm_pci_device_end"
++ATTR{vendor}=="0x1af4", ATTR{device}=="0x1050", RUN+="/usr/bin/touch /run/udev/gdm-machine-has-virtual-gpu", ENV{GDM_MACHINE_HAS_VIRTUAL_GPU}="1", GOTO="gdm_pci_device_end"
+ 
+ # disable Wayland on Hi1710 chipsets
+ ATTR{vendor}=="0x19e5", ATTR{device}=="0x1711", GOTO="gdm_disable_wayland"
+ 
+ # disable Wayland on Matrox chipsets
+ ATTR{vendor}=="0x102b", ATTR{device}=="0x0522", GOTO="gdm_disable_wayland"
+ ATTR{vendor}=="0x102b", ATTR{device}=="0x0524", GOTO="gdm_disable_wayland"
+ ATTR{vendor}=="0x102b", ATTR{device}=="0x0530", GOTO="gdm_disable_wayland"
+ ATTR{vendor}=="0x102b", ATTR{device}=="0x0532", GOTO="gdm_disable_wayland"
+ ATTR{vendor}=="0x102b", ATTR{device}=="0x0533", GOTO="gdm_disable_wayland"
+ ATTR{vendor}=="0x102b", ATTR{device}=="0x0534", GOTO="gdm_disable_wayland"
+ ATTR{vendor}=="0x102b", ATTR{device}=="0x0536", GOTO="gdm_disable_wayland"
+ ATTR{vendor}=="0x102b", ATTR{device}=="0x0538", GOTO="gdm_disable_wayland"
+ 
+ # disable Wayland on aspeed chipsets
+ ATTR{vendor}=="0x1a03", ATTR{device}=="0x2010", GOTO="gdm_disable_wayland"
+ ATTR{vendor}=="0x1a03", ATTR{device}=="0x2000", GOTO="gdm_disable_wayland"
+ 
+ LABEL="gdm_pci_device_end"
+ 
+ # disable Wayland if modesetting is disabled
+ KERNEL!="card[0-9]*", GOTO="gdm_nomodeset_end"
++KERNEL=="card[0-9]-*", GOTO="gdm_nomodeset_end"
+ SUBSYSTEM!="drm", GOTO="gdm_nomodeset_end"
+-IMPORT{parent}="GDM_MACHINE_HAS_VIRTUAL_GPU"
+-ENV{GDM_MACHINE_HAS_VIRTUAL_GPU}!="1", RUN+="/usr/bin/touch /run/udev/gdm-machine-has-hardware-gpu"
+ # but keep it enabled for simple framebuffer drivers
+ DRIVERS=="simple-framebuffer", GOTO="gdm_nomodeset_end"
++IMPORT{parent}="GDM_MACHINE_HAS_VIRTUAL_GPU"
++ENV{GDM_MACHINE_HAS_VIRTUAL_GPU}!="1", RUN+="/usr/bin/touch /run/udev/gdm-machine-has-hardware-gpu"
+ IMPORT{cmdline}="nomodeset", GOTO="gdm_disable_wayland"
+ LABEL="gdm_nomodeset_end"
+ 
+ # The vendor nvidia driver has multiple modules that need to be loaded before GDM can make an
+ # informed choice on which way to proceed, so force GDM to wait until NVidia's modules are
+ # loaded before starting up.
+ KERNEL!="nvidia", GOTO="gdm_nvidia_end"
+ SUBSYSTEM!="module", GOTO="gdm_nvidia_end"
+ ACTION!="add", GOTO="gdm_nvidia_end"
+ RUN+="/usr/bin/touch /run/udev/gdm-machine-has-vendor-nvidia-driver"
+ 
+ # Check if suspend/resume services necessary for working wayland support is available
+ TEST{0711}!="/usr/bin/nvidia-sleep.sh", GOTO="gdm_disable_wayland"
+ TEST{0711}!="/usr/lib/systemd/system-sleep/nvidia", GOTO="gdm_disable_wayland"
+ IMPORT{program}="/bin/sh -c \"sed -e 's/: /=/g' -e 's/\([^[:upper:]]\)\([[:upper:]]\)/\1_\2/g' -e 's/[[:lower:]]/\U&/g' -e 's/^/NVIDIA_/' /proc/driver/nvidia/params\""
+ ENV{NVIDIA_PRESERVE_VIDEO_MEMORY_ALLOCATIONS}!="1", GOTO="gdm_disable_wayland"
+ IMPORT{program}="/bin/sh -c 'echo NVIDIA_HIBERNATE=`systemctl is-enabled nvidia-hibernate`'"
+ ENV{NVIDIA_HIBERNATE}!="enabled", GOTO="gdm_disable_wayland"
+ IMPORT{program}="/bin/sh -c 'echo NVIDIA_RESUME=`systemctl is-enabled nvidia-resume`'"
+ ENV{NVIDIA_RESUME}!="enabled", GOTO="gdm_disable_wayland"
+ IMPORT{program}="/bin/sh -c 'echo NVIDIA_SUSPEND=`systemctl is-enabled nvidia-suspend`'"
+ ENV{NVIDIA_SUSPEND}!="enabled", GOTO="gdm_disable_wayland"
+ LABEL="gdm_nvidia_end"
+ 
+ # If this machine has an internal panel, take note, since it's probably a laptop
+ # FIXME: It could be "ghost connectors" make this pop positive for some workstations
+ # in the wild. If so, we may have to fallback to looking at the chassis type from
+ # dmi data or acpi
+ KERNEL!="card[0-9]-eDP-*", GOTO="gdm_laptop_check_end"
+ SUBSYSTEM!="drm", GOTO="gdm_laptop_check_end"
+ ACTION!="add", GOTO="gdm_laptop_check_end"
+ RUN+="/usr/bin/touch /run/udev/gdm-machine-is-laptop"
+ GOTO="gdm_hybrid_nvidia_laptop_check"
+ LABEL="gdm_laptop_check_end"
+ 
+ # If this is a hybrid graphics setup, take note
+ KERNEL!="card[1-9]*", GOTO="gdm_hybrid_graphics_check_end"
+ KERNEL=="card[1-9]-*", GOTO="gdm_hybrid_graphics_check_end"
+ SUBSYSTEM!="drm", GOTO="gdm_hybrid_graphics_check_end"
+ ACTION!="add", GOTO="gdm_hybrid_graphics_check_end"
+-RUN+="/usr/bin/touch /run/udev/gdm-machine-has-hybrid-graphics"
++IMPORT{program}="/bin/sh -c \"echo GDM_NUMBER_OF_GRAPHICS_CARDS=`ls -1d /sys/class/drm/card[0-9] | wc -l`\""
++ENV{GDM_NUMBER_OF_GRAPHICS_CARDS}=="1", RUN+="/usr/bin/rm -f /run/udev/gdm-machine-has-hybrid-graphics"
++ENV{GDM_NUMBER_OF_GRAPHICS_CARDS}!="1", RUN+="/usr/bin/touch /run/udev/gdm-machine-has-hybrid-graphics"
+ LABEL="gdm_hybrid_graphics_check_end"
+ 
+ # If this is a hybrid graphics laptop with vendor nvidia driver default to wayland
+ # It gives poor performance for multi-monitor, but Xorg fails some multi-monitor
+ # setups too, and this is what we did in F36.
+ LABEL="gdm_hybrid_nvidia_laptop_check"
+ TEST!="/run/udev/gdm-machine-is-laptop", GOTO="gdm_hybrid_nvidia_laptop_check_end"
+ TEST!="/run/udev/gdm-machine-has-hybrid-graphics", GOTO="gdm_hybrid_nvidia_laptop_check_end"
+ TEST!="/run/udev/gdm-machine-has-vendor-nvidia-driver", GOTO="gdm_hybrid_nvidia_laptop_check_end"
+ GOTO="gdm_end"
+ LABEL="gdm_hybrid_nvidia_laptop_check_end"
+ 
+ # Disable wayland in situation where we're in a guest with a virtual gpu and host passthrough gpu
+ LABEL="gdm_virt_passthrough_check"
+ TEST!="/run/udev/gdm-machine-has-hybrid-graphics", GOTO="gdm_virt_passthrough_check_end"
+ TEST!="/run/udev/gdm-machine-has-virtual-gpu", GOTO="gdm_virt_passthrough_check_end"
+ TEST!="/run/udev/gdm-machine-has-hardware-gpu", GOTO="gdm_virt_passthrough_check_end"
+ GOTO="gdm_disable_wayland"
+ LABEL="gdm_virt_passthrough_check_end"
+ 
+ # Disable wayland when there are multiple virtual gpus
+ LABEL="gdm_virt_multi_gpu_check"
+ TEST!="/run/udev/gdm-machine-has-hybrid-graphics", GOTO="gdm_virt_multi_gpu_check_end"
+ TEST!="/run/udev/gdm-machine-has-virtual-gpu", GOTO="gdm_virt_multi_gpu_check_end"
+ TEST=="/run/udev/gdm-machine-has-hardware-gpu", GOTO="gdm_virt_multi_gpu_check_end"
+ LABEL="gdm_virt_multi_gpu_check_end"
+ 
+ # Disable wayland when nvidia modeset is disabled or when drivers are a lower
+ # version than 470,
+ # For versions above 470 but lower than 510 prefer Xorg,
+-- 
+2.39.2
+

gdm.spec

@@ -11,7 +11,7 @@
 Name: gdm
 Epoch: 1
 Version: 43.0
-Release: 5%{?dist}
+Release: 6%{?dist}
 Summary: The GNOME Display Manager
 
 License: GPLv2+
@@ -29,6 +29,8 @@ Patch10001: 0001-pam-redhat-Remove-pam_console-from-service-files.patch
 
 # Downstream patches
 Patch70001: 0001-udev-Stick-with-wayland-on-hybrid-nvidia-with-vendor.patch
+Patch70002: 0002-udev-Try-to-detect-virtual-graphics-and-EFI-better.patch
+
 Patch80001: 0001-Honor-initial-setup-being-disabled-by-distro-install.patch
 Patch90001: 0001-data-add-system-dconf-databases-to-gdm-profile.patch
 
@@ -304,6 +306,10 @@ fi
 %{_libdir}/pkgconfig/gdm-pam-extensions.pc
 
 %changelog
+* Wed Mar 01 2023 Ray Strode <rstrode@redhat.com> - 1:43.0-6
+- Fix wayland on virt efi setups
+  Resolves: #2172291
+
 * Thu Feb 09 2023 Iker Pedrosa <ipedrosa@redhat.com> - 1:43.0-5
 - Remove pam_console from service file (#1822228)