From bc0aa7a7d4abda7bed9544706e7a4fb30fc89a97 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Thu, 12 Oct 2017 20:20:11 +0200 Subject: [PATCH] Fix vboxvideo causing gnome 3.26+ to not work under VirtualBox --- ...o-Fix-reporting-invalid-suggested-of.patch | 118 ++++++++++++++++++ kernel.spec | 6 + 2 files changed, 124 insertions(+) create mode 100644 0001-staging-vboxvideo-Fix-reporting-invalid-suggested-of.patch diff --git a/0001-staging-vboxvideo-Fix-reporting-invalid-suggested-of.patch b/0001-staging-vboxvideo-Fix-reporting-invalid-suggested-of.patch new file mode 100644 index 000000000..695fcda36 --- /dev/null +++ b/0001-staging-vboxvideo-Fix-reporting-invalid-suggested-of.patch @@ -0,0 +1,118 @@ +From 3b40f521aa2f42862203497a94ae77536f41ade2 Mon Sep 17 00:00:00 2001 +From: Hans de Goede +Date: Thu, 12 Oct 2017 19:44:48 +0200 +Subject: [PATCH] staging: vboxvideo: Fix reporting invalid + suggested-offset-properties + +The x and y hints receives from the host are unsigned 32 bit integers and +they get set to -1 (0xffffffff) when invalid. Before this commit the +vboxvideo driver was storing them in an u16 causing the -1 to be truncated +to 65535 which, once reported to userspace, was breaking gnome 3.26+ +in Wayland mode. + +This commit stores the host values in 32 bit variables, removing the +truncation and checks for -1, replacing it with 0 as -1 is not a valid +suggested-offset-property value. Likewise the properties are now +initialized to 0 instead of -1, since -1 is not a valid value. +This fixes gnome 3.26+ in Wayland mode not working with the vboxvideo +driver. + +Reported-by: Gianfranco Costamagna +Cc: stable@vger.kernel.org +Cc: Michael Thayer +Signed-off-by: Hans de Goede +--- + drivers/staging/vboxvideo/vbox_drv.h | 8 ++++---- + drivers/staging/vboxvideo/vbox_irq.c | 4 ++-- + drivers/staging/vboxvideo/vbox_mode.c | 26 ++++++++++++++++++-------- + 3 files changed, 24 insertions(+), 14 deletions(-) + +diff --git a/drivers/staging/vboxvideo/vbox_drv.h b/drivers/staging/vboxvideo/vbox_drv.h +index 4b9302703b36..eeac4f0cb2c6 100644 +--- a/drivers/staging/vboxvideo/vbox_drv.h ++++ b/drivers/staging/vboxvideo/vbox_drv.h +@@ -137,8 +137,8 @@ struct vbox_connector { + char name[32]; + struct vbox_crtc *vbox_crtc; + struct { +- u16 width; +- u16 height; ++ u32 width; ++ u32 height; + bool disconnected; + } mode_hint; + }; +@@ -150,8 +150,8 @@ struct vbox_crtc { + unsigned int crtc_id; + u32 fb_offset; + bool cursor_enabled; +- u16 x_hint; +- u16 y_hint; ++ u32 x_hint; ++ u32 y_hint; + }; + + struct vbox_encoder { +diff --git a/drivers/staging/vboxvideo/vbox_irq.c b/drivers/staging/vboxvideo/vbox_irq.c +index 3ca8bec62ac4..74abdf02d9fd 100644 +--- a/drivers/staging/vboxvideo/vbox_irq.c ++++ b/drivers/staging/vboxvideo/vbox_irq.c +@@ -150,8 +150,8 @@ static void vbox_update_mode_hints(struct vbox_private *vbox) + + disconnected = !(hints->enabled); + crtc_id = vbox_conn->vbox_crtc->crtc_id; +- vbox_conn->mode_hint.width = hints->cx & 0x8fff; +- vbox_conn->mode_hint.height = hints->cy & 0x8fff; ++ vbox_conn->mode_hint.width = hints->cx; ++ vbox_conn->mode_hint.height = hints->cy; + vbox_conn->vbox_crtc->x_hint = hints->dx; + vbox_conn->vbox_crtc->y_hint = hints->dy; + vbox_conn->mode_hint.disconnected = disconnected; +diff --git a/drivers/staging/vboxvideo/vbox_mode.c b/drivers/staging/vboxvideo/vbox_mode.c +index 257a77830410..6f08dc966719 100644 +--- a/drivers/staging/vboxvideo/vbox_mode.c ++++ b/drivers/staging/vboxvideo/vbox_mode.c +@@ -553,12 +553,22 @@ static int vbox_get_modes(struct drm_connector *connector) + ++num_modes; + } + vbox_set_edid(connector, preferred_width, preferred_height); +- drm_object_property_set_value( +- &connector->base, vbox->dev->mode_config.suggested_x_property, +- vbox_connector->vbox_crtc->x_hint); +- drm_object_property_set_value( +- &connector->base, vbox->dev->mode_config.suggested_y_property, +- vbox_connector->vbox_crtc->y_hint); ++ ++ if (vbox_connector->vbox_crtc->x_hint != -1) ++ drm_object_property_set_value(&connector->base, ++ vbox->dev->mode_config.suggested_x_property, ++ vbox_connector->vbox_crtc->x_hint); ++ else ++ drm_object_property_set_value(&connector->base, ++ vbox->dev->mode_config.suggested_x_property, 0); ++ ++ if (vbox_connector->vbox_crtc->y_hint != -1) ++ drm_object_property_set_value(&connector->base, ++ vbox->dev->mode_config.suggested_y_property, ++ vbox_connector->vbox_crtc->y_hint); ++ else ++ drm_object_property_set_value(&connector->base, ++ vbox->dev->mode_config.suggested_y_property, 0); + + return num_modes; + } +@@ -640,9 +650,9 @@ static int vbox_connector_init(struct drm_device *dev, + + drm_mode_create_suggested_offset_properties(dev); + drm_object_attach_property(&connector->base, +- dev->mode_config.suggested_x_property, -1); ++ dev->mode_config.suggested_x_property, 0); + drm_object_attach_property(&connector->base, +- dev->mode_config.suggested_y_property, -1); ++ dev->mode_config.suggested_y_property, 0); + drm_connector_register(connector); + + drm_mode_connector_attach_encoder(connector, encoder); +-- +2.14.2 + diff --git a/kernel.spec b/kernel.spec index 2e2ba0020..7837fd830 100644 --- a/kernel.spec +++ b/kernel.spec @@ -652,6 +652,9 @@ Patch617: Fix-for-module-sig-verification.patch # rhbz 1431375 Patch619: input-rmi4-remove-the-need-for-artifical-IRQ.patch +# fix gnome 3.26+ not working under VirtualBox, submitted upstream, Cc: Stable +Patch620: 0001-staging-vboxvideo-Fix-reporting-invalid-suggested-of.patch + # END OF PATCH DEFINITIONS %endif @@ -2206,6 +2209,9 @@ fi # # %changelog +* Thu Oct 12 2017 Hans de Goede +- Fix vboxvideo causing gnome 3.26+ to not work under VirtualBox + * Wed Oct 11 2017 Jeremy Cline - Fix incorrect updates of uninstantiated keys crash the kernel (rhbz 1498016 1498017)