Fix integer overflows in vmnc decoder (CVE-2026-52722)

Backport upstream commit 6c146775d784 to fix integer
overflows in the vmnc decoder when computing rectangle
positions and sizes (CVE-2026-52722). The patch adds
overflow-safe size computation using g_size_checked_mul()
for cursor data allocations in both CURSOR_COLOUR and
CURSOR_ALPHA code paths.

CVE: CVE-2026-52722
Upstream patches:
 - 6c146775d7.patch
Resolves: RHEL-184427

This commit was backported by Ymir, a Red Hat Enterprise Linux software maintenance AI agent.

Assisted-by: Ymir
This commit is contained in:
RHEL Packaging Agent 2026-07-29 09:43:53 +00:00
parent 7f62f1d9cc
commit dd99c2f139
2 changed files with 107 additions and 1 deletions

View File

@ -0,0 +1,99 @@
From 8b3522e88d99996928ab2827bf550906e37a0521 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= <sebastian@centricular.com>
Date: Tue, 16 Jun 2026 10:30:54 +0300
Subject: [PATCH] vnmdec: Avoid integer overflows when rectangle positions and
sizes
Patch based on a patch by Junyi Liu, who also reported this.
Fixes https://gitlab.freedesktop.org/gstreamer/gstreamer/-/work_items/5107
Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/11869>
---
.../gst-plugins-bad/gst/vmnc/vmncdec.c | 40 +++++++++++++++----
1 file changed, 32 insertions(+), 8 deletions(-)
diff --git a/subprojects/gst-plugins-bad/gst/vmnc/vmncdec.c b/subprojects/gst-plugins-bad/gst/vmnc/vmncdec.c
index 93c34e707e..32dd397893 100644
--- a/subprojects/gst-plugins-bad/gst/vmnc/vmncdec.c
+++ b/subprojects/gst-plugins-bad/gst/vmnc/vmncdec.c
@@ -155,6 +155,20 @@ struct RfbRectangle
typedef int (*rectangle_handler) (GstVMncDec * dec, struct RfbRectangle * rect,
const guint8 * data, int len, gboolean decode);
+static gboolean
+vmnc_rect_payload_size (struct RfbRectangle *rect, guint bytes_per_pixel,
+ gsize * size)
+{
+ gsize pixels;
+
+ if (!g_size_checked_mul (&pixels, rect->width, rect->height))
+ return FALSE;
+ if (!g_size_checked_mul (size, pixels, bytes_per_pixel))
+ return FALSE;
+
+ return TRUE;
+}
+
static int
vmnc_handle_wmvi_rectangle (GstVMncDec * dec, struct RfbRectangle *rect,
const guint8 * data, int len, gboolean decode)
@@ -395,7 +409,8 @@ vmnc_handle_wmvd_rectangle (GstVMncDec * dec, struct RfbRectangle *rect,
{
/* Cursor data. */
int datalen = 2;
- int type, size;
+ int type;
+ gsize size;
if (len < datalen) {
GST_LOG_OBJECT (dec, "Cursor data too short");
@@ -405,9 +420,19 @@ vmnc_handle_wmvd_rectangle (GstVMncDec * dec, struct RfbRectangle *rect,
type = RFB_GET_UINT8 (data);
if (type == CURSOR_COLOUR) {
- datalen += rect->width * rect->height * dec->format.bytes_per_pixel * 2;
+ if (!vmnc_rect_payload_size (rect, dec->format.bytes_per_pixel, &size) ||
+ size > ((gsize) G_MAXINT - datalen) / 2) {
+ GST_WARNING_OBJECT (dec, "Cursor data size overflow");
+ return ERROR_INVALID;
+ }
+ datalen += size * 2;
} else if (type == CURSOR_ALPHA) {
- datalen += rect->width * rect->height * 4;
+ if (!vmnc_rect_payload_size (rect, 4, &size) ||
+ size > (gsize) G_MAXINT - datalen) {
+ GST_WARNING_OBJECT (dec, "Cursor data size overflow");
+ return ERROR_INVALID;
+ }
+ datalen += size;
} else {
GST_WARNING_OBJECT (dec, "Unknown cursor type: %d", type);
return ERROR_INVALID;
@@ -422,22 +447,21 @@ vmnc_handle_wmvd_rectangle (GstVMncDec * dec, struct RfbRectangle *rect,
dec->cursor.type = type;
dec->cursor.width = rect->width;
dec->cursor.height = rect->height;
- dec->cursor.type = type;
dec->cursor.hot_x = rect->x;
dec->cursor.hot_y = rect->y;
g_free (dec->cursor.cursordata);
g_free (dec->cursor.cursormask);
- if (type == 0) {
- size = rect->width * rect->height * dec->format.bytes_per_pixel;
+ if (type == CURSOR_COLOUR) {
dec->cursor.cursordata = g_malloc (size);
dec->cursor.cursormask = g_malloc (size);
memcpy (dec->cursor.cursordata, data + 2, size);
memcpy (dec->cursor.cursormask, data + 2 + size, size);
} else {
- dec->cursor.cursordata = g_malloc (rect->width * rect->height * 4);
- memcpy (dec->cursor.cursordata, data + 2, rect->width * rect->height * 4);
+ dec->cursor.cursordata = g_malloc (size);
+ memcpy (dec->cursor.cursordata, data + 2, size);
+ dec->cursor.cursormask = NULL;
}
return datalen;

View File

@ -26,7 +26,7 @@
Name: gstreamer1-plugins-bad-free
Version: 1.26.7
Release: 5%{?dist}
Release: 6%{?dist}
Summary: GStreamer streaming media framework "bad" plugins
License: LGPLv2+ and LGPLv2
@ -53,6 +53,9 @@ Patch: gstreamer1-plugins-bad-free-1.26.7-CVE-2026-52720.patch
Patch: gstreamer1-plugins-bad-free-1.26.7-CVE-2026-59691.patch
# https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/12053
Patch: 0005-dtlsconnection-Allocate-large-enough-buffer-for-the-.patch
# https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/11869
# https://gitlab.freedesktop.org/gstreamer/gstreamer/-/commit/6c146775d784bbe91ff7afc6701ba351306282ce
Patch: gstreamer1-plugins-bad-free-1.26.7-CVE-2026-52722.patch
BuildRequires: meson >= 0.48.0
BuildRequires: gcc-c++
@ -880,6 +883,10 @@ EOF
%changelog
* Wed Jul 29 2026 RHEL Packaging Agent <redhat-ymir-agent@redhat.com> - 1.26.7-6
- Fix integer overflows in vmnc decoder (CVE-2026-52722)
Resolves: RHEL-184427
* Wed Jul 29 2026 RHEL Packaging Agent <redhat-ymir-agent@redhat.com> - 1.26.7-5
- Fix CVE-2026-59692: DTLS connection buffer overflow in certificate DN
Resolves: RHEL-193566