2652c5509d
Backport upstream fix (commit 6c146775) for CVE-2026-52722,
an integer overflow vulnerability in the VMnc decoder
(vmncdec). The patch adds a vmnc_rect_payload_size() helper
using g_size_checked_mul() for overflow-safe size computation
and adds overflow checks in vmnc_handle_wmvd_rectangle() for
CURSOR_COLOUR and CURSOR_ALPHA cursor types.
CVE: CVE-2026-52722
Upstream patches:
- 6c146775d7.patch
Resolves: RHEL-184414
This commit was backported by Ymir, a Red Hat Enterprise Linux software maintenance AI agent.
Assisted-by: Ymir
103 lines
3.5 KiB
Diff
103 lines
3.5 KiB
Diff
From 7087e0683b53a10492d9b7cc0bbfd7876a333e06 Mon Sep 17 00:00:00 2001
|
|
From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= <sebastian@centricular.com>
|
|
Date: Tue, 16 Jun 2026 10:30:54 +0300
|
|
Subject: [PATCH] vnmdec: Avoid integer overflows when rectangle positions and
|
|
sizes
|
|
|
|
Patch based on a patch by Junyi Liu, who also reported this.
|
|
|
|
Fixes https://gitlab.freedesktop.org/gstreamer/gstreamer/-/work_items/5107
|
|
|
|
Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/11869>
|
|
---
|
|
gst/vmnc/vmncdec.c | 40 ++++++++++++++++++++++++++++++++--------
|
|
1 file changed, 32 insertions(+), 8 deletions(-)
|
|
|
|
diff --git a/gst/vmnc/vmncdec.c b/gst/vmnc/vmncdec.c
|
|
index cbbaeb6..fe26d47 100644
|
|
--- a/gst/vmnc/vmncdec.c
|
|
+++ b/gst/vmnc/vmncdec.c
|
|
@@ -153,6 +153,20 @@ struct RfbRectangle
|
|
typedef int (*rectangle_handler) (GstVMncDec * dec, struct RfbRectangle * rect,
|
|
const guint8 * data, int len, gboolean decode);
|
|
|
|
+static gboolean
|
|
+vmnc_rect_payload_size (struct RfbRectangle *rect, guint bytes_per_pixel,
|
|
+ gsize * size)
|
|
+{
|
|
+ gsize pixels;
|
|
+
|
|
+ if (!g_size_checked_mul (&pixels, rect->width, rect->height))
|
|
+ return FALSE;
|
|
+ if (!g_size_checked_mul (size, pixels, bytes_per_pixel))
|
|
+ return FALSE;
|
|
+
|
|
+ return TRUE;
|
|
+}
|
|
+
|
|
static int
|
|
vmnc_handle_wmvi_rectangle (GstVMncDec * dec, struct RfbRectangle *rect,
|
|
const guint8 * data, int len, gboolean decode)
|
|
@@ -393,7 +407,8 @@ vmnc_handle_wmvd_rectangle (GstVMncDec * dec, struct RfbRectangle *rect,
|
|
{
|
|
/* Cursor data. */
|
|
int datalen = 2;
|
|
- int type, size;
|
|
+ int type;
|
|
+ gsize size;
|
|
|
|
if (len < datalen) {
|
|
GST_LOG_OBJECT (dec, "Cursor data too short");
|
|
@@ -403,9 +418,19 @@ vmnc_handle_wmvd_rectangle (GstVMncDec * dec, struct RfbRectangle *rect,
|
|
type = RFB_GET_UINT8 (data);
|
|
|
|
if (type == CURSOR_COLOUR) {
|
|
- datalen += rect->width * rect->height * dec->format.bytes_per_pixel * 2;
|
|
+ if (!vmnc_rect_payload_size (rect, dec->format.bytes_per_pixel, &size) ||
|
|
+ size > ((gsize) G_MAXINT - datalen) / 2) {
|
|
+ GST_WARNING_OBJECT (dec, "Cursor data size overflow");
|
|
+ return ERROR_INVALID;
|
|
+ }
|
|
+ datalen += size * 2;
|
|
} else if (type == CURSOR_ALPHA) {
|
|
- datalen += rect->width * rect->height * 4;
|
|
+ if (!vmnc_rect_payload_size (rect, 4, &size) ||
|
|
+ size > (gsize) G_MAXINT - datalen) {
|
|
+ GST_WARNING_OBJECT (dec, "Cursor data size overflow");
|
|
+ return ERROR_INVALID;
|
|
+ }
|
|
+ datalen += size;
|
|
} else {
|
|
GST_WARNING_OBJECT (dec, "Unknown cursor type: %d", type);
|
|
return ERROR_INVALID;
|
|
@@ -420,22 +445,21 @@ vmnc_handle_wmvd_rectangle (GstVMncDec * dec, struct RfbRectangle *rect,
|
|
dec->cursor.type = type;
|
|
dec->cursor.width = rect->width;
|
|
dec->cursor.height = rect->height;
|
|
- dec->cursor.type = type;
|
|
dec->cursor.hot_x = rect->x;
|
|
dec->cursor.hot_y = rect->y;
|
|
|
|
g_free (dec->cursor.cursordata);
|
|
g_free (dec->cursor.cursormask);
|
|
|
|
- if (type == 0) {
|
|
- size = rect->width * rect->height * dec->format.bytes_per_pixel;
|
|
+ if (type == CURSOR_COLOUR) {
|
|
dec->cursor.cursordata = g_malloc (size);
|
|
dec->cursor.cursormask = g_malloc (size);
|
|
memcpy (dec->cursor.cursordata, data + 2, size);
|
|
memcpy (dec->cursor.cursormask, data + 2 + size, size);
|
|
} else {
|
|
- dec->cursor.cursordata = g_malloc (rect->width * rect->height * 4);
|
|
- memcpy (dec->cursor.cursordata, data + 2, rect->width * rect->height * 4);
|
|
+ dec->cursor.cursordata = g_malloc (size);
|
|
+ memcpy (dec->cursor.cursordata, data + 2, size);
|
|
+ dec->cursor.cursormask = NULL;
|
|
}
|
|
|
|
return datalen;
|
|
--
|
|
2.52.0
|
|
|