238 lines
7.2 KiB
Diff
238 lines
7.2 KiB
Diff
From 64a0ffdc0bff2d55269c7e763824867d976506e0 Mon Sep 17 00:00:00 2001
|
|
From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= <sebastian@centricular.com>
|
|
Date: Sun, 14 Jun 2026 22:03:25 +0300
|
|
Subject: [PATCH] librfb: Validate framebuffer update rectangles against the
|
|
framebuffer size
|
|
|
|
Patch provided by Junyi Liu, who also reported this.
|
|
|
|
Fixes https://gitlab.freedesktop.org/gstreamer/gstreamer/-/work_items/5105
|
|
|
|
Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/11866>
|
|
---
|
|
gst/librfb/rfbdecoder.c | 136 +++++++++++++++++++++++++++++++++-------
|
|
1 file changed, 115 insertions(+), 21 deletions(-)
|
|
|
|
diff --git a/gst/librfb/rfbdecoder.c b/gst/librfb/rfbdecoder.c
|
|
index fa76331..efa3a84 100644
|
|
--- a/gst/librfb/rfbdecoder.c
|
|
+++ b/gst/librfb/rfbdecoder.c
|
|
@@ -45,6 +45,10 @@ static gboolean rfb_decoder_corre_encoding (RfbDecoder * decoder, gint start_x,
|
|
gint start_y, gint rect_w, gint rect_h);
|
|
static gboolean rfb_decoder_hextile_encoding (RfbDecoder * decoder,
|
|
gint start_x, gint start_y, gint rect_w, gint rect_h);
|
|
+static gboolean rfb_decoder_clip_rectangle (RfbDecoder * decoder, gint * x,
|
|
+ gint * y, gint * w, gint * h, gint * skip_x, gint * skip_y);
|
|
+static gboolean rfb_decoder_clip_copyrect (RfbDecoder * decoder, gint * dst_x,
|
|
+ gint * dst_y, gint * src_x, gint * src_y, gint * w, gint * h);
|
|
|
|
RfbDecoder *
|
|
rfb_decoder_new (void)
|
|
@@ -801,23 +805,20 @@ rfb_decoder_state_framebuffer_update_rectangle (RfbDecoder * decoder)
|
|
if (!rfb_decoder_read (decoder, 12))
|
|
return FALSE;
|
|
|
|
- x = RFB_GET_UINT16 (decoder->data + 0) - decoder->offset_x;
|
|
- y = RFB_GET_UINT16 (decoder->data + 2) - decoder->offset_y;
|
|
+ x = RFB_GET_UINT16 (decoder->data + 0);
|
|
+ y = RFB_GET_UINT16 (decoder->data + 2);
|
|
w = RFB_GET_UINT16 (decoder->data + 4);
|
|
h = RFB_GET_UINT16 (decoder->data + 6);
|
|
encoding = RFB_GET_UINT32 (decoder->data + 8);
|
|
|
|
+ x -= (gint) decoder->offset_x;
|
|
+ y -= (gint) decoder->offset_y;
|
|
+
|
|
GST_DEBUG ("update recieved");
|
|
GST_DEBUG ("x:%d y:%d", x, y);
|
|
GST_DEBUG ("w:%d h:%d", w, h);
|
|
GST_DEBUG ("encoding: %d", encoding);
|
|
|
|
- if (((w * h) + (x * y)) > (decoder->width * decoder->height)) {
|
|
- GST_ERROR ("Desktop resize is unsupported.");
|
|
- decoder->state = NULL;
|
|
- return TRUE;
|
|
- }
|
|
-
|
|
switch (encoding) {
|
|
case ENCODING_TYPE_RAW:
|
|
ret = rfb_decoder_raw_encoding (decoder, x, y, w, h);
|
|
@@ -856,11 +857,20 @@ static gboolean
|
|
rfb_decoder_raw_encoding (RfbDecoder * decoder, gint start_x, gint start_y,
|
|
gint rect_w, gint rect_h)
|
|
{
|
|
- gint size;
|
|
+ gint copy_x, copy_y, copy_w, copy_h;
|
|
+ gint skip_x = 0, skip_y = 0;
|
|
+ guint32 size;
|
|
guint8 *frame, *p;
|
|
guint32 raw_line_size;
|
|
+ guint32 copy_line_size;
|
|
+
|
|
+ if (rect_w <= 0 || rect_h <= 0)
|
|
+ return TRUE;
|
|
|
|
raw_line_size = rect_w * decoder->bytespp;
|
|
+ if (rect_h > 0 && raw_line_size > G_MAXUINT32 / rect_h)
|
|
+ return FALSE;
|
|
+
|
|
size = rect_h * raw_line_size;
|
|
|
|
GST_DEBUG ("Reading %d bytes (%dx%d)", size, rect_w, rect_h);
|
|
@@ -868,13 +878,23 @@ rfb_decoder_raw_encoding (RfbDecoder * decoder, gint start_x, gint start_y,
|
|
if (!rfb_decoder_read (decoder, size))
|
|
return FALSE;
|
|
|
|
+ copy_x = start_x;
|
|
+ copy_y = start_y;
|
|
+ copy_w = rect_w;
|
|
+ copy_h = rect_h;
|
|
+
|
|
+ if (!rfb_decoder_clip_rectangle (decoder, ©_x, ©_y, ©_w, ©_h,
|
|
+ &skip_x, &skip_y))
|
|
+ return TRUE;
|
|
+
|
|
frame =
|
|
- decoder->frame + (((start_y * decoder->rect_width) +
|
|
- start_x) * decoder->bytespp);
|
|
- p = decoder->data;
|
|
+ decoder->frame + (((copy_y * decoder->rect_width) +
|
|
+ copy_x) * decoder->bytespp);
|
|
+ p = decoder->data + (skip_y * raw_line_size) + (skip_x * decoder->bytespp);
|
|
+ copy_line_size = copy_w * decoder->bytespp;
|
|
|
|
- while (rect_h--) {
|
|
- memcpy (frame, p, raw_line_size);
|
|
+ while (copy_h--) {
|
|
+ memcpy (frame, p, copy_line_size);
|
|
p += raw_line_size;
|
|
frame += decoder->line_size;
|
|
}
|
|
@@ -886,7 +906,8 @@ static gboolean
|
|
rfb_decoder_copyrect_encoding (RfbDecoder * decoder, gint start_x, gint start_y,
|
|
gint rect_w, gint rect_h)
|
|
{
|
|
- guint16 src_x, src_y;
|
|
+ gint src_x, src_y;
|
|
+ gint copy_x, copy_y, copy_w, copy_h;
|
|
gint line_width, copyrect_width;
|
|
guint8 *src, *dst;
|
|
|
|
@@ -894,20 +915,28 @@ rfb_decoder_copyrect_encoding (RfbDecoder * decoder, gint start_x, gint start_y,
|
|
return FALSE;
|
|
|
|
/* don't forget the offset */
|
|
- src_x = RFB_GET_UINT16 (decoder->data) - decoder->offset_x;
|
|
- src_y = RFB_GET_UINT16 (decoder->data + 2) - decoder->offset_y;
|
|
+ src_x = RFB_GET_UINT16 (decoder->data) - (gint) decoder->offset_x;
|
|
+ src_y = RFB_GET_UINT16 (decoder->data + 2) - (gint) decoder->offset_y;
|
|
GST_DEBUG ("Copyrect from %d %d", src_x, src_y);
|
|
|
|
- copyrect_width = rect_w * decoder->bytespp;
|
|
+ copy_x = start_x;
|
|
+ copy_y = start_y;
|
|
+ copy_w = rect_w;
|
|
+ copy_h = rect_h;
|
|
+ if (!rfb_decoder_clip_copyrect (decoder, ©_x, ©_y, &src_x, &src_y,
|
|
+ ©_w, ©_h))
|
|
+ return TRUE;
|
|
+
|
|
+ copyrect_width = copy_w * decoder->bytespp;
|
|
line_width = decoder->line_size;
|
|
src =
|
|
decoder->prev_frame + ((src_y * decoder->rect_width) +
|
|
src_x) * decoder->bytespp;
|
|
dst =
|
|
- decoder->frame + ((start_y * decoder->rect_width) +
|
|
- start_x) * decoder->bytespp;
|
|
+ decoder->frame + ((copy_y * decoder->rect_width) +
|
|
+ copy_x) * decoder->bytespp;
|
|
|
|
- while (rect_h--) {
|
|
+ while (copy_h--) {
|
|
memcpy (dst, src, copyrect_width);
|
|
src += line_width;
|
|
dst += line_width;
|
|
@@ -925,6 +954,9 @@ rfb_decoder_fill_rectangle (RfbDecoder * decoder, gint x, gint y, gint w,
|
|
guint32 *offset;
|
|
gint i, j;
|
|
|
|
+ if (!rfb_decoder_clip_rectangle (decoder, &x, &y, &w, &h, NULL, NULL))
|
|
+ return;
|
|
+
|
|
for (i = 0; i < h; i++) {
|
|
offset =
|
|
(guint32 *) (decoder->frame + ((x + (y +
|
|
@@ -1100,6 +1132,68 @@ rfb_decoder_hextile_encoding (RfbDecoder * decoder, gint start_x, gint start_y,
|
|
return TRUE;
|
|
}
|
|
|
|
+static gboolean
|
|
+rfb_decoder_clip_rectangle (RfbDecoder * decoder, gint * x, gint * y, gint * w,
|
|
+ gint * h, gint * skip_x, gint * skip_y)
|
|
+{
|
|
+ gint x1, y1, x2, y2;
|
|
+ gint orig_x, orig_y;
|
|
+
|
|
+ if (*w <= 0 || *h <= 0)
|
|
+ return FALSE;
|
|
+
|
|
+ orig_x = *x;
|
|
+ orig_y = *y;
|
|
+ x1 = MAX (orig_x, 0);
|
|
+ y1 = MAX (orig_y, 0);
|
|
+ x2 = MIN (orig_x + *w, (gint) decoder->rect_width);
|
|
+ y2 = MIN (orig_y + *h, (gint) decoder->rect_height);
|
|
+
|
|
+ if (x2 <= x1 || y2 <= y1)
|
|
+ return FALSE;
|
|
+
|
|
+ *x = x1;
|
|
+ *y = y1;
|
|
+ *w = x2 - x1;
|
|
+ *h = y2 - y1;
|
|
+
|
|
+ if (skip_x)
|
|
+ *skip_x = x1 - orig_x;
|
|
+ if (skip_y)
|
|
+ *skip_y = y1 - orig_y;
|
|
+
|
|
+ return TRUE;
|
|
+}
|
|
+
|
|
+static gboolean
|
|
+rfb_decoder_clip_copyrect (RfbDecoder * decoder, gint * dst_x, gint * dst_y,
|
|
+ gint * src_x, gint * src_y, gint * w, gint * h)
|
|
+{
|
|
+ gint left, top, right, bottom;
|
|
+
|
|
+ if (*w <= 0 || *h <= 0)
|
|
+ return FALSE;
|
|
+
|
|
+ left = MAX (0, MAX (-*dst_x, -*src_x));
|
|
+ top = MAX (0, MAX (-*dst_y, -*src_y));
|
|
+ right = MIN (*w, MIN ((gint) decoder->rect_width - *dst_x,
|
|
+ (gint) decoder->rect_width - *src_x));
|
|
+ bottom = MIN (*h, MIN ((gint) decoder->rect_height - *dst_y,
|
|
+ (gint) decoder->rect_height - *src_y));
|
|
+
|
|
+ if (right <= left || bottom <= top)
|
|
+ return FALSE;
|
|
+
|
|
+ *dst_x += left;
|
|
+ *dst_y += top;
|
|
+ *src_x += left;
|
|
+ *src_y += top;
|
|
+ *w = right - left;
|
|
+ *h = bottom - top;
|
|
+
|
|
+ return TRUE;
|
|
+}
|
|
+
|
|
static gboolean
|
|
rfb_decoder_state_set_colour_map_entries (RfbDecoder * decoder)
|
|
{
|
|
--
|
|
2.52.0
|
|
|