From f3b66928a194b32b27fac3c3379d3d20e5966442 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= <sebastian@centricular.com>
Date: Sun, 14 Jun 2026 22:03:25 +0300
Subject: [PATCH] librfb: Validate framebuffer update rectangles against the
 framebuffer size

Patch provided by Junyi Liu, who also reported this.

Fixes https://gitlab.freedesktop.org/gstreamer/gstreamer/-/work_items/5105

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/11866>
---
 .../gst-plugins-bad/gst/librfb/rfbdecoder.c   | 136 +++++++++++++++---
 1 file changed, 115 insertions(+), 21 deletions(-)

diff --git a/subprojects/gst-plugins-bad/gst/librfb/rfbdecoder.c b/subprojects/gst-plugins-bad/gst/librfb/rfbdecoder.c
index f0507a36f46..7f94a28d319 100644
--- gst/librfb/rfbdecoder.c.orig
+++ gst/librfb/rfbdecoder.c
@@ -45,6 +45,10 @@ static gboolean rfb_decoder_corre_encoding (RfbDecoder * decoder, gint start_x,
     gint start_y, gint rect_w, gint rect_h);
 static gboolean rfb_decoder_hextile_encoding (RfbDecoder * decoder,
     gint start_x, gint start_y, gint rect_w, gint rect_h);
+static gboolean rfb_decoder_clip_rectangle (RfbDecoder * decoder, gint * x,
+    gint * y, gint * w, gint * h, gint * skip_x, gint * skip_y);
+static gboolean rfb_decoder_clip_copyrect (RfbDecoder * decoder, gint * dst_x,
+    gint * dst_y, gint * src_x, gint * src_y, gint * w, gint * h);
 
 RfbDecoder *
 rfb_decoder_new (void)
@@ -802,23 +806,20 @@ rfb_decoder_state_framebuffer_update_rectangle (RfbDecoder * decoder)
   if (!rfb_decoder_read (decoder, 12))
     return FALSE;
 
-  x = RFB_GET_UINT16 (decoder->data + 0) - decoder->offset_x;
-  y = RFB_GET_UINT16 (decoder->data + 2) - decoder->offset_y;
+  x = RFB_GET_UINT16 (decoder->data + 0);
+  y = RFB_GET_UINT16 (decoder->data + 2);
   w = RFB_GET_UINT16 (decoder->data + 4);
   h = RFB_GET_UINT16 (decoder->data + 6);
   encoding = RFB_GET_UINT32 (decoder->data + 8);
 
+  x -= (gint) decoder->offset_x;
+  y -= (gint) decoder->offset_y;
+
   GST_DEBUG ("update received");
   GST_DEBUG ("x:%d y:%d", x, y);
   GST_DEBUG ("w:%d h:%d", w, h);
   GST_DEBUG ("encoding: %d", encoding);
 
-  if (((w * h) + (x * y)) > (decoder->width * decoder->height)) {
-    GST_ERROR ("Desktop resize is unsupported.");
-    decoder->state = NULL;
-    return TRUE;
-  }
-
   switch (encoding) {
     case ENCODING_TYPE_RAW:
       ret = rfb_decoder_raw_encoding (decoder, x, y, w, h);
@@ -857,11 +858,20 @@ static gboolean
 rfb_decoder_raw_encoding (RfbDecoder * decoder, gint start_x, gint start_y,
     gint rect_w, gint rect_h)
 {
-  gint size;
+  gint copy_x, copy_y, copy_w, copy_h;
+  gint skip_x = 0, skip_y = 0;
+  guint32 size;
   guint8 *frame, *p;
   guint32 raw_line_size;
+  guint32 copy_line_size;
+
+  if (rect_w <= 0 || rect_h <= 0)
+    return TRUE;
 
   raw_line_size = rect_w * decoder->bytespp;
+  if (rect_h > 0 && raw_line_size > G_MAXUINT32 / rect_h)
+    return FALSE;
+
   size = rect_h * raw_line_size;
 
   GST_DEBUG ("Reading %d bytes (%dx%d)", size, rect_w, rect_h);
@@ -869,13 +879,23 @@ rfb_decoder_raw_encoding (RfbDecoder * decoder, gint start_x, gint start_y,
   if (!rfb_decoder_read (decoder, size))
     return FALSE;
 
+  copy_x = start_x;
+  copy_y = start_y;
+  copy_w = rect_w;
+  copy_h = rect_h;
+
+  if (!rfb_decoder_clip_rectangle (decoder, &copy_x, &copy_y, &copy_w, &copy_h,
+          &skip_x, &skip_y))
+    return TRUE;
+
   frame =
-      decoder->frame + (((start_y * decoder->rect_width) +
-          start_x) * decoder->bytespp);
-  p = decoder->data;
+      decoder->frame + (((copy_y * decoder->rect_width) +
+          copy_x) * decoder->bytespp);
+  p = decoder->data + (skip_y * raw_line_size) + (skip_x * decoder->bytespp);
+  copy_line_size = copy_w * decoder->bytespp;
 
-  while (rect_h--) {
-    memcpy (frame, p, raw_line_size);
+  while (copy_h--) {
+    memcpy (frame, p, copy_line_size);
     p += raw_line_size;
     frame += decoder->line_size;
   }
@@ -887,7 +907,8 @@ static gboolean
 rfb_decoder_copyrect_encoding (RfbDecoder * decoder, gint start_x, gint start_y,
     gint rect_w, gint rect_h)
 {
-  guint16 src_x, src_y;
+  gint src_x, src_y;
+  gint copy_x, copy_y, copy_w, copy_h;
   gint line_width, copyrect_width;
   guint8 *src, *dst;
 
@@ -895,20 +916,28 @@ rfb_decoder_copyrect_encoding (RfbDecoder * decoder, gint start_x, gint start_y,
     return FALSE;
 
   /* don't forget the offset */
-  src_x = RFB_GET_UINT16 (decoder->data) - decoder->offset_x;
-  src_y = RFB_GET_UINT16 (decoder->data + 2) - decoder->offset_y;
+  src_x = RFB_GET_UINT16 (decoder->data) - (gint) decoder->offset_x;
+  src_y = RFB_GET_UINT16 (decoder->data + 2) - (gint) decoder->offset_y;
   GST_DEBUG ("Copyrect from %d %d", src_x, src_y);
 
-  copyrect_width = rect_w * decoder->bytespp;
+  copy_x = start_x;
+  copy_y = start_y;
+  copy_w = rect_w;
+  copy_h = rect_h;
+  if (!rfb_decoder_clip_copyrect (decoder, &copy_x, &copy_y, &src_x, &src_y,
+          &copy_w, &copy_h))
+    return TRUE;
+
+  copyrect_width = copy_w * decoder->bytespp;
   line_width = decoder->line_size;
   src =
       decoder->prev_frame + ((src_y * decoder->rect_width) +
       src_x) * decoder->bytespp;
   dst =
-      decoder->frame + ((start_y * decoder->rect_width) +
-      start_x) * decoder->bytespp;
+      decoder->frame + ((copy_y * decoder->rect_width) +
+      copy_x) * decoder->bytespp;
 
-  while (rect_h--) {
+  while (copy_h--) {
     memcpy (dst, src, copyrect_width);
     src += line_width;
     dst += line_width;
@@ -926,6 +955,9 @@ rfb_decoder_fill_rectangle (RfbDecoder * decoder, gint x, gint y, gint w,
   guint32 *offset;
   gint i, j;
 
+  if (!rfb_decoder_clip_rectangle (decoder, &x, &y, &w, &h, NULL, NULL))
+    return;
+
   for (i = 0; i < h; i++) {
     offset =
         (guint32 *) (decoder->frame + ((x + (y +
@@ -1101,6 +1133,68 @@ rfb_decoder_hextile_encoding (RfbDecoder * decoder, gint start_x, gint start_y,
   return TRUE;
 }
 
+static gboolean
+rfb_decoder_clip_rectangle (RfbDecoder * decoder, gint * x, gint * y, gint * w,
+    gint * h, gint * skip_x, gint * skip_y)
+{
+  gint x1, y1, x2, y2;
+  gint orig_x, orig_y;
+
+  if (*w <= 0 || *h <= 0)
+    return FALSE;
+
+  orig_x = *x;
+  orig_y = *y;
+  x1 = MAX (orig_x, 0);
+  y1 = MAX (orig_y, 0);
+  x2 = MIN (orig_x + *w, (gint) decoder->rect_width);
+  y2 = MIN (orig_y + *h, (gint) decoder->rect_height);
+
+  if (x2 <= x1 || y2 <= y1)
+    return FALSE;
+
+  *x = x1;
+  *y = y1;
+  *w = x2 - x1;
+  *h = y2 - y1;
+
+  if (skip_x)
+    *skip_x = x1 - orig_x;
+  if (skip_y)
+    *skip_y = y1 - orig_y;
+
+  return TRUE;
+}
+
+static gboolean
+rfb_decoder_clip_copyrect (RfbDecoder * decoder, gint * dst_x, gint * dst_y,
+    gint * src_x, gint * src_y, gint * w, gint * h)
+{
+  gint left, top, right, bottom;
+
+  if (*w <= 0 || *h <= 0)
+    return FALSE;
+
+  left = MAX (0, MAX (-*dst_x, -*src_x));
+  top = MAX (0, MAX (-*dst_y, -*src_y));
+  right = MIN (*w, MIN ((gint) decoder->rect_width - *dst_x,
+          (gint) decoder->rect_width - *src_x));
+  bottom = MIN (*h, MIN ((gint) decoder->rect_height - *dst_y,
+          (gint) decoder->rect_height - *src_y));
+
+  if (right <= left || bottom <= top)
+    return FALSE;
+
+  *dst_x += left;
+  *dst_y += top;
+  *src_x += left;
+  *src_y += top;
+  *w = right - left;
+  *h = bottom - top;
+
+  return TRUE;
+}
+
 static gboolean
 rfb_decoder_state_set_colour_map_entries (RfbDecoder * decoder)
 {
-- 
GitLab

