freerdp/codec-dsp-add-format-checks.patch

From 74ff8a097c3606710327e3af2863aa52d2dac79e Mon Sep 17 00:00:00 2001
From: Ondrej Holy <oholy@redhat.com>
Date: Tue, 29 Apr 2026 15:10:00 +0000
Subject: [PATCH] [codec,dsp] add format checks

Backport of commit 03b48b3601d867afccac1cdc6081de7a275edce7.

Adapted for 2.11.x: uses `context->format` instead of
`context->common.format`, no `WINPR_RESTRICT`, added
`#include <winpr/assert.h>`. Kept C89 declaration style in
`freerdp_dsp_decode_mp3` and `freerdp_dsp_encode_mp3`. Omitted
LAME/mp3 variable modernization. DVI_ADPCM block in
`freerdp_dsp_context_reset` omits FramesPerPacket sizing from upstream 3.x
but adds `Stream_SetPosition(context->buffer, 0)` after validation (same
intent as upstream `Stream_ResetPosition`). `freerdp_dsp_decode_ima_adpcm`
matches upstream order: validate format before locals; `out_size` uses
`size * 4ull`. `Stream_BufferAs` replaced with
`Stream_Buffer` cast. `nullptr` replaced with `NULL`.

Made-with: Cursor
---
 libfreerdp/codec/dsp.c | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 93 insertions(+), 4 deletions(-)

diff --git a/libfreerdp/codec/dsp.c b/libfreerdp/codec/dsp.c
index a18d044..e79911d 100644
--- a/libfreerdp/codec/dsp.c
+++ b/libfreerdp/codec/dsp.c
@@ -27,6 +27,7 @@
 #include <string.h>
 
 #include <winpr/crt.h>
+#include <winpr/assert.h>
 
 #include <freerdp/types.h>
 #include <freerdp/log.h>
@@ -329,13 +330,30 @@
 	return (UINT16)d;
 }
 
+static BOOL valid_ima_adpcm_format(const FREERDP_DSP_CONTEXT* context)
+{
+	WINPR_ASSERT(context);
+	if (context->format.wFormatTag != WAVE_FORMAT_DVI_ADPCM)
+		return FALSE;
+	if (context->format.nBlockAlign <= 4ULL)
+		return FALSE;
+	if (context->format.nChannels < 1)
+		return FALSE;
+	if (context->format.wBitsPerSample == 0)
+		return FALSE;
+	return TRUE;
+}
+
 static BOOL freerdp_dsp_decode_ima_adpcm(FREERDP_DSP_CONTEXT* context, const BYTE* src, size_t size,
                                          wStream* out)
 {
+	if (!valid_ima_adpcm_format(context))
+		return FALSE;
+
 	BYTE* dst;
 	BYTE sample;
 	UINT16 decoded;
-	size_t out_size = size * 4;
+	size_t out_size = size * 4ull;
 	UINT32 channel;
 	const UINT32 block_size = context->format.nBlockAlign;
 	const UINT32 channels = context->format.nChannels;
@@ -464,6 +482,20 @@
 #endif
 
 #if defined(WITH_LAME)
+static BOOL valid_mp3_format(const FREERDP_DSP_CONTEXT* context)
+{
+	WINPR_ASSERT(context);
+	if (context->format.wFormatTag != WAVE_FORMAT_MPEGLAYER3)
+		return FALSE;
+	if (context->format.nChannels < 1)
+		return FALSE;
+	if (context->format.wBitsPerSample == 0)
+		return FALSE;
+	if (context->format.nSamplesPerSec == 0)
+		return FALSE;
+	return TRUE;
+}
+
 static BOOL freerdp_dsp_decode_mp3(FREERDP_DSP_CONTEXT* context, const BYTE* src, size_t size,
                                    wStream* out)
 {
@@ -474,6 +506,8 @@
 
 	if (!context || !src || !out)
 		return FALSE;
+	if (!valid_mp3_format(context))
+		return FALSE;
 
 	buffer_size = 2 * context->format.nChannels * context->format.nSamplesPerSec;
 
@@ -509,6 +543,9 @@
 	if (!context || !src || !out)
 		return FALSE;
 
+	if (!valid_mp3_format(context))
+		return FALSE;
+
 	samples_per_channel = size / context->format.nChannels / context->format.wBitsPerSample / 8;
 
 	/* Ensure worst case buffer size for mp3 stream taken from LAME header */
@@ -716,6 +753,10 @@
 	BYTE encoded;
 	size_t out_size;
 	size_t align;
+
+	if (!valid_ima_adpcm_format(context))
+		return FALSE;
+
 	out_size = size / 2;
 
 	if (!Stream_EnsureRemainingCapacity(out, size))
@@ -788,6 +829,20 @@
 
 static const INT32 ms_adpcm_coeffs2[7] = { 0, -256, 0, 64, 0, -208, -232 };
 
+static BOOL valid_ms_adpcm_format(const FREERDP_DSP_CONTEXT* context)
+{
+	WINPR_ASSERT(context);
+	if (context->format.wFormatTag != WAVE_FORMAT_ADPCM)
+		return FALSE;
+	if (context->format.nBlockAlign <= 4ULL)
+		return FALSE;
+	if (context->format.nChannels < 1)
+		return FALSE;
+	if (context->format.wBitsPerSample == 0)
+		return FALSE;
+	return TRUE;
+}
+
 static INLINE INT16 freerdp_dsp_decode_ms_adpcm_sample(ADPCM* adpcm, BYTE sample, int channel)
 {
 	INT8 nibble;
@@ -819,6 +874,9 @@
 	BYTE* dst;
 	BYTE sample;
 	const size_t out_size = size * 4;
+
+	if (!valid_ms_adpcm_format(context))
+		return FALSE;
 	const UINT32 channels = context->format.nChannels;
 	const UINT32 block_size = context->format.nBlockAlign;
 
@@ -947,6 +1005,10 @@
 	INT32 sample;
 	size_t out_size;
 	const size_t step = 8 + ((context->format.nChannels > 1) ? 4 : 0);
+
+	if (!valid_ms_adpcm_format(context))
+		return FALSE;
+
 	out_size = size / 2;
 
 	if (!Stream_EnsureRemainingCapacity(out, size))
@@ -1308,6 +1370,28 @@
 		return FALSE;
 
 	context->format = *targetFormat;
+
+	switch (context->format.wFormatTag)
+	{
+#if defined(WITH_LAME)
+		case WAVE_FORMAT_MPEGLAYER3:
+			if (!valid_mp3_format(context))
+				return FALSE;
+			break;
+#endif
+		case WAVE_FORMAT_ADPCM:
+			if (!valid_ms_adpcm_format(context))
+				return FALSE;
+			break;
+		case WAVE_FORMAT_DVI_ADPCM:
+			if (!valid_ima_adpcm_format(context))
+				return FALSE;
+			Stream_SetPosition(context->buffer, 0);
+			break;
+		default:
+			break;
+	}
+
 #if defined(WITH_FAAD2)
 	context->faadSetup = FALSE;
 #endif
@@ -1328,19 +1412,24 @@
 
 		cfg = faacEncGetCurrentConfiguration(context->faac);
 		cfg->bitRate = 10000;
-		faacEncSetConfiguration(context->faac, cfg);
+		{
+			const int frc = faacEncSetConfiguration(context->faac, cfg);
+			if (frc <= 0)
+				return FALSE;
+		}
 	}
 
 #endif
 #if defined(WITH_SOXR)
 	{
 		soxr_io_spec_t iospec = soxr_io_spec(SOXR_INT16, SOXR_INT16);
-		soxr_error_t error;
+		soxr_error_t error = NULL;
+
 		soxr_delete(context->sox);
 		context->sox = soxr_create(context->format.nSamplesPerSec, targetFormat->nSamplesPerSec,
 		                           targetFormat->nChannels, &error, &iospec, NULL, NULL);
 
-		if (!context->sox || (error != 0))
+		if (!context->sox || (error != NULL))
 			return FALSE;
 	}
 #endif