From 74ff8a097c3606710327e3af2863aa52d2dac79e Mon Sep 17 00:00:00 2001 From: Ondrej Holy Date: Tue, 29 Apr 2026 15:10:00 +0000 Subject: [PATCH] [codec,dsp] add format checks Backport of commit 03b48b3601d867afccac1cdc6081de7a275edce7. Adapted for 2.11.x: uses `context->format` instead of `context->common.format`, no `WINPR_RESTRICT`, added `#include `. Kept C89 declaration style in `freerdp_dsp_decode_mp3` and `freerdp_dsp_encode_mp3`. Omitted LAME/mp3 variable modernization. DVI_ADPCM block in `freerdp_dsp_context_reset` omits FramesPerPacket sizing from upstream 3.x but adds `Stream_SetPosition(context->buffer, 0)` after validation (same intent as upstream `Stream_ResetPosition`). `freerdp_dsp_decode_ima_adpcm` matches upstream order: validate format before locals; `out_size` uses `size * 4ull`. `Stream_BufferAs` replaced with `Stream_Buffer` cast. `nullptr` replaced with `NULL`. Made-with: Cursor --- libfreerdp/codec/dsp.c | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+), 4 deletions(-) diff --git a/libfreerdp/codec/dsp.c b/libfreerdp/codec/dsp.c index a18d044..e79911d 100644 --- a/libfreerdp/codec/dsp.c +++ b/libfreerdp/codec/dsp.c @@ -27,6 +27,7 @@ #include #include +#include #include #include @@ -329,13 +330,30 @@ return (UINT16)d; } +static BOOL valid_ima_adpcm_format(const FREERDP_DSP_CONTEXT* context) +{ + WINPR_ASSERT(context); + if (context->format.wFormatTag != WAVE_FORMAT_DVI_ADPCM) + return FALSE; + if (context->format.nBlockAlign <= 4ULL) + return FALSE; + if (context->format.nChannels < 1) + return FALSE; + if (context->format.wBitsPerSample == 0) + return FALSE; + return TRUE; +} + static BOOL freerdp_dsp_decode_ima_adpcm(FREERDP_DSP_CONTEXT* context, const BYTE* src, size_t size, wStream* out) { + if (!valid_ima_adpcm_format(context)) + return FALSE; + BYTE* dst; BYTE sample; UINT16 decoded; - size_t out_size = size * 4; + size_t out_size = size * 4ull; UINT32 channel; const UINT32 block_size = context->format.nBlockAlign; const UINT32 channels = context->format.nChannels; @@ -464,6 +482,20 @@ #endif #if defined(WITH_LAME) +static BOOL valid_mp3_format(const FREERDP_DSP_CONTEXT* context) +{ + WINPR_ASSERT(context); + if (context->format.wFormatTag != WAVE_FORMAT_MPEGLAYER3) + return FALSE; + if (context->format.nChannels < 1) + return FALSE; + if (context->format.wBitsPerSample == 0) + return FALSE; + if (context->format.nSamplesPerSec == 0) + return FALSE; + return TRUE; +} + static BOOL freerdp_dsp_decode_mp3(FREERDP_DSP_CONTEXT* context, const BYTE* src, size_t size, wStream* out) { @@ -474,6 +506,8 @@ if (!context || !src || !out) return FALSE; + if (!valid_mp3_format(context)) + return FALSE; buffer_size = 2 * context->format.nChannels * context->format.nSamplesPerSec; @@ -509,6 +543,9 @@ if (!context || !src || !out) return FALSE; + if (!valid_mp3_format(context)) + return FALSE; + samples_per_channel = size / context->format.nChannels / context->format.wBitsPerSample / 8; /* Ensure worst case buffer size for mp3 stream taken from LAME header */ @@ -716,6 +753,10 @@ BYTE encoded; size_t out_size; size_t align; + + if (!valid_ima_adpcm_format(context)) + return FALSE; + out_size = size / 2; if (!Stream_EnsureRemainingCapacity(out, size)) @@ -788,6 +829,20 @@ static const INT32 ms_adpcm_coeffs2[7] = { 0, -256, 0, 64, 0, -208, -232 }; +static BOOL valid_ms_adpcm_format(const FREERDP_DSP_CONTEXT* context) +{ + WINPR_ASSERT(context); + if (context->format.wFormatTag != WAVE_FORMAT_ADPCM) + return FALSE; + if (context->format.nBlockAlign <= 4ULL) + return FALSE; + if (context->format.nChannels < 1) + return FALSE; + if (context->format.wBitsPerSample == 0) + return FALSE; + return TRUE; +} + static INLINE INT16 freerdp_dsp_decode_ms_adpcm_sample(ADPCM* adpcm, BYTE sample, int channel) { INT8 nibble; @@ -819,6 +874,9 @@ BYTE* dst; BYTE sample; const size_t out_size = size * 4; + + if (!valid_ms_adpcm_format(context)) + return FALSE; const UINT32 channels = context->format.nChannels; const UINT32 block_size = context->format.nBlockAlign; @@ -947,6 +1005,10 @@ INT32 sample; size_t out_size; const size_t step = 8 + ((context->format.nChannels > 1) ? 4 : 0); + + if (!valid_ms_adpcm_format(context)) + return FALSE; + out_size = size / 2; if (!Stream_EnsureRemainingCapacity(out, size)) @@ -1308,6 +1370,28 @@ return FALSE; context->format = *targetFormat; + + switch (context->format.wFormatTag) + { +#if defined(WITH_LAME) + case WAVE_FORMAT_MPEGLAYER3: + if (!valid_mp3_format(context)) + return FALSE; + break; +#endif + case WAVE_FORMAT_ADPCM: + if (!valid_ms_adpcm_format(context)) + return FALSE; + break; + case WAVE_FORMAT_DVI_ADPCM: + if (!valid_ima_adpcm_format(context)) + return FALSE; + Stream_SetPosition(context->buffer, 0); + break; + default: + break; + } + #if defined(WITH_FAAD2) context->faadSetup = FALSE; #endif @@ -1328,19 +1412,24 @@ cfg = faacEncGetCurrentConfiguration(context->faac); cfg->bitRate = 10000; - faacEncSetConfiguration(context->faac, cfg); + { + const int frc = faacEncSetConfiguration(context->faac, cfg); + if (frc <= 0) + return FALSE; + } } #endif #if defined(WITH_SOXR) { soxr_io_spec_t iospec = soxr_io_spec(SOXR_INT16, SOXR_INT16); - soxr_error_t error; + soxr_error_t error = NULL; + soxr_delete(context->sox); context->sox = soxr_create(context->format.nSamplesPerSec, targetFormat->nSamplesPerSec, targetFormat->nChannels, &error, &iospec, NULL, NULL); - if (!context->sox || (error != 0)) + if (!context->sox || (error != NULL)) return FALSE; } #endif