From 81283063b678b6582244a0f32f2059c8d8bba8cb Mon Sep 17 00:00:00 2001 From: Ondrej Holy Date: Tue, 28 Apr 2026 04:41:15 +0000 Subject: [PATCH] [codec,dsp] add format checks Backport of commit 03b48b3601d867afccac1cdc6081de7a275edce7. Adapted for 3.10.x: `nullptr` replaced with `NULL`, `Stream_ResetPosition` replaced with `Stream_SetPosition`, `error != nullptr` replaced with `error != NULL`. Made-with: Cursor --- libfreerdp/codec/dsp.c | 148 +++++++++++++++++++++++++++++++---------- 1 file changed, 112 insertions(+), 36 deletions(-) diff --git a/libfreerdp/codec/dsp.c b/libfreerdp/codec/dsp.c index c5f5949..5e0f78a 100644 --- a/libfreerdp/codec/dsp.c +++ b/libfreerdp/codec/dsp.c @@ -354,11 +354,28 @@ static UINT16 dsp_decode_ima_adpcm_sample(ADPCM* WINPR_RESTRICT adpcm, unsigned return (UINT16)d; } +static BOOL valid_ima_adpcm_format(const FREERDP_DSP_CONTEXT* WINPR_RESTRICT context) +{ + WINPR_ASSERT(context); + if (context->common.format.wFormatTag != WAVE_FORMAT_DVI_ADPCM) + return FALSE; + if (context->common.format.nBlockAlign <= 4ULL) + return FALSE; + if (context->common.format.nChannels < 1) + return FALSE; + if (context->common.format.wBitsPerSample == 0) + return FALSE; + return TRUE; +} + static BOOL freerdp_dsp_decode_ima_adpcm(FREERDP_DSP_CONTEXT* WINPR_RESTRICT context, const BYTE* WINPR_RESTRICT src, size_t size, wStream* WINPR_RESTRICT out) { - size_t out_size = size * 4; + if (!valid_ima_adpcm_format(context)) + return FALSE; + + size_t out_size = size * 4ull; const UINT32 block_size = context->common.format.nBlockAlign; const UINT32 channels = context->common.format.nChannels; @@ -501,27 +518,38 @@ static BOOL freerdp_dsp_encode_gsm610(FREERDP_DSP_CONTEXT* WINPR_RESTRICT contex #endif #if defined(WITH_LAME) +static BOOL valid_mp3_format(const FREERDP_DSP_CONTEXT* WINPR_RESTRICT context) +{ + WINPR_ASSERT(context); + if (context->common.format.wFormatTag != WAVE_FORMAT_MPEGLAYER3) + return FALSE; + if (context->common.format.nChannels < 1) + return FALSE; + if (context->common.format.wBitsPerSample == 0) + return FALSE; + if (context->common.format.nSamplesPerSec == 0) + return FALSE; + return TRUE; +} + static BOOL freerdp_dsp_decode_mp3(FREERDP_DSP_CONTEXT* WINPR_RESTRICT context, const BYTE* WINPR_RESTRICT src, size_t size, wStream* WINPR_RESTRICT out) { - int rc; - short* pcm_l; - short* pcm_r; - size_t buffer_size; - if (!context || !src || !out) return FALSE; - - buffer_size = 2 * context->common.format.nChannels * context->common.format.nSamplesPerSec; + if (!valid_mp3_format(context)) + return FALSE; + const size_t buffer_size = + 2 * context->common.format.nChannels * context->common.format.nSamplesPerSec; if (!Stream_EnsureCapacity(context->common.buffer, 2 * buffer_size)) return FALSE; - pcm_l = Stream_BufferAs(context->common.buffer, short); - pcm_r = Stream_BufferAs(context->common.buffer, short) + buffer_size; - rc = hip_decode(context->hip, (unsigned char*)/* API is not modifying content */ src, size, - pcm_l, pcm_r); + short* pcm_l = Stream_BufferAs(context->common.buffer, short); + short* pcm_r = Stream_BufferAs(context->common.buffer, short) + buffer_size; + const int rc = hip_decode(context->hip, (unsigned char*)/* API is not modifying content */ src, + size, pcm_l, pcm_r); if (rc <= 0) return FALSE; @@ -542,13 +570,13 @@ static BOOL freerdp_dsp_encode_mp3(FREERDP_DSP_CONTEXT* WINPR_RESTRICT context, const BYTE* WINPR_RESTRICT src, size_t size, wStream* WINPR_RESTRICT out) { - size_t samples_per_channel; - int rc; - if (!context || !src || !out) return FALSE; - samples_per_channel = + if (!valid_mp3_format(context)) + return FALSE; + + size_t samples_per_channel = size / context->common.format.nChannels / context->common.format.wBitsPerSample / 8; /* Ensure worst case buffer size for mp3 stream taken from LAME header */ @@ -556,8 +584,9 @@ static BOOL freerdp_dsp_encode_mp3(FREERDP_DSP_CONTEXT* WINPR_RESTRICT context, return FALSE; samples_per_channel = size / 2 /* size of a sample */ / context->common.format.nChannels; - rc = lame_encode_buffer_interleaved(context->lame, (short*)src, samples_per_channel, - Stream_Pointer(out), Stream_GetRemainingCapacity(out)); + const int rc = + lame_encode_buffer_interleaved(context->lame, (short*)src, samples_per_channel, + Stream_Pointer(out), Stream_GetRemainingCapacity(out)); if (rc < 0) return FALSE; @@ -807,6 +836,8 @@ static BOOL freerdp_dsp_encode_ima_adpcm(FREERDP_DSP_CONTEXT* WINPR_RESTRICT con const BYTE* WINPR_RESTRICT src, size_t size, wStream* WINPR_RESTRICT out) { + if (!valid_ima_adpcm_format(context)) + return FALSE; if (!Stream_EnsureRemainingCapacity(out, size)) return FALSE; if (!Stream_EnsureRemainingCapacity(context->common.buffer, size + 64)) @@ -889,6 +920,20 @@ static const INT32 ms_adpcm_coeffs1[7] = { 256, 512, 0, 192, 240, 460, 392 }; static const INT32 ms_adpcm_coeffs2[7] = { 0, -256, 0, 64, 0, -208, -232 }; +static BOOL valid_ms_adpcm_format(const FREERDP_DSP_CONTEXT* WINPR_RESTRICT context) +{ + WINPR_ASSERT(context); + if (context->common.format.wFormatTag != WAVE_FORMAT_ADPCM) + return FALSE; + if (context->common.format.nBlockAlign <= 4ULL) + return FALSE; + if (context->common.format.nChannels < 1) + return FALSE; + if (context->common.format.wBitsPerSample == 0) + return FALSE; + return TRUE; +} + static INLINE INT16 freerdp_dsp_decode_ms_adpcm_sample(ADPCM* WINPR_RESTRICT adpcm, BYTE sample, int channel) { @@ -918,6 +963,8 @@ static BOOL freerdp_dsp_decode_ms_adpcm(FREERDP_DSP_CONTEXT* WINPR_RESTRICT cont const BYTE* WINPR_RESTRICT src, size_t size, wStream* WINPR_RESTRICT out) { + if (!valid_ms_adpcm_format(context)) + return FALSE; const size_t out_size = size * 4; const UINT32 channels = context->common.format.nChannels; const UINT32 block_size = context->common.format.nBlockAlign; @@ -1038,6 +1085,9 @@ static BOOL freerdp_dsp_encode_ms_adpcm(FREERDP_DSP_CONTEXT* WINPR_RESTRICT cont const BYTE* WINPR_RESTRICT src, size_t size, wStream* WINPR_RESTRICT out) { + if (!valid_ms_adpcm_format(context)) + return FALSE; + const size_t step = 8 + ((context->common.format.nChannels > 1) ? 4 : 0); if (!Stream_EnsureRemainingCapacity(out, size)) @@ -1482,21 +1532,44 @@ BOOL freerdp_dsp_context_reset(FREERDP_DSP_CONTEXT* WINPR_RESTRICT context, context->common.format = *targetFormat; - if (context->common.format.wFormatTag == WAVE_FORMAT_DVI_ADPCM) + switch (context->common.format.wFormatTag) { - size_t min_frame_data = 1ull * context->common.format.wBitsPerSample * - context->common.format.nChannels * FramesPerPacket; - size_t data_per_block = - (1ULL * context->common.format.nBlockAlign - 4ULL * context->common.format.nChannels) * - 8ULL; - size_t nb_block_per_packet = min_frame_data / data_per_block; - - if (min_frame_data % data_per_block) - nb_block_per_packet++; - - context->adpcm.ima.packet_size = nb_block_per_packet * context->common.format.nBlockAlign; - Stream_EnsureCapacity(context->common.buffer, context->adpcm.ima.packet_size); - Stream_SetPosition(context->common.buffer, 0); +#if defined(WITH_LAME) + case WAVE_FORMAT_MPEGLAYER3: + if (!valid_mp3_format(context)) + return FALSE; + break; +#endif + case WAVE_FORMAT_ADPCM: + if (!valid_ms_adpcm_format(context)) + return FALSE; + break; + case WAVE_FORMAT_DVI_ADPCM: + { + if (!valid_ima_adpcm_format(context)) + return FALSE; + if (FramesPerPacket == 0) + return FALSE; + + const size_t min_frame_data = 1ull * context->common.format.wBitsPerSample * + context->common.format.nChannels * FramesPerPacket; + const size_t data_per_block = (1ULL * context->common.format.nBlockAlign - + 4ULL * context->common.format.nChannels) * + 8ULL; + size_t nb_block_per_packet = min_frame_data / data_per_block; + + if (min_frame_data % data_per_block) + nb_block_per_packet++; + + context->adpcm.ima.packet_size = + nb_block_per_packet * context->common.format.nBlockAlign; + if (!Stream_EnsureCapacity(context->common.buffer, context->adpcm.ima.packet_size)) + return FALSE; + Stream_SetPosition(context->common.buffer, 0); + } + break; + default: + break; } #if defined(WITH_OPUS) @@ -1539,7 +1612,7 @@ BOOL freerdp_dsp_context_reset(FREERDP_DSP_CONTEXT* WINPR_RESTRICT context, if (context->common.encoder) { - faacEncConfigurationPtr cfg; + faacEncConfigurationPtr cfg = NULL; if (context->faac) faacEncClose(context->faac); @@ -1556,20 +1629,23 @@ BOOL freerdp_dsp_context_reset(FREERDP_DSP_CONTEXT* WINPR_RESTRICT context, cfg->mpegVersion = MPEG4; cfg->useTns = 1; cfg->bandWidth = targetFormat->nAvgBytesPerSec; - faacEncSetConfiguration(context->faac, cfg); + const int rc = faacEncSetConfiguration(context->faac, cfg); + if (rc <= 0) + return FALSE; } #endif #if defined(WITH_SOXR) { soxr_io_spec_t iospec = soxr_io_spec(SOXR_INT16, SOXR_INT16); - soxr_error_t error; + soxr_error_t error = NULL; + soxr_delete(context->sox); context->sox = soxr_create(context->common.format.nSamplesPerSec, targetFormat->nSamplesPerSec, targetFormat->nChannels, &error, &iospec, NULL, NULL); - if (!context->sox || (error != 0)) + if (!context->sox || (error != NULL)) return FALSE; } #endif -- 2.53.0