rpms/freerdp
7
0
Fork 0
Code Issues Pull Requests Releases Activity
c10
freerdp/codec-dsp-add-format-checks.patch
AlmaLinux RelEng Bot 6b0a8dd639 import Oracle_OSS freerdp-3.10.3-5.el10_1.8
2026-05-11 17:35:23 -04:00

278 lines
9.9 KiB
Diff
Raw Permalink Blame History

From 81283063b678b6582244a0f32f2059c8d8bba8cb Mon Sep 17 00:00:00 2001
From: Ondrej Holy <oholy@redhat.com>
Date: Tue, 28 Apr 2026 04:41:15 +0000
Subject: [PATCH] [codec,dsp] add format checks
Backport of commit 03b48b3601d867afccac1cdc6081de7a275edce7.
Adapted for 3.10.x: `nullptr` replaced with `NULL`,
`Stream_ResetPosition` replaced with `Stream_SetPosition`,
`error != nullptr` replaced with `error != NULL`.
Made-with: Cursor
---
libfreerdp/codec/dsp.c | 148 +++++++++++++++++++++++++++++++----------
1 file changed, 112 insertions(+), 36 deletions(-)
diff --git a/libfreerdp/codec/dsp.c b/libfreerdp/codec/dsp.c
index c5f5949..5e0f78a 100644
--- a/libfreerdp/codec/dsp.c
+++ b/libfreerdp/codec/dsp.c
@@ -354,11 +354,28 @@ static UINT16 dsp_decode_ima_adpcm_sample(ADPCM* WINPR_RESTRICT adpcm, unsigned
return (UINT16)d;
}
+static BOOL valid_ima_adpcm_format(const FREERDP_DSP_CONTEXT* WINPR_RESTRICT context)
+{
+ WINPR_ASSERT(context);
+ if (context->common.format.wFormatTag != WAVE_FORMAT_DVI_ADPCM)
+ return FALSE;
+ if (context->common.format.nBlockAlign <= 4ULL)
+ return FALSE;
+ if (context->common.format.nChannels < 1)
+ return FALSE;
+ if (context->common.format.wBitsPerSample == 0)
+ return FALSE;
+ return TRUE;
+}
+
static BOOL freerdp_dsp_decode_ima_adpcm(FREERDP_DSP_CONTEXT* WINPR_RESTRICT context,
const BYTE* WINPR_RESTRICT src, size_t size,
wStream* WINPR_RESTRICT out)
{
- size_t out_size = size * 4;
+ if (!valid_ima_adpcm_format(context))
+ return FALSE;
+
+ size_t out_size = size * 4ull;
const UINT32 block_size = context->common.format.nBlockAlign;
const UINT32 channels = context->common.format.nChannels;
@@ -501,27 +518,38 @@ static BOOL freerdp_dsp_encode_gsm610(FREERDP_DSP_CONTEXT* WINPR_RESTRICT contex
#endif
#if defined(WITH_LAME)
+static BOOL valid_mp3_format(const FREERDP_DSP_CONTEXT* WINPR_RESTRICT context)
+{
+ WINPR_ASSERT(context);
+ if (context->common.format.wFormatTag != WAVE_FORMAT_MPEGLAYER3)
+ return FALSE;
+ if (context->common.format.nChannels < 1)
+ return FALSE;
+ if (context->common.format.wBitsPerSample == 0)
+ return FALSE;
+ if (context->common.format.nSamplesPerSec == 0)
+ return FALSE;
+ return TRUE;
+}
+
static BOOL freerdp_dsp_decode_mp3(FREERDP_DSP_CONTEXT* WINPR_RESTRICT context,
const BYTE* WINPR_RESTRICT src, size_t size,
wStream* WINPR_RESTRICT out)
{
- int rc;
- short* pcm_l;
- short* pcm_r;
- size_t buffer_size;
-
if (!context || !src || !out)
return FALSE;
-
- buffer_size = 2 * context->common.format.nChannels * context->common.format.nSamplesPerSec;
+ if (!valid_mp3_format(context))
+ return FALSE;
+ const size_t buffer_size =
+ 2 * context->common.format.nChannels * context->common.format.nSamplesPerSec;
if (!Stream_EnsureCapacity(context->common.buffer, 2 * buffer_size))
return FALSE;
- pcm_l = Stream_BufferAs(context->common.buffer, short);
- pcm_r = Stream_BufferAs(context->common.buffer, short) + buffer_size;
- rc = hip_decode(context->hip, (unsigned char*)/* API is not modifying content */ src, size,
- pcm_l, pcm_r);
+ short* pcm_l = Stream_BufferAs(context->common.buffer, short);
+ short* pcm_r = Stream_BufferAs(context->common.buffer, short) + buffer_size;
+ const int rc = hip_decode(context->hip, (unsigned char*)/* API is not modifying content */ src,
+ size, pcm_l, pcm_r);
if (rc <= 0)
return FALSE;
@@ -542,13 +570,13 @@ static BOOL freerdp_dsp_encode_mp3(FREERDP_DSP_CONTEXT* WINPR_RESTRICT context,
const BYTE* WINPR_RESTRICT src, size_t size,
wStream* WINPR_RESTRICT out)
{
- size_t samples_per_channel;
- int rc;
-
if (!context || !src || !out)
return FALSE;
- samples_per_channel =
+ if (!valid_mp3_format(context))
+ return FALSE;
+
+ size_t samples_per_channel =
size / context->common.format.nChannels / context->common.format.wBitsPerSample / 8;
/* Ensure worst case buffer size for mp3 stream taken from LAME header */
@@ -556,8 +584,9 @@ static BOOL freerdp_dsp_encode_mp3(FREERDP_DSP_CONTEXT* WINPR_RESTRICT context,
return FALSE;
samples_per_channel = size / 2 /* size of a sample */ / context->common.format.nChannels;
- rc = lame_encode_buffer_interleaved(context->lame, (short*)src, samples_per_channel,
- Stream_Pointer(out), Stream_GetRemainingCapacity(out));
+ const int rc =
+ lame_encode_buffer_interleaved(context->lame, (short*)src, samples_per_channel,
+ Stream_Pointer(out), Stream_GetRemainingCapacity(out));
if (rc < 0)
return FALSE;
@@ -807,6 +836,8 @@ static BOOL freerdp_dsp_encode_ima_adpcm(FREERDP_DSP_CONTEXT* WINPR_RESTRICT con
const BYTE* WINPR_RESTRICT src, size_t size,
wStream* WINPR_RESTRICT out)
{
+ if (!valid_ima_adpcm_format(context))
+ return FALSE;
if (!Stream_EnsureRemainingCapacity(out, size))
return FALSE;
if (!Stream_EnsureRemainingCapacity(context->common.buffer, size + 64))
@@ -889,6 +920,20 @@ static const INT32 ms_adpcm_coeffs1[7] = { 256, 512, 0, 192, 240, 460, 392 };
static const INT32 ms_adpcm_coeffs2[7] = { 0, -256, 0, 64, 0, -208, -232 };
+static BOOL valid_ms_adpcm_format(const FREERDP_DSP_CONTEXT* WINPR_RESTRICT context)
+{
+ WINPR_ASSERT(context);
+ if (context->common.format.wFormatTag != WAVE_FORMAT_ADPCM)
+ return FALSE;
+ if (context->common.format.nBlockAlign <= 4ULL)
+ return FALSE;
+ if (context->common.format.nChannels < 1)
+ return FALSE;
+ if (context->common.format.wBitsPerSample == 0)
+ return FALSE;
+ return TRUE;
+}
+
static INLINE INT16 freerdp_dsp_decode_ms_adpcm_sample(ADPCM* WINPR_RESTRICT adpcm, BYTE sample,
int channel)
{
@@ -918,6 +963,8 @@ static BOOL freerdp_dsp_decode_ms_adpcm(FREERDP_DSP_CONTEXT* WINPR_RESTRICT cont
const BYTE* WINPR_RESTRICT src, size_t size,
wStream* WINPR_RESTRICT out)
{
+ if (!valid_ms_adpcm_format(context))
+ return FALSE;
const size_t out_size = size * 4;
const UINT32 channels = context->common.format.nChannels;
const UINT32 block_size = context->common.format.nBlockAlign;
@@ -1038,6 +1085,9 @@ static BOOL freerdp_dsp_encode_ms_adpcm(FREERDP_DSP_CONTEXT* WINPR_RESTRICT cont
const BYTE* WINPR_RESTRICT src, size_t size,
wStream* WINPR_RESTRICT out)
{
+ if (!valid_ms_adpcm_format(context))
+ return FALSE;
+
const size_t step = 8 + ((context->common.format.nChannels > 1) ? 4 : 0);
if (!Stream_EnsureRemainingCapacity(out, size))
@@ -1482,21 +1532,44 @@ BOOL freerdp_dsp_context_reset(FREERDP_DSP_CONTEXT* WINPR_RESTRICT context,
context->common.format = *targetFormat;
- if (context->common.format.wFormatTag == WAVE_FORMAT_DVI_ADPCM)
+ switch (context->common.format.wFormatTag)
{
- size_t min_frame_data = 1ull * context->common.format.wBitsPerSample *
- context->common.format.nChannels * FramesPerPacket;
- size_t data_per_block =
- (1ULL * context->common.format.nBlockAlign - 4ULL * context->common.format.nChannels) *
- 8ULL;
- size_t nb_block_per_packet = min_frame_data / data_per_block;
-
- if (min_frame_data % data_per_block)
- nb_block_per_packet++;
-
- context->adpcm.ima.packet_size = nb_block_per_packet * context->common.format.nBlockAlign;
- Stream_EnsureCapacity(context->common.buffer, context->adpcm.ima.packet_size);
- Stream_SetPosition(context->common.buffer, 0);
+#if defined(WITH_LAME)
+ case WAVE_FORMAT_MPEGLAYER3:
+ if (!valid_mp3_format(context))
+ return FALSE;
+ break;
+#endif
+ case WAVE_FORMAT_ADPCM:
+ if (!valid_ms_adpcm_format(context))
+ return FALSE;
+ break;
+ case WAVE_FORMAT_DVI_ADPCM:
+ {
+ if (!valid_ima_adpcm_format(context))
+ return FALSE;
+ if (FramesPerPacket == 0)
+ return FALSE;
+
+ const size_t min_frame_data = 1ull * context->common.format.wBitsPerSample *
+ context->common.format.nChannels * FramesPerPacket;
+ const size_t data_per_block = (1ULL * context->common.format.nBlockAlign -
+ 4ULL * context->common.format.nChannels) *
+ 8ULL;
+ size_t nb_block_per_packet = min_frame_data / data_per_block;
+
+ if (min_frame_data % data_per_block)
+ nb_block_per_packet++;
+
+ context->adpcm.ima.packet_size =
+ nb_block_per_packet * context->common.format.nBlockAlign;
+ if (!Stream_EnsureCapacity(context->common.buffer, context->adpcm.ima.packet_size))
+ return FALSE;
+ Stream_SetPosition(context->common.buffer, 0);
+ }
+ break;
+ default:
+ break;
}
#if defined(WITH_OPUS)
@@ -1539,7 +1612,7 @@ BOOL freerdp_dsp_context_reset(FREERDP_DSP_CONTEXT* WINPR_RESTRICT context,
if (context->common.encoder)
{
- faacEncConfigurationPtr cfg;
+ faacEncConfigurationPtr cfg = NULL;
if (context->faac)
faacEncClose(context->faac);
@@ -1556,20 +1629,23 @@ BOOL freerdp_dsp_context_reset(FREERDP_DSP_CONTEXT* WINPR_RESTRICT context,
cfg->mpegVersion = MPEG4;
cfg->useTns = 1;
cfg->bandWidth = targetFormat->nAvgBytesPerSec;
- faacEncSetConfiguration(context->faac, cfg);
+ const int rc = faacEncSetConfiguration(context->faac, cfg);
+ if (rc <= 0)
+ return FALSE;
}
#endif
#if defined(WITH_SOXR)
{
soxr_io_spec_t iospec = soxr_io_spec(SOXR_INT16, SOXR_INT16);
- soxr_error_t error;
+ soxr_error_t error = NULL;
+
soxr_delete(context->sox);
context->sox =
soxr_create(context->common.format.nSamplesPerSec, targetFormat->nSamplesPerSec,
targetFormat->nChannels, &error, &iospec, NULL, NULL);
- if (!context->sox || (error != 0))
+ if (!context->sox || (error != NULL))
return FALSE;
}
#endif
--
2.53.0
Reference in New Issue View Git Blame Copy Permalink