From eb6153583ce797bfdb13552737226834577c9849 Mon Sep 17 00:00:00 2001 From: eabdullin Date: Mon, 18 Nov 2024 10:21:24 +0000 Subject: [PATCH] Import from CS git --- ...x-bug-with-smaller-width-bigger-size.patch | 131 ++++++++++++++++++ ...overflows-in-calc-of-stride_in_bytes.patch | 119 ++++++++++++++++ ...01-VP8-disallow-thread-count-changes.patch | 34 +++++ ..._align-to-byte-count-not-pixel-count.patch | 34 +++++ ...ug-in-alloc_size-for-high-bit-depths.patch | 46 ++++++ SPECS/libvpx.spec | 30 +++- 6 files changed, 393 insertions(+), 1 deletion(-) create mode 100644 SOURCES/0001-Fix-bug-with-smaller-width-bigger-size.patch create mode 100644 SOURCES/0001-Fix-integer-overflows-in-calc-of-stride_in_bytes.patch create mode 100644 SOURCES/0001-VP8-disallow-thread-count-changes.patch create mode 100644 SOURCES/0002-Apply-stride_align-to-byte-count-not-pixel-count.patch create mode 100644 SOURCES/0003-Fix-a-bug-in-alloc_size-for-high-bit-depths.patch diff --git a/SOURCES/0001-Fix-bug-with-smaller-width-bigger-size.patch b/SOURCES/0001-Fix-bug-with-smaller-width-bigger-size.patch new file mode 100644 index 0000000..35a8cde --- /dev/null +++ b/SOURCES/0001-Fix-bug-with-smaller-width-bigger-size.patch @@ -0,0 +1,131 @@ +From bc54ed5c2d7b43a1904e04a59c268368c274169e Mon Sep 17 00:00:00 2001 +From: Jerome Jiang +Date: Thu, 30 Jun 2022 13:48:56 -0400 +Subject: [PATCH] Fix bug with smaller width bigger size + +Fixed previous patch that clusterfuzz failed on. + +Local fuzzing passing overnight. + +Bug: webm:1642 +Change-Id: If0e08e72abd2e042efe4dcfac21e4cc51afdfdb9 +(cherry picked from commit 263682c9a29395055f3b3afe2d97be1828a6223f) +--- + test/resize_test.cc | 9 +++------ + vp9/common/vp9_alloccommon.c | 13 ++++++------- + vp9/encoder/vp9_encoder.c | 27 +++++++++++++++++++++++++-- + 3 files changed, 34 insertions(+), 15 deletions(-) + +diff --git a/test/resize_test.cc b/test/resize_test.cc +index b2eadb1e8..ebc5f9b02 100644 +--- a/test/resize_test.cc ++++ b/test/resize_test.cc +@@ -101,11 +101,8 @@ void ScaleForFrameNumber(unsigned int frame, unsigned int initial_w, + *h = initial_h; + return; + } +- if (frame < 100) { +- *w = initial_w * 7 / 10; +- *h = initial_h * 16 / 10; +- return; +- } ++ *w = initial_w * 7 / 10; ++ *h = initial_h * 16 / 10; + return; + } + if (frame < 10) { +@@ -578,7 +575,7 @@ TEST_P(ResizeRealtimeTest, TestExternalResizeWorks) { + } + } + +-TEST_P(ResizeRealtimeTest, DISABLED_TestExternalResizeSmallerWidthBiggerSize) { ++TEST_P(ResizeRealtimeTest, TestExternalResizeSmallerWidthBiggerSize) { + ResizingVideoSource video; + video.flag_codec_ = true; + video.smaller_width_larger_size_ = true; +diff --git a/vp9/common/vp9_alloccommon.c b/vp9/common/vp9_alloccommon.c +index 5702dca71..0328f33a4 100644 +--- a/vp9/common/vp9_alloccommon.c ++++ b/vp9/common/vp9_alloccommon.c +@@ -131,13 +131,6 @@ int vp9_alloc_context_buffers(VP9_COMMON *cm, int width, int height) { + cm->free_mi(cm); + if (cm->alloc_mi(cm, new_mi_size)) goto fail; + } +- +- if (cm->seg_map_alloc_size < cm->mi_rows * cm->mi_cols) { +- // Create the segmentation map structure and set to 0. +- free_seg_map(cm); +- if (alloc_seg_map(cm, cm->mi_rows * cm->mi_cols)) goto fail; +- } +- + if (cm->above_context_alloc_cols < cm->mi_cols) { + vpx_free(cm->above_context); + cm->above_context = (ENTROPY_CONTEXT *)vpx_calloc( +@@ -152,6 +145,12 @@ int vp9_alloc_context_buffers(VP9_COMMON *cm, int width, int height) { + cm->above_context_alloc_cols = cm->mi_cols; + } + ++ if (cm->seg_map_alloc_size < cm->mi_rows * cm->mi_cols) { ++ // Create the segmentation map structure and set to 0. ++ free_seg_map(cm); ++ if (alloc_seg_map(cm, cm->mi_rows * cm->mi_cols)) goto fail; ++ } ++ + if (vp9_alloc_loop_filter(cm)) goto fail; + + return 0; +diff --git a/vp9/encoder/vp9_encoder.c b/vp9/encoder/vp9_encoder.c +index 20e8d494a..055e1def0 100644 +--- a/vp9/encoder/vp9_encoder.c ++++ b/vp9/encoder/vp9_encoder.c +@@ -1968,6 +1968,17 @@ static void alloc_copy_partition_data(VP9_COMP *cpi) { + } + } + ++static void free_copy_partition_data(VP9_COMP *cpi) { ++ vpx_free(cpi->prev_partition); ++ cpi->prev_partition = NULL; ++ vpx_free(cpi->prev_segment_id); ++ cpi->prev_segment_id = NULL; ++ vpx_free(cpi->prev_variance_low); ++ cpi->prev_variance_low = NULL; ++ vpx_free(cpi->copied_frame_cnt); ++ cpi->copied_frame_cnt = NULL; ++} ++ + void vp9_change_config(struct VP9_COMP *cpi, const VP9EncoderConfig *oxcf) { + VP9_COMMON *const cm = &cpi->common; + RATE_CONTROL *const rc = &cpi->rc; +@@ -2047,6 +2058,8 @@ void vp9_change_config(struct VP9_COMP *cpi, const VP9EncoderConfig *oxcf) { + new_mi_size = cm->mi_stride * calc_mi_size(cm->mi_rows); + if (cm->mi_alloc_size < new_mi_size) { + vp9_free_context_buffers(cm); ++ vp9_free_pc_tree(&cpi->td); ++ vpx_free(cpi->mbmi_ext_base); + alloc_compressor_data(cpi); + realloc_segmentation_maps(cpi); + cpi->initial_width = cpi->initial_height = 0; +@@ -2062,8 +2075,18 @@ void vp9_change_config(struct VP9_COMP *cpi, const VP9EncoderConfig *oxcf) { + update_frame_size(cpi); + + if (last_w != cpi->oxcf.width || last_h != cpi->oxcf.height) { +- memset(cpi->consec_zero_mv, 0, +- cm->mi_rows * cm->mi_cols * sizeof(*cpi->consec_zero_mv)); ++ vpx_free(cpi->consec_zero_mv); ++ CHECK_MEM_ERROR( ++ cm, cpi->consec_zero_mv, ++ vpx_calloc(cm->mi_rows * cm->mi_cols, sizeof(*cpi->consec_zero_mv))); ++ ++ vpx_free(cpi->skin_map); ++ CHECK_MEM_ERROR( ++ cm, cpi->skin_map, ++ vpx_calloc(cm->mi_rows * cm->mi_cols, sizeof(cpi->skin_map[0]))); ++ ++ free_copy_partition_data(cpi); ++ alloc_copy_partition_data(cpi); + if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) + vp9_cyclic_refresh_reset_resize(cpi); + rc->rc_1_frame = 0; +-- +2.41.0 + diff --git a/SOURCES/0001-Fix-integer-overflows-in-calc-of-stride_in_bytes.patch b/SOURCES/0001-Fix-integer-overflows-in-calc-of-stride_in_bytes.patch new file mode 100644 index 0000000..1cb31a8 --- /dev/null +++ b/SOURCES/0001-Fix-integer-overflows-in-calc-of-stride_in_bytes.patch @@ -0,0 +1,119 @@ +From 70e6813ca323789b6c937b9f6cbbfd3daf4b1692 Mon Sep 17 00:00:00 2001 +From: Wan-Teh Chang +Date: Wed, 10 Apr 2024 17:01:10 -0700 +Subject: [PATCH 1/3] Fix integer overflows in calc of stride_in_bytes + +A port of the libaom CL +https://aomedia-review.googlesource.com/c/aom/+/188761. + +Fix unsigned integer overflows in the calculation of stride_in_bytes in +img_alloc_helper() when d_w is huge. + +Change the type of stride_in_bytes from unsigned int to int because it +will be assigned to img->stride[VPX_PLANE_Y], which is of the int type. + +Test: +. ../libvpx/tools/set_analyzer_env.sh integer +../libvpx/configure --enable-debug --disable-optimizations +make -j +./test_libvpx --gtest_filter=VpxImageTest.VpxImgAllocHugeWidth + +Bug: chromium:332382766 +Change-Id: I3b39d78f61c7255e10cbf72ba2f4975425a05a82 +--- + vpx/src/vpx_image.c | 33 ++++++++++++++++++++------------- + 1 file changed, 20 insertions(+), 13 deletions(-) + +diff --git a/vpx/src/vpx_image.c b/vpx/src/vpx_image.c +index ff496b5d3..404c577d6 100644 +--- a/vpx/src/vpx_image.c ++++ b/vpx/src/vpx_image.c +@@ -10,6 +10,7 @@ + + #include + #include ++#include + + #include "vpx/vpx_image.h" + #include "vpx/vpx_integer.h" +@@ -20,9 +21,9 @@ static vpx_image_t *img_alloc_helper(vpx_image_t *img, vpx_img_fmt_t fmt, + unsigned int buf_align, + unsigned int stride_align, + unsigned char *img_data) { +- unsigned int h, w, s, xcs, ycs, bps; +- unsigned int stride_in_bytes; +- int align; ++ unsigned int h, w, xcs, ycs, bps; ++ uint64_t s; ++ int stride_in_bytes, align; + + /* Treat align==0 like align==1 */ + if (!buf_align) buf_align = 1; +@@ -77,9 +78,11 @@ static vpx_image_t *img_alloc_helper(vpx_image_t *img, vpx_img_fmt_t fmt, + * and height shouldn't be adjusted. */ + w = d_w; + h = d_h; +- s = (fmt & VPX_IMG_FMT_PLANAR) ? w : bps * w / 8; +- s = (s + stride_align - 1) & ~(stride_align - 1); +- stride_in_bytes = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s; ++ s = (fmt & VPX_IMG_FMT_PLANAR) ? w : (uint64_t)bps * w / 8; ++ s = (s + stride_align - 1) & ~((uint64_t)stride_align - 1); ++ s = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s; ++ if (s > INT_MAX) goto fail; ++ stride_in_bytes = (int)s; + + /* Allocate the new image */ + if (!img) { +@@ -102,9 +105,11 @@ static vpx_image_t *img_alloc_helper(vpx_image_t *img, vpx_img_fmt_t fmt, + align = (1 << ycs) - 1; + h = (d_h + align) & ~align; + +- s = (fmt & VPX_IMG_FMT_PLANAR) ? w : bps * w / 8; +- s = (s + stride_align - 1) & ~(stride_align - 1); +- stride_in_bytes = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s; ++ s = (fmt & VPX_IMG_FMT_PLANAR) ? w : (uint64_t)bps * w / 8; ++ s = (s + stride_align - 1) & ~((uint64_t)stride_align - 1); ++ s = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s; ++ if (s > INT_MAX) goto fail; ++ stride_in_bytes = (int)s; + alloc_size = (fmt & VPX_IMG_FMT_PLANAR) ? (uint64_t)h * s * bps / 8 + : (uint64_t)h * s; + +@@ -170,12 +175,12 @@ int vpx_img_set_rect(vpx_image_t *img, unsigned int x, unsigned int y, + if (img->fmt & VPX_IMG_FMT_HAS_ALPHA) { + img->planes[VPX_PLANE_ALPHA] = + data + x * bytes_per_sample + y * img->stride[VPX_PLANE_ALPHA]; +- data += img->h * img->stride[VPX_PLANE_ALPHA]; ++ data += (size_t)img->h * img->stride[VPX_PLANE_ALPHA]; + } + + img->planes[VPX_PLANE_Y] = + data + x * bytes_per_sample + y * img->stride[VPX_PLANE_Y]; +- data += img->h * img->stride[VPX_PLANE_Y]; ++ data += (size_t)img->h * img->stride[VPX_PLANE_Y]; + + if (img->fmt == VPX_IMG_FMT_NV12) { + img->planes[VPX_PLANE_U] = +@@ -186,7 +191,8 @@ int vpx_img_set_rect(vpx_image_t *img, unsigned int x, unsigned int y, + img->planes[VPX_PLANE_U] = + data + (x >> img->x_chroma_shift) * bytes_per_sample + + (y >> img->y_chroma_shift) * img->stride[VPX_PLANE_U]; +- data += (img->h >> img->y_chroma_shift) * img->stride[VPX_PLANE_U]; ++ data += ++ (size_t)(img->h >> img->y_chroma_shift) * img->stride[VPX_PLANE_U]; + img->planes[VPX_PLANE_V] = + data + (x >> img->x_chroma_shift) * bytes_per_sample + + (y >> img->y_chroma_shift) * img->stride[VPX_PLANE_V]; +@@ -194,7 +200,8 @@ int vpx_img_set_rect(vpx_image_t *img, unsigned int x, unsigned int y, + img->planes[VPX_PLANE_V] = + data + (x >> img->x_chroma_shift) * bytes_per_sample + + (y >> img->y_chroma_shift) * img->stride[VPX_PLANE_V]; +- data += (img->h >> img->y_chroma_shift) * img->stride[VPX_PLANE_V]; ++ data += ++ (size_t)(img->h >> img->y_chroma_shift) * img->stride[VPX_PLANE_V]; + img->planes[VPX_PLANE_U] = + data + (x >> img->x_chroma_shift) * bytes_per_sample + + (y >> img->y_chroma_shift) * img->stride[VPX_PLANE_U]; +-- +2.45.2 + diff --git a/SOURCES/0001-VP8-disallow-thread-count-changes.patch b/SOURCES/0001-VP8-disallow-thread-count-changes.patch new file mode 100644 index 0000000..d8583d1 --- /dev/null +++ b/SOURCES/0001-VP8-disallow-thread-count-changes.patch @@ -0,0 +1,34 @@ +From 793417d20735bb85fbd248ce364aaeeec6c12df2 Mon Sep 17 00:00:00 2001 +From: James Zern +Date: Mon, 25 Sep 2023 18:55:59 -0700 +Subject: [PATCH] VP8: disallow thread count changes + +Currently allocations are done at encoder creation time. Going from +threaded to non-threaded would cause a crash. + +Bug: chromium:1486441 +Change-Id: Ie301c2a70847dff2f0daae408fbef1e4d42e73d4 +(cherry picked from commit 3fbd1dca6a4d2dad332a2110d646e4ffef36d590) +--- + vp8/encoder/onyx_if.c | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/vp8/encoder/onyx_if.c b/vp8/encoder/onyx_if.c +index aeed719d1..369adbdcb 100644 +--- a/vp8/encoder/onyx_if.c ++++ b/vp8/encoder/onyx_if.c +@@ -1445,6 +1445,11 @@ void vp8_change_config(VP8_COMP *cpi, VP8_CONFIG *oxcf) { + last_h = cpi->oxcf.Height; + prev_number_of_layers = cpi->oxcf.number_of_layers; + ++ if (cpi->initial_width) { ++ // TODO(https://crbug.com/1486441): Allow changing thread counts; the ++ // allocation is done once in vp8_create_compressor(). ++ oxcf->multi_threaded = cpi->oxcf.multi_threaded; ++ } + cpi->oxcf = *oxcf; + + switch (cpi->oxcf.Mode) { +-- +2.41.0 + diff --git a/SOURCES/0002-Apply-stride_align-to-byte-count-not-pixel-count.patch b/SOURCES/0002-Apply-stride_align-to-byte-count-not-pixel-count.patch new file mode 100644 index 0000000..29db653 --- /dev/null +++ b/SOURCES/0002-Apply-stride_align-to-byte-count-not-pixel-count.patch @@ -0,0 +1,34 @@ +From 06ff2ce680ede4b727ca9fee0050e146736ec032 Mon Sep 17 00:00:00 2001 +From: Wan-Teh Chang +Date: Thu, 11 Apr 2024 16:38:45 -0700 +Subject: [PATCH 2/3] Apply stride_align to byte count, not pixel count + +A port of the libaom CL +https://aomedia-review.googlesource.com/c/aom/+/188962. + +stride_align is documented to be the "alignment, in bytes, of each row +in the image (stride)." + +Change-Id: I2184b50dc3607611f47719319fa5adb3adcef2fd +(cherry picked from commit 7d37ffacc6f7c45554b48ca867be4223248f1ed6) +--- + vpx/src/vpx_image.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/vpx/src/vpx_image.c b/vpx/src/vpx_image.c +index 404c577d6..c38b00aaf 100644 +--- a/vpx/src/vpx_image.c ++++ b/vpx/src/vpx_image.c +@@ -79,8 +79,8 @@ static vpx_image_t *img_alloc_helper(vpx_image_t *img, vpx_img_fmt_t fmt, + w = d_w; + h = d_h; + s = (fmt & VPX_IMG_FMT_PLANAR) ? w : (uint64_t)bps * w / 8; +- s = (s + stride_align - 1) & ~((uint64_t)stride_align - 1); + s = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s; ++ s = (s + stride_align - 1) & ~((uint64_t)stride_align - 1); + if (s > INT_MAX) goto fail; + stride_in_bytes = (int)s; + +-- +2.45.2 + diff --git a/SOURCES/0003-Fix-a-bug-in-alloc_size-for-high-bit-depths.patch b/SOURCES/0003-Fix-a-bug-in-alloc_size-for-high-bit-depths.patch new file mode 100644 index 0000000..ac95348 --- /dev/null +++ b/SOURCES/0003-Fix-a-bug-in-alloc_size-for-high-bit-depths.patch @@ -0,0 +1,46 @@ +From bc5ba1f95cd919fa13911d07b4f960ebbc0438bb Mon Sep 17 00:00:00 2001 +From: Wan-Teh Chang +Date: Fri, 12 Apr 2024 15:48:04 -0700 +Subject: [PATCH 3/3] Fix a bug in alloc_size for high bit depths + +I introduced this bug in commit 2e32276: +https://chromium-review.googlesource.com/c/webm/libvpx/+/5446333 + +I changed the line + + stride_in_bytes = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s; + +to three lines: + + s = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s; + if (s > INT_MAX) goto fail; + stride_in_bytes = (int)s; + +But I didn't realize that `s` is used later in the calculation of +alloc_size. + +As a quick fix, undo the effect of s * 2 for high bit depths after `s` +has been assigned to stride_in_bytes. + +Bug: chromium:332382766 +Change-Id: I53fbf405555645ab1d7254d31aadabe4f426be8c +(cherry picked from commit 74c70af01667733483dc69298b8921779f5f6ff3) +--- + vpx/src/vpx_image.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/vpx/src/vpx_image.c b/vpx/src/vpx_image.c +index c38b00aaf..693e7ac1f 100644 +--- a/vpx/src/vpx_image.c ++++ b/vpx/src/vpx_image.c +@@ -83,6 +83,7 @@ static vpx_image_t *img_alloc_helper(vpx_image_t *img, vpx_img_fmt_t fmt, + s = (s + stride_align - 1) & ~((uint64_t)stride_align - 1); + if (s > INT_MAX) goto fail; + stride_in_bytes = (int)s; ++ s = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s / 2 : s; + + /* Allocate the new image */ + if (!img) { +-- +2.45.2 + diff --git a/SPECS/libvpx.spec b/SPECS/libvpx.spec index d17c1c2..1c3880e 100644 --- a/SPECS/libvpx.spec +++ b/SPECS/libvpx.spec @@ -6,7 +6,7 @@ Name: libvpx Summary: VP8/VP9 Video Codec SDK Version: 1.9.0 -Release: 5%{?dist} +Release: 8%{?dist} License: BSD #Source0: http://downloads.webmproject.org/releases/webm/%{name}-%{version}.tar.bz2 Source0: https://github.com/webmproject/libvpx/archive/v%{version}.tar.gz @@ -23,6 +23,11 @@ BuildRequires: yasm BuildRequires: doxygen, php-cli, perl(Getopt::Long) # Do not disable FORTIFY_SOURCE=2 Patch0: libvpx-1.7.0-leave-fortify-source-on.patch +Patch1: 0001-Fix-bug-with-smaller-width-bigger-size.patch +Patch2: 0001-VP8-disallow-thread-count-changes.patch +Patch3: 0001-Fix-integer-overflows-in-calc-of-stride_in_bytes.patch +Patch4: 0002-Apply-stride_align-to-byte-count-not-pixel-count.patch +Patch5: 0003-Fix-a-bug-in-alloc_size-for-high-bit-depths.patch %description libvpx provides the VP8/VP9 SDK, which allows you to integrate your applications @@ -48,8 +53,18 @@ and decoder. %prep %setup -q -n libvpx-%{version} %patch0 -p1 -b .leave-fs-on +%patch1 -p1 -b .p1 +%patch2 -p1 -b .p2 +%patch3 -p1 -b .0003 +%patch4 -p1 -b .0004 +%patch5 -p1 -b .0005 %build +# This package fails to build with LTO due to undefined symbols. LTO +# was disabled in OpenSuSE as well, but with no real explanation why +# beyond the undefined symbols. It really shold be investigated further. +# Disable LTO +%define _lto_cflags %{nil} %ifarch %{ix86} %global vpxtarget x86-linux-gcc @@ -233,6 +248,19 @@ rm -rf %{buildroot}%{_prefix}/src %{_bindir}/* %changelog +* Fri Jul 5 2024 Wim Taymans - 1.9.0-8 +- Add patch to fix integer overflows. +- Disable LTO to fix build +- Resolves: RHEL-40655 + +* Thu Oct 5 2023 Wim Taymans - 1.9.0-7 +- Add patch for CVE-2023-5217 +- Resolves: RHEL-10631 + +* Tue Oct 3 2023 Wim Taymans - 1.9.0-6 +- Add patch for CVE-2023-44488 +- Resolves: RHEL-11636 + * Mon Aug 09 2021 Mohan Boddu - 1.9.0-5 - Rebuilt for IMA sigs, glibc 2.34, aarch64 flags Related: rhbz#1991688