From 7c4098a055594e3938dc14ba6d729a072346c53c Mon Sep 17 00:00:00 2001 From: Wim Taymans Date: Fri, 5 Jul 2024 15:29:50 +0200 Subject: [PATCH] Add patch to fix integer overflows. Resolves: RHEL-40655 --- ...overflows-in-calc-of-stride_in_bytes.patch | 119 ++++++++++++++++++ ..._align-to-byte-count-not-pixel-count.patch | 34 +++++ ...ug-in-alloc_size-for-high-bit-depths.patch | 46 +++++++ libvpx.spec | 18 ++- 4 files changed, 216 insertions(+), 1 deletion(-) create mode 100644 0001-Fix-integer-overflows-in-calc-of-stride_in_bytes.patch create mode 100644 0002-Apply-stride_align-to-byte-count-not-pixel-count.patch create mode 100644 0003-Fix-a-bug-in-alloc_size-for-high-bit-depths.patch diff --git a/0001-Fix-integer-overflows-in-calc-of-stride_in_bytes.patch b/0001-Fix-integer-overflows-in-calc-of-stride_in_bytes.patch new file mode 100644 index 0000000..1cb31a8 --- /dev/null +++ b/0001-Fix-integer-overflows-in-calc-of-stride_in_bytes.patch @@ -0,0 +1,119 @@ +From 70e6813ca323789b6c937b9f6cbbfd3daf4b1692 Mon Sep 17 00:00:00 2001 +From: Wan-Teh Chang +Date: Wed, 10 Apr 2024 17:01:10 -0700 +Subject: [PATCH 1/3] Fix integer overflows in calc of stride_in_bytes + +A port of the libaom CL +https://aomedia-review.googlesource.com/c/aom/+/188761. + +Fix unsigned integer overflows in the calculation of stride_in_bytes in +img_alloc_helper() when d_w is huge. + +Change the type of stride_in_bytes from unsigned int to int because it +will be assigned to img->stride[VPX_PLANE_Y], which is of the int type. + +Test: +. ../libvpx/tools/set_analyzer_env.sh integer +../libvpx/configure --enable-debug --disable-optimizations +make -j +./test_libvpx --gtest_filter=VpxImageTest.VpxImgAllocHugeWidth + +Bug: chromium:332382766 +Change-Id: I3b39d78f61c7255e10cbf72ba2f4975425a05a82 +--- + vpx/src/vpx_image.c | 33 ++++++++++++++++++++------------- + 1 file changed, 20 insertions(+), 13 deletions(-) + +diff --git a/vpx/src/vpx_image.c b/vpx/src/vpx_image.c +index ff496b5d3..404c577d6 100644 +--- a/vpx/src/vpx_image.c ++++ b/vpx/src/vpx_image.c +@@ -10,6 +10,7 @@ + + #include + #include ++#include + + #include "vpx/vpx_image.h" + #include "vpx/vpx_integer.h" +@@ -20,9 +21,9 @@ static vpx_image_t *img_alloc_helper(vpx_image_t *img, vpx_img_fmt_t fmt, + unsigned int buf_align, + unsigned int stride_align, + unsigned char *img_data) { +- unsigned int h, w, s, xcs, ycs, bps; +- unsigned int stride_in_bytes; +- int align; ++ unsigned int h, w, xcs, ycs, bps; ++ uint64_t s; ++ int stride_in_bytes, align; + + /* Treat align==0 like align==1 */ + if (!buf_align) buf_align = 1; +@@ -77,9 +78,11 @@ static vpx_image_t *img_alloc_helper(vpx_image_t *img, vpx_img_fmt_t fmt, + * and height shouldn't be adjusted. */ + w = d_w; + h = d_h; +- s = (fmt & VPX_IMG_FMT_PLANAR) ? w : bps * w / 8; +- s = (s + stride_align - 1) & ~(stride_align - 1); +- stride_in_bytes = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s; ++ s = (fmt & VPX_IMG_FMT_PLANAR) ? w : (uint64_t)bps * w / 8; ++ s = (s + stride_align - 1) & ~((uint64_t)stride_align - 1); ++ s = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s; ++ if (s > INT_MAX) goto fail; ++ stride_in_bytes = (int)s; + + /* Allocate the new image */ + if (!img) { +@@ -102,9 +105,11 @@ static vpx_image_t *img_alloc_helper(vpx_image_t *img, vpx_img_fmt_t fmt, + align = (1 << ycs) - 1; + h = (d_h + align) & ~align; + +- s = (fmt & VPX_IMG_FMT_PLANAR) ? w : bps * w / 8; +- s = (s + stride_align - 1) & ~(stride_align - 1); +- stride_in_bytes = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s; ++ s = (fmt & VPX_IMG_FMT_PLANAR) ? w : (uint64_t)bps * w / 8; ++ s = (s + stride_align - 1) & ~((uint64_t)stride_align - 1); ++ s = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s; ++ if (s > INT_MAX) goto fail; ++ stride_in_bytes = (int)s; + alloc_size = (fmt & VPX_IMG_FMT_PLANAR) ? (uint64_t)h * s * bps / 8 + : (uint64_t)h * s; + +@@ -170,12 +175,12 @@ int vpx_img_set_rect(vpx_image_t *img, unsigned int x, unsigned int y, + if (img->fmt & VPX_IMG_FMT_HAS_ALPHA) { + img->planes[VPX_PLANE_ALPHA] = + data + x * bytes_per_sample + y * img->stride[VPX_PLANE_ALPHA]; +- data += img->h * img->stride[VPX_PLANE_ALPHA]; ++ data += (size_t)img->h * img->stride[VPX_PLANE_ALPHA]; + } + + img->planes[VPX_PLANE_Y] = + data + x * bytes_per_sample + y * img->stride[VPX_PLANE_Y]; +- data += img->h * img->stride[VPX_PLANE_Y]; ++ data += (size_t)img->h * img->stride[VPX_PLANE_Y]; + + if (img->fmt == VPX_IMG_FMT_NV12) { + img->planes[VPX_PLANE_U] = +@@ -186,7 +191,8 @@ int vpx_img_set_rect(vpx_image_t *img, unsigned int x, unsigned int y, + img->planes[VPX_PLANE_U] = + data + (x >> img->x_chroma_shift) * bytes_per_sample + + (y >> img->y_chroma_shift) * img->stride[VPX_PLANE_U]; +- data += (img->h >> img->y_chroma_shift) * img->stride[VPX_PLANE_U]; ++ data += ++ (size_t)(img->h >> img->y_chroma_shift) * img->stride[VPX_PLANE_U]; + img->planes[VPX_PLANE_V] = + data + (x >> img->x_chroma_shift) * bytes_per_sample + + (y >> img->y_chroma_shift) * img->stride[VPX_PLANE_V]; +@@ -194,7 +200,8 @@ int vpx_img_set_rect(vpx_image_t *img, unsigned int x, unsigned int y, + img->planes[VPX_PLANE_V] = + data + (x >> img->x_chroma_shift) * bytes_per_sample + + (y >> img->y_chroma_shift) * img->stride[VPX_PLANE_V]; +- data += (img->h >> img->y_chroma_shift) * img->stride[VPX_PLANE_V]; ++ data += ++ (size_t)(img->h >> img->y_chroma_shift) * img->stride[VPX_PLANE_V]; + img->planes[VPX_PLANE_U] = + data + (x >> img->x_chroma_shift) * bytes_per_sample + + (y >> img->y_chroma_shift) * img->stride[VPX_PLANE_U]; +-- +2.45.2 + diff --git a/0002-Apply-stride_align-to-byte-count-not-pixel-count.patch b/0002-Apply-stride_align-to-byte-count-not-pixel-count.patch new file mode 100644 index 0000000..29db653 --- /dev/null +++ b/0002-Apply-stride_align-to-byte-count-not-pixel-count.patch @@ -0,0 +1,34 @@ +From 06ff2ce680ede4b727ca9fee0050e146736ec032 Mon Sep 17 00:00:00 2001 +From: Wan-Teh Chang +Date: Thu, 11 Apr 2024 16:38:45 -0700 +Subject: [PATCH 2/3] Apply stride_align to byte count, not pixel count + +A port of the libaom CL +https://aomedia-review.googlesource.com/c/aom/+/188962. + +stride_align is documented to be the "alignment, in bytes, of each row +in the image (stride)." + +Change-Id: I2184b50dc3607611f47719319fa5adb3adcef2fd +(cherry picked from commit 7d37ffacc6f7c45554b48ca867be4223248f1ed6) +--- + vpx/src/vpx_image.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/vpx/src/vpx_image.c b/vpx/src/vpx_image.c +index 404c577d6..c38b00aaf 100644 +--- a/vpx/src/vpx_image.c ++++ b/vpx/src/vpx_image.c +@@ -79,8 +79,8 @@ static vpx_image_t *img_alloc_helper(vpx_image_t *img, vpx_img_fmt_t fmt, + w = d_w; + h = d_h; + s = (fmt & VPX_IMG_FMT_PLANAR) ? w : (uint64_t)bps * w / 8; +- s = (s + stride_align - 1) & ~((uint64_t)stride_align - 1); + s = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s; ++ s = (s + stride_align - 1) & ~((uint64_t)stride_align - 1); + if (s > INT_MAX) goto fail; + stride_in_bytes = (int)s; + +-- +2.45.2 + diff --git a/0003-Fix-a-bug-in-alloc_size-for-high-bit-depths.patch b/0003-Fix-a-bug-in-alloc_size-for-high-bit-depths.patch new file mode 100644 index 0000000..ac95348 --- /dev/null +++ b/0003-Fix-a-bug-in-alloc_size-for-high-bit-depths.patch @@ -0,0 +1,46 @@ +From bc5ba1f95cd919fa13911d07b4f960ebbc0438bb Mon Sep 17 00:00:00 2001 +From: Wan-Teh Chang +Date: Fri, 12 Apr 2024 15:48:04 -0700 +Subject: [PATCH 3/3] Fix a bug in alloc_size for high bit depths + +I introduced this bug in commit 2e32276: +https://chromium-review.googlesource.com/c/webm/libvpx/+/5446333 + +I changed the line + + stride_in_bytes = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s; + +to three lines: + + s = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s; + if (s > INT_MAX) goto fail; + stride_in_bytes = (int)s; + +But I didn't realize that `s` is used later in the calculation of +alloc_size. + +As a quick fix, undo the effect of s * 2 for high bit depths after `s` +has been assigned to stride_in_bytes. + +Bug: chromium:332382766 +Change-Id: I53fbf405555645ab1d7254d31aadabe4f426be8c +(cherry picked from commit 74c70af01667733483dc69298b8921779f5f6ff3) +--- + vpx/src/vpx_image.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/vpx/src/vpx_image.c b/vpx/src/vpx_image.c +index c38b00aaf..693e7ac1f 100644 +--- a/vpx/src/vpx_image.c ++++ b/vpx/src/vpx_image.c +@@ -83,6 +83,7 @@ static vpx_image_t *img_alloc_helper(vpx_image_t *img, vpx_img_fmt_t fmt, + s = (s + stride_align - 1) & ~((uint64_t)stride_align - 1); + if (s > INT_MAX) goto fail; + stride_in_bytes = (int)s; ++ s = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s / 2 : s; + + /* Allocate the new image */ + if (!img) { +-- +2.45.2 + diff --git a/libvpx.spec b/libvpx.spec index bc68095..1c3880e 100644 --- a/libvpx.spec +++ b/libvpx.spec @@ -6,7 +6,7 @@ Name: libvpx Summary: VP8/VP9 Video Codec SDK Version: 1.9.0 -Release: 7%{?dist} +Release: 8%{?dist} License: BSD #Source0: http://downloads.webmproject.org/releases/webm/%{name}-%{version}.tar.bz2 Source0: https://github.com/webmproject/libvpx/archive/v%{version}.tar.gz @@ -25,6 +25,9 @@ BuildRequires: doxygen, php-cli, perl(Getopt::Long) Patch0: libvpx-1.7.0-leave-fortify-source-on.patch Patch1: 0001-Fix-bug-with-smaller-width-bigger-size.patch Patch2: 0001-VP8-disallow-thread-count-changes.patch +Patch3: 0001-Fix-integer-overflows-in-calc-of-stride_in_bytes.patch +Patch4: 0002-Apply-stride_align-to-byte-count-not-pixel-count.patch +Patch5: 0003-Fix-a-bug-in-alloc_size-for-high-bit-depths.patch %description libvpx provides the VP8/VP9 SDK, which allows you to integrate your applications @@ -52,8 +55,16 @@ and decoder. %patch0 -p1 -b .leave-fs-on %patch1 -p1 -b .p1 %patch2 -p1 -b .p2 +%patch3 -p1 -b .0003 +%patch4 -p1 -b .0004 +%patch5 -p1 -b .0005 %build +# This package fails to build with LTO due to undefined symbols. LTO +# was disabled in OpenSuSE as well, but with no real explanation why +# beyond the undefined symbols. It really shold be investigated further. +# Disable LTO +%define _lto_cflags %{nil} %ifarch %{ix86} %global vpxtarget x86-linux-gcc @@ -237,6 +248,11 @@ rm -rf %{buildroot}%{_prefix}/src %{_bindir}/* %changelog +* Fri Jul 5 2024 Wim Taymans - 1.9.0-8 +- Add patch to fix integer overflows. +- Disable LTO to fix build +- Resolves: RHEL-40655 + * Thu Oct 5 2023 Wim Taymans - 1.9.0-7 - Add patch for CVE-2023-5217 - Resolves: RHEL-10631