SOURCES/0001-Fix-bug-with-smaller-width-bigger-size.patch

@@ -0,0 +1,103 @@
+From 0b9c2b782ae87f820aa34a19be7be6268f258172 Mon Sep 17 00:00:00 2001
+From: Jerome Jiang <jianj@google.com>
+Date: Thu, 30 Jun 2022 13:48:56 -0400
+Subject: [PATCH] Fix bug with smaller width bigger size
+
+Fixed previous patch that clusterfuzz failed on.
+
+Local fuzzing passing overnight.
+
+Bug: webm:1642
+Change-Id: If0e08e72abd2e042efe4dcfac21e4cc51afdfdb9
+(cherry picked from commit 263682c9a29395055f3b3afe2d97be1828a6223f)
+---
+ vp9/common/vp9_alloccommon.c | 13 ++++++-------
+ vp9/encoder/vp9_encoder.c    | 27 +++++++++++++++++++++++++--
+ 2 files changed, 31 insertions(+), 9 deletions(-)
+
+diff --git a/vp9/common/vp9_alloccommon.c b/vp9/common/vp9_alloccommon.c
+index 7345e259b..2989ee015 100644
+--- a/vp9/common/vp9_alloccommon.c
++++ b/vp9/common/vp9_alloccommon.c
+@@ -122,13 +122,6 @@ int vp9_alloc_context_buffers(VP9_COMMON *cm, int width, int height) {
+     cm->free_mi(cm);
+     if (cm->alloc_mi(cm, new_mi_size)) goto fail;
+   }
+-
+-  if (cm->seg_map_alloc_size < cm->mi_rows * cm->mi_cols) {
+-    // Create the segmentation map structure and set to 0.
+-    free_seg_map(cm);
+-    if (alloc_seg_map(cm, cm->mi_rows * cm->mi_cols)) goto fail;
+-  }
+-
+   if (cm->above_context_alloc_cols < cm->mi_cols) {
+     vpx_free(cm->above_context);
+     cm->above_context = (ENTROPY_CONTEXT *)vpx_calloc(
+@@ -143,6 +136,12 @@ int vp9_alloc_context_buffers(VP9_COMMON *cm, int width, int height) {
+     cm->above_context_alloc_cols = cm->mi_cols;
+   }
+ 
++  if (cm->seg_map_alloc_size < cm->mi_rows * cm->mi_cols) {
++    // Create the segmentation map structure and set to 0.
++    free_seg_map(cm);
++    if (alloc_seg_map(cm, cm->mi_rows * cm->mi_cols)) goto fail;
++  }
++
+   if (vp9_alloc_loop_filter(cm)) goto fail;
+ 
+   return 0;
+diff --git a/vp9/encoder/vp9_encoder.c b/vp9/encoder/vp9_encoder.c
+index 2ae59dd98..01a6d907b 100644
+--- a/vp9/encoder/vp9_encoder.c
++++ b/vp9/encoder/vp9_encoder.c
+@@ -1751,6 +1751,17 @@ static void alloc_copy_partition_data(VP9_COMP *cpi) {
+   }
+ }
+ 
++static void free_copy_partition_data(VP9_COMP *cpi) {
++  vpx_free(cpi->prev_partition);
++  cpi->prev_partition = NULL;
++  vpx_free(cpi->prev_segment_id);
++  cpi->prev_segment_id = NULL;
++  vpx_free(cpi->prev_variance_low);
++  cpi->prev_variance_low = NULL;
++  vpx_free(cpi->copied_frame_cnt);
++  cpi->copied_frame_cnt = NULL;
++}
++
+ void vp9_change_config(struct VP9_COMP *cpi, const VP9EncoderConfig *oxcf) {
+   VP9_COMMON *const cm = &cpi->common;
+   RATE_CONTROL *const rc = &cpi->rc;
+@@ -1834,6 +1845,8 @@ void vp9_change_config(struct VP9_COMP *cpi, const VP9EncoderConfig *oxcf) {
+     new_mi_size = cm->mi_stride * calc_mi_size(cm->mi_rows);
+     if (cm->mi_alloc_size < new_mi_size) {
+       vp9_free_context_buffers(cm);
++      vp9_free_pc_tree(&cpi->td);
++      vpx_free(cpi->mbmi_ext_base);
+       alloc_compressor_data(cpi);
+       realloc_segmentation_maps(cpi);
+       cpi->initial_width = cpi->initial_height = 0;
+@@ -1849,8 +1862,18 @@ void vp9_change_config(struct VP9_COMP *cpi, const VP9EncoderConfig *oxcf) {
+     update_frame_size(cpi);
+ 
+   if (last_w != cpi->oxcf.width || last_h != cpi->oxcf.height) {
+-    memset(cpi->consec_zero_mv, 0,
+-           cm->mi_rows * cm->mi_cols * sizeof(*cpi->consec_zero_mv));
++    vpx_free(cpi->consec_zero_mv);
++    CHECK_MEM_ERROR(
++        cm, cpi->consec_zero_mv,
++        vpx_calloc(cm->mi_rows * cm->mi_cols, sizeof(*cpi->consec_zero_mv)));
++
++    vpx_free(cpi->skin_map);
++    CHECK_MEM_ERROR(
++        cm, cpi->skin_map,
++        vpx_calloc(cm->mi_rows * cm->mi_cols, sizeof(cpi->skin_map[0])));
++
++    free_copy_partition_data(cpi);
++    alloc_copy_partition_data(cpi);
+     if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ)
+       vp9_cyclic_refresh_reset_resize(cpi);
+     rc->rc_1_frame = 0;
+-- 
+2.41.0
+

SOURCES/0001-Fix-integer-overflows-in-calc-of-stride_in_bytes.patch

@@ -0,0 +1,109 @@
+From 6b98423e19a68b182cd50e3c640f9828b025818b Mon Sep 17 00:00:00 2001
+From: Wan-Teh Chang <wtc@google.com>
+Date: Wed, 10 Apr 2024 17:01:10 -0700
+Subject: [PATCH 1/3] Fix integer overflows in calc of stride_in_bytes
+
+A port of the libaom CL
+https://aomedia-review.googlesource.com/c/aom/+/188761.
+
+Fix unsigned integer overflows in the calculation of stride_in_bytes in
+img_alloc_helper() when d_w is huge.
+
+Change the type of stride_in_bytes from unsigned int to int because it
+will be assigned to img->stride[VPX_PLANE_Y], which is of the int type.
+
+Test:
+. ../libvpx/tools/set_analyzer_env.sh integer
+../libvpx/configure --enable-debug --disable-optimizations
+make -j
+./test_libvpx --gtest_filter=VpxImageTest.VpxImgAllocHugeWidth
+
+Bug: chromium:332382766
+Change-Id: I3b39d78f61c7255e10cbf72ba2f4975425a05a82
+---
+ vpx/src/vpx_image.c | 32 +++++++++++++++++++-------------
+ 1 file changed, 19 insertions(+), 13 deletions(-)
+
+diff --git a/vpx/src/vpx_image.c b/vpx/src/vpx_image.c
+index af7c529a7..a01aab29c 100644
+--- a/vpx/src/vpx_image.c
++++ b/vpx/src/vpx_image.c
+@@ -20,9 +20,9 @@ static vpx_image_t *img_alloc_helper(vpx_image_t *img, vpx_img_fmt_t fmt,
+                                      unsigned int buf_align,
+                                      unsigned int stride_align,
+                                      unsigned char *img_data) {
+-  unsigned int h, w, s, xcs, ycs, bps;
+-  unsigned int stride_in_bytes;
+-  int align;
++  unsigned int h, w, xcs, ycs, bps;
++  uint64_t s;
++  int stride_in_bytes, align;
+ 
+   /* Treat align==0 like align==1 */
+   if (!buf_align) buf_align = 1;
+@@ -92,9 +92,11 @@ static vpx_image_t *img_alloc_helper(vpx_image_t *img, vpx_img_fmt_t fmt,
+    * and height shouldn't be adjusted. */
+   w = d_w;
+   h = d_h;
+-  s = (fmt & VPX_IMG_FMT_PLANAR) ? w : bps * w / 8;
+-  s = (s + stride_align - 1) & ~(stride_align - 1);
+-  stride_in_bytes = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s;
++  s = (fmt & VPX_IMG_FMT_PLANAR) ? w : (uint64_t)bps * w / 8;
++  s = (s + stride_align - 1) & ~((uint64_t)stride_align - 1);
++  s = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s;
++  if (s > INT_MAX) goto fail;
++  stride_in_bytes = (int)s;
+ 
+   /* Allocate the new image */
+   if (!img) {
+@@ -117,9 +119,11 @@ static vpx_image_t *img_alloc_helper(vpx_image_t *img, vpx_img_fmt_t fmt,
+     align = (1 << ycs) - 1;
+     h = (d_h + align) & ~align;
+ 
+-    s = (fmt & VPX_IMG_FMT_PLANAR) ? w : bps * w / 8;
+-    s = (s + stride_align - 1) & ~(stride_align - 1);
+-    stride_in_bytes = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s;
++    s = (fmt & VPX_IMG_FMT_PLANAR) ? w : (uint64_t)bps * w / 8;
++    s = (s + stride_align - 1) & ~((uint64_t)stride_align - 1);
++    s = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s;
++    if (s > INT_MAX) goto fail;
++    stride_in_bytes = (int)s;
+     alloc_size = (fmt & VPX_IMG_FMT_PLANAR) ? (uint64_t)h * s * bps / 8
+                                             : (uint64_t)h * s;
+ 
+@@ -185,18 +189,19 @@ int vpx_img_set_rect(vpx_image_t *img, unsigned int x, unsigned int y,
+       if (img->fmt & VPX_IMG_FMT_HAS_ALPHA) {
+         img->planes[VPX_PLANE_ALPHA] =
+             data + x * bytes_per_sample + y * img->stride[VPX_PLANE_ALPHA];
+-        data += img->h * img->stride[VPX_PLANE_ALPHA];
++        data += (size_t)img->h * img->stride[VPX_PLANE_ALPHA];
+       }
+ 
+       img->planes[VPX_PLANE_Y] =
+           data + x * bytes_per_sample + y * img->stride[VPX_PLANE_Y];
+-      data += img->h * img->stride[VPX_PLANE_Y];
++      data += (size_t)img->h * img->stride[VPX_PLANE_Y];
+ 
+       if (!(img->fmt & VPX_IMG_FMT_UV_FLIP)) {
+         img->planes[VPX_PLANE_U] =
+             data + (x >> img->x_chroma_shift) * bytes_per_sample +
+             (y >> img->y_chroma_shift) * img->stride[VPX_PLANE_U];
+-        data += (img->h >> img->y_chroma_shift) * img->stride[VPX_PLANE_U];
++        data +=
++            (size_t)(img->h >> img->y_chroma_shift) * img->stride[VPX_PLANE_U];
+         img->planes[VPX_PLANE_V] =
+             data + (x >> img->x_chroma_shift) * bytes_per_sample +
+             (y >> img->y_chroma_shift) * img->stride[VPX_PLANE_V];
+@@ -204,7 +209,8 @@ int vpx_img_set_rect(vpx_image_t *img, unsigned int x, unsigned int y,
+         img->planes[VPX_PLANE_V] =
+             data + (x >> img->x_chroma_shift) * bytes_per_sample +
+             (y >> img->y_chroma_shift) * img->stride[VPX_PLANE_V];
+-        data += (img->h >> img->y_chroma_shift) * img->stride[VPX_PLANE_V];
++        data +=
++            (size_t)(img->h >> img->y_chroma_shift) * img->stride[VPX_PLANE_V];
+         img->planes[VPX_PLANE_U] =
+             data + (x >> img->x_chroma_shift) * bytes_per_sample +
+             (y >> img->y_chroma_shift) * img->stride[VPX_PLANE_U];
+-- 
+2.45.2
+

SOURCES/0001-VP8-disallow-thread-count-changes.patch

@@ -0,0 +1,34 @@
+From 3a03995efe86129cde5df3a7de32fecdfce3259e Mon Sep 17 00:00:00 2001
+From: James Zern <jzern@google.com>
+Date: Mon, 25 Sep 2023 18:55:59 -0700
+Subject: [PATCH] VP8: disallow thread count changes
+
+Currently allocations are done at encoder creation time. Going from
+threaded to non-threaded would cause a crash.
+
+Bug: chromium:1486441
+Change-Id: Ie301c2a70847dff2f0daae408fbef1e4d42e73d4
+(cherry picked from commit 3fbd1dca6a4d2dad332a2110d646e4ffef36d590)
+---
+ vp8/encoder/onyx_if.c | 5 +++++
+ 1 file changed, 5 insertions(+)
+
+diff --git a/vp8/encoder/onyx_if.c b/vp8/encoder/onyx_if.c
+index 224318242..e0c22070c 100644
+--- a/vp8/encoder/onyx_if.c
++++ b/vp8/encoder/onyx_if.c
+@@ -1449,6 +1449,11 @@ void vp8_change_config(VP8_COMP *cpi, VP8_CONFIG *oxcf) {
+   last_h = cpi->oxcf.Height;
+   prev_number_of_layers = cpi->oxcf.number_of_layers;
+ 
++  if (cpi->initial_width) {
++    // TODO(https://crbug.com/1486441): Allow changing thread counts; the
++    // allocation is done once in vp8_create_compressor().
++    oxcf->multi_threaded = cpi->oxcf.multi_threaded;
++  }
+   cpi->oxcf = *oxcf;
+ 
+   switch (cpi->oxcf.Mode) {
+-- 
+2.41.0
+

SOURCES/0002-Apply-stride_align-to-byte-count-not-pixel-count.patch

@@ -0,0 +1,34 @@
+From f8472f581ed1b4dd0d205efdec72e43742f579fb Mon Sep 17 00:00:00 2001
+From: Wan-Teh Chang <wtc@google.com>
+Date: Thu, 11 Apr 2024 16:38:45 -0700
+Subject: [PATCH 2/3] Apply stride_align to byte count, not pixel count
+
+A port of the libaom CL
+https://aomedia-review.googlesource.com/c/aom/+/188962.
+
+stride_align is documented to be the "alignment, in bytes, of each row
+in the image (stride)."
+
+Change-Id: I2184b50dc3607611f47719319fa5adb3adcef2fd
+(cherry picked from commit 7d37ffacc6f7c45554b48ca867be4223248f1ed6)
+---
+ vpx/src/vpx_image.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/vpx/src/vpx_image.c b/vpx/src/vpx_image.c
+index a01aab29c..0c84562ae 100644
+--- a/vpx/src/vpx_image.c
++++ b/vpx/src/vpx_image.c
+@@ -93,8 +93,8 @@ static vpx_image_t *img_alloc_helper(vpx_image_t *img, vpx_img_fmt_t fmt,
+   w = d_w;
+   h = d_h;
+   s = (fmt & VPX_IMG_FMT_PLANAR) ? w : (uint64_t)bps * w / 8;
+-  s = (s + stride_align - 1) & ~((uint64_t)stride_align - 1);
+   s = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s;
++  s = (s + stride_align - 1) & ~((uint64_t)stride_align - 1);
+   if (s > INT_MAX) goto fail;
+   stride_in_bytes = (int)s;
+ 
+-- 
+2.45.2
+

SOURCES/0003-Fix-a-bug-in-alloc_size-for-high-bit-depths.patch

@@ -0,0 +1,46 @@
+From 0a68a93729ab879251ad63f833a327d20dbbbc23 Mon Sep 17 00:00:00 2001
+From: Wan-Teh Chang <wtc@google.com>
+Date: Fri, 12 Apr 2024 15:48:04 -0700
+Subject: [PATCH 3/3] Fix a bug in alloc_size for high bit depths
+
+I introduced this bug in commit 2e32276:
+https://chromium-review.googlesource.com/c/webm/libvpx/+/5446333
+
+I changed the line
+
+  stride_in_bytes = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s;
+
+to three lines:
+
+  s = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s;
+  if (s > INT_MAX) goto fail;
+  stride_in_bytes = (int)s;
+
+But I didn't realize that `s` is used later in the calculation of
+alloc_size.
+
+As a quick fix, undo the effect of s * 2 for high bit depths after `s`
+has been assigned to stride_in_bytes.
+
+Bug: chromium:332382766
+Change-Id: I53fbf405555645ab1d7254d31aadabe4f426be8c
+(cherry picked from commit 74c70af01667733483dc69298b8921779f5f6ff3)
+---
+ vpx/src/vpx_image.c | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/vpx/src/vpx_image.c b/vpx/src/vpx_image.c
+index 0c84562ae..38d4c1ce3 100644
+--- a/vpx/src/vpx_image.c
++++ b/vpx/src/vpx_image.c
+@@ -97,6 +97,7 @@ static vpx_image_t *img_alloc_helper(vpx_image_t *img, vpx_img_fmt_t fmt,
+   s = (s + stride_align - 1) & ~((uint64_t)stride_align - 1);
+   if (s > INT_MAX) goto fail;
+   stride_in_bytes = (int)s;
++  s = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s / 2 : s;
+ 
+   /* Allocate the new image */
+   if (!img) {
+-- 
+2.45.2
+

SOURCES/include-limits.patch

@@ -0,0 +1,11 @@
+diff -ru libvpx-1.7.0/vpx/src/vpx_image.c libvpx-1.7.0.new/vpx/src/vpx_image.c
+--- libvpx-1.7.0/vpx/src/vpx_image.c	2018-01-24 23:25:44.000000000 +0100
++++ libvpx-1.7.0.new/vpx/src/vpx_image.c	2024-08-27 15:22:25.886886526 +0200
+@@ -10,6 +10,7 @@
+ 
+ #include <stdlib.h>
+ #include <string.h>
++#include <limits.h>
+ 
+ #include "vpx/vpx_image.h"
+ #include "vpx/vpx_integer.h"

SPECS/libvpx.spec

@@ -6,7 +6,7 @@
 Name:			libvpx
 Summary:		VP8/VP9 Video Codec SDK
 Version:		1.7.0
-Release:		8%{?dist}
+Release:		11%{?dist}
 License:		BSD
 Group:			System Environment/Libraries
 #Source0:		http://downloads.webmproject.org/releases/webm/%{name}-%{version}.tar.bz2
@@ -25,6 +25,12 @@ Patch1:			0001-CVE-2019-9232-Fix-OOB-memory-access-on-fuzzed-data.patch
 Patch2:			0002-CVE-2019-9433-VP8-Fix-use-after-free-in-postproc.patch
 Patch3:			0003-CVE-2019-9371-update-libwebm.patch
 Patch4:			0004-CVE-2019-2126-update-libwebm-to-libwebm-1.0.0.27-361.patch
+Patch5:			0001-Fix-bug-with-smaller-width-bigger-size.patch
+Patch6:			0001-VP8-disallow-thread-count-changes.patch
+Patch7:			0001-Fix-integer-overflows-in-calc-of-stride_in_bytes.patch
+Patch8:			0002-Apply-stride_align-to-byte-count-not-pixel-count.patch
+Patch9:			0003-Fix-a-bug-in-alloc_size-for-high-bit-depths.patch
+Patch10:		include-limits.patch
 
 %description
 libvpx provides the VP8/VP9 SDK, which allows you to integrate your applications 
@@ -56,6 +62,12 @@ and decoder.
 %patch2 -p1 -b .0002
 %patch3 -p1 -b .0003
 %patch4 -p1 -b .0004
+%patch5 -p1 -b .0005
+%patch6 -p1 -b .0006
+%patch7 -p1 -b .0007
+%patch8 -p1 -b .0008
+%patch9 -p1 -b .0009
+%patch10 -p1 -b .0010
 
 %build
 %ifarch %{ix86}
@@ -245,6 +257,19 @@ rm -rf %{buildroot}%{_prefix}/src
 %{_bindir}/*
 
 %changelog
+* Fri Jul 5 2024 Wim Taymans <wtaymans@redhat.com> - 1.7.0-11
+- Add patch to fix integer overflows.
+- Fix compilation by including limits.h
+- Resolves: RHEL-40650
+
+* Thu Oct 5 2023 Wim Taymans <wtaymans@redhat.com> - 1.7.0-10
+- Add patch for CVE-2023-5217
+- Resolves: RHEL-10612
+
+* Tue Oct 3 2023 Wim Taymans <wtaymans@redhat.com> - 1.7.0-9
+- Add patch for CVE-2023-44488
+- Resolves: RHEL-11615
+
 * Wed Apr 1 2020 Wim Taymans <wtaymans@redhat.com> - 1.7.0-8
 - Resolves: rhbz#1796086, rhbz#1796100, rhbz#1796448, rhbz#1796454
 - Enable webm-io explicitly