Add patch for CVE-2023-44488

Resolves: RHEL-11636

0001-Fix-bug-with-smaller-width-bigger-size.patch

@@ -0,0 +1,131 @@
+From bc54ed5c2d7b43a1904e04a59c268368c274169e Mon Sep 17 00:00:00 2001
+From: Jerome Jiang <jianj@google.com>
+Date: Thu, 30 Jun 2022 13:48:56 -0400
+Subject: [PATCH] Fix bug with smaller width bigger size
+
+Fixed previous patch that clusterfuzz failed on.
+
+Local fuzzing passing overnight.
+
+Bug: webm:1642
+Change-Id: If0e08e72abd2e042efe4dcfac21e4cc51afdfdb9
+(cherry picked from commit 263682c9a29395055f3b3afe2d97be1828a6223f)
+---
+ test/resize_test.cc          |  9 +++------
+ vp9/common/vp9_alloccommon.c | 13 ++++++-------
+ vp9/encoder/vp9_encoder.c    | 27 +++++++++++++++++++++++++--
+ 3 files changed, 34 insertions(+), 15 deletions(-)
+
+diff --git a/test/resize_test.cc b/test/resize_test.cc
+index b2eadb1e8..ebc5f9b02 100644
+--- a/test/resize_test.cc
++++ b/test/resize_test.cc
+@@ -101,11 +101,8 @@ void ScaleForFrameNumber(unsigned int frame, unsigned int initial_w,
+       *h = initial_h;
+       return;
+     }
+-    if (frame < 100) {
+-      *w = initial_w * 7 / 10;
+-      *h = initial_h * 16 / 10;
+-      return;
+-    }
++    *w = initial_w * 7 / 10;
++    *h = initial_h * 16 / 10;
+     return;
+   }
+   if (frame < 10) {
+@@ -578,7 +575,7 @@ TEST_P(ResizeRealtimeTest, TestExternalResizeWorks) {
+   }
+ }
+ 
+-TEST_P(ResizeRealtimeTest, DISABLED_TestExternalResizeSmallerWidthBiggerSize) {
++TEST_P(ResizeRealtimeTest, TestExternalResizeSmallerWidthBiggerSize) {
+   ResizingVideoSource video;
+   video.flag_codec_ = true;
+   video.smaller_width_larger_size_ = true;
+diff --git a/vp9/common/vp9_alloccommon.c b/vp9/common/vp9_alloccommon.c
+index 5702dca71..0328f33a4 100644
+--- a/vp9/common/vp9_alloccommon.c
++++ b/vp9/common/vp9_alloccommon.c
+@@ -131,13 +131,6 @@ int vp9_alloc_context_buffers(VP9_COMMON *cm, int width, int height) {
+     cm->free_mi(cm);
+     if (cm->alloc_mi(cm, new_mi_size)) goto fail;
+   }
+-
+-  if (cm->seg_map_alloc_size < cm->mi_rows * cm->mi_cols) {
+-    // Create the segmentation map structure and set to 0.
+-    free_seg_map(cm);
+-    if (alloc_seg_map(cm, cm->mi_rows * cm->mi_cols)) goto fail;
+-  }
+-
+   if (cm->above_context_alloc_cols < cm->mi_cols) {
+     vpx_free(cm->above_context);
+     cm->above_context = (ENTROPY_CONTEXT *)vpx_calloc(
+@@ -152,6 +145,12 @@ int vp9_alloc_context_buffers(VP9_COMMON *cm, int width, int height) {
+     cm->above_context_alloc_cols = cm->mi_cols;
+   }
+ 
++  if (cm->seg_map_alloc_size < cm->mi_rows * cm->mi_cols) {
++    // Create the segmentation map structure and set to 0.
++    free_seg_map(cm);
++    if (alloc_seg_map(cm, cm->mi_rows * cm->mi_cols)) goto fail;
++  }
++
+   if (vp9_alloc_loop_filter(cm)) goto fail;
+ 
+   return 0;
+diff --git a/vp9/encoder/vp9_encoder.c b/vp9/encoder/vp9_encoder.c
+index 20e8d494a..055e1def0 100644
+--- a/vp9/encoder/vp9_encoder.c
++++ b/vp9/encoder/vp9_encoder.c
+@@ -1968,6 +1968,17 @@ static void alloc_copy_partition_data(VP9_COMP *cpi) {
+   }
+ }
+ 
++static void free_copy_partition_data(VP9_COMP *cpi) {
++  vpx_free(cpi->prev_partition);
++  cpi->prev_partition = NULL;
++  vpx_free(cpi->prev_segment_id);
++  cpi->prev_segment_id = NULL;
++  vpx_free(cpi->prev_variance_low);
++  cpi->prev_variance_low = NULL;
++  vpx_free(cpi->copied_frame_cnt);
++  cpi->copied_frame_cnt = NULL;
++}
++
+ void vp9_change_config(struct VP9_COMP *cpi, const VP9EncoderConfig *oxcf) {
+   VP9_COMMON *const cm = &cpi->common;
+   RATE_CONTROL *const rc = &cpi->rc;
+@@ -2047,6 +2058,8 @@ void vp9_change_config(struct VP9_COMP *cpi, const VP9EncoderConfig *oxcf) {
+     new_mi_size = cm->mi_stride * calc_mi_size(cm->mi_rows);
+     if (cm->mi_alloc_size < new_mi_size) {
+       vp9_free_context_buffers(cm);
++      vp9_free_pc_tree(&cpi->td);
++      vpx_free(cpi->mbmi_ext_base);
+       alloc_compressor_data(cpi);
+       realloc_segmentation_maps(cpi);
+       cpi->initial_width = cpi->initial_height = 0;
+@@ -2062,8 +2075,18 @@ void vp9_change_config(struct VP9_COMP *cpi, const VP9EncoderConfig *oxcf) {
+     update_frame_size(cpi);
+ 
+   if (last_w != cpi->oxcf.width || last_h != cpi->oxcf.height) {
+-    memset(cpi->consec_zero_mv, 0,
+-           cm->mi_rows * cm->mi_cols * sizeof(*cpi->consec_zero_mv));
++    vpx_free(cpi->consec_zero_mv);
++    CHECK_MEM_ERROR(
++        cm, cpi->consec_zero_mv,
++        vpx_calloc(cm->mi_rows * cm->mi_cols, sizeof(*cpi->consec_zero_mv)));
++
++    vpx_free(cpi->skin_map);
++    CHECK_MEM_ERROR(
++        cm, cpi->skin_map,
++        vpx_calloc(cm->mi_rows * cm->mi_cols, sizeof(cpi->skin_map[0])));
++
++    free_copy_partition_data(cpi);
++    alloc_copy_partition_data(cpi);
+     if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ)
+       vp9_cyclic_refresh_reset_resize(cpi);
+     rc->rc_1_frame = 0;
+-- 
+2.41.0
+

libvpx.spec

@@ -6,7 +6,7 @@
 Name:			libvpx
 Summary:		VP8/VP9 Video Codec SDK
 Version:		1.9.0
-Release:		5%{?dist}
+Release:		6%{?dist}
 License:		BSD
 #Source0:		http://downloads.webmproject.org/releases/webm/%{name}-%{version}.tar.bz2
 Source0:		https://github.com/webmproject/libvpx/archive/v%{version}.tar.gz
@@ -23,6 +23,7 @@ BuildRequires:		yasm
 BuildRequires:		doxygen, php-cli, perl(Getopt::Long)
 # Do not disable FORTIFY_SOURCE=2
 Patch0:			libvpx-1.7.0-leave-fortify-source-on.patch
+Patch1:			0001-Fix-bug-with-smaller-width-bigger-size.patch
 
 %description
 libvpx provides the VP8/VP9 SDK, which allows you to integrate your applications 
@@ -48,6 +49,7 @@ and decoder.
 %prep
 %setup -q -n libvpx-%{version}
 %patch0 -p1 -b .leave-fs-on
+%patch1 -p1 -b .p1
 
 %build
 
@@ -233,6 +235,10 @@ rm -rf %{buildroot}%{_prefix}/src
 %{_bindir}/*
 
 %changelog
+* Tue Oct 3 2023 Wim Taymans <wtaymans@redhat.com> - 1.9.0-6
+- Add patch for CVE-2023-44488
+- Resolves: RHEL-11636
+
 * Mon Aug 09 2021 Mohan Boddu <mboddu@redhat.com> - 1.9.0-5
 - Rebuilt for IMA sigs, glibc 2.34, aarch64 flags
   Related: rhbz#1991688