libvpx/0001-Fix-bug-with-smaller-width-bigger-size.patch

132 lines
4.5 KiB
Diff
Raw Permalink Normal View History

From bc54ed5c2d7b43a1904e04a59c268368c274169e Mon Sep 17 00:00:00 2001
From: Jerome Jiang <jianj@google.com>
Date: Thu, 30 Jun 2022 13:48:56 -0400
Subject: [PATCH] Fix bug with smaller width bigger size
Fixed previous patch that clusterfuzz failed on.
Local fuzzing passing overnight.
Bug: webm:1642
Change-Id: If0e08e72abd2e042efe4dcfac21e4cc51afdfdb9
(cherry picked from commit 263682c9a29395055f3b3afe2d97be1828a6223f)
---
test/resize_test.cc | 9 +++------
vp9/common/vp9_alloccommon.c | 13 ++++++-------
vp9/encoder/vp9_encoder.c | 27 +++++++++++++++++++++++++--
3 files changed, 34 insertions(+), 15 deletions(-)
diff --git a/test/resize_test.cc b/test/resize_test.cc
index b2eadb1e8..ebc5f9b02 100644
--- a/test/resize_test.cc
+++ b/test/resize_test.cc
@@ -101,11 +101,8 @@ void ScaleForFrameNumber(unsigned int frame, unsigned int initial_w,
*h = initial_h;
return;
}
- if (frame < 100) {
- *w = initial_w * 7 / 10;
- *h = initial_h * 16 / 10;
- return;
- }
+ *w = initial_w * 7 / 10;
+ *h = initial_h * 16 / 10;
return;
}
if (frame < 10) {
@@ -578,7 +575,7 @@ TEST_P(ResizeRealtimeTest, TestExternalResizeWorks) {
}
}
-TEST_P(ResizeRealtimeTest, DISABLED_TestExternalResizeSmallerWidthBiggerSize) {
+TEST_P(ResizeRealtimeTest, TestExternalResizeSmallerWidthBiggerSize) {
ResizingVideoSource video;
video.flag_codec_ = true;
video.smaller_width_larger_size_ = true;
diff --git a/vp9/common/vp9_alloccommon.c b/vp9/common/vp9_alloccommon.c
index 5702dca71..0328f33a4 100644
--- a/vp9/common/vp9_alloccommon.c
+++ b/vp9/common/vp9_alloccommon.c
@@ -131,13 +131,6 @@ int vp9_alloc_context_buffers(VP9_COMMON *cm, int width, int height) {
cm->free_mi(cm);
if (cm->alloc_mi(cm, new_mi_size)) goto fail;
}
-
- if (cm->seg_map_alloc_size < cm->mi_rows * cm->mi_cols) {
- // Create the segmentation map structure and set to 0.
- free_seg_map(cm);
- if (alloc_seg_map(cm, cm->mi_rows * cm->mi_cols)) goto fail;
- }
-
if (cm->above_context_alloc_cols < cm->mi_cols) {
vpx_free(cm->above_context);
cm->above_context = (ENTROPY_CONTEXT *)vpx_calloc(
@@ -152,6 +145,12 @@ int vp9_alloc_context_buffers(VP9_COMMON *cm, int width, int height) {
cm->above_context_alloc_cols = cm->mi_cols;
}
+ if (cm->seg_map_alloc_size < cm->mi_rows * cm->mi_cols) {
+ // Create the segmentation map structure and set to 0.
+ free_seg_map(cm);
+ if (alloc_seg_map(cm, cm->mi_rows * cm->mi_cols)) goto fail;
+ }
+
if (vp9_alloc_loop_filter(cm)) goto fail;
return 0;
diff --git a/vp9/encoder/vp9_encoder.c b/vp9/encoder/vp9_encoder.c
index 20e8d494a..055e1def0 100644
--- a/vp9/encoder/vp9_encoder.c
+++ b/vp9/encoder/vp9_encoder.c
@@ -1968,6 +1968,17 @@ static void alloc_copy_partition_data(VP9_COMP *cpi) {
}
}
+static void free_copy_partition_data(VP9_COMP *cpi) {
+ vpx_free(cpi->prev_partition);
+ cpi->prev_partition = NULL;
+ vpx_free(cpi->prev_segment_id);
+ cpi->prev_segment_id = NULL;
+ vpx_free(cpi->prev_variance_low);
+ cpi->prev_variance_low = NULL;
+ vpx_free(cpi->copied_frame_cnt);
+ cpi->copied_frame_cnt = NULL;
+}
+
void vp9_change_config(struct VP9_COMP *cpi, const VP9EncoderConfig *oxcf) {
VP9_COMMON *const cm = &cpi->common;
RATE_CONTROL *const rc = &cpi->rc;
@@ -2047,6 +2058,8 @@ void vp9_change_config(struct VP9_COMP *cpi, const VP9EncoderConfig *oxcf) {
new_mi_size = cm->mi_stride * calc_mi_size(cm->mi_rows);
if (cm->mi_alloc_size < new_mi_size) {
vp9_free_context_buffers(cm);
+ vp9_free_pc_tree(&cpi->td);
+ vpx_free(cpi->mbmi_ext_base);
alloc_compressor_data(cpi);
realloc_segmentation_maps(cpi);
cpi->initial_width = cpi->initial_height = 0;
@@ -2062,8 +2075,18 @@ void vp9_change_config(struct VP9_COMP *cpi, const VP9EncoderConfig *oxcf) {
update_frame_size(cpi);
if (last_w != cpi->oxcf.width || last_h != cpi->oxcf.height) {
- memset(cpi->consec_zero_mv, 0,
- cm->mi_rows * cm->mi_cols * sizeof(*cpi->consec_zero_mv));
+ vpx_free(cpi->consec_zero_mv);
+ CHECK_MEM_ERROR(
+ cm, cpi->consec_zero_mv,
+ vpx_calloc(cm->mi_rows * cm->mi_cols, sizeof(*cpi->consec_zero_mv)));
+
+ vpx_free(cpi->skin_map);
+ CHECK_MEM_ERROR(
+ cm, cpi->skin_map,
+ vpx_calloc(cm->mi_rows * cm->mi_cols, sizeof(cpi->skin_map[0])));
+
+ free_copy_partition_data(cpi);
+ alloc_copy_partition_data(cpi);
if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ)
vp9_cyclic_refresh_reset_resize(cpi);
rc->rc_1_frame = 0;
--
2.41.0