Added VA-API fix for mozbz#1645671

firefox.spec

@@ -114,7 +114,7 @@ ExcludeArch: s390x
 Summary:        Mozilla Firefox Web browser
 Name:           firefox
 Version:        79.0
-Release:        2%{?dist}
+Release:        3%{?dist}
 URL:            https://www.mozilla.org/firefox/
 License:        MPLv1.1 or GPLv2+ or LGPLv2+
 Source0:        https://archive.mozilla.org/pub/firefox/releases/%{version}%{?pre_version}/source/firefox-%{version}%{?pre_version}.source.tar.xz
@@ -183,6 +183,7 @@ Patch575:        firefox-pipewire-0-3.patch
 #VA-API patches
 Patch584:        firefox-disable-ffvpx-with-vapi.patch
 Patch585:        firefox-vaapi-extra-frames.patch
+Patch586:        mozilla-1645671.patch
 
 # PGO/LTO patches
 Patch600:        pgo.patch
@@ -388,6 +389,7 @@ This package contains results of tests executed during build.
 
 %patch584 -p1 -b .firefox-disable-ffvpx-with-vapi
 %patch585 -p1 -b .firefox-vaapi-extra-frames
+%patch586 -p1 -b .mozilla-1645671
 
 # PGO patches
 %patch600 -p1 -b .pgo
@@ -956,6 +958,9 @@ gtk-update-icon-cache %{_datadir}/icons/hicolor &>/dev/null || :
 #---------------------------------------------------------------------
 
 %changelog
+* Tue Jul 30 2020 Martin Stransky <stransky@redhat.com> - 79.0-3
+- Added VA-API fix for mozbz#1645671
+
 * Wed Jul 29 2020 Martin Stransky <stransky@redhat.com> - 79.0-2
 - Try to enable armv7hl again.
 - Disabled ppc64le due to cargo crash (rhbz#1862012).

mozilla-1645671.patch

@@ -0,0 +1,94 @@
+diff --git a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h
+--- a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h
++++ b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h
+@@ -156,7 +156,7 @@
+                                 MediaDataDecoder::DecodedData& aResults);
+ 
+   void ReleaseUnusedVAAPIFrames();
+-  DMABufSurfaceWrapper* GetUnusedDMABufSurfaceWrapper();
++  int GetUnusedDMABufSurfaceWrapperIndex();
+   void ReleaseDMABufSurfaces();
+ #endif
+ 
+@@ -174,7 +174,7 @@
+   const bool mDisableHardwareDecoding;
+   VADisplay mDisplay;
+   bool mUseDMABufSurfaces;
+-  nsTArray<DMABufSurfaceWrapper> mDMABufSurfaces;
++  nsTArray<UniquePtr<DMABufSurfaceWrapper>> mDMABufSurfaces;
+ #endif
+   RefPtr<KnowsCompositor> mImageAllocator;
+   RefPtr<ImageContainer> mImageContainer;
+diff --git a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp
+--- a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp
++++ b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp
+@@ -698,21 +698,20 @@
+ void FFmpegVideoDecoder<LIBAV_VER>::ReleaseUnusedVAAPIFrames() {
+   int len = mDMABufSurfaces.Length();
+   for (int i = 0; i < len; i++) {
+-    if (!mDMABufSurfaces[i].IsUsed()) {
+-      mDMABufSurfaces[i].ReleaseVAAPIData();
++    if (!mDMABufSurfaces[i]->IsUsed()) {
++      mDMABufSurfaces[i]->ReleaseVAAPIData();
+     }
+   }
+ }
+ 
+-DMABufSurfaceWrapper*
+-FFmpegVideoDecoder<LIBAV_VER>::GetUnusedDMABufSurfaceWrapper() {
++int FFmpegVideoDecoder<LIBAV_VER>::GetUnusedDMABufSurfaceWrapperIndex() {
+   int len = mDMABufSurfaces.Length();
+   for (int i = 0; i < len; i++) {
+-    if (!mDMABufSurfaces[i].IsUsed()) {
+-      return &(mDMABufSurfaces[i]);
++    if (!mDMABufSurfaces[i]->IsUsed()) {
++      return i;
+     }
+   }
+-  return nullptr;
++  return -1;
+ }
+ 
+ void FFmpegVideoDecoder<LIBAV_VER>::ReleaseDMABufSurfaces() {
+@@ -763,8 +762,8 @@
+ 
+   RefPtr<DMABufSurfaceYUV> surface;
+ 
+-  DMABufSurfaceWrapper* surfaceWrapper = GetUnusedDMABufSurfaceWrapper();
+-  if (!surfaceWrapper) {
++  int surfaceWrapperIndex = GetUnusedDMABufSurfaceWrapperIndex();
++  if (surfaceWrapperIndex < 0) {
+     if (mVAAPIDeviceContext) {
+       surface = DMABufSurfaceYUV::CreateYUVSurface(vaDesc);
+     } else {
+@@ -776,16 +775,16 @@
+       return MediaResult(NS_ERROR_OUT_OF_MEMORY,
+                          RESULT_DETAIL("Unable to get DMABufSurfaceYUV"));
+     }
+-
+ #  ifdef MOZ_LOGGING
+     static int uid = 0;
+     surface->SetUID(++uid);
+     FFMPEG_LOG("Created new DMABufSurface UID = %d", uid);
+ #  endif
+-    mDMABufSurfaces.AppendElement(DMABufSurfaceWrapper(surface, mLib));
+-    surfaceWrapper = &(mDMABufSurfaces[mDMABufSurfaces.Length() - 1]);
++    mDMABufSurfaces.AppendElement(
++        MakeUnique<DMABufSurfaceWrapper>(surface, mLib));
++    surfaceWrapperIndex = mDMABufSurfaces.Length() - 1;
+   } else {
+-    surface = surfaceWrapper->GetDMABufSurface();
++    surface = mDMABufSurfaces[surfaceWrapperIndex]->GetDMABufSurface();
+     bool ret;
+ 
+     if (mVAAPIDeviceContext) {
+@@ -803,7 +802,7 @@
+   }
+ 
+   if (mVAAPIDeviceContext) {
+-    surfaceWrapper->LockVAAPIData(mCodecContext, mFrame);
++    mDMABufSurfaces[surfaceWrapperIndex]->LockVAAPIData(mCodecContext, mFrame);
+   }
+ 
+   surface->SetYUVColorSpace(GetFrameColorSpace());
+