firefox/mozilla-1630754.patch

diff --git a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h
--- a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h
+++ b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h
@@ -116,9 +116,9 @@
 #ifdef MOZ_WAYLAND
   const AVCodecHWConfig* (*avcodec_get_hw_config)(const AVCodec* codec,
                                                   int index);
-  int (*av_hwdevice_ctx_create)(AVBufferRef** device_ctx, int type,
-                                const char* device, AVDictionary* opts,
-                                int flags);
+  AVBufferRef* (*av_hwdevice_ctx_alloc)(int);
+  int (*av_hwdevice_ctx_init)(AVBufferRef* ref);
+
   AVBufferRef* (*av_buffer_ref)(AVBufferRef* buf);
   void (*av_buffer_unref)(AVBufferRef** buf);
   int (*av_hwframe_transfer_get_formats)(AVBufferRef* hwframe_ctx, int dir,
@@ -132,12 +132,16 @@
 
   int (*vaExportSurfaceHandle)(void*, unsigned int, uint32_t, uint32_t, void*);
   int (*vaSyncSurface)(void*, unsigned int);
+  int (*vaInitialize)(void* dpy, int* major_version, int* minor_version);
+  int (*vaTerminate)(void* dpy);
+  void* (*vaGetDisplayWl)(struct wl_display* display);
 #endif
 
   PRLibrary* mAVCodecLib;
   PRLibrary* mAVUtilLib;
 #ifdef MOZ_WAYLAND
   PRLibrary* mVALib;
+  PRLibrary* mVALibWayland;
 #endif
 
  private:
diff --git a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp
--- a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp
+++ b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp
@@ -159,7 +159,8 @@
   AV_FUNC_OPTION(av_frame_get_color_range, AV_FUNC_AVUTIL_ALL)
 #ifdef MOZ_WAYLAND
   AV_FUNC_OPTION_SILENT(avcodec_get_hw_config, AV_FUNC_58)
-  AV_FUNC_OPTION_SILENT(av_hwdevice_ctx_create, AV_FUNC_58)
+  AV_FUNC_OPTION_SILENT(av_hwdevice_ctx_init, AV_FUNC_58)
+  AV_FUNC_OPTION_SILENT(av_hwdevice_ctx_alloc, AV_FUNC_58)
   AV_FUNC_OPTION_SILENT(av_buffer_ref, AV_FUNC_AVUTIL_58)
   AV_FUNC_OPTION_SILENT(av_buffer_unref, AV_FUNC_AVUTIL_58)
   AV_FUNC_OPTION_SILENT(av_hwframe_transfer_get_formats, AV_FUNC_58)
@@ -181,8 +182,21 @@
   if (mVALib) {
     VA_FUNC_OPTION_SILENT(vaExportSurfaceHandle)
     VA_FUNC_OPTION_SILENT(vaSyncSurface)
+    VA_FUNC_OPTION_SILENT(vaInitialize)
+    VA_FUNC_OPTION_SILENT(vaTerminate)
   }
-#  undef VA_FUNC_OPTION
+#  undef VA_FUNC_OPTION_SILENT
+
+#  define VAW_FUNC_OPTION_SILENT(func)                                   \
+    if (!(func = (decltype(func))PR_FindSymbol(mVALibWayland, #func))) { \
+      FFMPEG_LOG("Couldn't load function " #func);                       \
+    }
+
+  // mVALibWayland is optional and may not be present.
+  if (mVALibWayland) {
+    VAW_FUNC_OPTION_SILENT(vaGetDisplayWl)
+  }
+#  undef VAW_FUNC_OPTION_SILENT
 #endif
 
   avcodec_register_all();
@@ -218,6 +232,9 @@
   if (mVALib) {
     PR_UnloadLibrary(mVALib);
   }
+  if (mVALibWayland) {
+    PR_UnloadLibrary(mVALibWayland);
+  }
 #endif
   PodZero(this);
 }
@@ -226,13 +243,16 @@
 bool FFmpegLibWrapper::IsVAAPIAvailable() {
 #  define VA_FUNC_LOADED(func) (func != nullptr)
   return VA_FUNC_LOADED(avcodec_get_hw_config) &&
-         VA_FUNC_LOADED(av_hwdevice_ctx_create) &&
+         VA_FUNC_LOADED(av_hwdevice_ctx_alloc) &&
+         VA_FUNC_LOADED(av_hwdevice_ctx_init) &&
          VA_FUNC_LOADED(av_buffer_ref) && VA_FUNC_LOADED(av_buffer_unref) &&
          VA_FUNC_LOADED(av_hwframe_transfer_get_formats) &&
          VA_FUNC_LOADED(av_hwdevice_ctx_create_derived) &&
          VA_FUNC_LOADED(av_hwframe_ctx_alloc) && VA_FUNC_LOADED(av_dict_set) &&
          VA_FUNC_LOADED(av_dict_free) &&
-         VA_FUNC_LOADED(vaExportSurfaceHandle) && VA_FUNC_LOADED(vaSyncSurface);
+         VA_FUNC_LOADED(vaExportSurfaceHandle) &&
+         VA_FUNC_LOADED(vaSyncSurface) && VA_FUNC_LOADED(vaInitialize) &&
+         VA_FUNC_LOADED(vaTerminate) && VA_FUNC_LOADED(vaGetDisplayWl);
 }
 #endif
 
diff --git a/dom/media/platforms/ffmpeg/FFmpegRuntimeLinker.cpp b/dom/media/platforms/ffmpeg/FFmpegRuntimeLinker.cpp
--- a/dom/media/platforms/ffmpeg/FFmpegRuntimeLinker.cpp
+++ b/dom/media/platforms/ffmpeg/FFmpegRuntimeLinker.cpp
@@ -9,6 +9,9 @@
 #include "mozilla/ArrayUtils.h"
 #include "FFmpegLog.h"
 #include "prlink.h"
+#ifdef MOZ_WAYLAND
+#  include "gfxPlatformGtk.h"
+#endif
 
 namespace mozilla {
 
@@ -54,21 +57,33 @@
   }
 
 #ifdef MOZ_WAYLAND
-  {
-    const char* lib = "libva.so.2";
+  if (gfxPlatformGtk::GetPlatform()->UseWaylandHardwareVideoDecoding()) {
+    const char* libWayland = "libva-wayland.so.2";
     PRLibSpec lspec;
     lspec.type = PR_LibSpec_Pathname;
-    lspec.value.pathname = lib;
-    sLibAV.mVALib = PR_LoadLibraryWithFlags(lspec, PR_LD_NOW | PR_LD_LOCAL);
-    // Don't use libva when it's missing vaExportSurfaceHandle.
-    if (sLibAV.mVALib &&
-        !PR_FindSymbol(sLibAV.mVALib, "vaExportSurfaceHandle")) {
-      PR_UnloadLibrary(sLibAV.mVALib);
-      sLibAV.mVALib = nullptr;
+    lspec.value.pathname = libWayland;
+    sLibAV.mVALibWayland =
+        PR_LoadLibraryWithFlags(lspec, PR_LD_NOW | PR_LD_LOCAL);
+    if (!sLibAV.mVALibWayland) {
+      FFMPEG_LOG("VA-API support: Missing or old %s library.\n", libWayland);
     }
-    if (!sLibAV.mVALib) {
-      FFMPEG_LOG("VA-API support: Missing or old %s library.\n", lib);
+
+    if (sLibAV.mVALibWayland) {
+      const char* lib = "libva.so.2";
+      lspec.value.pathname = lib;
+      sLibAV.mVALib = PR_LoadLibraryWithFlags(lspec, PR_LD_NOW | PR_LD_LOCAL);
+      // Don't use libva when it's missing vaExportSurfaceHandle.
+      if (sLibAV.mVALib &&
+          !PR_FindSymbol(sLibAV.mVALib, "vaExportSurfaceHandle")) {
+        PR_UnloadLibrary(sLibAV.mVALib);
+        sLibAV.mVALib = nullptr;
+      }
+      if (!sLibAV.mVALib) {
+        FFMPEG_LOG("VA-API support: Missing or old %s library.\n", lib);
+      }
     }
+  } else {
+    FFMPEG_LOG("VA-API FFmpeg is disabled by platform");
   }
 #endif
 
diff --git a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h
--- a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h
+++ b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h
@@ -111,6 +111,7 @@
 #ifdef MOZ_WAYLAND_USE_VAAPI
   AVBufferRef* mVAAPIDeviceContext;
   const bool mDisableHardwareDecoding;
+  VADisplay mDisplay;
 #endif
   RefPtr<KnowsCompositor> mImageAllocator;
   RefPtr<ImageContainer> mImageContainer;
diff --git a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp
--- a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp
+++ b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp
@@ -154,14 +154,44 @@
   return nullptr;
 }
 
+class VAAPIDisplayHolder {
+ public:
+  VAAPIDisplayHolder(FFmpegLibWrapper* aLib, VADisplay aDisplay)
+      : mLib(aLib), mDisplay(aDisplay){};
+  ~VAAPIDisplayHolder() { mLib->vaTerminate(mDisplay); }
+
+ private:
+  FFmpegLibWrapper* mLib;
+  VADisplay mDisplay;
+};
+
+static void VAAPIDisplayReleaseCallback(struct AVHWDeviceContext* hwctx) {
+  auto displayHolder = static_cast<VAAPIDisplayHolder*>(hwctx->user_opaque);
+  delete displayHolder;
+}
+
 bool FFmpegVideoDecoder<LIBAV_VER>::CreateVAAPIDeviceContext() {
-  AVDictionary* opts = nullptr;
-  mLib->av_dict_set(&opts, "connection_type", "drm", 0);
-  bool ret =
-      (mLib->av_hwdevice_ctx_create(
-           &mVAAPIDeviceContext, AV_HWDEVICE_TYPE_VAAPI, NULL, NULL, 0) == 0);
-  mLib->av_dict_free(&opts);
-  if (!ret) {
+  mVAAPIDeviceContext = mLib->av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VAAPI);
+  if (!mVAAPIDeviceContext) {
+    return false;
+  }
+  AVHWDeviceContext* hwctx = (AVHWDeviceContext*)mVAAPIDeviceContext->data;
+  AVVAAPIDeviceContext* vactx = (AVVAAPIDeviceContext*)hwctx->hwctx;
+
+  mDisplay = mLib->vaGetDisplayWl(widget::WaylandDisplayGet()->GetDisplay());
+
+  hwctx->user_opaque = new VAAPIDisplayHolder(mLib, mDisplay);
+  hwctx->free = VAAPIDisplayReleaseCallback;
+
+  int major, minor;
+  int status = mLib->vaInitialize(mDisplay, &major, &minor);
+  if (status != VA_STATUS_SUCCESS) {
+    return false;
+  }
+
+  vactx->display = mDisplay;
+
+  if (mLib->av_hwdevice_ctx_init(mVAAPIDeviceContext) < 0) {
     return false;
   }
 
@@ -172,6 +202,11 @@
 MediaResult FFmpegVideoDecoder<LIBAV_VER>::InitVAAPIDecoder() {
   FFMPEG_LOG("Initialising VA-API FFmpeg decoder");
 
+  if (!mLib->IsVAAPIAvailable()) {
+    FFMPEG_LOG("libva library or symbols are missing.");
+    return NS_ERROR_NOT_AVAILABLE;
+  }
+
   auto layersBackend = mImageAllocator
                            ? mImageAllocator->GetCompositorBackendType()
                            : layers::LayersBackend::LAYERS_BASIC;
@@ -181,16 +216,6 @@
     return NS_ERROR_NOT_AVAILABLE;
   }
 
-  if (!mLib->IsVAAPIAvailable()) {
-    FFMPEG_LOG("libva library or symbols are missing.");
-    return NS_ERROR_NOT_AVAILABLE;
-  }
-
-  if (!gfxPlatformGtk::GetPlatform()->UseWaylandHardwareVideoDecoding()) {
-    FFMPEG_LOG("VA-API FFmpeg is disabled by platform");
-    return NS_ERROR_NOT_AVAILABLE;
-  }
-
   AVCodec* codec = FindVAAPICodec();
   if (!codec) {
     FFMPEG_LOG("Couldn't find ffmpeg VA-API decoder");
@@ -275,6 +300,7 @@
 #ifdef MOZ_WAYLAND_USE_VAAPI
       mVAAPIDeviceContext(nullptr),
       mDisableHardwareDecoding(aDisableHardwareDecoding),
+      mDisplay(nullptr),
 #endif
       mImageAllocator(aAllocator),
       mImageContainer(aImageContainer),
@@ -606,16 +632,10 @@
              " duration=%" PRId64 " opaque=%" PRId64,
              aPts, mFrame->pkt_dts, aDuration, mCodecContext->reordered_opaque);
 
-  AVHWDeviceContext* device_ctx = (AVHWDeviceContext*)mVAAPIDeviceContext->data;
-  AVVAAPIDeviceContext* VAAPIDeviceContext =
-      (AVVAAPIDeviceContext*)device_ctx->hwctx;
   VADRMPRIMESurfaceDescriptor va_desc;
-
   VASurfaceID surface_id = (VASurfaceID)(uintptr_t)mFrame->data[3];
-
   VAStatus vas = mLib->vaExportSurfaceHandle(
-      VAAPIDeviceContext->display, surface_id,
-      VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2,
+      mDisplay, surface_id, VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2,
       VA_EXPORT_SURFACE_READ_ONLY | VA_EXPORT_SURFACE_SEPARATE_LAYERS,
       &va_desc);
   if (vas != VA_STATUS_SUCCESS) {
@@ -623,7 +643,7 @@
         NS_ERROR_OUT_OF_MEMORY,
         RESULT_DETAIL("Unable to get frame by vaExportSurfaceHandle()"));
   }
-  vas = mLib->vaSyncSurface(VAAPIDeviceContext->display, surface_id);
+  vas = mLib->vaSyncSurface(mDisplay, surface_id);
   if (vas != VA_STATUS_SUCCESS) {
     NS_WARNING("vaSyncSurface() failed.");
   }
diff --git a/dom/media/platforms/ffmpeg/moz.build b/dom/media/platforms/ffmpeg/moz.build
--- a/dom/media/platforms/ffmpeg/moz.build
+++ b/dom/media/platforms/ffmpeg/moz.build
@@ -20,4 +20,7 @@
     'FFmpegRuntimeLinker.cpp',
 ]
 
+if CONFIG['MOZ_WAYLAND']:
+  include('/ipc/chromium/chromium-config.mozbuild')
+
 FINAL_LIBRARY = 'xul'