Bug 1979100 - Add infrastructure for importing video frames into wgpu textures. r=webgpu-reviewers,webidl,teoxoy,smaug

In order to support webgpu's GPUExternalTexture we must first be able
to import video frames into wgpu. To do so we introduce the concept of
an external texture "source". This consists of between 1 and 3
textures and texture views, in order to support single-plane RGBA or
multiplanar YCbCr data. The source is implemented as a distinct
concept from the external texture itself so that multiple external
textures can be created from the same source.

An external texture source has both a client-side and host-side
representation. The client side is responsible for managing the
lifecycle of the host side. It obtains a SurfaceDescriptor from the
current layers::Image of the video element or frame it was imported
from, and sends this to the host side. The host side is responsible
for creating the wgpu Textures and TextureViews it needs from the
SurfaceDescriptor. It also holds various pieces of information derived
from the imported frames that will be required to construct the
wgpu::ExternalTextureDescriptor required to create a
wgpu::ExternalTexture.

For now, we always call Image::BuildSurfaceDescriptorBuffer to obtain
the surface descriptor, and therefore only have to handle
SurfaceDescriptorBuffers. In subsequent patches we will handle
GPUVideo surface descriptors to directly import hardware decoded video
frames without requiring a readback.

Differential Revision: https://phabricator.services.mozilla.com/D259819

Author: jamienicol

dom/webgpu/ExternalTexture.cpp

@@ -6,10 +6,28 @@
 #ifndef ExternalTexture_H_
 #define ExternalTexture_H_
 
+#include <array>
+
 #include "ObjectModel.h"
+#include "mozilla/Span.h"
+#include "mozilla/gfx/Types.h"
+#include "mozilla/webgpu/WebGPUTypes.h"
 #include "nsIGlobalObject.h"
+#include "nsTArrayForwardDeclare.h"
+
+namespace mozilla {
+namespace dom {
+class OwningHTMLVideoElementOrVideoFrame;
+}
+namespace layers {
+class BufferDescriptor;
+class Image;
+}  // namespace layers
+
+namespace webgpu {
 
-namespace mozilla::webgpu {
+class Device;
+class WebGPUParent;
 
 // NOTE: Incomplete. Follow-up to complete implementation is at
 // <https://bugzilla.mozilla.org/show_bug.cgi?id=1827116>.
@@ -29,6 +47,96 @@ class ExternalTexture : public ObjectBase {
   void Cleanup() {}
 };
 
-}  // namespace mozilla::webgpu
+// The client side of an imported external texture source. This gets imported
+// from either an HTMLVideoElement or a VideoFrame. ExternalTextures can then
+// be created from a source. It is important to separate the source from the
+// external texture as multiple external textures can be created from the same
+// source.
+// The client side is responsible for creating and destroying the host side.
+// Any external texture created from this source must ensure the source remains
+// alive as long as it is required by the external texture, by holding a strong
+// reference.
+class ExternalTextureSourceClient {
+  NS_INLINE_DECL_REFCOUNTING(ExternalTextureSourceClient)
+
+ public:
+  // Creates an ExternalTextureSourceClient from a video element or video frame.
+  // Returns nullptr on failure. Throws security error if the source is not
+  // origin-clean.
+  static already_AddRefed<ExternalTextureSourceClient> Create(
+      Device* aDevice, const dom::OwningHTMLVideoElementOrVideoFrame& aSource,
+      ErrorResult& aRv);
+
+  const RawId mId;
+
+  // External texture sources can consist of up to 3 planes of texture data, but
+  // on the client side we do not know how many planes will actually be
+  // required. We therefore unconditionally make IDs for 3 textures and 3
+  // texture views, and the host side will only use the IDs that it requires.
+  const std::array<RawId, 3> mTextureIds;
+  const std::array<RawId, 3> mViewIds;
+
+ private:
+  ExternalTextureSourceClient(WebGPUChild* aBridge, RawId aId,
+                              const std::array<RawId, 3>& aTextureIds,
+                              const std::array<RawId, 3>& aViewIds);
+  ~ExternalTextureSourceClient();
+
+  // Used to free resources on the host side when we are destroyed, if the
+  // bridge is still valid.
+  const WeakPtr<WebGPUChild> mBridge;
+};
+
+// Host side of an external texture source. This is responsible for creating
+// and managing the lifecycle of the wgpu textures and texture views created
+// from the provided SurfaceDescriptor.
+class ExternalTextureSourceHost {
+ public:
+  // Creates an external texture source from a descriptor. If this fails it
+  // will create an external texture source in an error state, which will be
+  // propagated to any external textures created from it.
+  static ExternalTextureSourceHost Create(
+      WebGPUParent* aParent, RawId aDeviceId, RawId aQueueId,
+      const ExternalTextureSourceDescriptor& aDesc);
+
+  // Texture and TextureView IDs used by the source. These will be a subset of
+  // the IDs provided by the client in the descriptor.
+  Span<const RawId> TextureIds() const { return mTextureIds; }
+  Span<const RawId> ViewIds() const { return mViewIds; }
+
+ private:
+  ExternalTextureSourceHost(Span<const RawId> aTextureIds,
+                            Span<const RawId> aViewIds, gfx::IntSize aSize,
+                            gfx::SurfaceFormat aFormat,
+                            gfx::YUVRangedColorSpace aColorSpace,
+                            const std::array<float, 6>& aSampleTransform,
+                            const std::array<float, 6>& aLoadTransform);
+
+  static ExternalTextureSourceHost CreateFromBufferDesc(
+      WebGPUParent* aParent, RawId aDeviceId, RawId aQueueId,
+      const ExternalTextureSourceDescriptor& aDesc,
+      const layers::BufferDescriptor& aSd, uint8_t* aBuffer);
+
+  // Creates an external texture source in an error state that will be
+  // propagated to any external textures created from it.
+  static ExternalTextureSourceHost CreateError();
+
+  // These should be const but can't be else we wouldn't be move constructible.
+  // While we are always provided with 3 texture IDs and 3 view IDs by the
+  // client, we only store here the IDs that are actually used. For example an
+  // RGBA format source will only require 1 texture and 1 view. NV12 will
+  // require 2 views, and either 1 or 2 textures depending on whether the
+  // platform natively supports NV12 format textures.
+  AutoTArray<RawId, 3> mTextureIds;
+  AutoTArray<RawId, 3> mViewIds;
+  const gfx::IntSize mSize;
+  MOZ_MAYBE_UNUSED const gfx::SurfaceFormat mFormat;
+  MOZ_MAYBE_UNUSED const gfx::YUVRangedColorSpace mColorSpace;
+  MOZ_MAYBE_UNUSED const std::array<float, 6> mSampleTransform;
+  MOZ_MAYBE_UNUSED const std::array<float, 6> mLoadTransform;
+};
+
+}  // namespace webgpu
+}  // namespace mozilla
 
 #endif  // GPU_ExternalTexture_H_

dom/webgpu/ExternalTexture.h

@@ -15,6 +15,7 @@ using mozilla::dom::GPUErrorFilter from "mozilla/dom/WebGPUBinding.h";
 using mozilla::dom::GPURequestAdapterOptions from "mozilla/dom/WebGPUBinding.h";
 using mozilla::dom::GPUCommandBufferDescriptor from "mozilla/dom/WebGPUBinding.h";
 using mozilla::dom::GPUBufferDescriptor from "mozilla/dom/WebGPUBinding.h";
+using mozilla::webgpu::ExternalTextureSourceDescriptor from "mozilla/webgpu/WebGPUTypes.h";
 using mozilla::webgpu::PopErrorScopeResult from "mozilla/webgpu/WebGPUTypes.h";
 using mozilla::webgpu::WebGPUCompilationMessage from "mozilla/webgpu/WebGPUTypes.h";
 [MoveOnly] using mozilla::ipc::MutableSharedMemoryHandle from "mozilla/ipc/SharedMemoryHandle.h";
@@ -42,6 +43,16 @@ async protocol PWebGPU
 parent:
   async Messages(uint32_t nrOfMessages, ByteBuf serializedMessages, ByteBuf[] dataBuffers, MutableSharedMemoryHandle[] shmems);
 
+  // Ideally this would be sent in Messages(), but we need to implement
+  // SurfaceDescriptor serialization first.
+  async CreateExternalTextureSource(
+    RawId deviceId,
+    RawId queueId,
+    RawId externalTextureSourceId,
+    ExternalTextureSourceDescriptor aDesc
+  );
+
+
 child:
   async ServerMessage(ByteBuf buf);
 

dom/webgpu/ipc/PWebGPU.ipdl

@@ -7,6 +7,7 @@
 
 #include <unordered_set>
 
+#include "ExternalTexture.h"
 #include "mozilla/PodOperations.h"
 #include "mozilla/ScopeExit.h"
 #include "mozilla/dom/WebGPUBinding.h"
@@ -171,6 +172,18 @@ extern void wgpu_server_remove_shared_texture(WGPUWebGPUParentPtr aParent,
   parent->RemoveSharedTexture(aId);
 }
 
+extern void wgpu_parent_destroy_external_texture_source(
+    WGPUWebGPUParentPtr aParent, WGPUExternalTextureSourceId aId) {
+  auto* const parent = static_cast<WebGPUParent*>(aParent);
+  parent->DestroyExternalTextureSource(aId);
+}
+
+extern void wgpu_parent_drop_external_texture_source(
+    WGPUWebGPUParentPtr aParent, WGPUExternalTextureSourceId aId) {
+  auto* const parent = static_cast<WebGPUParent*>(aParent);
+  parent->DropExternalTextureSource(aId);
+}
+
 extern void wgpu_server_dealloc_buffer_shmem(WGPUWebGPUParentPtr aParent,
                                              WGPUBufferId aId) {
   auto* parent = static_cast<WebGPUParent*>(aParent);
@@ -779,6 +792,28 @@ void WebGPUParent::RemoveSharedTexture(RawId aTextureId) {
   }
 }
 
+void WebGPUParent::DestroyExternalTextureSource(RawId aSourceId) {
+  auto it = mExternalTextureSources.find(aSourceId);
+  if (it != mExternalTextureSources.end()) {
+    for (const auto textureId : it->second.TextureIds()) {
+      ffi::wgpu_server_texture_destroy(mContext.get(), textureId);
+    }
+  }
+}
+
+void WebGPUParent::DropExternalTextureSource(RawId aSourceId) {
+  auto it = mExternalTextureSources.find(aSourceId);
+  if (it != mExternalTextureSources.end()) {
+    for (const auto viewId : it->second.ViewIds()) {
+      ffi::wgpu_server_texture_view_drop(mContext.get(), viewId);
+    }
+    for (const auto textureId : it->second.TextureIds()) {
+      ffi::wgpu_server_texture_drop(mContext.get(), textureId);
+    }
+    mExternalTextureSources.erase(it);
+  }
+}
+
 void WebGPUParent::QueueSubmit(RawId aQueueId, RawId aDeviceId,
                                Span<const RawId> aCommandBuffers,
                                Span<const RawId> aTextureIds) {
@@ -1529,6 +1564,18 @@ ipc::IPCResult WebGPUParent::RecvMessages(
   return IPC_OK();
 }
 
+ipc::IPCResult WebGPUParent::RecvCreateExternalTextureSource(
+    RawId aDeviceId, RawId aQueueId, RawId aExternalTextureSourceId,
+    const ExternalTextureSourceDescriptor& aDesc) {
+  MOZ_RELEASE_ASSERT(mExternalTextureSources.find(aExternalTextureSourceId) ==
+                     mExternalTextureSources.end());
+  mExternalTextureSources.emplace(
+      aExternalTextureSourceId,
+      ExternalTextureSourceHost::Create(this, aDeviceId, aQueueId, aDesc));
+
+  return IPC_OK();
+}
+
 void WebGPUParent::DevicePushErrorScope(RawId aDeviceId,
                                         const dom::GPUErrorFilter aFilter) {
   const auto& itr = mErrorScopeStackByDevice.find(aDeviceId);

dom/webgpu/ipc/WebGPUParent.cpp

@@ -12,6 +12,7 @@
 #include "base/timer.h"
 #include "mozilla/WeakPtr.h"
 #include "mozilla/ipc/SharedMemoryHandle.h"
+#include "mozilla/webgpu/ExternalTexture.h"
 #include "mozilla/webgpu/PWebGPUParent.h"
 #include "mozilla/webgpu/ffi/wgpu.h"
 #include "mozilla/webrender/WebRenderAPI.h"
@@ -102,6 +103,9 @@ class WebGPUParent final : public PWebGPUParent, public SupportsWeakPtr {
                               ipc::ByteBuf&& aSerializedMessages,
                               nsTArray<ipc::ByteBuf>&& aDataBuffers,
                               nsTArray<MutableSharedMemoryHandle>&& aShmems);
+  ipc::IPCResult RecvCreateExternalTextureSource(
+      RawId aDeviceId, RawId aQueueId, RawId aExternalTextureSourceId,
+      const ExternalTextureSourceDescriptor& aDesc);
   void QueueSubmit(RawId aQueueId, RawId aDeviceId,
                    Span<const RawId> aCommandBuffers,
                    Span<const RawId> aTextureIds);
@@ -179,6 +183,10 @@ class WebGPUParent final : public PWebGPUParent, public SupportsWeakPtr {
   RefPtr<gfx::FileHandleWrapper> GetDeviceFenceHandle(const RawId aDeviceId);
 
   void RemoveSharedTexture(RawId aTextureId);
+
+  void DestroyExternalTextureSource(RawId aId);
+  void DropExternalTextureSource(RawId aId);
+
   void DeallocBufferShmem(RawId aBufferId);
   void PreDeviceDrop(RawId aDeviceId);
 
@@ -241,6 +249,8 @@ class WebGPUParent final : public PWebGPUParent, public SupportsWeakPtr {
   std::unordered_map<ffi::WGPUTextureId, std::shared_ptr<SharedTexture>>
       mSharedTextures;
 
+  std::unordered_map<RawId, ExternalTextureSourceHost> mExternalTextureSources;
+
   // Store a set of DeviceIds that have been SendDeviceLost. We use this to
   // limit each Device to one DeviceLost message.
   nsTHashSet<RawId> mLostDeviceIds;

dom/webgpu/ipc/WebGPUParent.h

@@ -8,8 +8,12 @@
 
 #include <cstdint>
 
+#include "mozilla/GfxMessageUtils.h"
 #include "mozilla/Maybe.h"
+#include "mozilla/ParamTraits_STL.h"
 #include "mozilla/dom/BindingDeclarations.h"
+#include "mozilla/ipc/IPDLParamTraits.h"
+#include "mozilla/layers/LayersSurfaces.h"
 #include "nsString.h"
 
 namespace mozilla::dom {
@@ -78,6 +82,42 @@ class StringHelper {
   Maybe<NS_ConvertUTF16toUTF8> mNarrow;
 };
 
+// Used to create an ExternalTextureSourceHost.
+struct ExternalTextureSourceDescriptor {
+  std::array<RawId, 3> mTextureIds;
+  std::array<RawId, 3> mViewIds;
+  layers::SurfaceDescriptor mSurfaceDescriptor;
+  gfx::IntSize mSize;
+  std::array<float, 6> mSampleTransform;
+  std::array<float, 6> mLoadTransform;
+};
+
 }  // namespace mozilla::webgpu
 
+namespace IPC {
+template <>
+struct ParamTraits<mozilla::webgpu::ExternalTextureSourceDescriptor> {
+  using ParamType = mozilla::webgpu::ExternalTextureSourceDescriptor;
+
+  static void Write(MessageWriter* aWriter, const ParamType& aParam) {
+    WriteParam(aWriter, aParam.mTextureIds);
+    WriteParam(aWriter, aParam.mViewIds);
+    WriteParam(aWriter, aParam.mSurfaceDescriptor);
+    WriteParam(aWriter, aParam.mSize);
+    WriteParam(aWriter, aParam.mSampleTransform);
+    WriteParam(aWriter, aParam.mLoadTransform);
+  }
+
+  static bool Read(MessageReader* aReader, ParamType* aResult) {
+    return ReadParam(aReader, &aResult->mTextureIds) &&
+           ReadParam(aReader, &aResult->mViewIds) &&
+           ReadParam(aReader, &aResult->mSurfaceDescriptor) &&
+           ReadParam(aReader, &aResult->mSize) &&
+           ReadParam(aReader, &aResult->mSampleTransform) &&
+           ReadParam(aReader, &aResult->mLoadTransform);
+  }
+};
+
+}  // namespace IPC
+
 #endif  // WEBGPU_TYPES_H_

dom/webgpu/ipc/WebGPUTypes.h

@@ -456,6 +456,12 @@ interface GPUExternalTexture {
 };
 GPUExternalTexture includes GPUObjectBase;
 
+dictionary GPUExternalTextureDescriptor
+         : GPUObjectDescriptorBase {
+    required (HTMLVideoElement or VideoFrame) source;
+    PredefinedColorSpace colorSpace = "srgb";
+};
+
 [Func="mozilla::webgpu::Instance::PrefEnabled",
  Exposed=(Window, Worker), SecureContext]
 interface GPUSampler {