Use GPU texture in desktop capture This change adds a FrameTexture member in the DesktopFrame class, so that WGC desktop capture can pass DXGIResource handles to supported frame consumers to save CPU-GPU copies. Bug: chromium:40929600 Change-Id: Ide143596272e4b72c099afa2d366f58e5ef835c4 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/446781 Reviewed-by: Henrik Andreassson <henrika@webrtc.org> Reviewed-by: Markus Handell <handellm@webrtc.org> Commit-Queue: Wang, Zhibo1 <zhibo1.wang@intel.com> Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org> Reviewed-by: Harald Alvestrand <hta@webrtc.org> Reviewed-by: Alexander Cooper <alcooper@chromium.org> Cr-Commit-Position: refs/heads/main@{#47254}
diff --git a/modules/desktop_capture/BUILD.gn b/modules/desktop_capture/BUILD.gn index 78d6fc4..3da5304 100644 --- a/modules/desktop_capture/BUILD.gn +++ b/modules/desktop_capture/BUILD.gn
@@ -28,6 +28,8 @@ "desktop_geometry.h", "desktop_region.cc", "desktop_region.h", + "frame_texture.cc", + "frame_texture.h", "shared_desktop_frame.cc", "shared_desktop_frame.h", "shared_memory.cc", @@ -41,6 +43,7 @@ "../../rtc_base:refcount", "../../rtc_base:stringutils", "../../rtc_base/system:rtc_export", + "//third_party/abseil-cpp/absl/base:nullability", "//third_party/libyuv", ] } @@ -140,6 +143,7 @@ "desktop_and_cursor_composer_unittest.cc", "desktop_capturer_differ_wrapper_unittest.cc", "desktop_frame_rotation_unittest.cc", + "desktop_frame_texture_unittest.cc", "desktop_frame_unittest.cc", "desktop_geometry_unittest.cc", "desktop_region_unittest.cc", @@ -591,6 +595,8 @@ if (rtc_enable_win_wgc) { sources += [ + "win/dxgi_desktop_frame.cc", + "win/dxgi_desktop_frame.h", "win/wgc_capture_session.cc", "win/wgc_capture_session.h", "win/wgc_capture_source.cc",
diff --git a/modules/desktop_capture/desktop_capture_options.h b/modules/desktop_capture/desktop_capture_options.h index bc1342a..3ce4678 100644 --- a/modules/desktop_capture/desktop_capture_options.h +++ b/modules/desktop_capture/desktop_capture_options.h
@@ -227,6 +227,13 @@ void set_wgc_include_secondary_windows(bool include) { wgc_include_secondary_windows_ = include; } + + // This flag enables native texture of frame with the WGC capturer. + // The flag has no effect if the allow_wgc_capturer flag is false. + bool allow_wgc_using_texture() const { return allow_wgc_using_texture_; } + void set_allow_wgc_using_texture(bool allow) { + allow_wgc_using_texture_ = allow; + } #endif // defined(RTC_ENABLE_WIN_WGC) #endif // defined(WEBRTC_WIN) @@ -285,6 +292,7 @@ bool allow_wgc_zero_hertz_ = false; bool wgc_require_border_ = false; bool wgc_include_secondary_windows_ = false; + bool allow_wgc_using_texture_ = false; #endif #endif #if defined(WEBRTC_USE_X11)
diff --git a/modules/desktop_capture/desktop_frame.cc b/modules/desktop_capture/desktop_frame.cc index 1f0f604..bfba612 100644 --- a/modules/desktop_capture/desktop_frame.cc +++ b/modules/desktop_capture/desktop_frame.cc
@@ -29,9 +29,11 @@ int stride, FourCC pixel_format, uint8_t* data, - SharedMemory* shared_memory) + SharedMemory* shared_memory, + FrameTexture* texture) : data_(data), shared_memory_(shared_memory), + texture_(texture), size_(size), stride_(stride), pixel_format_(pixel_format), @@ -134,6 +136,7 @@ } uint8_t* DesktopFrame::GetFrameDataAtPos(const DesktopVector& pos) const { + RTC_DCHECK(data()); return data() + stride() * pos.y() + DesktopFrame::kBytesPerPixel * pos.x(); } @@ -160,6 +163,7 @@ } bool DesktopFrame::FrameDataIsBlack() const { + RTC_DCHECK(data()); if (size().is_empty()) return false; @@ -172,6 +176,7 @@ } void DesktopFrame::SetFrameDataToBlack() { + RTC_DCHECK(data()); const uint8_t kBlackPixelValue = 0x00; memset(data(), kBlackPixelValue, stride() * size().height()); }
diff --git a/modules/desktop_capture/desktop_frame.h b/modules/desktop_capture/desktop_frame.h index 104fe63..bda6b7f 100644 --- a/modules/desktop_capture/desktop_frame.h +++ b/modules/desktop_capture/desktop_frame.h
@@ -17,9 +17,11 @@ #include <optional> #include <vector> +#include "absl/base/nullability.h" #include "media/base/video_common.h" #include "modules/desktop_capture/desktop_geometry.h" #include "modules/desktop_capture/desktop_region.h" +#include "modules/desktop_capture/frame_texture.h" #include "modules/desktop_capture/shared_memory.h" #include "rtc_base/system/rtc_export.h" @@ -28,6 +30,8 @@ const float kStandardDPI = 96.0f; // DesktopFrame represents a video frame captured from the screen. +// TODO(crbug.com/40929600): Make data-dependent methods virtual so +// texture-backed frames can provide their own implementations. class RTC_EXPORT DesktopFrame { public: // DesktopFrame objects always hold BGRA data. @@ -62,8 +66,11 @@ // The pixel format the `DesktopFrame` is stored in. FourCC pixel_format() const { return pixel_format_; } - // Data buffer used for the frame. - uint8_t* data() const { return data_; } + // Data buffer used for the frame. May be nullptr for texture-backed frames. + uint8_t* absl_nullable data() const { return data_; } + + // Texture used for the frame. + FrameTexture* texture() const { return texture_; } // SharedMemory used for the buffer or NULL if memory is allocated on the // heap. The result is guaranteed to be deleted only after the frame is @@ -172,13 +179,15 @@ int stride, FourCC pixel_format, uint8_t* data, - SharedMemory* shared_memory); + SharedMemory* shared_memory, + FrameTexture* texture = nullptr); // Ownership of the buffers is defined by the classes that inherit from this // class. They must guarantee that the buffer is not deleted before the frame // is deleted. uint8_t* const data_; SharedMemory* const shared_memory_; + FrameTexture* texture_; private: const DesktopSize size_;
diff --git a/modules/desktop_capture/desktop_frame_texture_unittest.cc b/modules/desktop_capture/desktop_frame_texture_unittest.cc new file mode 100644 index 0000000..6e0f790 --- /dev/null +++ b/modules/desktop_capture/desktop_frame_texture_unittest.cc
@@ -0,0 +1,146 @@ +/* + * Copyright (c) 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include <memory> +#include <utility> + +#include "modules/desktop_capture/desktop_frame.h" +#include "modules/desktop_capture/desktop_geometry.h" +#include "modules/desktop_capture/frame_texture.h" +#include "modules/desktop_capture/shared_desktop_frame.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +// Returns a non-null, non-invalid sentinel FrameTexture::Handle for testing. +// On Windows Handle is void*, on other platforms it is int. +FrameTexture::Handle MakeHandle() { + constexpr uintptr_t kValue = 0x1234; +#if defined(WEBRTC_WIN) + return reinterpret_cast<FrameTexture::Handle>(kValue); +#else + return static_cast<FrameTexture::Handle>(kValue); +#endif +} + +// A concrete FrameTexture subclass for testing purposes. Unlike the +// platform-specific DXGIFrameTexture, this doesn't own any real GPU resources. +class FakeFrameTexture : public FrameTexture { + public: + explicit FakeFrameTexture(Handle handle) : FrameTexture(handle) {} + ~FakeFrameTexture() override = default; +}; + +// A texture-backed DesktopFrame with nullptr data, mimicking DXGIDesktopFrame. +class TextureDesktopFrame : public DesktopFrame { + public: + TextureDesktopFrame(DesktopSize size, + std::unique_ptr<FakeFrameTexture> texture) + : DesktopFrame(size, + /*stride=*/0, + FOURCC_ARGB, + /*data=*/nullptr, + /*shared_memory=*/nullptr, + texture.get()), + owned_texture_(std::move(texture)) {} + + ~TextureDesktopFrame() override = default; + + private: + std::unique_ptr<FakeFrameTexture> owned_texture_; +}; + +} // namespace + +TEST(FrameTextureTest, StoresHandle) { + const FrameTexture::Handle handle = MakeHandle(); + FakeFrameTexture texture(handle); + EXPECT_EQ(texture.handle(), handle); +} + +TEST(DesktopFrameTest, DefaultTextureIsNull) { + auto frame = std::make_unique<BasicDesktopFrame>(DesktopSize(10, 10)); + EXPECT_EQ(frame->texture(), nullptr); +} + +TEST(DesktopFrameTest, TextureFrameHasValidTexture) { + const FrameTexture::Handle handle = MakeHandle(); + auto texture = std::make_unique<FakeFrameTexture>(handle); + FrameTexture* raw_texture = texture.get(); + + auto frame = std::make_unique<TextureDesktopFrame>(DesktopSize(10, 10), + std::move(texture)); + EXPECT_EQ(frame->data(), nullptr); + EXPECT_NE(frame->texture(), nullptr); + EXPECT_EQ(frame->texture(), raw_texture); + EXPECT_EQ(frame->texture()->handle(), handle); +} + +TEST(DesktopFrameTest, TextureFrameHasNullData) { + // Verify that a texture-backed frame with nullptr data() still works for + // metadata access (size, texture handle, etc.). + const FrameTexture::Handle handle = MakeHandle(); + auto texture = std::make_unique<FakeFrameTexture>(handle); + + auto frame = std::make_unique<TextureDesktopFrame>(DesktopSize(10, 10), + std::move(texture)); + EXPECT_EQ(frame->data(), nullptr); + EXPECT_NE(frame->texture(), nullptr); + EXPECT_EQ(frame->texture()->handle(), handle); + EXPECT_EQ(frame->size().width(), 10); + EXPECT_EQ(frame->size().height(), 10); + + // Metadata operations should work on texture-only frames. + frame->set_dpi(DesktopVector(96, 96)); + EXPECT_EQ(frame->dpi().x(), 96); + frame->set_capture_time_ms(123); + EXPECT_EQ(frame->capture_time_ms(), 123); + frame->mutable_updated_region()->SetRect(DesktopRect::MakeWH(10, 10)); + EXPECT_FALSE(frame->updated_region().is_empty()); +} + +TEST(SharedDesktopFrameTest, TexturePropagatedToSharedFrame) { + const FrameTexture::Handle handle = MakeHandle(); + auto texture = std::make_unique<FakeFrameTexture>(handle); + FrameTexture* raw_texture = texture.get(); + + auto frame = std::make_unique<TextureDesktopFrame>(DesktopSize(10, 10), + std::move(texture)); + auto shared_frame = SharedDesktopFrame::Wrap(std::move(frame)); + ASSERT_NE(shared_frame, nullptr); + EXPECT_EQ(shared_frame->texture(), raw_texture); + EXPECT_EQ(shared_frame->texture()->handle(), handle); +} + +TEST(SharedDesktopFrameTest, TexturePropagatedToClone) { + const FrameTexture::Handle handle = MakeHandle(); + auto texture = std::make_unique<FakeFrameTexture>(handle); + FrameTexture* raw_texture = texture.get(); + + auto frame = std::make_unique<TextureDesktopFrame>(DesktopSize(10, 10), + std::move(texture)); + auto shared_frame = SharedDesktopFrame::Wrap(std::move(frame)); + auto clone = shared_frame->Share(); + ASSERT_NE(clone, nullptr); + EXPECT_EQ(clone->texture(), raw_texture); + EXPECT_EQ(clone->texture()->handle(), handle); + EXPECT_TRUE(shared_frame->ShareFrameWith(*clone)); +} + +TEST(SharedDesktopFrameTest, NullTextureForNormalFrame) { + auto frame = std::make_unique<BasicDesktopFrame>(DesktopSize(10, 10)); + auto shared_frame = SharedDesktopFrame::Wrap(std::move(frame)); + ASSERT_NE(shared_frame, nullptr); + EXPECT_EQ(shared_frame->texture(), nullptr); +} + +} // namespace webrtc
diff --git a/modules/desktop_capture/frame_texture.cc b/modules/desktop_capture/frame_texture.cc new file mode 100644 index 0000000..c9934c7 --- /dev/null +++ b/modules/desktop_capture/frame_texture.cc
@@ -0,0 +1,18 @@ +/* + * Copyright (c) 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/desktop_capture/frame_texture.h" + +namespace webrtc { + +FrameTexture::FrameTexture(Handle handle) : handle_(handle) {} +FrameTexture::~FrameTexture() = default; + +} // namespace webrtc
diff --git a/modules/desktop_capture/frame_texture.h b/modules/desktop_capture/frame_texture.h new file mode 100644 index 0000000..4051b31 --- /dev/null +++ b/modules/desktop_capture/frame_texture.h
@@ -0,0 +1,48 @@ +/* + * Copyright (c) 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_DESKTOP_CAPTURE_FRAME_TEXTURE_H_ +#define MODULES_DESKTOP_CAPTURE_FRAME_TEXTURE_H_ + +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// FrameTexture is a base class for platform-specific texture handles. It stores +// the texture handle. +class RTC_EXPORT FrameTexture { + public: + // Platform-specific handle type for the GPU texture resource. + // On Windows, this is a HANDLE (e.g., DXGI shared resource handle). + // On other platforms, this is an integer file descriptor. +#if defined(WEBRTC_WIN) + typedef void* Handle; + static constexpr Handle kInvalidHandle = nullptr; +#else + typedef int Handle; + static constexpr Handle kInvalidHandle = -1; +#endif + // Platform-specific handle of the texture. + Handle handle() const { return handle_; } + + virtual ~FrameTexture(); + + FrameTexture(const FrameTexture&) = delete; + FrameTexture& operator=(const FrameTexture&) = delete; + + protected: + explicit FrameTexture(Handle handle); + + Handle handle_; +}; + +} // namespace webrtc + +#endif // MODULES_DESKTOP_CAPTURE_FRAME_TEXTURE_H_
diff --git a/modules/desktop_capture/shared_desktop_frame.cc b/modules/desktop_capture/shared_desktop_frame.cc index 57a98d7..d147264 100644 --- a/modules/desktop_capture/shared_desktop_frame.cc +++ b/modules/desktop_capture/shared_desktop_frame.cc
@@ -54,7 +54,8 @@ (*core)->stride(), (*core)->pixel_format(), (*core)->data(), - (*core)->shared_memory()), + (*core)->shared_memory(), + (*core)->texture()), core_(core) { CopyFrameInfoFrom(*(core_->get())); }
diff --git a/modules/desktop_capture/win/dxgi_desktop_frame.cc b/modules/desktop_capture/win/dxgi_desktop_frame.cc new file mode 100644 index 0000000..2a929c3 --- /dev/null +++ b/modules/desktop_capture/win/dxgi_desktop_frame.cc
@@ -0,0 +1,89 @@ +/* + * Copyright (c) 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/desktop_capture/win/dxgi_desktop_frame.h" + +#include <comdef.h> +#include <d3d11_4.h> +#include <dxgi1_2.h> + +#include <utility> + +#include "modules/desktop_capture/win/desktop_capture_utils.h" +#include "rtc_base/logging.h" + +using Microsoft::WRL::ComPtr; + +namespace webrtc { + +namespace { + +class DXGIFrameTexture : public FrameTexture { + public: + DXGIFrameTexture(Handle handle, ComPtr<IUnknown> prevent_release) + : FrameTexture(handle), prevent_release_(std::move(prevent_release)) {} + + ~DXGIFrameTexture() override { + if (handle_ != kInvalidHandle) { + CloseHandle(handle_); + } + } + + private: + // Prevents the WGC frame pool from recycling the captured texture. Typically + // holds a reference to the IDirect3D11CaptureFrame. When this pointer (and + // hence this DXGIFrameTexture) is destroyed, the capture frame is released + // and the frame pool may reuse the underlying texture. + ComPtr<IUnknown> prevent_release_; +}; + +} // namespace + +DXGIDesktopFrame::DXGIDesktopFrame(DesktopSize size, + int stride, + uint8_t* data, + std::unique_ptr<FrameTexture> frame_texture) + : DesktopFrame(size, + stride, + FOURCC_ARGB, + data, + /*shared_memory=*/nullptr, + frame_texture.get()), + owned_frame_texture_(std::move(frame_texture)) {} + +// static +std::unique_ptr<DXGIDesktopFrame> DXGIDesktopFrame::Create( + DesktopSize size, + ComPtr<ID3D11Texture2D> texture, + ComPtr<IUnknown> prevent_release) { + ComPtr<IDXGIResource1> dxgi_resource; + HRESULT hr = texture.As(&dxgi_resource); + if (FAILED(hr)) { + RTC_LOG(LS_ERROR) << "Failed to query IDXGIResource1: " + << desktop_capture::utils::ComErrorToString(hr); + return nullptr; + } + + HANDLE texture_handle; + hr = dxgi_resource->CreateSharedHandle(nullptr, DXGI_SHARED_RESOURCE_READ, + nullptr, &texture_handle); + if (FAILED(hr)) { + RTC_LOG(LS_ERROR) << "Failed to create shared handle for texture: " + << desktop_capture::utils::ComErrorToString(hr); + return nullptr; + } + + auto frame_texture = std::make_unique<DXGIFrameTexture>( + texture_handle, std::move(prevent_release)); + return std::unique_ptr<DXGIDesktopFrame>( + new DXGIDesktopFrame(size, 0, nullptr, std::move(frame_texture))); +} + +} // namespace webrtc
diff --git a/modules/desktop_capture/win/dxgi_desktop_frame.h b/modules/desktop_capture/win/dxgi_desktop_frame.h new file mode 100644 index 0000000..6ab7b71 --- /dev/null +++ b/modules/desktop_capture/win/dxgi_desktop_frame.h
@@ -0,0 +1,51 @@ +/* + * Copyright (c) 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_DESKTOP_CAPTURE_WIN_DXGI_DESKTOP_FRAME_H_ +#define MODULES_DESKTOP_CAPTURE_WIN_DXGI_DESKTOP_FRAME_H_ + +#include <d3d11.h> +#include <wrl/client.h> + +#include <memory> + +#include "modules/desktop_capture/desktop_frame.h" +#include "modules/desktop_capture/frame_texture.h" + +namespace webrtc { + +// DesktopFrame implementation used by DXGI captures on Windows. +// Frame texture is stored in the handle. +class DXGIDesktopFrame : public DesktopFrame { + public: + DXGIDesktopFrame(const DXGIDesktopFrame&) = delete; + DXGIDesktopFrame& operator=(const DXGIDesktopFrame&) = delete; + + // Creates a DXGIDesktopFrame. If `prevent_release` is provided (typically + // an IDirect3D11CaptureFrame), it prevents the WGC frame pool from recycling + // the underlying texture while downstream consumers still reference this + // frame. + static std::unique_ptr<DXGIDesktopFrame> Create( + DesktopSize size, + Microsoft::WRL::ComPtr<ID3D11Texture2D> texture, + Microsoft::WRL::ComPtr<IUnknown> prevent_release); + + private: + DXGIDesktopFrame(DesktopSize size, + int stride, + uint8_t* data, + std::unique_ptr<FrameTexture> frame_texture); + + const std::unique_ptr<FrameTexture> owned_frame_texture_; +}; + +} // namespace webrtc + +#endif // MODULES_DESKTOP_CAPTURE_WIN_DXGI_DESKTOP_FRAME_H_
diff --git a/modules/desktop_capture/win/wgc_capture_session.cc b/modules/desktop_capture/win/wgc_capture_session.cc index 7d02d41..c7eaf54 100644 --- a/modules/desktop_capture/win/wgc_capture_session.cc +++ b/modules/desktop_capture/win/wgc_capture_session.cc
@@ -8,6 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include "modules/desktop_capture/win/wgc_capture_session.h" + #include <DispatcherQueue.h> #include <windows.graphics.capture.interop.h> #include <windows.graphics.directX.direct3d11.interop.h> @@ -26,8 +28,8 @@ #include "modules/desktop_capture/desktop_frame.h" #include "modules/desktop_capture/desktop_geometry.h" #include "modules/desktop_capture/shared_desktop_frame.h" +#include "modules/desktop_capture/win/dxgi_desktop_frame.h" #include "modules/desktop_capture/win/screen_capture_utils.h" -#include "modules/desktop_capture/win/wgc_capture_session.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/thread.h" @@ -181,6 +183,24 @@ } } +bool WgcCaptureSession::MayContainCursor() const { + RTC_DCHECK_RUN_ON(&sequence_checker_); + if (!session_) { + return false; + } + // IsCursorCaptureEnabled was introduced in IGraphicsCaptureSession2. + // Default to true (cursor captured) if the interface is not available. + ComPtr<ABI::Windows::Graphics::Capture::IGraphicsCaptureSession2> session2; + HRESULT hr = session_->QueryInterface( + ABI::Windows::Graphics::Capture::IID_IGraphicsCaptureSession2, &session2); + if (FAILED(hr)) { + return true; + } + boolean cursor_enabled = true; + session2->get_IsCursorCaptureEnabled(&cursor_enabled); + return cursor_enabled; +} + HRESULT WgcCaptureSession::StartCapture(const DesktopCaptureOptions& options) { RTC_DCHECK_RUN_ON(&sequence_checker_); RTC_DCHECK(!is_capture_started_); @@ -249,8 +269,17 @@ return hr; } + allow_zero_hertz_ = options.allow_wgc_zero_hertz(); + // Texture mode requires Windows 11 24H2+ where WGC natively skips static + // frames, since we cannot perform CPU-side frame comparison on GPU textures. + allow_using_texture_ = + options.allow_wgc_using_texture() && DoesWgcSkipStaticFrames(); + RTC_LOG(LS_INFO) << "WGC capture session is outputing " + << (allow_using_texture_ ? "GPU texture" : "CPU memory") + << "."; + hr = frame_pool_statics2->CreateFreeThreaded( - direct3d_device_.Get(), kPixelFormat, kNumBuffers, size_, &frame_pool_); + direct3d_device_.Get(), kPixelFormat, num_buffers(), size_, &frame_pool_); if (FAILED(hr)) { RecordStartCaptureResult(StartCaptureResult::kCreateFramePoolFailed); return hr; @@ -262,7 +291,11 @@ return hr; } - if (!options.prefer_cursor_embedded()) { + // We cannot do cursor composition on texture for now, so we need to enable + // cursor embedding when texture is allowed. + const bool embed_cursor = + options.prefer_cursor_embedded() || allow_using_texture_; + if (!embed_cursor) { ComPtr<ABI::Windows::Graphics::Capture::IGraphicsCaptureSession2> session2; if (SUCCEEDED(session_->QueryInterface( ABI::Windows::Graphics::Capture::IID_IGraphicsCaptureSession2, @@ -295,8 +328,6 @@ options.wgc_include_secondary_windows()); } - allow_zero_hertz_ = options.allow_wgc_zero_hertz(); - hr = session_->StartCapture(); if (FAILED(hr)) { RTC_LOG(LS_ERROR) << "Failed to start CaptureSession: " << hr; @@ -316,7 +347,7 @@ ComPtr<WGC::IDirect3D11CaptureFrame> capture_frame = nullptr; // Flush the `frame_pool_` buffers so that we can receive the most recent // frames. - for (int i = 0; i < kNumBuffers; ++i) { + for (int i = 0; i < num_buffers(); ++i) { HRESULT hr = frame_pool_->TryGetNextFrame(&capture_frame); if (FAILED(hr)) { RTC_LOG(LS_ERROR) << "TryGetNextFrame failed: " << hr; @@ -456,6 +487,9 @@ UINT width, UINT height) { RTC_DCHECK_RUN_ON(&sequence_checker_); + if (allow_using_texture_) { + return S_OK; + } D3D11_TEXTURE2D_DESC src_desc; src_texture->GetDesc(&src_desc); @@ -562,13 +596,19 @@ } hr = frame_pool_->Recreate(direct3d_device_.Get(), kPixelFormat, - kNumBuffers, new_size); + num_buffers(), new_size); if (FAILED(hr)) { RecordGetFrameResult(GetFrameResult::kRecreateFramePoolFailed); return hr; } } + if (allow_using_texture_) { + ComPtr<IUnknown> prevent_release; + capture_frame.As(&prevent_release); + return ProcessTexture(texture_2D, std::move(prevent_release)); + } + // If the size has changed since the last capture, we must be sure to use // the smaller dimensions. Otherwise we might overrun our buffer, or // read stale data from the last frame. @@ -742,6 +782,61 @@ return hr; } +HRESULT WgcCaptureSession::ProcessTexture(ComPtr<ID3D11Texture2D> texture, + ComPtr<IUnknown> capture_frame) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + + D3D11_TEXTURE2D_DESC desc; + texture->GetDesc(&desc); + DesktopSize size(desc.Width, desc.Height); + + std::unique_ptr<DesktopFrame> texture_frame = DXGIDesktopFrame::Create( + size, std::move(texture), std::move(capture_frame)); + if (!texture_frame) { + return E_FAIL; + } + + if (queue_.current_frame() && queue_.current_frame()->IsShared()) { + RTC_LOG(LS_VERBOSE) << "Overwriting texture frame that is still shared."; + } + + // Update the monitor and scale factor information for the new frame. + if (is_window_source_) { + monitor_ = ::MonitorFromWindow(reinterpret_cast<HWND>(source_id_), + MONITOR_DEFAULTTONEAREST); + } else if (!monitor_.has_value()) { + HMONITOR monitor; + if (GetHmonitorFromDeviceIndex(source_id_, &monitor)) { + monitor_ = monitor; + } + } + + if (monitor_.has_value()) { + DEVICE_SCALE_FACTOR device_scale_factor = DEVICE_SCALE_FACTOR_INVALID; + HRESULT hr = + GetScaleFactorForMonitor(monitor_.value(), &device_scale_factor); + if (SUCCEEDED(hr) && device_scale_factor != DEVICE_SCALE_FACTOR_INVALID) { + texture_frame->set_device_scale_factor( + static_cast<float>(device_scale_factor) / 100.0f); + } + } + + queue_.ReplaceCurrentFrame( + SharedDesktopFrame::Wrap(std::move(texture_frame))); + + // We use ProcessTexture() only when WGC supports skipping static frames + // (Windows 11 24H2+), so we can directly mark the whole frame as damaged + // without doing any content comparison. + damage_region_.SetRect(DesktopRect::MakeSize(size)); + + // Sync the session size with the captured texture size. + size_.Width = size.width(); + size_.Height = size.height(); + + RecordGetFrameResult(GetFrameResult::kSuccess); + return S_OK; +} + HRESULT WgcCaptureSession::OnItemClosed(WGC::IGraphicsCaptureItem* sender, IInspectable* event_args) { RTC_DCHECK_RUN_ON(&sequence_checker_);
diff --git a/modules/desktop_capture/win/wgc_capture_session.h b/modules/desktop_capture/win/wgc_capture_session.h index 0e5fbd3..74ef4c0 100644 --- a/modules/desktop_capture/win/wgc_capture_session.h +++ b/modules/desktop_capture/win/wgc_capture_session.h
@@ -63,6 +63,10 @@ return is_capture_started_; } + // Returns true if the captured frame may contain the cursor. Reads the + // actual IsCursorCaptureEnabled state from the WGC capture session. + bool MayContainCursor() const; + // We keep 2 buffers in the frame pool since it results in a good compromise // between latency/capture-rate and the rate at which // Direct3D11CaptureFramePool.TryGetNextFrame returns NULL and we have to fall @@ -70,6 +74,14 @@ // We make this public for tests. static constexpr int kNumBuffers = 2; + // When using the texture path (allow_using_texture_), each in-flight frame + // holds a reference to the WGC capture frame (via prevent_release) to prevent + // the frame pool from recycling the texture. We need more buffers so that the + // frame pool has free slots for new captures while downstream consumers still + // hold older frames. The queue holds 2 frames, and the downstream consumer + // (encoder) may hold 1-2 more, so we use 5 to provide sufficient headroom. + static constexpr int kNumBuffersForTexture = 5; + private: class RefCountedEvent : public RefCountedNonVirtual<RefCountedEvent>, public Event { @@ -135,6 +147,13 @@ // Process the captured frame and copy it to the `queue_`. HRESULT ProcessFrame(); + // Process texture and copy frame with texture to the `queue_`. + // `capture_frame` is stored in the DXGIDesktopFrame's FrameTexture to keep + // the WGC frame alive, preventing the frame pool from recycling the texture + // while downstream consumers still reference this frame. + HRESULT ProcessTexture(Microsoft::WRL::ComPtr<ID3D11Texture2D> texture, + Microsoft::WRL::ComPtr<IUnknown> capture_frame); + void RemoveEventHandlers(); void RemoveItemClosedEventHandler(); void RemoveFrameArrivedEventHandler(); @@ -144,6 +163,10 @@ bool allow_zero_hertz() const { return allow_zero_hertz_; } + int num_buffers() const { + return allow_using_texture_ ? kNumBuffersForTexture : kNumBuffers; + } + std::unique_ptr<EventRegistrationToken> item_closed_token_; std::unique_ptr<EventRegistrationToken> frame_arrived_token_; @@ -197,6 +220,12 @@ // adds complexity since memcmp() is performed on two successive frames. bool allow_zero_hertz_ = false; + // Caches the value of DesktopCaptureOptions.allow_wgc_using_texture() in + // StartCapture(). Store texture handle of captured frame in DesktopFrame + // instead of mapping texture data which can reduce copy if clients process + // textures directly. + bool allow_using_texture_ = false; + // Tracks damage region updates that were reported since the last time a frame // was captured. Currently only supports either the complete rect being // captured or an empty region. Will always be empty if `allow_zero_hertz_` is
diff --git a/modules/desktop_capture/win/wgc_capturer_win.cc b/modules/desktop_capture/win/wgc_capturer_win.cc index 2f9abc0..cc9ea52 100644 --- a/modules/desktop_capture/win/wgc_capturer_win.cc +++ b/modules/desktop_capture/win/wgc_capturer_win.cc
@@ -431,7 +431,7 @@ capture_time_ms); frame->set_capture_time_ms(capture_time_ms); frame->set_capturer_id(DesktopCapturerId::kWgcCapturerWin); - frame->set_may_contain_cursor(options_.prefer_cursor_embedded()); + frame->set_may_contain_cursor(capture_session->MayContainCursor()); frame->set_top_left(capture_source_->GetTopLeft()); RecordWgcCapturerResult(WgcCapturerResult::kSuccess); callback_->OnCaptureResult(DesktopCapturer::Result::SUCCESS,
diff --git a/modules/desktop_capture/win/wgc_capturer_win_unittest.cc b/modules/desktop_capture/win/wgc_capturer_win_unittest.cc index 46e8e14..8119c70 100644 --- a/modules/desktop_capture/win/wgc_capturer_win_unittest.cc +++ b/modules/desktop_capture/win/wgc_capturer_win_unittest.cc
@@ -21,6 +21,7 @@ #include "modules/desktop_capture/desktop_capture_options.h" #include "modules/desktop_capture/desktop_capture_types.h" #include "modules/desktop_capture/desktop_capturer.h" +#include "modules/desktop_capture/frame_texture.h" #include "modules/desktop_capture/full_screen_window_detector.h" #include "modules/desktop_capture/win/full_screen_win_application_handler.h" #include "modules/desktop_capture/win/screen_capture_utils.h" @@ -789,4 +790,273 @@ 0); } +// Tests for the WGC texture mode (allow_wgc_using_texture). +class WgcCapturerTextureTest : public ::testing::TestWithParam<CaptureType>, + public DesktopCapturer::Callback { + public: + void SetUp() override { + com_initializer_ = + std::make_unique<ScopedCOMInitializer>(ScopedCOMInitializer::kMTA); + ASSERT_TRUE(com_initializer_->Succeeded()); + + if (!IsWgcSupported(GetParam())) { + RTC_LOG(LS_INFO) + << "Skipping WgcCapturerTextureTests on unsupported platforms."; + GTEST_SKIP(); + } + } + + void SetUpForWindowCapture(int window_width = kMediumWindowWidth, + int window_height = kMediumWindowHeight) { + auto options = DesktopCaptureOptions::CreateDefault(); + options.set_allow_wgc_using_texture(true); + capturer_ = WgcCapturerWin::CreateRawWindowCapturer(options); + CreateWindowOnSeparateThread(window_width, window_height); + StartWindowThreadMessageLoop(); + source_id_ = GetTestWindowIdFromSourceList(); + } + + void SetUpForScreenCapture() { + auto options = DesktopCaptureOptions::CreateDefault(); + options.set_allow_wgc_using_texture(true); + capturer_ = WgcCapturerWin::CreateRawScreenCapturer(options); + source_id_ = GetScreenIdFromSourceList(); + } + + void TearDown() override { + if (window_open_) { + CloseTestWindow(); + } + } + + void CreateWindowOnSeparateThread(int window_width, int window_height) { + window_thread_ = Thread::Create(); + window_thread_->SetName(kWindowThreadName, nullptr); + window_thread_->Start(); + SendTask(window_thread_.get(), [this, window_width, window_height]() { + window_thread_id_ = GetCurrentThreadId(); + window_info_ = + CreateTestWindow(kWindowTitle, window_height, window_width); + window_open_ = true; + + while (!IsWindowResponding(window_info_.hwnd)) { + RTC_LOG(LS_INFO) << "Waiting for test window to become responsive in " + "WgcCapturerTextureTest."; + } + + while (!IsWindowValidAndVisible(window_info_.hwnd)) { + RTC_LOG(LS_INFO) << "Waiting for test window to be visible in " + "WgcCapturerTextureTest."; + } + }); + + ASSERT_TRUE(window_thread_->RunningForTest()); + ASSERT_FALSE(window_thread_->IsCurrent()); + } + + void StartWindowThreadMessageLoop() { + window_thread_->PostTask([this]() { + MSG msg; + BOOL gm; + while ((gm = ::GetMessage(&msg, NULL, 0, 0)) != 0 && gm != -1) { + ::DispatchMessage(&msg); + if (msg.message == kDestroyWindow) { + DestroyTestWindow(window_info_); + } + if (msg.message == kQuitRunning) { + PostQuitMessage(0); + } + } + }); + } + + void CloseTestWindow() { + ::PostThreadMessage(window_thread_id_, kDestroyWindow, 0, 0); + ::PostThreadMessage(window_thread_id_, kQuitRunning, 0, 0); + window_thread_->Stop(); + window_open_ = false; + } + + DesktopCapturer::SourceId GetTestWindowIdFromSourceList() { + intptr_t src_id = 0; + do { + DesktopCapturer::SourceList sources; + EXPECT_TRUE(capturer_->GetSourceList(&sources)); + auto it = std::find_if( + sources.begin(), sources.end(), + [&](const DesktopCapturer::Source& src) { + return src.id == reinterpret_cast<intptr_t>(window_info_.hwnd); + }); + if (it != sources.end()) + src_id = it->id; + } while (src_id != reinterpret_cast<intptr_t>(window_info_.hwnd)); + return src_id; + } + + DesktopCapturer::SourceId GetScreenIdFromSourceList() { + DesktopCapturer::SourceList sources; + EXPECT_TRUE(capturer_->GetSourceList(&sources)); + EXPECT_GT(sources.size(), 0ULL); + return sources[0].id; + } + + void DoCapture(int num_captures = 1) { + const int max_tries = num_captures == 1 ? 1 : kMaxTries; + int success_count = 0; + for (int i = 0; success_count < num_captures && i < max_tries; i++) { + capturer_->CaptureFrame(); + if (result_ == DesktopCapturer::Result::ERROR_PERMANENT) + break; + if (result_ == DesktopCapturer::Result::SUCCESS) + success_count++; + } + + total_successful_captures_ += success_count; + EXPECT_EQ(success_count, num_captures); + EXPECT_EQ(result_, DesktopCapturer::Result::SUCCESS); + EXPECT_TRUE(frame_); + } + + // DesktopCapturer::Callback interface + void OnCaptureResult(DesktopCapturer::Result result, + std::unique_ptr<DesktopFrame> frame) override { + result_ = result; + frame_ = std::move(frame); + } + + protected: + std::unique_ptr<ScopedCOMInitializer> com_initializer_; + DWORD window_thread_id_; + std::unique_ptr<Thread> window_thread_; + WindowInfo window_info_; + intptr_t source_id_; + bool window_open_ = false; + DesktopCapturer::Result result_; + int total_successful_captures_ = 0; + std::unique_ptr<DesktopFrame> frame_; + std::unique_ptr<DesktopCapturer> capturer_; +}; + +// Verifies that the captured frame in texture mode has a non-null texture with +// a valid handle, and that the CPU data pointer is null (texture-only frame). +TEST_P(WgcCapturerTextureTest, CapturedFrameHasTexture) { + if (GetParam() == CaptureType::kWindow) { + SetUpForWindowCapture(); + } else { + SetUpForScreenCapture(); + } + + EXPECT_TRUE(capturer_->SelectSource(source_id_)); + capturer_->Start(this); + DoCapture(); + + ASSERT_NE(frame_, nullptr); + EXPECT_GT(frame_->size().width(), 0); + EXPECT_GT(frame_->size().height(), 0); + ASSERT_NE(frame_->texture(), nullptr); + EXPECT_NE(frame_->texture()->handle(), FrameTexture::kInvalidHandle); + // In texture mode, frame data should be null (no CPU copy). + EXPECT_EQ(frame_->data(), nullptr); +} + +// Verifies that consecutive captures in texture mode each produce frames with +// valid texture handles. +TEST_P(WgcCapturerTextureTest, MultipleCapturesProduceValidTextures) { + if (GetParam() == CaptureType::kWindow) { + SetUpForWindowCapture(); + } else { + SetUpForScreenCapture(); + } + + EXPECT_TRUE(capturer_->SelectSource(source_id_)); + capturer_->Start(this); + + for (int i = 0; i < 3; ++i) { + DoCapture(); + ASSERT_NE(frame_, nullptr); + ASSERT_NE(frame_->texture(), nullptr); + EXPECT_NE(frame_->texture()->handle(), FrameTexture::kInvalidHandle); + } +} + +// Verifies that the frame pool uses `kNumBuffersForTexture` buffers when +// texture mode is enabled (verified indirectly by successfully capturing more +// frames than kNumBuffers without starvation). +TEST_P(WgcCapturerTextureTest, CaptureDoesNotStarveWithTextureBuffers) { + if (GetParam() == CaptureType::kWindow) { + SetUpForWindowCapture(); + } else { + SetUpForScreenCapture(); + } + + EXPECT_TRUE(capturer_->SelectSource(source_id_)); + capturer_->Start(this); + + // Capture more frames than kNumBuffers to ensure the texture buffer count + // (kNumBuffersForTexture) is sufficient. + constexpr int kCaptures = WgcCaptureSession::kNumBuffersForTexture + 1; + for (int i = 0; i < kCaptures; ++i) { + DoCapture(); + ASSERT_NE(frame_, nullptr); + ASSERT_NE(frame_->texture(), nullptr); + } +} + +INSTANTIATE_TEST_SUITE_P(SourceAgnosticTexture, + WgcCapturerTextureTest, + ::testing::Values(CaptureType::kWindow, + CaptureType::kScreen)); + +// Window-specific texture mode tests. +class WgcCapturerTextureWindowTest : public WgcCapturerTextureTest { + public: + void SetUp() override { + com_initializer_ = + std::make_unique<ScopedCOMInitializer>(ScopedCOMInitializer::kMTA); + ASSERT_TRUE(com_initializer_->Succeeded()); + + if (!IsWgcSupported(CaptureType::kWindow)) { + RTC_LOG(LS_INFO) << "Skipping WgcCapturerTextureWindowTest on " + "unsupported platforms."; + GTEST_SKIP(); + } + } +}; + +// Verifies that the cursor is embedded in the frame when texture mode is +// enabled (since we cannot do cursor composition on texture). +TEST_F(WgcCapturerTextureWindowTest, CursorEmbeddedInTextureMode) { + SetUpForWindowCapture(); + EXPECT_TRUE(capturer_->SelectSource(source_id_)); + capturer_->Start(this); + DoCapture(); + ASSERT_NE(frame_, nullptr); + // When texture mode is enabled, cursor embedding should be forced on, + // so the frame should indicate it may contain the cursor. + EXPECT_TRUE(frame_->may_contain_cursor()); +} + +// Verifies capture continues to work after a window resize in texture mode. +TEST_F(WgcCapturerTextureWindowTest, ResizeWindowMidTextureCapture) { + SetUpForWindowCapture(kSmallWindowWidth, kSmallWindowHeight); + EXPECT_TRUE(capturer_->SelectSource(source_id_)); + capturer_->Start(this); + DoCapture(); + ASSERT_NE(frame_, nullptr); + ASSERT_NE(frame_->texture(), nullptr); + + // Resize the window. + ResizeTestWindow(window_info_.hwnd, kLargeWindowWidth, kLargeWindowHeight); + + // We need more captures to flush the buffers and pick up the new size. + constexpr int kNumCapturesToFlushTextureBuffers = + WgcCaptureSession::kNumBuffersForTexture * 2 + 1; + DoCapture(kNumCapturesToFlushTextureBuffers); + ASSERT_NE(frame_, nullptr); + ASSERT_NE(frame_->texture(), nullptr); + EXPECT_NE(frame_->texture()->handle(), FrameTexture::kInvalidHandle); + EXPECT_GT(frame_->size().width(), 0); + EXPECT_GT(frame_->size().height(), 0); +} + } // namespace webrtc