Can use the given IOSurfaceRef to reach higher capture framerate on Mac
The given IOSurfaceRef was ignored until now. Wrap it into the new
DesktopFrameIOSurface. The new DesktopFrameProvider object is there
to manage them as it has to be done per display id.
From initial measurement this speed-up the frame capture by 2.
Disabled by default for now but it can be enabled by calling
options.set_use_iosurface. This CL will allow to do some advanced
tests.
Bug: webrtc:8652
Change-Id: Ia9ac0b69b30098774941cb378804b45cb1710119
Reviewed-on: https://webrtc-review.googlesource.com/33014
Commit-Queue: Zijie He <zijiehe@chromium.org>
Reviewed-by: Zijie He <zijiehe@chromium.org>
Cr-Commit-Position: refs/heads/master@{#22801}diff --git a/modules/desktop_capture/BUILD.gn b/modules/desktop_capture/BUILD.gn
index 778d155..7c07c9e 100644
--- a/modules/desktop_capture/BUILD.gn
+++ b/modules/desktop_capture/BUILD.gn
@@ -177,6 +177,10 @@
"mac/desktop_configuration.mm",
"mac/desktop_frame_cgimage.h",
"mac/desktop_frame_cgimage.mm",
+ "mac/desktop_frame_iosurface.h",
+ "mac/desktop_frame_iosurface.mm",
+ "mac/desktop_frame_provider.h",
+ "mac/desktop_frame_provider.mm",
"mac/screen_capturer_mac.h",
"mac/screen_capturer_mac.mm",
"mouse_cursor_monitor_mac.mm",
@@ -191,11 +195,13 @@
"../../rtc_base:checks",
"../../rtc_base:rtc_base",
"../../rtc_base:rtc_base_approved",
+ "../../rtc_base/synchronization:rw_lock_wrapper",
"../../sdk:common_objc",
]
libs = [
"AppKit.framework",
"IOKit.framework",
+ "IOSurface.framework",
]
}
}
diff --git a/modules/desktop_capture/DEPS b/modules/desktop_capture/DEPS
index 61bf938..8c894c4 100644
--- a/modules/desktop_capture/DEPS
+++ b/modules/desktop_capture/DEPS
@@ -7,6 +7,12 @@
"desktop_frame_cgimage\.h": [
"+sdk/objc",
],
+ "desktop_frame_iosurface\.h": [
+ "+sdk/objc",
+ ],
+ "desktop_frame_provider\.h": [
+ "+sdk/objc",
+ ],
"screen_capturer_mac\.mm": [
"+sdk/objc",
],
diff --git a/modules/desktop_capture/desktop_capture_options.h b/modules/desktop_capture/desktop_capture_options.h
index fec5ff0..a945d27 100644
--- a/modules/desktop_capture/desktop_capture_options.h
+++ b/modules/desktop_capture/desktop_capture_options.h
@@ -72,6 +72,9 @@
rtc::scoped_refptr<FullScreenChromeWindowDetector> detector) {
full_screen_window_detector_ = detector;
}
+
+ bool allow_iosurface() const { return allow_iosurface_; }
+ void set_allow_iosurface(bool allow) { allow_iosurface_ = allow; }
#endif
// Flag indicating that the capturer should use screen change notifications.
@@ -122,6 +125,7 @@
rtc::scoped_refptr<DesktopConfigurationMonitor> configuration_monitor_;
rtc::scoped_refptr<FullScreenChromeWindowDetector>
full_screen_window_detector_;
+ bool allow_iosurface_ = false;
#endif
#if defined(WEBRTC_WIN)
diff --git a/modules/desktop_capture/mac/desktop_frame_iosurface.h b/modules/desktop_capture/mac/desktop_frame_iosurface.h
new file mode 100644
index 0000000..ced60f8
--- /dev/null
+++ b/modules/desktop_capture/mac/desktop_frame_iosurface.h
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_DESKTOP_CAPTURE_MAC_DESKTOP_FRAME_IOSURFACE_H_
+#define MODULES_DESKTOP_CAPTURE_MAC_DESKTOP_FRAME_IOSURFACE_H_
+
+#include <CoreGraphics/CoreGraphics.h>
+#include <IOSurface/IOSurface.h>
+
+#include <memory>
+
+#include "modules/desktop_capture/desktop_frame.h"
+#include "sdk/objc/Framework/Classes/Common/scoped_cftyperef.h"
+
+namespace webrtc {
+
+class DesktopFrameIOSurface final : public DesktopFrame {
+ public:
+ // Lock an IOSurfaceRef containing a snapshot of a display. Return NULL if
+ // failed to lock.
+ static std::unique_ptr<DesktopFrameIOSurface> Wrap(
+ rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface);
+
+ ~DesktopFrameIOSurface() override;
+
+ private:
+ // This constructor expects |io_surface| to hold a non-null IOSurfaceRef.
+ explicit DesktopFrameIOSurface(rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface);
+
+ const rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface_;
+
+ RTC_DISALLOW_COPY_AND_ASSIGN(DesktopFrameIOSurface);
+};
+
+} // namespace webrtc
+
+#endif // MODULES_DESKTOP_CAPTURE_MAC_DESKTOP_FRAME_IOSURFACE_H_
diff --git a/modules/desktop_capture/mac/desktop_frame_iosurface.mm b/modules/desktop_capture/mac/desktop_frame_iosurface.mm
new file mode 100644
index 0000000..7df01b7
--- /dev/null
+++ b/modules/desktop_capture/mac/desktop_frame_iosurface.mm
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/desktop_capture/mac/desktop_frame_iosurface.h"
+
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+
+// static
+std::unique_ptr<DesktopFrameIOSurface> DesktopFrameIOSurface::Wrap(
+ rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface) {
+ if (!io_surface) {
+ return nullptr;
+ }
+
+ IOSurfaceIncrementUseCount(io_surface.get());
+ IOReturn status = IOSurfaceLock(io_surface.get(), kIOSurfaceLockReadOnly, nullptr);
+ if (status != kIOReturnSuccess) {
+ RTC_LOG(LS_ERROR) << "Failed to lock the IOSurface with status " << status;
+ IOSurfaceDecrementUseCount(io_surface.get());
+ return nullptr;
+ }
+
+ // Verify that the image has 32-bit depth.
+ int bytes_per_pixel = IOSurfaceGetBytesPerElement(io_surface.get());
+ if (bytes_per_pixel != DesktopFrame::kBytesPerPixel) {
+ RTC_LOG(LS_ERROR) << "CGDisplayStream handler returned IOSurface with " << (8 * bytes_per_pixel)
+ << " bits per pixel. Only 32-bit depth is supported.";
+ return nullptr;
+ }
+
+ return std::unique_ptr<DesktopFrameIOSurface>(new DesktopFrameIOSurface(io_surface));
+}
+
+DesktopFrameIOSurface::DesktopFrameIOSurface(rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface)
+ : DesktopFrame(
+ DesktopSize(IOSurfaceGetWidth(io_surface.get()), IOSurfaceGetHeight(io_surface.get())),
+ IOSurfaceGetBytesPerRow(io_surface.get()),
+ static_cast<uint8_t*>(IOSurfaceGetBaseAddress(io_surface.get())),
+ nullptr),
+ io_surface_(io_surface) {
+ RTC_DCHECK(io_surface_);
+}
+
+DesktopFrameIOSurface::~DesktopFrameIOSurface() {
+ IOSurfaceUnlock(io_surface_.get(), kIOSurfaceLockReadOnly, nullptr);
+ IOSurfaceDecrementUseCount(io_surface_.get());
+}
+
+} // namespace webrtc
diff --git a/modules/desktop_capture/mac/desktop_frame_provider.h b/modules/desktop_capture/mac/desktop_frame_provider.h
new file mode 100644
index 0000000..d4bbdb1
--- /dev/null
+++ b/modules/desktop_capture/mac/desktop_frame_provider.h
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_DESKTOP_CAPTURE_MAC_DESKTOP_FRAME_PROVIDER_H_
+#define MODULES_DESKTOP_CAPTURE_MAC_DESKTOP_FRAME_PROVIDER_H_
+
+#include <CoreGraphics/CoreGraphics.h>
+#include <IOSurface/IOSurface.h>
+
+#include <map>
+#include <memory>
+
+#include "modules/desktop_capture/desktop_frame.h"
+#include "rtc_base/synchronization/rw_lock_wrapper.h"
+#include "sdk/objc/Framework/Classes/Common/scoped_cftyperef.h"
+
+namespace webrtc {
+
+class DesktopFrameProvider {
+ public:
+ explicit DesktopFrameProvider(bool allow_iosurface);
+ ~DesktopFrameProvider();
+
+ // The caller takes ownership of the returned desktop frame. Otherwise
+ // returns null if |display_id| is invalid or not ready.
+ std::unique_ptr<DesktopFrame> TakeLatestFrameForDisplay(
+ CGDirectDisplayID display_id);
+
+ // OS sends the latest IOSurfaceRef through
+ // CGDisplayStreamFrameAvailableHandler callback; we store it here.
+ void InvalidateIOSurface(CGDirectDisplayID display_id,
+ rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface);
+
+ // Expected to be called before stopping the CGDisplayStreamRef streams.
+ void Release();
+
+ private:
+ const bool allow_iosurface_;
+
+ // A lock protecting |io_surfaces_| across threads.
+ const std::unique_ptr<RWLockWrapper> io_surfaces_lock_;
+
+ // Most recent IOSurface that contains a capture of matching display.
+ std::map<CGDirectDisplayID, std::unique_ptr<DesktopFrame>> io_surfaces_;
+
+ RTC_DISALLOW_COPY_AND_ASSIGN(DesktopFrameProvider);
+};
+
+} // namespace webrtc
+
+#endif // MODULES_DESKTOP_CAPTURE_MAC_DESKTOP_FRAME_PROVIDER_H_
diff --git a/modules/desktop_capture/mac/desktop_frame_provider.mm b/modules/desktop_capture/mac/desktop_frame_provider.mm
new file mode 100644
index 0000000..3e13125
--- /dev/null
+++ b/modules/desktop_capture/mac/desktop_frame_provider.mm
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/desktop_capture/mac/desktop_frame_provider.h"
+
+#include <utility>
+
+#include "modules/desktop_capture/mac/desktop_frame_cgimage.h"
+#include "modules/desktop_capture/mac/desktop_frame_iosurface.h"
+
+namespace webrtc {
+
+DesktopFrameProvider::DesktopFrameProvider(bool allow_iosurface)
+ : allow_iosurface_(allow_iosurface), io_surfaces_lock_(RWLockWrapper::CreateRWLock()) {}
+
+DesktopFrameProvider::~DesktopFrameProvider() {
+ // Might be called from a thread which is not the one running the CGDisplayStream
+ // handler. Indeed chromium's content destroys it from a dedicated thread.
+ Release();
+}
+
+std::unique_ptr<DesktopFrame> DesktopFrameProvider::TakeLatestFrameForDisplay(
+ CGDirectDisplayID display_id) {
+ if (!allow_iosurface_) {
+ // Regenerate a snapshot.
+ return DesktopFrameCGImage::CreateForDisplay(display_id);
+ }
+
+ // Might be called from a thread which is not the one running the CGDisplayStream
+ // handler. Indeed chromium's content uses a dedicates thread.
+ WriteLockScoped scoped_io_surfaces_lock(*io_surfaces_lock_);
+ if (io_surfaces_[display_id]) {
+ return std::move(io_surfaces_[display_id]);
+ }
+
+ return nullptr;
+}
+
+void DesktopFrameProvider::InvalidateIOSurface(CGDirectDisplayID display_id,
+ rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface) {
+ if (!allow_iosurface_) {
+ return;
+ }
+
+ std::unique_ptr<DesktopFrameIOSurface> desktop_frame_iosurface =
+ DesktopFrameIOSurface::Wrap(io_surface);
+
+ // Call from the thread which runs the CGDisplayStream handler.
+ WriteLockScoped scoped_io_surfaces_lock(*io_surfaces_lock_);
+ io_surfaces_[display_id] = std::move(desktop_frame_iosurface);
+}
+
+void DesktopFrameProvider::Release() {
+ if (!allow_iosurface_) {
+ return;
+ }
+
+ WriteLockScoped scoped_io_surfaces_lock(*io_surfaces_lock_);
+ io_surfaces_.clear();
+}
+
+} // namespace webrtc
diff --git a/modules/desktop_capture/mac/screen_capturer_mac.h b/modules/desktop_capture/mac/screen_capturer_mac.h
index 0191317..8a40670 100644
--- a/modules/desktop_capture/mac/screen_capturer_mac.h
+++ b/modules/desktop_capture/mac/screen_capturer_mac.h
@@ -22,6 +22,7 @@
#include "modules/desktop_capture/desktop_region.h"
#include "modules/desktop_capture/mac/desktop_configuration.h"
#include "modules/desktop_capture/mac/desktop_configuration_monitor.h"
+#include "modules/desktop_capture/mac/desktop_frame_provider.h"
#include "modules/desktop_capture/screen_capture_frame_queue.h"
#include "modules/desktop_capture/screen_capturer_helper.h"
#include "modules/desktop_capture/shared_desktop_frame.h"
@@ -33,9 +34,10 @@
// A class to perform video frame capturing for mac.
class ScreenCapturerMac final : public DesktopCapturer {
public:
- explicit ScreenCapturerMac(
+ ScreenCapturerMac(
rtc::scoped_refptr<DesktopConfigurationMonitor> desktop_config_monitor,
- bool detect_updated_region);
+ bool detect_updated_region,
+ bool allow_iosurface);
~ScreenCapturerMac() override;
// TODO(julien.isorce): Remove Init() or make it private.
@@ -58,9 +60,11 @@
bool RegisterRefreshAndMoveHandlers();
void UnregisterRefreshAndMoveHandlers();
- void ScreenRefresh(CGRectCount count,
+ void ScreenRefresh(CGDirectDisplayID display_id,
+ CGRectCount count,
const CGRect* rect_array,
- DesktopVector display_origin);
+ DesktopVector display_origin,
+ IOSurfaceRef io_surface);
void ReleaseBuffers();
std::unique_ptr<DesktopFrame> CreateFrame();
@@ -101,6 +105,9 @@
// all display streams have been destroyed..
DisplayStreamManager* display_stream_manager_;
+ // Container holding latest state of the snapshot per displays.
+ DesktopFrameProvider desktop_frame_provider_;
+
RTC_DISALLOW_COPY_AND_ASSIGN(ScreenCapturerMac);
};
diff --git a/modules/desktop_capture/mac/screen_capturer_mac.mm b/modules/desktop_capture/mac/screen_capturer_mac.mm
index df18777..c06723c 100644
--- a/modules/desktop_capture/mac/screen_capturer_mac.mm
+++ b/modules/desktop_capture/mac/screen_capturer_mac.mm
@@ -12,7 +12,7 @@
#include "modules/desktop_capture/mac/screen_capturer_mac.h"
-#include "modules/desktop_capture/mac/desktop_frame_cgimage.h"
+#include "modules/desktop_capture/mac/desktop_frame_provider.h"
#include "rtc_base/checks.h"
#include "rtc_base/constructormagic.h"
#include "rtc_base/logging.h"
@@ -215,10 +215,14 @@
ScreenCapturerMac::ScreenCapturerMac(
rtc::scoped_refptr<DesktopConfigurationMonitor> desktop_config_monitor,
- bool detect_updated_region)
+ bool detect_updated_region,
+ bool allow_iosurface)
: detect_updated_region_(detect_updated_region),
- desktop_config_monitor_(desktop_config_monitor) {
+ desktop_config_monitor_(desktop_config_monitor),
+ desktop_frame_provider_(allow_iosurface) {
display_stream_manager_ = new DisplayStreamManager;
+
+ RTC_LOG(LS_INFO) << "Allow IOSurface: " << allow_iosurface;
}
ScreenCapturerMac::~ScreenCapturerMac() {
@@ -406,8 +410,8 @@
}
}
- std::unique_ptr<DesktopFrameCGImage> frame_source =
- DesktopFrameCGImage::CreateForDisplay(display_config.id);
+ std::unique_ptr<DesktopFrame> frame_source =
+ desktop_frame_provider_.TakeLatestFrameForDisplay(display_config.id);
if (!frame_source) {
continue;
}
@@ -518,7 +522,7 @@
if (count != 0) {
// According to CGDisplayStream.h, it's safe to call
// CGDisplayStreamStop() from within the callback.
- ScreenRefresh(count, rects, display_origin);
+ ScreenRefresh(display_id, count, rects, display_origin, frame_surface);
}
};
@@ -547,12 +551,17 @@
}
void ScreenCapturerMac::UnregisterRefreshAndMoveHandlers() {
+ // Release obsolete io surfaces.
+ desktop_frame_provider_.Release();
+
display_stream_manager_->UnregisterActiveStreams();
}
-void ScreenCapturerMac::ScreenRefresh(CGRectCount count,
+void ScreenCapturerMac::ScreenRefresh(CGDirectDisplayID display_id,
+ CGRectCount count,
const CGRect* rect_array,
- DesktopVector display_origin) {
+ DesktopVector display_origin,
+ IOSurfaceRef io_surface) {
if (screen_pixel_bounds_.is_empty()) ScreenConfigurationChanged();
// The refresh rects are in display coordinates. We want to translate to
@@ -576,6 +585,9 @@
}
helper_.InvalidateRegion(region);
+
+ desktop_frame_provider_.InvalidateIOSurface(
+ display_id, rtc::ScopedCFTypeRef<IOSurfaceRef>(io_surface, rtc::RetainPolicy::RETAIN));
}
std::unique_ptr<DesktopFrame> ScreenCapturerMac::CreateFrame() {
diff --git a/modules/desktop_capture/screen_capturer_darwin.mm b/modules/desktop_capture/screen_capturer_darwin.mm
index ca26f32..d5a7bb0 100644
--- a/modules/desktop_capture/screen_capturer_darwin.mm
+++ b/modules/desktop_capture/screen_capturer_darwin.mm
@@ -21,8 +21,8 @@
return nullptr;
}
- std::unique_ptr<ScreenCapturerMac> capturer(
- new ScreenCapturerMac(options.configuration_monitor(), options.detect_updated_region()));
+ std::unique_ptr<ScreenCapturerMac> capturer(new ScreenCapturerMac(
+ options.configuration_monitor(), options.detect_updated_region(), options.allow_iosurface()));
if (!capturer.get()->Init()) {
return nullptr;
}
