Android: Add video processing interface
This CL adds an API for injecting video processing after the WebRTC
CPU and QP scaling step.
Bug: webrtc:10247
Change-Id: I776498e1e9113f50e953ee411bbb31f181863312
Reviewed-on: https://webrtc-review.googlesource.com/c/119953
Commit-Queue: Magnus Jedvert <magjed@webrtc.org>
Reviewed-by: Sami Kalliomäki <sakal@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26740}diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn
index cbae015..01acfc5 100644
--- a/sdk/android/BUILD.gn
+++ b/sdk/android/BUILD.gn
@@ -326,6 +326,7 @@
"api/org/webrtc/StatsReport.java",
"api/org/webrtc/TurnCustomizer.java",
"api/org/webrtc/VideoSource.java",
+ "api/org/webrtc/VideoProcessor.java",
"api/org/webrtc/VideoTrack.java",
"src/java/org/webrtc/NativeLibrary.java",
"src/java/org/webrtc/NativeCapturerObserver.java",
diff --git a/sdk/android/api/org/webrtc/VideoProcessor.java b/sdk/android/api/org/webrtc/VideoProcessor.java
new file mode 100644
index 0000000..1a2aca5
--- /dev/null
+++ b/sdk/android/api/org/webrtc/VideoProcessor.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+import android.support.annotation.Nullable;
+
+/**
+ * Lightweight abstraction for an object that can receive video frames, process them, and pass them
+ * on to another object. This object is also allowed to observe capturer start/stop.
+ */
+public interface VideoProcessor extends CapturerObserver {
+ /**
+ * Set the sink that receives the output from this processor. Null can be passed in to unregister
+ * a sink. After this call returns, no frames should be delivered to an unregistered sink.
+ */
+ void setSink(@Nullable VideoSink sink);
+}
diff --git a/sdk/android/api/org/webrtc/VideoSource.java b/sdk/android/api/org/webrtc/VideoSource.java
index 0939457..847a7bd 100644
--- a/sdk/android/api/org/webrtc/VideoSource.java
+++ b/sdk/android/api/org/webrtc/VideoSource.java
@@ -30,15 +30,31 @@
}
private final NativeAndroidVideoTrackSource nativeAndroidVideoTrackSource;
+ private final Object videoProcessorLock = new Object();
+ @Nullable private VideoProcessor videoProcessor;
+ private boolean isCapturerRunning;
+
private final CapturerObserver capturerObserver = new CapturerObserver() {
@Override
public void onCapturerStarted(boolean success) {
nativeAndroidVideoTrackSource.setState(success);
+ synchronized (videoProcessorLock) {
+ isCapturerRunning = success;
+ if (videoProcessor != null) {
+ videoProcessor.onCapturerStarted(success);
+ }
+ }
}
@Override
public void onCapturerStopped() {
nativeAndroidVideoTrackSource.setState(/* isLive= */ false);
+ synchronized (videoProcessorLock) {
+ isCapturerRunning = false;
+ if (videoProcessor != null) {
+ videoProcessor.onCapturerStopped();
+ }
+ }
}
@Override
@@ -53,9 +69,17 @@
final VideoFrame.Buffer adaptedBuffer =
frame.getBuffer().cropAndScale(parameters.cropX, parameters.cropY, parameters.cropWidth,
parameters.cropHeight, parameters.scaleWidth, parameters.scaleHeight);
- // TODO(magjed): Add video processing hook here.
- nativeAndroidVideoTrackSource.onFrameCaptured(
- new VideoFrame(adaptedBuffer, frame.getRotation(), parameters.timestampNs));
+ final VideoFrame adaptedFrame =
+ new VideoFrame(adaptedBuffer, frame.getRotation(), parameters.timestampNs);
+
+ synchronized (videoProcessorLock) {
+ if (videoProcessor != null) {
+ videoProcessor.onFrameCaptured(adaptedFrame);
+ adaptedBuffer.release();
+ return;
+ }
+ }
+ nativeAndroidVideoTrackSource.onFrameCaptured(adaptedFrame);
adaptedBuffer.release();
}
};
@@ -98,6 +122,31 @@
maxLandscapePixelCount, targetPortraitAspectRatio, maxPortraitPixelCount, maxFps);
}
+ /**
+ * Hook for injecting a custom video processor before frames are passed onto WebRTC. The frames
+ * will be cropped and scaled depending on CPU and network conditions before they are passed to
+ * the video processor. Frames will be delivered to the video processor on the same thread they
+ * are passed to this object. The video processor is allowed to deliver the processed frames
+ * back on any thread.
+ */
+ public void setVideoProcessor(@Nullable VideoProcessor newVideoProcessor) {
+ synchronized (videoProcessorLock) {
+ if (videoProcessor != null) {
+ videoProcessor.setSink(/* sink= */ null);
+ if (isCapturerRunning) {
+ videoProcessor.onCapturerStopped();
+ }
+ }
+ videoProcessor = newVideoProcessor;
+ if (newVideoProcessor != null) {
+ newVideoProcessor.setSink(nativeAndroidVideoTrackSource::onFrameCaptured);
+ if (isCapturerRunning) {
+ newVideoProcessor.onCapturerStarted(/* success= */ true);
+ }
+ }
+ }
+ }
+
public CapturerObserver getCapturerObserver() {
return capturerObserver;
}
@@ -106,4 +155,10 @@
long getNativeVideoTrackSource() {
return getNativeMediaSource();
}
+
+ @Override
+ public void dispose() {
+ setVideoProcessor(/* newVideoProcessor= */ null);
+ super.dispose();
+ }
}
