Move frame adaptation inside video processor.
Bug: webrtc:10530
Change-Id: Iba6a91bf3e1ec4b2821b554e9e28fd2ead662723
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/131947
Reviewed-by: Magnus Jedvert <magjed@webrtc.org>
Commit-Queue: Sami Kalliomäki <sakal@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27542}diff --git a/sdk/android/api/org/webrtc/VideoProcessor.java b/sdk/android/api/org/webrtc/VideoProcessor.java
index 1a2aca5..3a89090 100644
--- a/sdk/android/api/org/webrtc/VideoProcessor.java
+++ b/sdk/android/api/org/webrtc/VideoProcessor.java
@@ -17,9 +17,60 @@
* on to another object. This object is also allowed to observe capturer start/stop.
*/
public interface VideoProcessor extends CapturerObserver {
+ public static class FrameAdaptationParameters {
+ public final int cropX;
+ public final int cropY;
+ public final int cropWidth;
+ public final int cropHeight;
+ public final int scaleWidth;
+ public final int scaleHeight;
+ public final long timestampNs;
+ public final boolean drop;
+
+ public FrameAdaptationParameters(int cropX, int cropY, int cropWidth, int cropHeight,
+ int scaleWidth, int scaleHeight, long timestampNs, boolean drop) {
+ this.cropX = cropX;
+ this.cropY = cropY;
+ this.cropWidth = cropWidth;
+ this.cropHeight = cropHeight;
+ this.scaleWidth = scaleWidth;
+ this.scaleHeight = scaleHeight;
+ this.timestampNs = timestampNs;
+ this.drop = drop;
+ }
+ }
+
+ /**
+ * This is a chance to access an unadapted frame. The default implementation applies the
+ * adaptation and forwards the frame to {@link #onFrameCaptured(VideoFrame)}.
+ */
+ default void onFrameCaptured(VideoFrame frame, FrameAdaptationParameters parameters) {
+ VideoFrame adaptedFrame = applyFrameAdaptationParameters(frame, parameters);
+ if (adaptedFrame != null) {
+ onFrameCaptured(adaptedFrame);
+ adaptedFrame.release();
+ }
+ }
+
/**
* Set the sink that receives the output from this processor. Null can be passed in to unregister
* a sink. After this call returns, no frames should be delivered to an unregistered sink.
*/
void setSink(@Nullable VideoSink sink);
+
+ /**
+ * Applies the frame adaptation parameters to a frame. Returns null if the frame is meant to be
+ * dropped. Returns a new frame. The caller is responsible for releasing the returned frame.
+ */
+ public static @Nullable VideoFrame applyFrameAdaptationParameters(
+ VideoFrame frame, FrameAdaptationParameters parameters) {
+ if (parameters.drop) {
+ return null;
+ }
+
+ final VideoFrame.Buffer adaptedBuffer =
+ frame.getBuffer().cropAndScale(parameters.cropX, parameters.cropY, parameters.cropWidth,
+ parameters.cropHeight, parameters.scaleWidth, parameters.scaleHeight);
+ return new VideoFrame(adaptedBuffer, frame.getRotation(), parameters.timestampNs);
+ }
}
diff --git a/sdk/android/api/org/webrtc/VideoSource.java b/sdk/android/api/org/webrtc/VideoSource.java
index 847a7bd..1b40231 100644
--- a/sdk/android/api/org/webrtc/VideoSource.java
+++ b/sdk/android/api/org/webrtc/VideoSource.java
@@ -59,28 +59,20 @@
@Override
public void onFrameCaptured(VideoFrame frame) {
- final NativeAndroidVideoTrackSource.FrameAdaptationParameters parameters =
+ final VideoProcessor.FrameAdaptationParameters parameters =
nativeAndroidVideoTrackSource.adaptFrame(frame);
- if (parameters == null) {
- // Drop frame.
- return;
- }
-
- final VideoFrame.Buffer adaptedBuffer =
- frame.getBuffer().cropAndScale(parameters.cropX, parameters.cropY, parameters.cropWidth,
- parameters.cropHeight, parameters.scaleWidth, parameters.scaleHeight);
- final VideoFrame adaptedFrame =
- new VideoFrame(adaptedBuffer, frame.getRotation(), parameters.timestampNs);
-
synchronized (videoProcessorLock) {
if (videoProcessor != null) {
- videoProcessor.onFrameCaptured(adaptedFrame);
- adaptedBuffer.release();
+ videoProcessor.onFrameCaptured(frame, parameters);
return;
}
}
- nativeAndroidVideoTrackSource.onFrameCaptured(adaptedFrame);
- adaptedBuffer.release();
+
+ VideoFrame adaptedFrame = VideoProcessor.applyFrameAdaptationParameters(frame, parameters);
+ if (adaptedFrame != null) {
+ nativeAndroidVideoTrackSource.onFrameCaptured(adaptedFrame);
+ adaptedFrame.release();
+ }
}
};
diff --git a/sdk/android/src/java/org/webrtc/NativeAndroidVideoTrackSource.java b/sdk/android/src/java/org/webrtc/NativeAndroidVideoTrackSource.java
index d19f6f0..48690fb 100644
--- a/sdk/android/src/java/org/webrtc/NativeAndroidVideoTrackSource.java
+++ b/sdk/android/src/java/org/webrtc/NativeAndroidVideoTrackSource.java
@@ -12,6 +12,7 @@
import android.support.annotation.Nullable;
import org.webrtc.VideoFrame;
+import org.webrtc.VideoProcessor;
/**
* This class is meant to be a simple layer that only handles the JNI wrapping of a C++
@@ -25,28 +26,6 @@
// Pointer to webrtc::jni::AndroidVideoTrackSource.
private final long nativeAndroidVideoTrackSource;
- public static class FrameAdaptationParameters {
- public final int cropX;
- public final int cropY;
- public final int cropWidth;
- public final int cropHeight;
- public final int scaleWidth;
- public final int scaleHeight;
- public final long timestampNs;
-
- @CalledByNative("FrameAdaptationParameters")
- FrameAdaptationParameters(int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth,
- int scaleHeight, long timestampNs) {
- this.cropX = cropX;
- this.cropY = cropY;
- this.cropWidth = cropWidth;
- this.cropHeight = cropHeight;
- this.scaleWidth = scaleWidth;
- this.scaleHeight = scaleHeight;
- this.timestampNs = timestampNs;
- }
- }
-
public NativeAndroidVideoTrackSource(long nativeAndroidVideoTrackSource) {
this.nativeAndroidVideoTrackSource = nativeAndroidVideoTrackSource;
}
@@ -66,7 +45,7 @@
* adaptation parameters before calling onFrameCaptured().
*/
@Nullable
- public FrameAdaptationParameters adaptFrame(VideoFrame frame) {
+ public VideoProcessor.FrameAdaptationParameters adaptFrame(VideoFrame frame) {
return nativeAdaptFrame(nativeAndroidVideoTrackSource, frame.getBuffer().getWidth(),
frame.getBuffer().getHeight(), frame.getRotation(), frame.getTimestampNs());
}
@@ -93,13 +72,21 @@
targetPortraitAspectRatio.height, maxPortraitPixelCount, maxFps);
}
+ @CalledByNative
+ static VideoProcessor.FrameAdaptationParameters createFrameAdaptationParameters(int cropX,
+ int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight, long timestampNs,
+ boolean drop) {
+ return new VideoProcessor.FrameAdaptationParameters(
+ cropX, cropY, cropWidth, cropHeight, scaleWidth, scaleHeight, timestampNs, drop);
+ }
+
private static native void nativeSetState(long nativeAndroidVideoTrackSource, boolean isLive);
private static native void nativeAdaptOutputFormat(long nativeAndroidVideoTrackSource,
int landscapeWidth, int landscapeHeight, @Nullable Integer maxLandscapePixelCount,
int portraitWidth, int portraitHeight, @Nullable Integer maxPortraitPixelCount,
@Nullable Integer maxFps);
@Nullable
- private static native FrameAdaptationParameters nativeAdaptFrame(
+ private static native VideoProcessor.FrameAdaptationParameters nativeAdaptFrame(
long nativeAndroidVideoTrackSource, int width, int height, int rotation, long timestampNs);
private static native void nativeOnFrameCaptured(
long nativeAndroidVideoTrackSource, int rotation, long timestampNs, VideoFrame.Buffer buffer);
diff --git a/sdk/android/src/java/org/webrtc/NativeCapturerObserver.java b/sdk/android/src/java/org/webrtc/NativeCapturerObserver.java
index f0c27c5..c195fb3 100644
--- a/sdk/android/src/java/org/webrtc/NativeCapturerObserver.java
+++ b/sdk/android/src/java/org/webrtc/NativeCapturerObserver.java
@@ -36,7 +36,7 @@
@Override
public void onFrameCaptured(VideoFrame frame) {
- final NativeAndroidVideoTrackSource.FrameAdaptationParameters parameters =
+ final VideoProcessor.FrameAdaptationParameters parameters =
nativeAndroidVideoTrackSource.adaptFrame(frame);
if (parameters == null) {
// Drop frame.
diff --git a/sdk/android/src/jni/android_video_track_source.cc b/sdk/android/src/jni/android_video_track_source.cc
index dbcbd6a..973167a 100644
--- a/sdk/android/src/jni/android_video_track_source.cc
+++ b/sdk/android/src/jni/android_video_track_source.cc
@@ -102,33 +102,30 @@
camera_time_us, rtc::TimeMicros())
: j_timestamp_ns;
- int adapted_width;
- int adapted_height;
- int crop_width;
- int crop_height;
- int crop_x;
- int crop_y;
+ int adapted_width = 0;
+ int adapted_height = 0;
+ int crop_width = 0;
+ int crop_height = 0;
+ int crop_x = 0;
+ int crop_y = 0;
+ bool drop;
// TODO(magjed): Move this logic to users of NativeAndroidVideoTrackSource
// instead, in order to keep this native wrapping layer as thin as possible.
if (rotation % 180 == 0) {
- if (!rtc::AdaptedVideoTrackSource::AdaptFrame(
- j_width, j_height, camera_time_us, &adapted_width, &adapted_height,
- &crop_width, &crop_height, &crop_x, &crop_y)) {
- return nullptr;
- }
+ drop = !rtc::AdaptedVideoTrackSource::AdaptFrame(
+ j_width, j_height, camera_time_us, &adapted_width, &adapted_height,
+ &crop_width, &crop_height, &crop_x, &crop_y);
} else {
// Swap all width/height and x/y.
- if (!rtc::AdaptedVideoTrackSource::AdaptFrame(
- j_height, j_width, camera_time_us, &adapted_height, &adapted_width,
- &crop_height, &crop_width, &crop_y, &crop_x)) {
- return nullptr;
- }
+ drop = !rtc::AdaptedVideoTrackSource::AdaptFrame(
+ j_height, j_width, camera_time_us, &adapted_height, &adapted_width,
+ &crop_height, &crop_width, &crop_y, &crop_x);
}
- return Java_FrameAdaptationParameters_Constructor(
+ return Java_NativeAndroidVideoTrackSource_createFrameAdaptationParameters(
env, crop_x, crop_y, crop_width, crop_height, adapted_width,
- adapted_height, aligned_timestamp_ns);
+ adapted_height, aligned_timestamp_ns, drop);
}
void AndroidVideoTrackSource::OnFrameCaptured(
