Add resolution bitrate thresholds to EncoderInfo.
When provided, these thresholds will be used instead of WebRTC default
limits specified in DropDueToSize() and GetMaxDefaultVideoBitrateKbps().
Bug: none
Change-Id: Ida45ea832041963b8b8475d69114b5c60a172fb7
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/142170
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Alex Glaznev <glaznev@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#28390}
diff --git a/api/video_codecs/video_encoder.h b/api/video_codecs/video_encoder.h
index d2e4877..5609e93 100644
--- a/api/video_codecs/video_encoder.h
+++ b/api/video_codecs/video_encoder.h
@@ -122,6 +122,26 @@
ScalingSettings();
};
+ // Bitrate thresholds for resolution.
+ struct ResolutionBitrateThresholds {
+ ResolutionBitrateThresholds(int frame_size_pixels,
+ int min_start_bitrate_bps,
+ int min_bitrate_bps,
+ int max_bitrate_bps)
+ : frame_size_pixels(frame_size_pixels),
+ min_start_bitrate_bps(min_start_bitrate_bps),
+ min_bitrate_bps(min_bitrate_bps),
+ max_bitrate_bps(max_bitrate_bps) {}
+ // Size of video frame, in pixels, the bitrate thresholds are intended for.
+ int frame_size_pixels = 0;
+ // Recommended minimum bitrate to start encoding.
+ int min_start_bitrate_bps = 0;
+ // Recommended minimum bitrate.
+ int min_bitrate_bps = 0;
+ // Recommended maximum bitrate.
+ int max_bitrate_bps = 0;
+ };
+
// Struct containing metadata about the encoder implementing this interface.
struct EncoderInfo {
static constexpr uint8_t kMaxFramerateFraction =
@@ -192,6 +212,9 @@
// with a 100% frame rate fraction.
absl::InlinedVector<uint8_t, kMaxTemporalStreams>
fps_allocation[kMaxSpatialLayers];
+
+ // Recommended bitrate thresholds for different resolutions.
+ std::vector<ResolutionBitrateThresholds> resolution_bitrate_thresholds;
};
struct RateControlParameters {
diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn
index ec3fbb8..2253f53 100644
--- a/sdk/android/BUILD.gn
+++ b/sdk/android/BUILD.gn
@@ -1507,6 +1507,7 @@
"native_unittests/org/webrtc/ApplicationContextProvider.java",
"native_unittests/org/webrtc/BuildInfo.java",
"native_unittests/org/webrtc/CodecsWrapperTestHelper.java",
+ "native_unittests/org/webrtc/FakeVideoEncoder.java",
"native_unittests/org/webrtc/JavaTypesTestHelper.java",
"native_unittests/org/webrtc/JavaVideoSourceTestHelper.java",
"native_unittests/org/webrtc/PeerConnectionFactoryInitializationHelper.java",
diff --git a/sdk/android/api/org/webrtc/VideoEncoder.java b/sdk/android/api/org/webrtc/VideoEncoder.java
index a5b2c4d..eeafa6b 100644
--- a/sdk/android/api/org/webrtc/VideoEncoder.java
+++ b/sdk/android/api/org/webrtc/VideoEncoder.java
@@ -11,6 +11,8 @@
package org.webrtc;
import android.support.annotation.Nullable;
+import java.util.Collections;
+import java.util.List;
import org.webrtc.EncodedImage;
/**
@@ -181,6 +183,59 @@
}
}
+ /**
+ * Bitrate thresholds for resolution.
+ */
+ public class ResolutionBitrateThresholds {
+ /**
+ * Maximum size of video frame, in pixels, the bitrate thresholds are intended for.
+ */
+ public final int frameSizePixels;
+
+ /**
+ * Recommended minimum bitrate to start encoding.
+ */
+ public final int minStartBitrateBps;
+
+ /**
+ * Recommended minimum bitrate.
+ */
+ public final int minBitrateBps;
+
+ /**
+ * Recommended maximum bitrate.
+ */
+ public final int maxBitrateBps;
+
+ public ResolutionBitrateThresholds(
+ int frameSizePixels, int minStartBitrateBps, int minBitrateBps, int maxBitrateBps) {
+ this.frameSizePixels = frameSizePixels;
+ this.minStartBitrateBps = minStartBitrateBps;
+ this.minBitrateBps = minBitrateBps;
+ this.maxBitrateBps = maxBitrateBps;
+ }
+
+ @CalledByNative("ResolutionBitrateThresholds")
+ public int getFrameSizePixels() {
+ return frameSizePixels;
+ }
+
+ @CalledByNative("ResolutionBitrateThresholds")
+ public int getMinStartBitrateBps() {
+ return minStartBitrateBps;
+ }
+
+ @CalledByNative("ResolutionBitrateThresholds")
+ public int getMinBitrateBps() {
+ return minBitrateBps;
+ }
+
+ @CalledByNative("ResolutionBitrateThresholds")
+ public int getMaxBitrateBps() {
+ return maxBitrateBps;
+ }
+ }
+
public interface Callback {
/**
* Call to return an encoded frame. It is safe to assume the byte buffer held by |frame| is not
@@ -240,6 +295,14 @@
/** Any encoder that wants to use WebRTC provided quality scaler must implement this method. */
@CalledByNative ScalingSettings getScalingSettings();
+ /** Returns the list of resolution bitrate thresholds. */
+ @CalledByNative
+ default ResolutionBitrateThresholds[] getResolutionBitrateThresholds() {
+ // TODO(ssilkin): Update downstream projects and remove default implementation.
+ ResolutionBitrateThresholds thresholds[] = {};
+ return thresholds;
+ }
+
/**
* Should return a descriptive name for the implementation. Gets called once and cached. May be
* called from arbitrary thread.
diff --git a/sdk/android/native_unittests/codecs/wrapper_unittest.cc b/sdk/android/native_unittests/codecs/wrapper_unittest.cc
index d9f268d..d5a29d1 100644
--- a/sdk/android/native_unittests/codecs/wrapper_unittest.cc
+++ b/sdk/android/native_unittests/codecs/wrapper_unittest.cc
@@ -8,9 +8,13 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include "sdk/android/native_api/codecs/wrapper.h"
+#include <memory>
+
+#include "absl/memory/memory.h"
#include "media/base/media_constants.h"
#include "sdk/android/generated_native_unittests_jni/CodecsWrapperTestHelper_jni.h"
+#include "sdk/android/native_api/codecs/wrapper.h"
+#include "sdk/android/src/jni/video_encoder_wrapper.h"
#include "test/gtest.h"
namespace webrtc {
@@ -30,6 +34,25 @@
ASSERT_NE(it, video_format.parameters.end());
EXPECT_EQ(cricket::kH264ProfileLevelConstrainedBaseline, it->second);
}
+
+TEST(JavaCodecsWrapperTest, JavaToNativeResolutionBitrateThresholds) {
+ JNIEnv* env = AttachCurrentThreadIfNeeded();
+ ScopedJavaLocalRef<jobject> j_fake_encoder =
+ jni::Java_CodecsWrapperTestHelper_createFakeVideoEncoder(env);
+
+ auto encoder = jni::JavaToNativeVideoEncoder(env, j_fake_encoder);
+ ASSERT_TRUE(encoder);
+
+ // Check that the resolution bitrate thresholds are correctly passed from Java
+ // to native.
+ const std::vector<VideoEncoder::ResolutionBitrateThresholds> thresholds =
+ encoder->GetEncoderInfo().resolution_bitrate_thresholds;
+ ASSERT_EQ(thresholds.size(), 1u);
+ EXPECT_EQ(thresholds[0].frame_size_pixels, 640 * 360);
+ EXPECT_EQ(thresholds[0].min_start_bitrate_bps, 300000);
+ EXPECT_EQ(thresholds[0].min_bitrate_bps, 200000);
+ EXPECT_EQ(thresholds[0].max_bitrate_bps, 1000000);
+}
} // namespace
} // namespace test
} // namespace webrtc
diff --git a/sdk/android/native_unittests/org/webrtc/CodecsWrapperTestHelper.java b/sdk/android/native_unittests/org/webrtc/CodecsWrapperTestHelper.java
index 0213574..70151d3 100644
--- a/sdk/android/native_unittests/org/webrtc/CodecsWrapperTestHelper.java
+++ b/sdk/android/native_unittests/org/webrtc/CodecsWrapperTestHelper.java
@@ -23,4 +23,9 @@
VideoCodecInfo codec_info = new VideoCodecInfo("H264", params);
return codec_info;
}
+
+ @CalledByNative
+ public static VideoEncoder createFakeVideoEncoder() {
+ return new FakeVideoEncoder();
+ }
}
diff --git a/sdk/android/native_unittests/org/webrtc/FakeVideoEncoder.java b/sdk/android/native_unittests/org/webrtc/FakeVideoEncoder.java
new file mode 100644
index 0000000..32a3ae4
--- /dev/null
+++ b/sdk/android/native_unittests/org/webrtc/FakeVideoEncoder.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+/**
+ * An implementation of VideoEncoder that is used for testing of functionalities of
+ * VideoEncoderWrapper.
+ */
+class FakeVideoEncoder implements VideoEncoder {
+ @Override
+ public VideoCodecStatus initEncode(Settings settings, Callback encodeCallback) {
+ return VideoCodecStatus.OK;
+ }
+
+ @Override
+ public VideoCodecStatus release() {
+ return VideoCodecStatus.OK;
+ }
+
+ @Override
+ public VideoCodecStatus encode(VideoFrame frame, EncodeInfo info) {
+ return VideoCodecStatus.OK;
+ }
+
+ @Override
+ public VideoCodecStatus setRateAllocation(BitrateAllocation allocation, int framerate) {
+ return VideoCodecStatus.OK;
+ }
+
+ @Override
+ public ScalingSettings getScalingSettings() {
+ return ScalingSettings.OFF;
+ }
+
+ @Override
+ public ResolutionBitrateThresholds[] getResolutionBitrateThresholds() {
+ ResolutionBitrateThresholds resolution_bitrate_thresholds[] = {
+ new ResolutionBitrateThresholds(/* frameSizePixels = */ 640 * 360,
+ /* minStartBitrateBps = */ 300000,
+ /* minBitrateBps = */ 200000,
+ /* maxBitrateBps = */ 1000000)};
+
+ return resolution_bitrate_thresholds;
+ }
+
+ @Override
+ public String getImplementationName() {
+ return "FakeVideoEncoder";
+ }
+}
diff --git a/sdk/android/src/jni/video_encoder_wrapper.cc b/sdk/android/src/jni/video_encoder_wrapper.cc
index 4f4cce7..02c9216 100644
--- a/sdk/android/src/jni/video_encoder_wrapper.cc
+++ b/sdk/android/src/jni/video_encoder_wrapper.cc
@@ -35,6 +35,11 @@
: encoder_(jni, j_encoder), int_array_class_(GetClass(jni, "[I")) {
initialized_ = false;
num_resets_ = 0;
+
+ // Get bitrate thresholds in the constructor. This is a static property of the
+ // encoder and is expected to be available before it is initialized.
+ encoder_info_.resolution_bitrate_thresholds =
+ GetResolutionBitrateThresholds(jni);
}
VideoEncoderWrapper::~VideoEncoderWrapper() = default;
@@ -214,6 +219,38 @@
}
}
+std::vector<VideoEncoder::ResolutionBitrateThresholds>
+VideoEncoderWrapper::GetResolutionBitrateThresholds(JNIEnv* jni) const {
+ std::vector<VideoEncoder::ResolutionBitrateThresholds>
+ resolution_bitrate_thresholds;
+
+ ScopedJavaLocalRef<jobjectArray> j_thresholds_array =
+ Java_VideoEncoder_getResolutionBitrateThresholds(jni, encoder_);
+
+ const jsize num_thresholds = jni->GetArrayLength(j_thresholds_array.obj());
+ for (int i = 0; i < num_thresholds; ++i) {
+ ScopedJavaLocalRef<jobject> j_thresholds = ScopedJavaLocalRef<jobject>(
+ jni, jni->GetObjectArrayElement(j_thresholds_array.obj(), i));
+
+ jint frame_size_pixels =
+ Java_ResolutionBitrateThresholds_getFrameSizePixels(jni, j_thresholds);
+ jint min_start_bitrate_bps =
+ Java_ResolutionBitrateThresholds_getMinStartBitrateBps(jni,
+ j_thresholds);
+ jint min_bitrate_bps =
+ Java_ResolutionBitrateThresholds_getMinBitrateBps(jni, j_thresholds);
+ jint max_bitrate_bps =
+ Java_ResolutionBitrateThresholds_getMaxBitrateBps(jni, j_thresholds);
+
+ resolution_bitrate_thresholds.push_back(
+ VideoEncoder::ResolutionBitrateThresholds(
+ frame_size_pixels, min_start_bitrate_bps, min_bitrate_bps,
+ max_bitrate_bps));
+ }
+
+ return resolution_bitrate_thresholds;
+}
+
void VideoEncoderWrapper::OnEncodedFrame(
JNIEnv* jni,
const JavaRef<jobject>& j_caller,
diff --git a/sdk/android/src/jni/video_encoder_wrapper.h b/sdk/android/src/jni/video_encoder_wrapper.h
index 685c6da..1564b67 100644
--- a/sdk/android/src/jni/video_encoder_wrapper.h
+++ b/sdk/android/src/jni/video_encoder_wrapper.h
@@ -79,6 +79,9 @@
ScalingSettings GetScalingSettingsInternal(JNIEnv* jni) const;
+ std::vector<ResolutionBitrateThresholds> GetResolutionBitrateThresholds(
+ JNIEnv* jni) const;
+
const ScopedJavaGlobalRef<jobject> encoder_;
const ScopedJavaGlobalRef<jclass> int_array_class_;
