Reland of Add experimental simulcast screen content mode
The original CL was reverted because of a bug discovered by the
chromium bots. Description of that CL:
> Review-Url: https://codereview.webrtc.org/2636443002
> Cr-Commit-Position: refs/heads/master@{#16135}
> Committed: https://chromium.googlesource.com/external/webrtc/+/a28e971e3bc8a06f366f0ef82cc7168b05a83b59
The first patch set of this CL is the same as r16135.
Subsequence patch sets are the fixes applied.
Some new test cases have been added, which reveal a few more bugs that
have also been fixed.
BUG=webrtc:4172
Review-Url: https://codereview.webrtc.org/2641133002
Cr-Commit-Position: refs/heads/master@{#16299}
diff --git a/webrtc/media/engine/simulcast.cc b/webrtc/media/engine/simulcast.cc
index a89ac52..62587f6 100644
--- a/webrtc/media/engine/simulcast.cc
+++ b/webrtc/media/engine/simulcast.cc
@@ -13,6 +13,7 @@
#include "webrtc/base/arraysize.h"
#include "webrtc/base/logging.h"
#include "webrtc/media/base/streamparams.h"
+#include "webrtc/media/engine/constants.h"
#include "webrtc/media/engine/simulcast.h"
#include "webrtc/system_wrappers/include/field_trial.h"
@@ -48,6 +49,8 @@
{0, 0, 1, 200, 150, 30}
};
+const int kDefaultScreenshareSimulcastStreams = 2;
+
// Multiway: Number of temporal layers for each simulcast stream, for maximum
// possible number of simulcast streams |kMaxSimulcastStreams|. The array
// goes from lowest resolution at position 0 to highest resolution.
@@ -78,8 +81,8 @@
MaybeExchangeWidthHeight(&width, &height);
for (uint32_t i = 0; i < arraysize(kSimulcastFormats); ++i) {
- if (width >= kSimulcastFormats[i].width &&
- height >= kSimulcastFormats[i].height) {
+ if (width * height >=
+ kSimulcastFormats[i].width * kSimulcastFormats[i].height) {
return i;
}
}
@@ -90,8 +93,8 @@
MaybeExchangeWidthHeight(&width, &height);
for (uint32_t i = 0; i < arraysize(kSimulcastFormats); ++i) {
- if (width >= kSimulcastFormats[i].width &&
- height >= kSimulcastFormats[i].height &&
+ if (width * height >=
+ kSimulcastFormats[i].width * kSimulcastFormats[i].height &&
max_layers == kSimulcastFormats[i].max_layers) {
return i;
}
@@ -117,7 +120,7 @@
// TODO(marpan): Investigate if we should return 0 instead of -1 in
// FindSimulcast[Max/Target/Min]Bitrate functions below, since the
// codec struct max/min/targeBitrates are unsigned.
-int FindSimulcastMaxBitrateBps(int width, int height, size_t max_layers) {
+int FindSimulcastMaxBitrateBps(int width, int height) {
const int format_index = FindSimulcastFormatIndex(width, height);
if (format_index == -1) {
return -1;
@@ -125,9 +128,7 @@
return kSimulcastFormats[format_index].max_bitrate_kbps * 1000;
}
-int FindSimulcastTargetBitrateBps(int width,
- int height,
- size_t max_layers) {
+int FindSimulcastTargetBitrateBps(int width, int height) {
const int format_index = FindSimulcastFormatIndex(width, height);
if (format_index == -1) {
return -1;
@@ -135,7 +136,7 @@
return kSimulcastFormats[format_index].target_bitrate_kbps * 1000;
}
-int FindSimulcastMinBitrateBps(int width, int height, size_t max_layers) {
+int FindSimulcastMinBitrateBps(int width, int height) {
const int format_index = FindSimulcastFormatIndex(width, height);
if (format_index == -1) {
return -1;
@@ -166,52 +167,73 @@
return total_max_bitrate_bps;
}
-std::vector<webrtc::VideoStream> GetSimulcastConfig(
- size_t max_streams,
- int width,
- int height,
- int max_bitrate_bps,
- int max_qp,
- int max_framerate) {
- size_t simulcast_layers = FindSimulcastMaxLayers(width, height);
- if (simulcast_layers > max_streams) {
+std::vector<webrtc::VideoStream> GetSimulcastConfig(size_t max_streams,
+ int width,
+ int height,
+ int max_bitrate_bps,
+ int max_qp,
+ int max_framerate,
+ bool is_screencast) {
+ size_t num_simulcast_layers;
+ if (is_screencast) {
+ num_simulcast_layers =
+ UseSimulcastScreenshare() ? kDefaultScreenshareSimulcastStreams : 1;
+ } else {
+ num_simulcast_layers = FindSimulcastMaxLayers(width, height);
+ }
+
+ if (num_simulcast_layers > max_streams) {
// If the number of SSRCs in the group differs from our target
// number of simulcast streams for current resolution, switch down
// to a resolution that matches our number of SSRCs.
if (!SlotSimulcastMaxResolution(max_streams, &width, &height)) {
return std::vector<webrtc::VideoStream>();
}
- simulcast_layers = max_streams;
+ num_simulcast_layers = max_streams;
}
std::vector<webrtc::VideoStream> streams;
- streams.resize(simulcast_layers);
+ streams.resize(num_simulcast_layers);
- // Format width and height has to be divisible by |2 ^ number_streams - 1|.
- width = NormalizeSimulcastSize(width, simulcast_layers);
- height = NormalizeSimulcastSize(height, simulcast_layers);
+ if (!is_screencast) {
+ // Format width and height has to be divisible by |2 ^ number_streams - 1|.
+ width = NormalizeSimulcastSize(width, num_simulcast_layers);
+ height = NormalizeSimulcastSize(height, num_simulcast_layers);
+ }
// Add simulcast sub-streams from lower resolution to higher resolutions.
// Add simulcast streams, from highest resolution (|s| = number_streams -1)
// to lowest resolution at |s| = 0.
- for (size_t s = simulcast_layers - 1;; --s) {
+ for (size_t s = num_simulcast_layers - 1;; --s) {
streams[s].width = width;
streams[s].height = height;
// TODO(pbos): Fill actual temporal-layer bitrate thresholds.
- streams[s].temporal_layer_thresholds_bps.resize(
- kDefaultConferenceNumberOfTemporalLayers[s] - 1);
- streams[s].max_bitrate_bps =
- FindSimulcastMaxBitrateBps(width, height, simulcast_layers);
- streams[s].target_bitrate_bps =
- FindSimulcastTargetBitrateBps(width, height, simulcast_layers);
- streams[s].min_bitrate_bps =
- FindSimulcastMinBitrateBps(width, height, simulcast_layers);
streams[s].max_qp = max_qp;
- streams[s].max_framerate = max_framerate;
+ if (is_screencast && s == 0) {
+ ScreenshareLayerConfig config = ScreenshareLayerConfig::GetDefault();
+ // For legacy screenshare in conference mode, tl0 and tl1 bitrates are
+ // piggybacked on the VideoCodec struct as target and max bitrates,
+ // respectively. See eg. webrtc::VP8EncoderImpl::SetRates().
+ streams[s].min_bitrate_bps = kMinVideoBitrateKbps * 1000;
+ streams[s].target_bitrate_bps = config.tl0_bitrate_kbps * 1000;
+ streams[s].max_bitrate_bps = config.tl1_bitrate_kbps * 1000;
+ streams[s].temporal_layer_thresholds_bps.clear();
+ streams[s].temporal_layer_thresholds_bps.push_back(
+ config.tl0_bitrate_kbps * 1000);
+ streams[s].max_framerate = 5;
+ } else {
+ streams[s].temporal_layer_thresholds_bps.resize(
+ kDefaultConferenceNumberOfTemporalLayers[s] - 1);
+ streams[s].max_bitrate_bps = FindSimulcastMaxBitrateBps(width, height);
+ streams[s].target_bitrate_bps =
+ FindSimulcastTargetBitrateBps(width, height);
+ streams[s].min_bitrate_bps = FindSimulcastMinBitrateBps(width, height);
+ streams[s].max_framerate = max_framerate;
+ }
+
width /= 2;
height /= 2;
- if (s == 0) {
+ if (s == 0)
break;
- }
}
// Spend additional bits to boost the max stream.
@@ -230,6 +252,8 @@
static const char* kScreencastLayerFieldTrialName =
"WebRTC-ScreenshareLayerRates";
+static const char* kSimulcastScreenshareFieldTrialName =
+ "WebRTC-SimulcastScreenshare";
ScreenshareLayerConfig::ScreenshareLayerConfig(int tl0_bitrate, int tl1_bitrate)
: tl0_bitrate_kbps(tl0_bitrate), tl1_bitrate_kbps(tl1_bitrate) {
@@ -272,4 +296,9 @@
return true;
}
+bool UseSimulcastScreenshare() {
+ return webrtc::field_trial::FindFullName(
+ kSimulcastScreenshareFieldTrialName) == "Enabled";
+}
+
} // namespace cricket
diff --git a/webrtc/media/engine/simulcast.h b/webrtc/media/engine/simulcast.h
index 20be4c4..3aae70f 100644
--- a/webrtc/media/engine/simulcast.h
+++ b/webrtc/media/engine/simulcast.h
@@ -19,6 +19,7 @@
namespace cricket {
struct StreamParams;
+// TODO(sprang): Remove this, as we're moving away from temporal layer mode.
// Config for use with screen cast when temporal layers are enabled.
struct ScreenshareLayerConfig {
public:
@@ -45,12 +46,16 @@
void GetSimulcastSsrcs(const StreamParams& sp, std::vector<uint32_t>* ssrcs);
// Get simulcast settings.
+// TODO(sprang): Remove default parameter when it's not longer referenced.
std::vector<webrtc::VideoStream> GetSimulcastConfig(size_t max_streams,
int width,
int height,
int max_bitrate_bps,
int max_qp,
- int max_framerate);
+ int max_framerate,
+ bool is_screencast = false);
+
+bool UseSimulcastScreenshare();
} // namespace cricket
diff --git a/webrtc/media/engine/webrtcvideoengine2.cc b/webrtc/media/engine/webrtcvideoengine2.cc
index 9e7f5c4..0cd3193 100644
--- a/webrtc/media/engine/webrtcvideoengine2.cc
+++ b/webrtc/media/engine/webrtcvideoengine2.cc
@@ -301,11 +301,16 @@
int width,
int height,
const webrtc::VideoEncoderConfig& encoder_config) override {
- RTC_DCHECK(encoder_config.number_of_streams > 1 ? !is_screencast_ : true);
- if (encoder_config.number_of_streams > 1) {
+ if (is_screencast_ &&
+ (!conference_mode_ || !cricket::UseSimulcastScreenshare())) {
+ RTC_DCHECK_EQ(1, encoder_config.number_of_streams);
+ }
+ if (encoder_config.number_of_streams > 1 ||
+ (CodecNamesEq(codec_name_, kVp8CodecName) && is_screencast_ &&
+ conference_mode_)) {
return GetSimulcastConfig(encoder_config.number_of_streams, width, height,
encoder_config.max_bitrate_bps, max_qp_,
- max_framerate_);
+ max_framerate_, is_screencast_);
}
// For unset max bitrates set default bitrate for non-simulcast.
@@ -322,20 +327,6 @@
stream.target_bitrate_bps = stream.max_bitrate_bps = max_bitrate_bps;
stream.max_qp = max_qp_;
- // Conference mode screencast uses 2 temporal layers split at 100kbit.
- if (conference_mode_ && is_screencast_) {
- ScreenshareLayerConfig config = ScreenshareLayerConfig::GetDefault();
- // For screenshare in conference mode, tl0 and tl1 bitrates are
- // piggybacked
- // on the VideoCodec struct as target and max bitrates, respectively.
- // See eg. webrtc::VP8EncoderImpl::SetRates().
- stream.target_bitrate_bps = config.tl0_bitrate_kbps * 1000;
- stream.max_bitrate_bps = config.tl1_bitrate_kbps * 1000;
- stream.temporal_layer_thresholds_bps.clear();
- stream.temporal_layer_thresholds_bps.push_back(config.tl0_bitrate_kbps *
- 1000);
- }
-
if (CodecNamesEq(codec_name_, kVp9CodecName) && !is_screencast_) {
stream.temporal_layer_thresholds_bps.resize(
GetDefaultVp9TemporalLayers() - 1);
@@ -1551,6 +1542,7 @@
enable_cpu_overuse_detection_(enable_cpu_overuse_detection),
source_(nullptr),
external_encoder_factory_(external_encoder_factory),
+ internal_encoder_factory_(new InternalEncoderFactory()),
stream_(nullptr),
encoder_sink_(nullptr),
parameters_(std::move(config), options, max_bitrate_bps, codec_settings),
@@ -1678,10 +1670,20 @@
}
// Try creating internal encoder.
- InternalEncoderFactory internal_encoder_factory;
- if (FindMatchingCodec(internal_encoder_factory.supported_codecs(), codec)) {
- return AllocatedEncoder(internal_encoder_factory.CreateVideoEncoder(codec),
- codec, false /* is_external */);
+ if (FindMatchingCodec(internal_encoder_factory_->supported_codecs(), codec)) {
+ if (parameters_.encoder_config.content_type ==
+ webrtc::VideoEncoderConfig::ContentType::kScreen &&
+ parameters_.conference_mode && UseSimulcastScreenshare()) {
+ // TODO(sprang): Remove this adapter once libvpx supports simulcast with
+ // same-resolution substreams.
+ WebRtcSimulcastEncoderFactory adapter_factory(
+ internal_encoder_factory_.get());
+ return AllocatedEncoder(adapter_factory.CreateVideoEncoder(codec), codec,
+ false /* is_external */);
+ }
+ return AllocatedEncoder(
+ internal_encoder_factory_->CreateVideoEncoder(codec), codec,
+ false /* is_external */);
}
// This shouldn't happen, we should not be trying to create something we don't
@@ -1858,9 +1860,11 @@
// By default, the stream count for the codec configuration should match the
// number of negotiated ssrcs. But if the codec is blacklisted for simulcast
- // or a screencast, only configure a single stream.
+ // or a screencast (and not in simulcast screenshare experiment), only
+ // configure a single stream.
encoder_config.number_of_streams = parameters_.config.rtp.ssrcs.size();
- if (IsCodecBlacklistedForSimulcast(codec.name) || is_screencast) {
+ if (IsCodecBlacklistedForSimulcast(codec.name) ||
+ (is_screencast && !UseSimulcastScreenshare())) {
encoder_config.number_of_streams = 1;
}
diff --git a/webrtc/media/engine/webrtcvideoengine2.h b/webrtc/media/engine/webrtcvideoengine2.h
index 2ee9bc7..7043e03 100644
--- a/webrtc/media/engine/webrtcvideoengine2.h
+++ b/webrtc/media/engine/webrtcvideoengine2.h
@@ -330,6 +330,8 @@
ACCESS_ON(&thread_checker_);
WebRtcVideoEncoderFactory* const external_encoder_factory_
ACCESS_ON(&thread_checker_);
+ const std::unique_ptr<WebRtcVideoEncoderFactory> internal_encoder_factory_
+ ACCESS_ON(&thread_checker_);
webrtc::VideoSendStream* stream_ ACCESS_ON(&thread_checker_);
rtc::VideoSinkInterface<webrtc::VideoFrame>* encoder_sink_
diff --git a/webrtc/media/engine/webrtcvideoengine2_unittest.cc b/webrtc/media/engine/webrtcvideoengine2_unittest.cc
index f72d7ae..a93d2d9 100644
--- a/webrtc/media/engine/webrtcvideoengine2_unittest.cc
+++ b/webrtc/media/engine/webrtcvideoengine2_unittest.cc
@@ -21,6 +21,7 @@
#include "webrtc/media/base/mediaconstants.h"
#include "webrtc/media/base/testutils.h"
#include "webrtc/media/base/videoengine_unittest.h"
+#include "webrtc/media/engine/constants.h"
#include "webrtc/media/engine/fakewebrtccall.h"
#include "webrtc/media/engine/fakewebrtcvideoengine.h"
#include "webrtc/media/engine/simulcast.h"
@@ -3960,7 +3961,7 @@
class WebRtcVideoChannel2SimulcastTest : public testing::Test {
public:
WebRtcVideoChannel2SimulcastTest()
- : fake_call_(webrtc::Call::Config(&event_log_)) {}
+ : fake_call_(webrtc::Call::Config(&event_log_)), last_ssrc_(0) {}
void SetUp() override {
engine_.Init();
@@ -3975,9 +3976,16 @@
int capture_width,
int capture_height,
size_t num_configured_streams,
- size_t expected_num_streams) {
+ size_t expected_num_streams,
+ bool screenshare,
+ bool conference_mode) {
cricket::VideoSendParameters parameters;
+ VideoOptions options;
parameters.codecs.push_back(codec);
+ parameters.conference_mode = conference_mode;
+ if (screenshare) {
+ options.is_screencast = rtc::Optional<bool>(screenshare);
+ }
ASSERT_TRUE(channel_->SetSendParameters(parameters));
std::vector<uint32_t> ssrcs = MAKE_VECTOR(kSsrcs3);
@@ -3990,7 +3998,7 @@
// expected simulcast layers.
cricket::FakeVideoCapturer capturer;
EXPECT_TRUE(
- channel_->SetVideoSend(ssrcs.front(), true, nullptr, &capturer));
+ channel_->SetVideoSend(ssrcs.front(), true, &options, &capturer));
EXPECT_EQ(cricket::CS_RUNNING, capturer.Start(cricket::VideoFormat(
capture_width, capture_height,
cricket::VideoFormat::FpsToInterval(30),
@@ -4001,9 +4009,32 @@
std::vector<webrtc::VideoStream> video_streams = stream->GetVideoStreams();
ASSERT_EQ(expected_num_streams, video_streams.size());
- std::vector<webrtc::VideoStream> expected_streams = GetSimulcastConfig(
- num_configured_streams, capture_width, capture_height, 0, kDefaultQpMax,
- kDefaultVideoMaxFramerate);
+ std::vector<webrtc::VideoStream> expected_streams;
+ if (conference_mode) {
+ expected_streams = GetSimulcastConfig(
+ num_configured_streams, capture_width, capture_height, 0,
+ kDefaultQpMax, kDefaultVideoMaxFramerate, screenshare);
+ } else {
+ webrtc::VideoStream stream;
+ stream.width = capture_width;
+ stream.height = capture_height;
+ stream.max_framerate = kDefaultVideoMaxFramerate;
+ stream.min_bitrate_bps = cricket::kMinVideoBitrateKbps * 1000;
+ int max_bitrate_kbps;
+ if (capture_width * capture_height <= 320 * 240) {
+ max_bitrate_kbps = 600;
+ } else if (capture_width * capture_height <= 640 * 480) {
+ max_bitrate_kbps = 1700;
+ } else if (capture_width * capture_height <= 960 * 540) {
+ max_bitrate_kbps = 2000;
+ } else {
+ max_bitrate_kbps = 2500;
+ }
+ stream.target_bitrate_bps = stream.max_bitrate_bps =
+ max_bitrate_kbps * 1000;
+ stream.max_qp = kDefaultQpMax;
+ expected_streams.push_back(stream);
+ }
ASSERT_EQ(expected_streams.size(), video_streams.size());
@@ -4032,7 +4063,8 @@
EXPECT_GT(video_streams[i].max_qp, 0);
EXPECT_EQ(expected_streams[i].max_qp, video_streams[i].max_qp);
- EXPECT_FALSE(expected_streams[i].temporal_layer_thresholds_bps.empty());
+ EXPECT_EQ(!conference_mode,
+ expected_streams[i].temporal_layer_thresholds_bps.empty());
EXPECT_EQ(expected_streams[i].temporal_layer_thresholds_bps,
video_streams[i].temporal_layer_thresholds_bps);
@@ -4086,15 +4118,37 @@
};
TEST_F(WebRtcVideoChannel2SimulcastTest, SetSendCodecsWith2SimulcastStreams) {
- VerifySimulcastSettings(cricket::VideoCodec("VP8"), 640, 360, 2, 2);
+ VerifySimulcastSettings(cricket::VideoCodec("VP8"), 640, 360, 2, 2, false,
+ true);
}
TEST_F(WebRtcVideoChannel2SimulcastTest, SetSendCodecsWith3SimulcastStreams) {
- VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 3);
+ VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 3, false,
+ true);
}
// Test that we normalize send codec format size in simulcast.
TEST_F(WebRtcVideoChannel2SimulcastTest, SetSendCodecsWithOddSizeInSimulcast) {
- VerifySimulcastSettings(cricket::VideoCodec("VP8"), 541, 271, 2, 2);
+ VerifySimulcastSettings(cricket::VideoCodec("VP8"), 541, 271, 2, 2, false,
+ true);
}
+
+TEST_F(WebRtcVideoChannel2SimulcastTest, SetSendCodecsForScreenshare) {
+ VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 1, true,
+ false);
+}
+
+TEST_F(WebRtcVideoChannel2SimulcastTest,
+ SetSendCodecsForConferenceModeScreenshare) {
+ VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 1, true,
+ true);
+}
+
+TEST_F(WebRtcVideoChannel2SimulcastTest, SetSendCodecsForSimulcastScreenshare) {
+ webrtc::test::ScopedFieldTrials override_field_trials_(
+ "WebRTC-SimulcastScreenshare/Enabled/");
+ VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 2, true,
+ true);
+}
+
} // namespace cricket
diff --git a/webrtc/modules/video_coding/BUILD.gn b/webrtc/modules/video_coding/BUILD.gn
index fc4d3af..e09fd9f 100644
--- a/webrtc/modules/video_coding/BUILD.gn
+++ b/webrtc/modules/video_coding/BUILD.gn
@@ -371,6 +371,7 @@
"utility/moving_average_unittest.cc",
"utility/quality_scaler_unittest.cc",
"utility/simulcast_rate_allocator_unittest.cc",
+ "video_codec_initializer_unittest.cc",
"video_coding_robustness_unittest.cc",
"video_packet_buffer_unittest.cc",
"video_receiver_unittest.cc",
diff --git a/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc b/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc
index 7807d45..f842fb4 100644
--- a/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc
+++ b/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc
@@ -94,11 +94,11 @@
timestamp_(0),
last_base_layer_sync_(0),
layer_ids_length_(0),
- layer_ids_(NULL),
+ layer_ids_(nullptr),
encode_flags_length_(0),
- encode_flags_(NULL) {
+ encode_flags_(nullptr) {
RTC_CHECK_GE(max_temporal_layers_, 1);
- RTC_CHECK_GE(max_temporal_layers_, 3);
+ RTC_CHECK_LE(max_temporal_layers_, 3);
}
virtual ~RealTimeTemporalLayers() {}
diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc
index feb8da9..aca4514 100644
--- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc
+++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc
@@ -30,6 +30,8 @@
const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5;
const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0;
+constexpr int ScreenshareLayers::kMaxNumTemporalLayers;
+
// Since this is TL0 we only allow updating and predicting from the LAST
// reference frame.
const int ScreenshareLayers::kTl0Flags =
@@ -55,8 +57,14 @@
int simulcast_id,
int num_temporal_layers,
uint8_t initial_tl0_pic_idx) const {
- webrtc::TemporalLayers* tl = new webrtc::ScreenshareLayers(
- num_temporal_layers, rand(), webrtc::Clock::GetRealTimeClock());
+ webrtc::TemporalLayers* tl;
+ if (simulcast_id == 0) {
+ tl = new webrtc::ScreenshareLayers(num_temporal_layers, rand(),
+ webrtc::Clock::GetRealTimeClock());
+ } else {
+ RealTimeTemporalLayersFactory rt_tl_factory;
+ tl = rt_tl_factory.Create(simulcast_id, num_temporal_layers, rand());
+ }
if (listener_)
listener_->OnTemporalLayersCreated(simulcast_id, tl);
return tl;
@@ -66,7 +74,8 @@
uint8_t initial_tl0_pic_idx,
Clock* clock)
: clock_(clock),
- number_of_temporal_layers_(num_temporal_layers),
+ number_of_temporal_layers_(
+ std::min(kMaxNumTemporalLayers, num_temporal_layers)),
last_base_layer_sync_(false),
tl0_pic_idx_(initial_tl0_pic_idx),
active_layer_(-1),
@@ -78,8 +87,8 @@
max_debt_bytes_(0),
encode_framerate_(1000.0f, 1000.0f), // 1 second window, second scale.
bitrate_updated_(false) {
- RTC_CHECK_GT(num_temporal_layers, 0);
- RTC_CHECK_LE(num_temporal_layers, 2);
+ RTC_CHECK_GT(number_of_temporal_layers_, 0);
+ RTC_CHECK_LE(number_of_temporal_layers_, kMaxNumTemporalLayers);
}
ScreenshareLayers::~ScreenshareLayers() {
diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h
index de7665c..53818a2 100644
--- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h
+++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h
@@ -84,7 +84,7 @@
RateStatistics encode_framerate_;
bool bitrate_updated_;
- static const int kMaxNumTemporalLayers = 2;
+ static constexpr int kMaxNumTemporalLayers = 2;
struct TemporalLayer {
TemporalLayer()
: state(State::kNormal),
diff --git a/webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc b/webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc
index 311568b..0a81550 100644
--- a/webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc
+++ b/webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc
@@ -106,23 +106,24 @@
const int num_temporal_streams = std::max<uint8_t>(
1, codec_.numberOfSimulcastStreams == 0
? codec_.VP8().numberOfTemporalLayers
- : codec_.simulcastStream[0].numberOfTemporalLayers);
+ : codec_.simulcastStream[simulcast_id].numberOfTemporalLayers);
uint32_t max_bitrate_kbps;
- if (num_spatial_streams == 1) {
- max_bitrate_kbps = codec_.maxBitrate;
-
- // TODO(holmer): This is a temporary hack for screensharing, where we
+ // Legacy temporal-layered only screenshare, or simulcast screenshare
+ // with legacy mode for simulcast stream 0.
+ if (codec_.mode == kScreensharing && codec_.targetBitrate > 0 &&
+ ((num_spatial_streams == 1 && num_temporal_streams == 2) || // Legacy.
+ (num_spatial_streams > 1 && simulcast_id == 0))) { // Simulcast.
+ // TODO(holmer): This is a "temporary" hack for screensharing, where we
// interpret the startBitrate as the encoder target bitrate. This is
// to allow for a different max bitrate, so if the codec can't meet
// the target we still allow it to overshoot up to the max before dropping
// frames. This hack should be improved.
- if (codec_.mode == kScreensharing && codec_.targetBitrate > 0 &&
- num_temporal_streams == 2) {
- int tl0_bitrate = std::min(codec_.targetBitrate, target_bitrate_kbps);
- max_bitrate_kbps = std::min(codec_.maxBitrate, target_bitrate_kbps);
- target_bitrate_kbps = tl0_bitrate;
- }
+ int tl0_bitrate = std::min(codec_.targetBitrate, target_bitrate_kbps);
+ max_bitrate_kbps = std::min(codec_.maxBitrate, target_bitrate_kbps);
+ target_bitrate_kbps = tl0_bitrate;
+ } else if (num_spatial_streams == 1) {
+ max_bitrate_kbps = codec_.maxBitrate;
} else {
max_bitrate_kbps = codec_.simulcastStream[simulcast_id].maxBitrate;
}
diff --git a/webrtc/modules/video_coding/video_codec_initializer.cc b/webrtc/modules/video_coding/video_codec_initializer.cc
index dbdede0..96a0033 100644
--- a/webrtc/modules/video_coding/video_codec_initializer.cc
+++ b/webrtc/modules/video_coding/video_codec_initializer.cc
@@ -38,8 +38,9 @@
case kVideoCodecVP8: {
if (!codec->VP8()->tl_factory) {
if (codec->mode == kScreensharing &&
- codec->numberOfSimulcastStreams == 1 &&
- codec->VP8()->numberOfTemporalLayers == 2) {
+ (codec->numberOfSimulcastStreams > 1 ||
+ (codec->numberOfSimulcastStreams == 1 &&
+ codec->VP8()->numberOfTemporalLayers == 2))) {
// Conference mode temporal layering for screen content.
tl_factory.reset(new ScreenshareTemporalLayersFactory());
} else {
@@ -102,7 +103,7 @@
break;
case VideoEncoderConfig::ContentType::kScreen:
video_codec.mode = kScreensharing;
- if (streams.size() == 1 &&
+ if (streams.size() >= 1 &&
streams[0].temporal_layer_thresholds_bps.size() == 1) {
video_codec.targetBitrate =
streams[0].temporal_layer_thresholds_bps[0] / 1000;
@@ -180,8 +181,12 @@
RTC_DCHECK_GT(streams[i].width, 0);
RTC_DCHECK_GT(streams[i].height, 0);
RTC_DCHECK_GT(streams[i].max_framerate, 0);
- // Different framerates not supported per stream at the moment.
- RTC_DCHECK_EQ(streams[i].max_framerate, streams[0].max_framerate);
+ // Different framerates not supported per stream at the moment, unless it's
+ // screenshare where there is an exception and a simulcast encoder adapter,
+ // which supports different framerates, is used instead.
+ if (config.content_type != VideoEncoderConfig::ContentType::kScreen) {
+ RTC_DCHECK_EQ(streams[i].max_framerate, streams[0].max_framerate);
+ }
RTC_DCHECK_GE(streams[i].min_bitrate_bps, 0);
RTC_DCHECK_GE(streams[i].target_bitrate_bps, streams[i].min_bitrate_bps);
RTC_DCHECK_GE(streams[i].max_bitrate_bps, streams[i].target_bitrate_bps);
diff --git a/webrtc/modules/video_coding/video_codec_initializer_unittest.cc b/webrtc/modules/video_coding/video_codec_initializer_unittest.cc
new file mode 100644
index 0000000..d9d1d03
--- /dev/null
+++ b/webrtc/modules/video_coding/video_codec_initializer_unittest.cc
@@ -0,0 +1,217 @@
+/*
+ * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/common_video/include/video_bitrate_allocator.h"
+#include "webrtc/common_types.h"
+#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
+#include "webrtc/modules/video_coding/include/video_codec_initializer.h"
+#include "webrtc/test/gtest.h"
+#include "webrtc/video_encoder.h"
+
+namespace webrtc {
+
+namespace {
+static const char* kVp8PayloadName = "VP8";
+static const int kVp8PayloadType = 100;
+static const int kDefaultWidth = 1280;
+static const int kDefaultHeight = 720;
+static const int kDefaultFrameRate = 30;
+static const uint32_t kDefaultMinBitrateBps = 60000;
+static const uint32_t kDefaultTargetBitrateBps = 2000000;
+static const uint32_t kDefaultMaxBitrateBps = 2000000;
+static const uint32_t kDefaultMinTransmitBitrateBps = 400000;
+static const int kDefaultMaxQp = 48;
+static const uint32_t kScreenshareTl0BitrateBps = 100000;
+static const uint32_t kScreenshareCodecTargetBitrateBps = 200000;
+static const uint32_t kScreenshareDefaultFramerate = 5;
+// Bitrates for the temporal layers of the higher screenshare simulcast stream.
+static const uint32_t kHighScreenshareTl0Bps = 800000;
+static const uint32_t kHighScreenshareTl1Bps = 1200000;
+} // namespace
+
+/*
+ * static bool SetupCodec(
+ const VideoEncoderConfig& config,
+ const VideoSendStream::Config::EncoderSettings settings,
+ const std::vector<VideoStream>& streams,
+ bool nack_enabled,
+ VideoCodec* codec,
+ std::unique_ptr<VideoBitrateAllocator>* bitrate_allocator);
+
+ // Create a bitrate allocator for the specified codec. |tl_factory| is
+ // optional, if it is populated, ownership of that instance will be
+ // transferred to the VideoBitrateAllocator instance.
+ static std::unique_ptr<VideoBitrateAllocator> CreateBitrateAllocator(
+ const VideoCodec& codec,
+ std::unique_ptr<TemporalLayersFactory> tl_factory);
+ */
+
+// TODO(sprang): Extend coverage to handle the rest of the codec initializer.
+class VideoCodecInitializerTest : public ::testing::Test {
+ public:
+ VideoCodecInitializerTest() : nack_enabled_(false) {}
+ virtual ~VideoCodecInitializerTest() {}
+
+ protected:
+ void SetUpFor(VideoCodecType type,
+ int num_spatial_streams,
+ int num_temporal_streams,
+ bool screenshare) {
+ config_ = VideoEncoderConfig();
+ if (screenshare) {
+ config_.min_transmit_bitrate_bps = kDefaultMinTransmitBitrateBps;
+ config_.content_type = VideoEncoderConfig::ContentType::kScreen;
+ }
+
+ if (type == VideoCodecType::kVideoCodecVP8) {
+ config_.number_of_streams = num_spatial_streams;
+ VideoCodecVP8 vp8_settings = VideoEncoder::GetDefaultVp8Settings();
+ vp8_settings.numberOfTemporalLayers = num_temporal_streams;
+ config_.encoder_specific_settings = new rtc::RefCountedObject<
+ webrtc::VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8_settings);
+ settings_.payload_name = kVp8PayloadName;
+ settings_.payload_type = kVp8PayloadType;
+ } else {
+ ADD_FAILURE() << "Unexpected codec type: " << type;
+ }
+ }
+
+ bool InitializeCodec() {
+ codec_out_ = VideoCodec();
+ bitrate_allocator_out_.reset();
+ temporal_layers_.clear();
+ if (!VideoCodecInitializer::SetupCodec(config_, settings_, streams_,
+ nack_enabled_, &codec_out_,
+ &bitrate_allocator_out_)) {
+ return false;
+ }
+
+ // Make sure temporal layers instances have been created.
+ if (codec_out_.codecType == VideoCodecType::kVideoCodecVP8) {
+ if (!codec_out_.VP8()->tl_factory)
+ return false;
+
+ for (int i = 0; i < codec_out_.numberOfSimulcastStreams; ++i) {
+ temporal_layers_.emplace_back(codec_out_.VP8()->tl_factory->Create(
+ i, streams_[i].temporal_layer_thresholds_bps.size() + 1, 0));
+ }
+ }
+ return true;
+ }
+
+ VideoStream DefaultStream() {
+ VideoStream stream;
+ stream.width = kDefaultWidth;
+ stream.height = kDefaultHeight;
+ stream.max_framerate = kDefaultFrameRate;
+ stream.min_bitrate_bps = kDefaultMinBitrateBps;
+ stream.target_bitrate_bps = kDefaultTargetBitrateBps;
+ stream.max_bitrate_bps = kDefaultMaxBitrateBps;
+ stream.max_qp = kDefaultMaxQp;
+ return stream;
+ }
+
+ VideoStream DefaultScreenshareStream() {
+ VideoStream stream = DefaultStream();
+ stream.min_bitrate_bps = 30000;
+ stream.target_bitrate_bps = kScreenshareTl0BitrateBps;
+ stream.max_bitrate_bps = 1000000;
+ stream.max_framerate = kScreenshareDefaultFramerate;
+ stream.temporal_layer_thresholds_bps.push_back(kScreenshareTl0BitrateBps);
+ return stream;
+ }
+
+ // Input settings.
+ VideoEncoderConfig config_;
+ VideoSendStream::Config::EncoderSettings settings_;
+ std::vector<VideoStream> streams_;
+ bool nack_enabled_;
+
+ // Output.
+ VideoCodec codec_out_;
+ std::unique_ptr<VideoBitrateAllocator> bitrate_allocator_out_;
+ std::vector<std::unique_ptr<TemporalLayers>> temporal_layers_;
+};
+
+TEST_F(VideoCodecInitializerTest, SingleStreamVp8Screenshare) {
+ SetUpFor(VideoCodecType::kVideoCodecVP8, 1, 1, true);
+ streams_.push_back(DefaultStream());
+ EXPECT_TRUE(InitializeCodec());
+
+ BitrateAllocation bitrate_allocation = bitrate_allocator_out_->GetAllocation(
+ kDefaultTargetBitrateBps, kDefaultFrameRate);
+ EXPECT_EQ(1u, codec_out_.numberOfSimulcastStreams);
+ EXPECT_EQ(1u, codec_out_.VP8()->numberOfTemporalLayers);
+ EXPECT_EQ(kDefaultTargetBitrateBps, bitrate_allocation.get_sum_bps());
+}
+
+TEST_F(VideoCodecInitializerTest, TemporalLayeredVp8Screenshare) {
+ SetUpFor(VideoCodecType::kVideoCodecVP8, 1, 2, true);
+ streams_.push_back(DefaultScreenshareStream());
+ EXPECT_TRUE(InitializeCodec());
+
+ EXPECT_EQ(1u, codec_out_.numberOfSimulcastStreams);
+ EXPECT_EQ(2u, codec_out_.VP8()->numberOfTemporalLayers);
+ BitrateAllocation bitrate_allocation = bitrate_allocator_out_->GetAllocation(
+ kScreenshareCodecTargetBitrateBps, kScreenshareDefaultFramerate);
+ EXPECT_EQ(kScreenshareCodecTargetBitrateBps,
+ bitrate_allocation.get_sum_bps());
+ EXPECT_EQ(kScreenshareTl0BitrateBps, bitrate_allocation.GetBitrate(0, 0));
+}
+
+TEST_F(VideoCodecInitializerTest, SimlucastVp8Screenshare) {
+ SetUpFor(VideoCodecType::kVideoCodecVP8, 2, 1, true);
+ streams_.push_back(DefaultScreenshareStream());
+ VideoStream video_stream = DefaultStream();
+ video_stream.max_framerate = kScreenshareDefaultFramerate;
+ streams_.push_back(video_stream);
+ EXPECT_TRUE(InitializeCodec());
+
+ EXPECT_EQ(2u, codec_out_.numberOfSimulcastStreams);
+ EXPECT_EQ(1u, codec_out_.VP8()->numberOfTemporalLayers);
+ const uint32_t max_bitrate_bps =
+ streams_[0].target_bitrate_bps + streams_[1].max_bitrate_bps;
+ BitrateAllocation bitrate_allocation = bitrate_allocator_out_->GetAllocation(
+ max_bitrate_bps, kScreenshareDefaultFramerate);
+ EXPECT_EQ(max_bitrate_bps, bitrate_allocation.get_sum_bps());
+ EXPECT_EQ(static_cast<uint32_t>(streams_[0].target_bitrate_bps),
+ bitrate_allocation.GetSpatialLayerSum(0));
+ EXPECT_EQ(static_cast<uint32_t>(streams_[1].max_bitrate_bps),
+ bitrate_allocation.GetSpatialLayerSum(1));
+}
+
+TEST_F(VideoCodecInitializerTest, HighFpsSimlucastVp8Screenshare) {
+ // Two simulcast streams, the lower one using legacy settings (two temporal
+ // streams, 5fps), the higher one using 3 temporal streams and 30fps.
+ SetUpFor(VideoCodecType::kVideoCodecVP8, 2, 3, true);
+ streams_.push_back(DefaultScreenshareStream());
+ VideoStream video_stream = DefaultStream();
+ video_stream.temporal_layer_thresholds_bps.push_back(kHighScreenshareTl0Bps);
+ video_stream.temporal_layer_thresholds_bps.push_back(kHighScreenshareTl1Bps);
+ streams_.push_back(video_stream);
+ EXPECT_TRUE(InitializeCodec());
+
+ EXPECT_EQ(2u, codec_out_.numberOfSimulcastStreams);
+ EXPECT_EQ(3u, codec_out_.VP8()->numberOfTemporalLayers);
+ const uint32_t max_bitrate_bps =
+ streams_[0].target_bitrate_bps + streams_[1].max_bitrate_bps;
+ BitrateAllocation bitrate_allocation =
+ bitrate_allocator_out_->GetAllocation(max_bitrate_bps, kDefaultFrameRate);
+ EXPECT_EQ(max_bitrate_bps, bitrate_allocation.get_sum_bps());
+ EXPECT_EQ(static_cast<uint32_t>(streams_[0].target_bitrate_bps),
+ bitrate_allocation.GetSpatialLayerSum(0));
+ EXPECT_EQ(static_cast<uint32_t>(streams_[1].max_bitrate_bps),
+ bitrate_allocation.GetSpatialLayerSum(1));
+ EXPECT_EQ(kHighScreenshareTl0Bps, bitrate_allocation.GetBitrate(1, 0));
+ EXPECT_EQ(kHighScreenshareTl1Bps - kHighScreenshareTl0Bps,
+ bitrate_allocation.GetBitrate(1, 1));
+}
+
+} // namespace webrtc
