Reland of Add experimental simulcast screen content mode
The original CL was reverted because of a bug discovered by the
chromium bots. Description of that CL:
> Review-Url: https://codereview.webrtc.org/2636443002
> Cr-Commit-Position: refs/heads/master@{#16135}
> Committed: https://chromium.googlesource.com/external/webrtc/+/a28e971e3bc8a06f366f0ef82cc7168b05a83b59
The first patch set of this CL is the same as r16135.
Subsequence patch sets are the fixes applied.
Some new test cases have been added, which reveal a few more bugs that
have also been fixed.
BUG=webrtc:4172
Review-Url: https://codereview.webrtc.org/2641133002
Cr-Original-Commit-Position: refs/heads/master@{#16299}
Cr-Mirrored-From: https://chromium.googlesource.com/external/webrtc
Cr-Mirrored-Commit: 429600d7d00b526b3b35b6387b7d72a053709860
diff --git a/media/engine/simulcast.cc b/media/engine/simulcast.cc
index a89ac52..62587f6 100644
--- a/media/engine/simulcast.cc
+++ b/media/engine/simulcast.cc
@@ -13,6 +13,7 @@
#include "webrtc/base/arraysize.h"
#include "webrtc/base/logging.h"
#include "webrtc/media/base/streamparams.h"
+#include "webrtc/media/engine/constants.h"
#include "webrtc/media/engine/simulcast.h"
#include "webrtc/system_wrappers/include/field_trial.h"
@@ -48,6 +49,8 @@
{0, 0, 1, 200, 150, 30}
};
+const int kDefaultScreenshareSimulcastStreams = 2;
+
// Multiway: Number of temporal layers for each simulcast stream, for maximum
// possible number of simulcast streams |kMaxSimulcastStreams|. The array
// goes from lowest resolution at position 0 to highest resolution.
@@ -78,8 +81,8 @@
MaybeExchangeWidthHeight(&width, &height);
for (uint32_t i = 0; i < arraysize(kSimulcastFormats); ++i) {
- if (width >= kSimulcastFormats[i].width &&
- height >= kSimulcastFormats[i].height) {
+ if (width * height >=
+ kSimulcastFormats[i].width * kSimulcastFormats[i].height) {
return i;
}
}
@@ -90,8 +93,8 @@
MaybeExchangeWidthHeight(&width, &height);
for (uint32_t i = 0; i < arraysize(kSimulcastFormats); ++i) {
- if (width >= kSimulcastFormats[i].width &&
- height >= kSimulcastFormats[i].height &&
+ if (width * height >=
+ kSimulcastFormats[i].width * kSimulcastFormats[i].height &&
max_layers == kSimulcastFormats[i].max_layers) {
return i;
}
@@ -117,7 +120,7 @@
// TODO(marpan): Investigate if we should return 0 instead of -1 in
// FindSimulcast[Max/Target/Min]Bitrate functions below, since the
// codec struct max/min/targeBitrates are unsigned.
-int FindSimulcastMaxBitrateBps(int width, int height, size_t max_layers) {
+int FindSimulcastMaxBitrateBps(int width, int height) {
const int format_index = FindSimulcastFormatIndex(width, height);
if (format_index == -1) {
return -1;
@@ -125,9 +128,7 @@
return kSimulcastFormats[format_index].max_bitrate_kbps * 1000;
}
-int FindSimulcastTargetBitrateBps(int width,
- int height,
- size_t max_layers) {
+int FindSimulcastTargetBitrateBps(int width, int height) {
const int format_index = FindSimulcastFormatIndex(width, height);
if (format_index == -1) {
return -1;
@@ -135,7 +136,7 @@
return kSimulcastFormats[format_index].target_bitrate_kbps * 1000;
}
-int FindSimulcastMinBitrateBps(int width, int height, size_t max_layers) {
+int FindSimulcastMinBitrateBps(int width, int height) {
const int format_index = FindSimulcastFormatIndex(width, height);
if (format_index == -1) {
return -1;
@@ -166,52 +167,73 @@
return total_max_bitrate_bps;
}
-std::vector<webrtc::VideoStream> GetSimulcastConfig(
- size_t max_streams,
- int width,
- int height,
- int max_bitrate_bps,
- int max_qp,
- int max_framerate) {
- size_t simulcast_layers = FindSimulcastMaxLayers(width, height);
- if (simulcast_layers > max_streams) {
+std::vector<webrtc::VideoStream> GetSimulcastConfig(size_t max_streams,
+ int width,
+ int height,
+ int max_bitrate_bps,
+ int max_qp,
+ int max_framerate,
+ bool is_screencast) {
+ size_t num_simulcast_layers;
+ if (is_screencast) {
+ num_simulcast_layers =
+ UseSimulcastScreenshare() ? kDefaultScreenshareSimulcastStreams : 1;
+ } else {
+ num_simulcast_layers = FindSimulcastMaxLayers(width, height);
+ }
+
+ if (num_simulcast_layers > max_streams) {
// If the number of SSRCs in the group differs from our target
// number of simulcast streams for current resolution, switch down
// to a resolution that matches our number of SSRCs.
if (!SlotSimulcastMaxResolution(max_streams, &width, &height)) {
return std::vector<webrtc::VideoStream>();
}
- simulcast_layers = max_streams;
+ num_simulcast_layers = max_streams;
}
std::vector<webrtc::VideoStream> streams;
- streams.resize(simulcast_layers);
+ streams.resize(num_simulcast_layers);
- // Format width and height has to be divisible by |2 ^ number_streams - 1|.
- width = NormalizeSimulcastSize(width, simulcast_layers);
- height = NormalizeSimulcastSize(height, simulcast_layers);
+ if (!is_screencast) {
+ // Format width and height has to be divisible by |2 ^ number_streams - 1|.
+ width = NormalizeSimulcastSize(width, num_simulcast_layers);
+ height = NormalizeSimulcastSize(height, num_simulcast_layers);
+ }
// Add simulcast sub-streams from lower resolution to higher resolutions.
// Add simulcast streams, from highest resolution (|s| = number_streams -1)
// to lowest resolution at |s| = 0.
- for (size_t s = simulcast_layers - 1;; --s) {
+ for (size_t s = num_simulcast_layers - 1;; --s) {
streams[s].width = width;
streams[s].height = height;
// TODO(pbos): Fill actual temporal-layer bitrate thresholds.
- streams[s].temporal_layer_thresholds_bps.resize(
- kDefaultConferenceNumberOfTemporalLayers[s] - 1);
- streams[s].max_bitrate_bps =
- FindSimulcastMaxBitrateBps(width, height, simulcast_layers);
- streams[s].target_bitrate_bps =
- FindSimulcastTargetBitrateBps(width, height, simulcast_layers);
- streams[s].min_bitrate_bps =
- FindSimulcastMinBitrateBps(width, height, simulcast_layers);
streams[s].max_qp = max_qp;
- streams[s].max_framerate = max_framerate;
+ if (is_screencast && s == 0) {
+ ScreenshareLayerConfig config = ScreenshareLayerConfig::GetDefault();
+ // For legacy screenshare in conference mode, tl0 and tl1 bitrates are
+ // piggybacked on the VideoCodec struct as target and max bitrates,
+ // respectively. See eg. webrtc::VP8EncoderImpl::SetRates().
+ streams[s].min_bitrate_bps = kMinVideoBitrateKbps * 1000;
+ streams[s].target_bitrate_bps = config.tl0_bitrate_kbps * 1000;
+ streams[s].max_bitrate_bps = config.tl1_bitrate_kbps * 1000;
+ streams[s].temporal_layer_thresholds_bps.clear();
+ streams[s].temporal_layer_thresholds_bps.push_back(
+ config.tl0_bitrate_kbps * 1000);
+ streams[s].max_framerate = 5;
+ } else {
+ streams[s].temporal_layer_thresholds_bps.resize(
+ kDefaultConferenceNumberOfTemporalLayers[s] - 1);
+ streams[s].max_bitrate_bps = FindSimulcastMaxBitrateBps(width, height);
+ streams[s].target_bitrate_bps =
+ FindSimulcastTargetBitrateBps(width, height);
+ streams[s].min_bitrate_bps = FindSimulcastMinBitrateBps(width, height);
+ streams[s].max_framerate = max_framerate;
+ }
+
width /= 2;
height /= 2;
- if (s == 0) {
+ if (s == 0)
break;
- }
}
// Spend additional bits to boost the max stream.
@@ -230,6 +252,8 @@
static const char* kScreencastLayerFieldTrialName =
"WebRTC-ScreenshareLayerRates";
+static const char* kSimulcastScreenshareFieldTrialName =
+ "WebRTC-SimulcastScreenshare";
ScreenshareLayerConfig::ScreenshareLayerConfig(int tl0_bitrate, int tl1_bitrate)
: tl0_bitrate_kbps(tl0_bitrate), tl1_bitrate_kbps(tl1_bitrate) {
@@ -272,4 +296,9 @@
return true;
}
+bool UseSimulcastScreenshare() {
+ return webrtc::field_trial::FindFullName(
+ kSimulcastScreenshareFieldTrialName) == "Enabled";
+}
+
} // namespace cricket
diff --git a/media/engine/simulcast.h b/media/engine/simulcast.h
index 20be4c4..3aae70f 100644
--- a/media/engine/simulcast.h
+++ b/media/engine/simulcast.h
@@ -19,6 +19,7 @@
namespace cricket {
struct StreamParams;
+// TODO(sprang): Remove this, as we're moving away from temporal layer mode.
// Config for use with screen cast when temporal layers are enabled.
struct ScreenshareLayerConfig {
public:
@@ -45,12 +46,16 @@
void GetSimulcastSsrcs(const StreamParams& sp, std::vector<uint32_t>* ssrcs);
// Get simulcast settings.
+// TODO(sprang): Remove default parameter when it's not longer referenced.
std::vector<webrtc::VideoStream> GetSimulcastConfig(size_t max_streams,
int width,
int height,
int max_bitrate_bps,
int max_qp,
- int max_framerate);
+ int max_framerate,
+ bool is_screencast = false);
+
+bool UseSimulcastScreenshare();
} // namespace cricket
diff --git a/media/engine/webrtcvideoengine2.cc b/media/engine/webrtcvideoengine2.cc
index 9e7f5c4..0cd3193 100644
--- a/media/engine/webrtcvideoengine2.cc
+++ b/media/engine/webrtcvideoengine2.cc
@@ -301,11 +301,16 @@
int width,
int height,
const webrtc::VideoEncoderConfig& encoder_config) override {
- RTC_DCHECK(encoder_config.number_of_streams > 1 ? !is_screencast_ : true);
- if (encoder_config.number_of_streams > 1) {
+ if (is_screencast_ &&
+ (!conference_mode_ || !cricket::UseSimulcastScreenshare())) {
+ RTC_DCHECK_EQ(1, encoder_config.number_of_streams);
+ }
+ if (encoder_config.number_of_streams > 1 ||
+ (CodecNamesEq(codec_name_, kVp8CodecName) && is_screencast_ &&
+ conference_mode_)) {
return GetSimulcastConfig(encoder_config.number_of_streams, width, height,
encoder_config.max_bitrate_bps, max_qp_,
- max_framerate_);
+ max_framerate_, is_screencast_);
}
// For unset max bitrates set default bitrate for non-simulcast.
@@ -322,20 +327,6 @@
stream.target_bitrate_bps = stream.max_bitrate_bps = max_bitrate_bps;
stream.max_qp = max_qp_;
- // Conference mode screencast uses 2 temporal layers split at 100kbit.
- if (conference_mode_ && is_screencast_) {
- ScreenshareLayerConfig config = ScreenshareLayerConfig::GetDefault();
- // For screenshare in conference mode, tl0 and tl1 bitrates are
- // piggybacked
- // on the VideoCodec struct as target and max bitrates, respectively.
- // See eg. webrtc::VP8EncoderImpl::SetRates().
- stream.target_bitrate_bps = config.tl0_bitrate_kbps * 1000;
- stream.max_bitrate_bps = config.tl1_bitrate_kbps * 1000;
- stream.temporal_layer_thresholds_bps.clear();
- stream.temporal_layer_thresholds_bps.push_back(config.tl0_bitrate_kbps *
- 1000);
- }
-
if (CodecNamesEq(codec_name_, kVp9CodecName) && !is_screencast_) {
stream.temporal_layer_thresholds_bps.resize(
GetDefaultVp9TemporalLayers() - 1);
@@ -1551,6 +1542,7 @@
enable_cpu_overuse_detection_(enable_cpu_overuse_detection),
source_(nullptr),
external_encoder_factory_(external_encoder_factory),
+ internal_encoder_factory_(new InternalEncoderFactory()),
stream_(nullptr),
encoder_sink_(nullptr),
parameters_(std::move(config), options, max_bitrate_bps, codec_settings),
@@ -1678,10 +1670,20 @@
}
// Try creating internal encoder.
- InternalEncoderFactory internal_encoder_factory;
- if (FindMatchingCodec(internal_encoder_factory.supported_codecs(), codec)) {
- return AllocatedEncoder(internal_encoder_factory.CreateVideoEncoder(codec),
- codec, false /* is_external */);
+ if (FindMatchingCodec(internal_encoder_factory_->supported_codecs(), codec)) {
+ if (parameters_.encoder_config.content_type ==
+ webrtc::VideoEncoderConfig::ContentType::kScreen &&
+ parameters_.conference_mode && UseSimulcastScreenshare()) {
+ // TODO(sprang): Remove this adapter once libvpx supports simulcast with
+ // same-resolution substreams.
+ WebRtcSimulcastEncoderFactory adapter_factory(
+ internal_encoder_factory_.get());
+ return AllocatedEncoder(adapter_factory.CreateVideoEncoder(codec), codec,
+ false /* is_external */);
+ }
+ return AllocatedEncoder(
+ internal_encoder_factory_->CreateVideoEncoder(codec), codec,
+ false /* is_external */);
}
// This shouldn't happen, we should not be trying to create something we don't
@@ -1858,9 +1860,11 @@
// By default, the stream count for the codec configuration should match the
// number of negotiated ssrcs. But if the codec is blacklisted for simulcast
- // or a screencast, only configure a single stream.
+ // or a screencast (and not in simulcast screenshare experiment), only
+ // configure a single stream.
encoder_config.number_of_streams = parameters_.config.rtp.ssrcs.size();
- if (IsCodecBlacklistedForSimulcast(codec.name) || is_screencast) {
+ if (IsCodecBlacklistedForSimulcast(codec.name) ||
+ (is_screencast && !UseSimulcastScreenshare())) {
encoder_config.number_of_streams = 1;
}
diff --git a/media/engine/webrtcvideoengine2.h b/media/engine/webrtcvideoengine2.h
index 2ee9bc7..7043e03 100644
--- a/media/engine/webrtcvideoengine2.h
+++ b/media/engine/webrtcvideoengine2.h
@@ -330,6 +330,8 @@
ACCESS_ON(&thread_checker_);
WebRtcVideoEncoderFactory* const external_encoder_factory_
ACCESS_ON(&thread_checker_);
+ const std::unique_ptr<WebRtcVideoEncoderFactory> internal_encoder_factory_
+ ACCESS_ON(&thread_checker_);
webrtc::VideoSendStream* stream_ ACCESS_ON(&thread_checker_);
rtc::VideoSinkInterface<webrtc::VideoFrame>* encoder_sink_
diff --git a/media/engine/webrtcvideoengine2_unittest.cc b/media/engine/webrtcvideoengine2_unittest.cc
index f72d7ae..a93d2d9 100644
--- a/media/engine/webrtcvideoengine2_unittest.cc
+++ b/media/engine/webrtcvideoengine2_unittest.cc
@@ -21,6 +21,7 @@
#include "webrtc/media/base/mediaconstants.h"
#include "webrtc/media/base/testutils.h"
#include "webrtc/media/base/videoengine_unittest.h"
+#include "webrtc/media/engine/constants.h"
#include "webrtc/media/engine/fakewebrtccall.h"
#include "webrtc/media/engine/fakewebrtcvideoengine.h"
#include "webrtc/media/engine/simulcast.h"
@@ -3960,7 +3961,7 @@
class WebRtcVideoChannel2SimulcastTest : public testing::Test {
public:
WebRtcVideoChannel2SimulcastTest()
- : fake_call_(webrtc::Call::Config(&event_log_)) {}
+ : fake_call_(webrtc::Call::Config(&event_log_)), last_ssrc_(0) {}
void SetUp() override {
engine_.Init();
@@ -3975,9 +3976,16 @@
int capture_width,
int capture_height,
size_t num_configured_streams,
- size_t expected_num_streams) {
+ size_t expected_num_streams,
+ bool screenshare,
+ bool conference_mode) {
cricket::VideoSendParameters parameters;
+ VideoOptions options;
parameters.codecs.push_back(codec);
+ parameters.conference_mode = conference_mode;
+ if (screenshare) {
+ options.is_screencast = rtc::Optional<bool>(screenshare);
+ }
ASSERT_TRUE(channel_->SetSendParameters(parameters));
std::vector<uint32_t> ssrcs = MAKE_VECTOR(kSsrcs3);
@@ -3990,7 +3998,7 @@
// expected simulcast layers.
cricket::FakeVideoCapturer capturer;
EXPECT_TRUE(
- channel_->SetVideoSend(ssrcs.front(), true, nullptr, &capturer));
+ channel_->SetVideoSend(ssrcs.front(), true, &options, &capturer));
EXPECT_EQ(cricket::CS_RUNNING, capturer.Start(cricket::VideoFormat(
capture_width, capture_height,
cricket::VideoFormat::FpsToInterval(30),
@@ -4001,9 +4009,32 @@
std::vector<webrtc::VideoStream> video_streams = stream->GetVideoStreams();
ASSERT_EQ(expected_num_streams, video_streams.size());
- std::vector<webrtc::VideoStream> expected_streams = GetSimulcastConfig(
- num_configured_streams, capture_width, capture_height, 0, kDefaultQpMax,
- kDefaultVideoMaxFramerate);
+ std::vector<webrtc::VideoStream> expected_streams;
+ if (conference_mode) {
+ expected_streams = GetSimulcastConfig(
+ num_configured_streams, capture_width, capture_height, 0,
+ kDefaultQpMax, kDefaultVideoMaxFramerate, screenshare);
+ } else {
+ webrtc::VideoStream stream;
+ stream.width = capture_width;
+ stream.height = capture_height;
+ stream.max_framerate = kDefaultVideoMaxFramerate;
+ stream.min_bitrate_bps = cricket::kMinVideoBitrateKbps * 1000;
+ int max_bitrate_kbps;
+ if (capture_width * capture_height <= 320 * 240) {
+ max_bitrate_kbps = 600;
+ } else if (capture_width * capture_height <= 640 * 480) {
+ max_bitrate_kbps = 1700;
+ } else if (capture_width * capture_height <= 960 * 540) {
+ max_bitrate_kbps = 2000;
+ } else {
+ max_bitrate_kbps = 2500;
+ }
+ stream.target_bitrate_bps = stream.max_bitrate_bps =
+ max_bitrate_kbps * 1000;
+ stream.max_qp = kDefaultQpMax;
+ expected_streams.push_back(stream);
+ }
ASSERT_EQ(expected_streams.size(), video_streams.size());
@@ -4032,7 +4063,8 @@
EXPECT_GT(video_streams[i].max_qp, 0);
EXPECT_EQ(expected_streams[i].max_qp, video_streams[i].max_qp);
- EXPECT_FALSE(expected_streams[i].temporal_layer_thresholds_bps.empty());
+ EXPECT_EQ(!conference_mode,
+ expected_streams[i].temporal_layer_thresholds_bps.empty());
EXPECT_EQ(expected_streams[i].temporal_layer_thresholds_bps,
video_streams[i].temporal_layer_thresholds_bps);
@@ -4086,15 +4118,37 @@
};
TEST_F(WebRtcVideoChannel2SimulcastTest, SetSendCodecsWith2SimulcastStreams) {
- VerifySimulcastSettings(cricket::VideoCodec("VP8"), 640, 360, 2, 2);
+ VerifySimulcastSettings(cricket::VideoCodec("VP8"), 640, 360, 2, 2, false,
+ true);
}
TEST_F(WebRtcVideoChannel2SimulcastTest, SetSendCodecsWith3SimulcastStreams) {
- VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 3);
+ VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 3, false,
+ true);
}
// Test that we normalize send codec format size in simulcast.
TEST_F(WebRtcVideoChannel2SimulcastTest, SetSendCodecsWithOddSizeInSimulcast) {
- VerifySimulcastSettings(cricket::VideoCodec("VP8"), 541, 271, 2, 2);
+ VerifySimulcastSettings(cricket::VideoCodec("VP8"), 541, 271, 2, 2, false,
+ true);
}
+
+TEST_F(WebRtcVideoChannel2SimulcastTest, SetSendCodecsForScreenshare) {
+ VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 1, true,
+ false);
+}
+
+TEST_F(WebRtcVideoChannel2SimulcastTest,
+ SetSendCodecsForConferenceModeScreenshare) {
+ VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 1, true,
+ true);
+}
+
+TEST_F(WebRtcVideoChannel2SimulcastTest, SetSendCodecsForSimulcastScreenshare) {
+ webrtc::test::ScopedFieldTrials override_field_trials_(
+ "WebRTC-SimulcastScreenshare/Enabled/");
+ VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 2, true,
+ true);
+}
+
} // namespace cricket
