[Adaptation] Add ability to inject resources on the PeerConnection.
This unblocks injecting platform-specific resources, such as power
usage signals in Chrome.
This CL adds AddAdaptationResource to PeerConnectionInterface and
integration tests verifying that if an injected resource is overusing,
resolution will soon be reduced.
To aid testing, some testing-only classes have been updated.
Bug: webrtc:11525
Change-Id: I820099e79f18d910fd641ee1412ad064b99ebce9
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/177003
Reviewed-by: Evan Shrubsole <eshr@google.com>
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#31505}
diff --git a/api/BUILD.gn b/api/BUILD.gn
index 2121744..30e414c 100644
--- a/api/BUILD.gn
+++ b/api/BUILD.gn
@@ -173,6 +173,7 @@
":rtp_parameters",
":rtp_transceiver_direction",
":scoped_refptr",
+ "adaptation:resource_adaptation_api",
"audio:audio_mixer_api",
"audio_codecs:audio_codecs_api",
"crypto:frame_decryptor_interface",
diff --git a/api/peer_connection_interface.h b/api/peer_connection_interface.h
index 0ae47b2..3293910 100644
--- a/api/peer_connection_interface.h
+++ b/api/peer_connection_interface.h
@@ -73,6 +73,7 @@
#include <string>
#include <vector>
+#include "api/adaptation/resource.h"
#include "api/async_resolver_factory.h"
#include "api/audio/audio_mixer.h"
#include "api/audio_codecs/audio_decoder_factory.h"
@@ -1116,6 +1117,14 @@
return absl::nullopt;
}
+ // When a resource is overused, the PeerConnection will try to reduce the load
+ // on the sysem, for example by reducing the resolution or frame rate of
+ // encoded streams. The Resource API allows injecting platform-specific usage
+ // measurements. The conditions to trigger kOveruse or kUnderuse are up to the
+ // implementation.
+ // TODO(hbos): Make pure virtual when implemented by downstream projects.
+ virtual void AddAdaptationResource(rtc::scoped_refptr<Resource> resource) {}
+
// Start RtcEventLog using an existing output-sink. Takes ownership of
// |output| and passes it on to Call, which will take the ownership. If the
// operation fails the output will be closed and deallocated. The event log
diff --git a/api/peer_connection_proxy.h b/api/peer_connection_proxy.h
index c278308..23887e5 100644
--- a/api/peer_connection_proxy.h
+++ b/api/peer_connection_proxy.h
@@ -132,6 +132,7 @@
PROXY_METHOD0(PeerConnectionState, peer_connection_state)
PROXY_METHOD0(IceGatheringState, ice_gathering_state)
PROXY_METHOD0(absl::optional<bool>, can_trickle_ice_candidates)
+PROXY_METHOD1(void, AddAdaptationResource, rtc::scoped_refptr<Resource>)
PROXY_METHOD2(bool,
StartRtcEventLog,
std::unique_ptr<RtcEventLogOutput>,
diff --git a/pc/BUILD.gn b/pc/BUILD.gn
index 1e83273..12a7fcc 100644
--- a/pc/BUILD.gn
+++ b/pc/BUILD.gn
@@ -522,6 +522,7 @@
"jsep_session_description_unittest.cc",
"local_audio_source_unittest.cc",
"media_stream_unittest.cc",
+ "peer_connection_adaptation_integrationtest.cc",
"peer_connection_bundle_unittest.cc",
"peer_connection_crypto_unittest.cc",
"peer_connection_data_channel_unittest.cc",
@@ -589,6 +590,7 @@
"../api/transport/rtp:rtp_source",
"../api/units:time_delta",
"../api/video:builtin_video_bitrate_allocator_factory",
+ "../call/adaptation:resource_adaptation_test_utilities",
"../logging:fake_rtc_event_log",
"../media:rtc_media_config",
"../media:rtc_media_engine_defaults",
diff --git a/pc/peer_connection.cc b/pc/peer_connection.cc
index 059d5dd..c46eaa2 100644
--- a/pc/peer_connection.cc
+++ b/pc/peer_connection.cc
@@ -4369,6 +4369,21 @@
return nullptr;
}
+void PeerConnection::AddAdaptationResource(
+ rtc::scoped_refptr<Resource> resource) {
+ if (!worker_thread()->IsCurrent()) {
+ return worker_thread()->Invoke<void>(RTC_FROM_HERE, [this, resource]() {
+ return AddAdaptationResource(resource);
+ });
+ }
+ RTC_DCHECK_RUN_ON(worker_thread());
+ if (!call_) {
+ // The PeerConnection has been closed.
+ return;
+ }
+ call_->AddAdaptationResource(resource);
+}
+
bool PeerConnection::StartRtcEventLog(std::unique_ptr<RtcEventLogOutput> output,
int64_t output_period_ms) {
return worker_thread()->Invoke<bool>(
diff --git a/pc/peer_connection.h b/pc/peer_connection.h
index f310257..3bb962b 100644
--- a/pc/peer_connection.h
+++ b/pc/peer_connection.h
@@ -237,6 +237,8 @@
rtc::scoped_refptr<SctpTransportInterface> GetSctpTransport() const override;
+ void AddAdaptationResource(rtc::scoped_refptr<Resource> resource) override;
+
bool StartRtcEventLog(std::unique_ptr<RtcEventLogOutput> output,
int64_t output_period_ms) override;
bool StartRtcEventLog(std::unique_ptr<RtcEventLogOutput> output) override;
diff --git a/pc/peer_connection_adaptation_integrationtest.cc b/pc/peer_connection_adaptation_integrationtest.cc
new file mode 100644
index 0000000..71d054e
--- /dev/null
+++ b/pc/peer_connection_adaptation_integrationtest.cc
@@ -0,0 +1,161 @@
+/*
+ * Copyright 2020 The WebRTC Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <memory>
+
+#include "api/audio_codecs/builtin_audio_decoder_factory.h"
+#include "api/audio_codecs/builtin_audio_encoder_factory.h"
+#include "api/rtp_parameters.h"
+#include "api/scoped_refptr.h"
+#include "call/adaptation/test/fake_resource.h"
+#include "pc/test/fake_periodic_video_source.h"
+#include "pc/test/fake_periodic_video_track_source.h"
+#include "pc/test/peer_connection_test_wrapper.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/gunit.h"
+#include "rtc_base/ref_counted_object.h"
+#include "rtc_base/thread.h"
+#include "rtc_base/virtual_socket_server.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+
+const int64_t kDefaultTimeoutMs = 5000;
+
+struct TrackWithPeriodicSource {
+ rtc::scoped_refptr<VideoTrackInterface> track;
+ rtc::scoped_refptr<FakePeriodicVideoTrackSource> periodic_track_source;
+};
+
+// Performs an O/A exchange and waits until the signaling state is stable again.
+void Negotiate(rtc::scoped_refptr<PeerConnectionTestWrapper> caller,
+ rtc::scoped_refptr<PeerConnectionTestWrapper> callee) {
+ // Wire up callbacks and listeners such that a full O/A is performed in
+ // response to CreateOffer().
+ PeerConnectionTestWrapper::Connect(caller.get(), callee.get());
+ caller->CreateOffer(PeerConnectionInterface::RTCOfferAnswerOptions());
+ caller->WaitForNegotiation();
+}
+
+TrackWithPeriodicSource CreateTrackWithPeriodicSource(
+ rtc::scoped_refptr<PeerConnectionFactoryInterface> factory) {
+ FakePeriodicVideoSource::Config periodic_track_source_config;
+ periodic_track_source_config.frame_interval_ms = 100;
+ periodic_track_source_config.timestamp_offset_ms = rtc::TimeMillis();
+ rtc::scoped_refptr<FakePeriodicVideoTrackSource> periodic_track_source =
+ new rtc::RefCountedObject<FakePeriodicVideoTrackSource>(
+ periodic_track_source_config, /* remote */ false);
+ TrackWithPeriodicSource track_with_source;
+ track_with_source.track =
+ factory->CreateVideoTrack("PeriodicTrack", periodic_track_source);
+ track_with_source.periodic_track_source = periodic_track_source;
+ return track_with_source;
+}
+
+// Triggers overuse and obtains VideoSinkWants. Adaptation processing happens in
+// parallel and this function makes no guarantee that the returnd VideoSinkWants
+// have yet to reflect the overuse signal. Used together with EXPECT_TRUE_WAIT
+// to "spam overuse until a change is observed".
+rtc::VideoSinkWants TriggerOveruseAndGetSinkWants(
+ rtc::scoped_refptr<FakeResource> fake_resource,
+ const FakePeriodicVideoSource& source) {
+ fake_resource->SetUsageState(ResourceUsageState::kOveruse);
+ return source.wants();
+}
+
+class PeerConnectionAdaptationIntegrationTest : public ::testing::Test {
+ public:
+ PeerConnectionAdaptationIntegrationTest()
+ : virtual_socket_server_(),
+ network_thread_(new rtc::Thread(&virtual_socket_server_)),
+ worker_thread_(rtc::Thread::Create()) {
+ RTC_CHECK(network_thread_->Start());
+ RTC_CHECK(worker_thread_->Start());
+ }
+
+ rtc::scoped_refptr<PeerConnectionTestWrapper> CreatePcWrapper(
+ const char* name) {
+ rtc::scoped_refptr<PeerConnectionTestWrapper> pc_wrapper =
+ new rtc::RefCountedObject<PeerConnectionTestWrapper>(
+ name, network_thread_.get(), worker_thread_.get());
+ PeerConnectionInterface::RTCConfiguration config;
+ config.sdp_semantics = SdpSemantics::kUnifiedPlan;
+ EXPECT_TRUE(pc_wrapper->CreatePc(config, CreateBuiltinAudioEncoderFactory(),
+ CreateBuiltinAudioDecoderFactory()));
+ return pc_wrapper;
+ }
+
+ protected:
+ rtc::VirtualSocketServer virtual_socket_server_;
+ std::unique_ptr<rtc::Thread> network_thread_;
+ std::unique_ptr<rtc::Thread> worker_thread_;
+};
+
+TEST_F(PeerConnectionAdaptationIntegrationTest,
+ ResouceInjectedAfterNegotiationCausesReductionInResolution) {
+ auto caller_wrapper = CreatePcWrapper("caller");
+ auto caller = caller_wrapper->pc();
+ auto callee_wrapper = CreatePcWrapper("callee");
+
+ // Adding a track and negotiating ensures that a VideoSendStream exists.
+ TrackWithPeriodicSource track_with_source =
+ CreateTrackWithPeriodicSource(caller_wrapper->pc_factory());
+ auto sender = caller->AddTrack(track_with_source.track, {}).value();
+ Negotiate(caller_wrapper, callee_wrapper);
+ // Prefer degrading resolution.
+ auto parameters = sender->GetParameters();
+ parameters.degradation_preference = DegradationPreference::MAINTAIN_FRAMERATE;
+ sender->SetParameters(parameters);
+
+ const auto& source =
+ track_with_source.periodic_track_source->fake_periodic_source();
+ int pixel_count_before_overuse = source.wants().max_pixel_count;
+
+ // Inject a fake resource and spam kOveruse until resolution becomes limited.
+ auto fake_resource = FakeResource::Create("FakeResource");
+ caller->AddAdaptationResource(fake_resource);
+ EXPECT_TRUE_WAIT(
+ TriggerOveruseAndGetSinkWants(fake_resource, source).max_pixel_count <
+ pixel_count_before_overuse,
+ kDefaultTimeoutMs);
+}
+
+TEST_F(PeerConnectionAdaptationIntegrationTest,
+ ResouceInjectedBeforeNegotiationCausesReductionInResolution) {
+ auto caller_wrapper = CreatePcWrapper("caller");
+ auto caller = caller_wrapper->pc();
+ auto callee_wrapper = CreatePcWrapper("callee");
+
+ // Inject a fake resource before adding any tracks or negotiating.
+ auto fake_resource = FakeResource::Create("FakeResource");
+ caller->AddAdaptationResource(fake_resource);
+
+ // Adding a track and negotiating ensures that a VideoSendStream exists.
+ TrackWithPeriodicSource track_with_source =
+ CreateTrackWithPeriodicSource(caller_wrapper->pc_factory());
+ auto sender = caller->AddTrack(track_with_source.track, {}).value();
+ Negotiate(caller_wrapper, callee_wrapper);
+ // Prefer degrading resolution.
+ auto parameters = sender->GetParameters();
+ parameters.degradation_preference = DegradationPreference::MAINTAIN_FRAMERATE;
+ sender->SetParameters(parameters);
+
+ const auto& source =
+ track_with_source.periodic_track_source->fake_periodic_source();
+ int pixel_count_before_overuse = source.wants().max_pixel_count;
+
+ // Spam kOveruse until resolution becomes limited.
+ EXPECT_TRUE_WAIT(
+ TriggerOveruseAndGetSinkWants(fake_resource, source).max_pixel_count <
+ pixel_count_before_overuse,
+ kDefaultTimeoutMs);
+}
+
+} // namespace webrtc
diff --git a/pc/test/fake_periodic_video_source.h b/pc/test/fake_periodic_video_source.h
index 1684ca4..b1cff4e 100644
--- a/pc/test/fake_periodic_video_source.h
+++ b/pc/test/fake_periodic_video_source.h
@@ -16,6 +16,7 @@
#include "api/video/video_source_interface.h"
#include "media/base/fake_frame_source.h"
#include "media/base/video_broadcaster.h"
+#include "rtc_base/critical_section.h"
#include "rtc_base/task_queue_for_test.h"
#include "rtc_base/task_utils/repeating_task.h"
@@ -59,6 +60,11 @@
});
}
+ rtc::VideoSinkWants wants() const {
+ rtc::CritScope cs(&crit_);
+ return wants_;
+ }
+
void RemoveSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override {
RTC_DCHECK(thread_checker_.IsCurrent());
broadcaster_.RemoveSink(sink);
@@ -67,6 +73,10 @@
void AddOrUpdateSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink,
const rtc::VideoSinkWants& wants) override {
RTC_DCHECK(thread_checker_.IsCurrent());
+ {
+ rtc::CritScope cs(&crit_);
+ wants_ = wants;
+ }
broadcaster_.AddOrUpdateSink(sink, wants);
}
@@ -80,6 +90,8 @@
rtc::VideoBroadcaster broadcaster_;
cricket::FakeFrameSource frame_source_;
+ rtc::CriticalSection crit_;
+ rtc::VideoSinkWants wants_ RTC_GUARDED_BY(&crit_);
std::unique_ptr<TaskQueueForTest> task_queue_;
};
diff --git a/pc/test/fake_periodic_video_track_source.h b/pc/test/fake_periodic_video_track_source.h
index cc406d6..98a456f 100644
--- a/pc/test/fake_periodic_video_track_source.h
+++ b/pc/test/fake_periodic_video_track_source.h
@@ -29,6 +29,10 @@
~FakePeriodicVideoTrackSource() = default;
+ const FakePeriodicVideoSource& fake_periodic_source() const {
+ return source_;
+ }
+
protected:
rtc::VideoSourceInterface<VideoFrame>* source() override { return &source_; }
diff --git a/pc/test/peer_connection_test_wrapper.cc b/pc/test/peer_connection_test_wrapper.cc
index 4f0d72e..946f459 100644
--- a/pc/test/peer_connection_test_wrapper.cc
+++ b/pc/test/peer_connection_test_wrapper.cc
@@ -80,7 +80,8 @@
rtc::Thread* worker_thread)
: name_(name),
network_thread_(network_thread),
- worker_thread_(worker_thread) {
+ worker_thread_(worker_thread),
+ pending_negotiation_(false) {
pc_thread_checker_.Detach();
}
@@ -135,6 +136,17 @@
return peer_connection_->CreateDataChannel(label, &init);
}
+void PeerConnectionTestWrapper::WaitForNegotiation() {
+ EXPECT_TRUE_WAIT(!pending_negotiation_, kMaxWait);
+}
+
+void PeerConnectionTestWrapper::OnSignalingChange(
+ webrtc::PeerConnectionInterface::SignalingState new_state) {
+ if (new_state == webrtc::PeerConnectionInterface::SignalingState::kStable) {
+ pending_negotiation_ = false;
+ }
+}
+
void PeerConnectionTestWrapper::OnAddTrack(
rtc::scoped_refptr<RtpReceiverInterface> receiver,
const std::vector<rtc::scoped_refptr<MediaStreamInterface>>& streams) {
@@ -182,6 +194,7 @@
void PeerConnectionTestWrapper::CreateOffer(
const webrtc::PeerConnectionInterface::RTCOfferAnswerOptions& options) {
RTC_LOG(LS_INFO) << "PeerConnectionTestWrapper " << name_ << ": CreateOffer.";
+ pending_negotiation_ = true;
peer_connection_->CreateOffer(this, options);
}
@@ -189,6 +202,7 @@
const webrtc::PeerConnectionInterface::RTCOfferAnswerOptions& options) {
RTC_LOG(LS_INFO) << "PeerConnectionTestWrapper " << name_
<< ": CreateAnswer.";
+ pending_negotiation_ = true;
peer_connection_->CreateAnswer(this, options);
}
diff --git a/pc/test/peer_connection_test_wrapper.h b/pc/test/peer_connection_test_wrapper.h
index 2dc88e9..92599b7 100644
--- a/pc/test/peer_connection_test_wrapper.h
+++ b/pc/test/peer_connection_test_wrapper.h
@@ -49,15 +49,21 @@
rtc::scoped_refptr<webrtc::AudioEncoderFactory> audio_encoder_factory,
rtc::scoped_refptr<webrtc::AudioDecoderFactory> audio_decoder_factory);
+ rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> pc_factory()
+ const {
+ return peer_connection_factory_;
+ }
webrtc::PeerConnectionInterface* pc() { return peer_connection_.get(); }
rtc::scoped_refptr<webrtc::DataChannelInterface> CreateDataChannel(
const std::string& label,
const webrtc::DataChannelInit& init);
+ void WaitForNegotiation();
+
// Implements PeerConnectionObserver.
void OnSignalingChange(
- webrtc::PeerConnectionInterface::SignalingState new_state) override {}
+ webrtc::PeerConnectionInterface::SignalingState new_state) override;
void OnAddTrack(
rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver,
const std::vector<rtc::scoped_refptr<webrtc::MediaStreamInterface>>&
@@ -121,6 +127,7 @@
rtc::scoped_refptr<FakeAudioCaptureModule> fake_audio_capture_module_;
std::unique_ptr<webrtc::FakeVideoTrackRenderer> renderer_;
int num_get_user_media_calls_ = 0;
+ bool pending_negotiation_;
};
#endif // PC_TEST_PEER_CONNECTION_TEST_WRAPPER_H_
