Route min/max FPS constraints to VideoStreamEncoder.
This change
- adds new type VideoTrackSourceConstraints expressing min/max FPS
constraints.
- adds new method VideoTrackSourceInterface::ProcessConstraints.
- adds new method VideoSinkInterface<>::OnConstraintsChanged.
- updates AdaptedVideoTrackSource and VideoBroadcaster to forward
the constraints to sinks.
- adds several unit tests for the added functionality.
- and finally, implements OnConstraintsChanged in VideoStreamEncoder.
Chromium will be updated in coming CLs to supply constraints set
through the MediaStream module.
go/rtc-0hz-present
Bug: chromium:1255737
No-Try: true
Change-Id: Iffef239217269c332a1aaa902ddeae2440929e22
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/235040
Commit-Queue: Markus Handell <handellm@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Henrik Andreassson <henrika@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35197}
diff --git a/api/BUILD.gn b/api/BUILD.gn
index e969fec..0352f99 100644
--- a/api/BUILD.gn
+++ b/api/BUILD.gn
@@ -98,6 +98,12 @@
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
+rtc_source_set("video_track_source_constraints") {
+ visibility = [ "*" ]
+ sources = [ "video_track_source_constraints.h" ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+}
+
rtc_library("media_stream_interface") {
visibility = [ "*" ]
sources = [
@@ -110,6 +116,7 @@
":audio_options_api",
":rtp_parameters",
":scoped_refptr",
+ ":video_track_source_constraints",
"../modules/audio_processing:audio_processing_statistics",
"../rtc_base:checks",
"../rtc_base:refcount",
diff --git a/api/media_stream_interface.h b/api/media_stream_interface.h
index 874b4db..ad497d9 100644
--- a/api/media_stream_interface.h
+++ b/api/media_stream_interface.h
@@ -28,6 +28,7 @@
#include "api/video/video_frame.h"
#include "api/video/video_sink_interface.h"
#include "api/video/video_source_interface.h"
+#include "api/video_track_source_constraints.h"
#include "modules/audio_processing/include/audio_processing_statistics.h"
#include "rtc_base/ref_count.h"
#include "rtc_base/system/rtc_export.h"
@@ -146,8 +147,6 @@
// Add an encoded video sink to the source and additionally cause
// a key frame to be generated from the source. The sink will be
// invoked from a decoder queue.
- // TODO(bugs.webrtc.org/11114): make pure virtual once downstream project
- // adapts.
virtual void AddEncodedSink(
rtc::VideoSinkInterface<RecordableEncodedFrame>* sink) = 0;
@@ -155,6 +154,13 @@
virtual void RemoveEncodedSink(
rtc::VideoSinkInterface<RecordableEncodedFrame>* sink) = 0;
+ // Notify about constraints set on the source. The information eventually gets
+ // routed to attached sinks via VideoSinkInterface<>::OnConstraintsChanged.
+ // The call is expected to happen on the network thread.
+ // TODO(crbug/1255737): make pure virtual once downstream project adapts.
+ virtual void ProcessConstraints(
+ const webrtc::VideoTrackSourceConstraints& constraints) {}
+
protected:
~VideoTrackSourceInterface() override = default;
};
diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn
index ec74869..e6052fe 100644
--- a/api/video/BUILD.gn
+++ b/api/video/BUILD.gn
@@ -60,6 +60,7 @@
"..:array_view",
"..:rtp_packet_info",
"..:scoped_refptr",
+ "..:video_track_source_constraints",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
"../../rtc_base/memory:aligned_malloc",
diff --git a/api/video/video_sink_interface.h b/api/video/video_sink_interface.h
index 88cf9d9..9c1f5f3 100644
--- a/api/video/video_sink_interface.h
+++ b/api/video/video_sink_interface.h
@@ -11,6 +11,8 @@
#ifndef API_VIDEO_VIDEO_SINK_INTERFACE_H_
#define API_VIDEO_VIDEO_SINK_INTERFACE_H_
+#include "absl/types/optional.h"
+#include "api/video_track_source_constraints.h"
#include "rtc_base/checks.h"
namespace rtc {
@@ -25,6 +27,11 @@
// Should be called by the source when it discards the frame due to rate
// limiting.
virtual void OnDiscardedFrame() {}
+
+ // Called on the network thread when video constraints change.
+ // TODO(crbug/1255737): make pure virtual once downstream project adapts.
+ virtual void OnConstraintsChanged(
+ const webrtc::VideoTrackSourceConstraints& constraints) {}
};
} // namespace rtc
diff --git a/api/video_track_source_constraints.h b/api/video_track_source_constraints.h
new file mode 100644
index 0000000..55e5396
--- /dev/null
+++ b/api/video_track_source_constraints.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+// This file contains interfaces for MediaStream, MediaTrack and MediaSource.
+// These interfaces are used for implementing MediaStream and MediaTrack as
+// defined in http://dev.w3.org/2011/webrtc/editor/webrtc.html#stream-api. These
+// interfaces must be used only with PeerConnection.
+
+#ifndef API_VIDEO_TRACK_SOURCE_CONSTRAINTS_H_
+#define API_VIDEO_TRACK_SOURCE_CONSTRAINTS_H_
+
+#include "absl/types/optional.h"
+
+namespace webrtc {
+
+// This struct definition describes constraints on the video source that may be
+// set with VideoTrackSourceInterface::ProcessConstraints.
+struct VideoTrackSourceConstraints {
+ absl::optional<double> min_fps;
+ absl::optional<double> max_fps;
+};
+
+} // namespace webrtc
+
+#endif // API_VIDEO_TRACK_SOURCE_CONSTRAINTS_H_
diff --git a/media/base/adapted_video_track_source.cc b/media/base/adapted_video_track_source.cc
index 2fce973..f8f8f2d 100644
--- a/media/base/adapted_video_track_source.cc
+++ b/media/base/adapted_video_track_source.cc
@@ -114,4 +114,9 @@
return true;
}
+void AdaptedVideoTrackSource::ProcessConstraints(
+ const webrtc::VideoTrackSourceConstraints& constraints) {
+ broadcaster_.ProcessConstraints(constraints);
+}
+
} // namespace rtc
diff --git a/media/base/adapted_video_track_source.h b/media/base/adapted_video_track_source.h
index d40baef..1386fbd 100644
--- a/media/base/adapted_video_track_source.h
+++ b/media/base/adapted_video_track_source.h
@@ -86,6 +86,8 @@
rtc::VideoSinkInterface<webrtc::RecordableEncodedFrame>* sink) override {}
void RemoveEncodedSink(
rtc::VideoSinkInterface<webrtc::RecordableEncodedFrame>* sink) override {}
+ void ProcessConstraints(
+ const webrtc::VideoTrackSourceConstraints& constraints) override;
cricket::VideoAdapter video_adapter_;
diff --git a/media/base/video_broadcaster.cc b/media/base/video_broadcaster.cc
index 1b55786..1167d7f 100644
--- a/media/base/video_broadcaster.cc
+++ b/media/base/video_broadcaster.cc
@@ -32,6 +32,13 @@
if (!FindSinkPair(sink)) {
// `Sink` is a new sink, which didn't receive previous frame.
previous_frame_sent_to_all_sinks_ = false;
+
+ if (last_constraints_.has_value()) {
+ RTC_LOG(LS_INFO) << __func__ << " forwarding stored constraints min_fps "
+ << last_constraints_->min_fps.value_or(-1) << " max_fps "
+ << last_constraints_->max_fps.value_or(-1);
+ sink->OnConstraintsChanged(*last_constraints_);
+ }
}
VideoSourceBase::AddOrUpdateSink(sink, wants);
UpdateWants();
@@ -100,6 +107,18 @@
}
}
+void VideoBroadcaster::ProcessConstraints(
+ const webrtc::VideoTrackSourceConstraints& constraints) {
+ webrtc::MutexLock lock(&sinks_and_wants_lock_);
+ RTC_LOG(LS_INFO) << __func__ << " min_fps "
+ << constraints.min_fps.value_or(-1) << " max_fps "
+ << constraints.max_fps.value_or(-1) << " broadcasting to "
+ << sink_pairs().size() << " sinks.";
+ last_constraints_ = constraints;
+ for (auto& sink_pair : sink_pairs())
+ sink_pair.sink->OnConstraintsChanged(constraints);
+}
+
void VideoBroadcaster::UpdateWants() {
VideoSinkWants wants;
wants.rotation_applied = false;
diff --git a/media/base/video_broadcaster.h b/media/base/video_broadcaster.h
index 2f4e578..c253d44 100644
--- a/media/base/video_broadcaster.h
+++ b/media/base/video_broadcaster.h
@@ -11,6 +11,7 @@
#ifndef MEDIA_BASE_VIDEO_BROADCASTER_H_
#define MEDIA_BASE_VIDEO_BROADCASTER_H_
+#include "api/media_stream_interface.h"
#include "api/scoped_refptr.h"
#include "api/sequence_checker.h"
#include "api/video/video_frame_buffer.h"
@@ -31,6 +32,11 @@
public:
VideoBroadcaster();
~VideoBroadcaster() override;
+
+ // Adds a new, or updates an already existing sink. If the sink is new and
+ // ProcessConstraints has been called previously, the new sink's
+ // OnConstraintsCalled method will be invoked with the most recent
+ // constraints.
void AddOrUpdateSink(VideoSinkInterface<webrtc::VideoFrame>* sink,
const VideoSinkWants& wants) override;
void RemoveSink(VideoSinkInterface<webrtc::VideoFrame>* sink) override;
@@ -50,6 +56,11 @@
void OnDiscardedFrame() override;
+ // Called on the network thread when constraints change. Forwards the
+ // constraints to sinks added with AddOrUpdateSink via OnConstraintsChanged.
+ void ProcessConstraints(
+ const webrtc::VideoTrackSourceConstraints& constraints);
+
protected:
void UpdateWants() RTC_EXCLUSIVE_LOCKS_REQUIRED(sinks_and_wants_lock_);
const rtc::scoped_refptr<webrtc::VideoFrameBuffer>& GetBlackFrameBuffer(
@@ -62,6 +73,8 @@
rtc::scoped_refptr<webrtc::VideoFrameBuffer> black_frame_buffer_;
bool previous_frame_sent_to_all_sinks_ RTC_GUARDED_BY(sinks_and_wants_lock_) =
true;
+ absl::optional<webrtc::VideoTrackSourceConstraints> last_constraints_
+ RTC_GUARDED_BY(sinks_and_wants_lock_);
};
} // namespace rtc
diff --git a/media/base/video_broadcaster_unittest.cc b/media/base/video_broadcaster_unittest.cc
index b007278..b967237 100644
--- a/media/base/video_broadcaster_unittest.cc
+++ b/media/base/video_broadcaster_unittest.cc
@@ -16,13 +16,31 @@
#include "api/video/i420_buffer.h"
#include "api/video/video_frame.h"
#include "api/video/video_rotation.h"
+#include "api/video/video_source_interface.h"
#include "media/base/fake_video_renderer.h"
+#include "test/gmock.h"
#include "test/gtest.h"
using cricket::FakeVideoRenderer;
using rtc::VideoBroadcaster;
using rtc::VideoSinkWants;
+using ::testing::AllOf;
+using ::testing::Eq;
+using ::testing::Field;
+using ::testing::Mock;
+using ::testing::Optional;
+
+class MockSink : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
+ public:
+ void OnFrame(const webrtc::VideoFrame&) override {}
+
+ MOCK_METHOD(void,
+ OnConstraintsChanged,
+ (const webrtc::VideoTrackSourceConstraints& constraints),
+ (override));
+};
+
TEST(VideoBroadcasterTest, frame_wanted) {
VideoBroadcaster broadcaster;
EXPECT_FALSE(broadcaster.frame_wanted());
@@ -232,3 +250,83 @@
EXPECT_TRUE(sink2.black_frame());
EXPECT_EQ(30, sink2.timestamp_us());
}
+
+TEST(VideoBroadcasterTest, ConstraintsChangedNotCalledOnSinkAddition) {
+ MockSink sink;
+ VideoBroadcaster broadcaster;
+ EXPECT_CALL(sink, OnConstraintsChanged).Times(0);
+ broadcaster.AddOrUpdateSink(&sink, VideoSinkWants());
+}
+
+TEST(VideoBroadcasterTest, ForwardsLastConstraintsOnAdd) {
+ MockSink sink;
+ VideoBroadcaster broadcaster;
+ broadcaster.ProcessConstraints(webrtc::VideoTrackSourceConstraints{2, 3});
+ broadcaster.ProcessConstraints(webrtc::VideoTrackSourceConstraints{1, 4});
+ EXPECT_CALL(
+ sink,
+ OnConstraintsChanged(AllOf(
+ Field(&webrtc::VideoTrackSourceConstraints::min_fps, Optional(1)),
+ Field(&webrtc::VideoTrackSourceConstraints::max_fps, Optional(4)))));
+ broadcaster.AddOrUpdateSink(&sink, VideoSinkWants());
+}
+
+TEST(VideoBroadcasterTest, UpdatesOnlyNewSinksWithConstraints) {
+ MockSink sink1;
+ VideoBroadcaster broadcaster;
+ broadcaster.AddOrUpdateSink(&sink1, VideoSinkWants());
+ broadcaster.ProcessConstraints(webrtc::VideoTrackSourceConstraints{1, 4});
+ Mock::VerifyAndClearExpectations(&sink1);
+ EXPECT_CALL(sink1, OnConstraintsChanged).Times(0);
+ MockSink sink2;
+ EXPECT_CALL(
+ sink2,
+ OnConstraintsChanged(AllOf(
+ Field(&webrtc::VideoTrackSourceConstraints::min_fps, Optional(1)),
+ Field(&webrtc::VideoTrackSourceConstraints::max_fps, Optional(4)))));
+ broadcaster.AddOrUpdateSink(&sink2, VideoSinkWants());
+}
+
+TEST(VideoBroadcasterTest, ForwardsConstraintsToSink) {
+ MockSink sink;
+ VideoBroadcaster broadcaster;
+ EXPECT_CALL(sink, OnConstraintsChanged).Times(0);
+ broadcaster.AddOrUpdateSink(&sink, VideoSinkWants());
+ Mock::VerifyAndClearExpectations(&sink);
+
+ EXPECT_CALL(sink, OnConstraintsChanged(AllOf(
+ Field(&webrtc::VideoTrackSourceConstraints::min_fps,
+ Eq(absl::nullopt)),
+ Field(&webrtc::VideoTrackSourceConstraints::max_fps,
+ Eq(absl::nullopt)))));
+ broadcaster.ProcessConstraints(
+ webrtc::VideoTrackSourceConstraints{absl::nullopt, absl::nullopt});
+ Mock::VerifyAndClearExpectations(&sink);
+
+ EXPECT_CALL(
+ sink,
+ OnConstraintsChanged(AllOf(
+ Field(&webrtc::VideoTrackSourceConstraints::min_fps,
+ Eq(absl::nullopt)),
+ Field(&webrtc::VideoTrackSourceConstraints::max_fps, Optional(3)))));
+ broadcaster.ProcessConstraints(
+ webrtc::VideoTrackSourceConstraints{absl::nullopt, 3});
+ Mock::VerifyAndClearExpectations(&sink);
+
+ EXPECT_CALL(
+ sink,
+ OnConstraintsChanged(AllOf(
+ Field(&webrtc::VideoTrackSourceConstraints::min_fps, Optional(2)),
+ Field(&webrtc::VideoTrackSourceConstraints::max_fps,
+ Eq(absl::nullopt)))));
+ broadcaster.ProcessConstraints(
+ webrtc::VideoTrackSourceConstraints{2, absl::nullopt});
+ Mock::VerifyAndClearExpectations(&sink);
+
+ EXPECT_CALL(
+ sink,
+ OnConstraintsChanged(AllOf(
+ Field(&webrtc::VideoTrackSourceConstraints::min_fps, Optional(2)),
+ Field(&webrtc::VideoTrackSourceConstraints::max_fps, Optional(3)))));
+ broadcaster.ProcessConstraints(webrtc::VideoTrackSourceConstraints{2, 3});
+}
diff --git a/pc/video_track_source_proxy.h b/pc/video_track_source_proxy.h
index 8914dd0..6e71bb1 100644
--- a/pc/video_track_source_proxy.h
+++ b/pc/video_track_source_proxy.h
@@ -42,6 +42,9 @@
PROXY_SECONDARY_METHOD1(void,
RemoveEncodedSink,
rtc::VideoSinkInterface<RecordableEncodedFrame>*)
+PROXY_SECONDARY_METHOD1(void,
+ ProcessConstraints,
+ const webrtc::VideoTrackSourceConstraints&)
END_PROXY_MAP(VideoTrackSource)
} // namespace webrtc
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index 9180bbb..f3872cc 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -1372,6 +1372,18 @@
VideoStreamEncoderObserver::DropReason::kSource);
}
+void VideoStreamEncoder::OnConstraintsChanged(
+ const webrtc::VideoTrackSourceConstraints& constraints) {
+ // This method is called on the network thread.
+ RTC_LOG(LS_INFO) << __func__ << " min_fps "
+ << constraints.min_fps.value_or(-1) << " max_fps "
+ << constraints.max_fps.value_or(-1);
+ main_queue_->PostTask(ToQueuedTask(task_safety_, [this, constraints] {
+ RTC_DCHECK_RUN_ON(main_queue_);
+ source_constraints_ = constraints;
+ }));
+}
+
bool VideoStreamEncoder::EncoderPaused() const {
RTC_DCHECK_RUN_ON(&encoder_queue_);
// Pause video if paused by caller or as long as the network is down or the
diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h
index 2b18155..93c7903 100644
--- a/video/video_stream_encoder.h
+++ b/video/video_stream_encoder.h
@@ -176,6 +176,8 @@
// Implements VideoSinkInterface.
void OnFrame(const VideoFrame& video_frame) override;
void OnDiscardedFrame() override;
+ void OnConstraintsChanged(
+ const webrtc::VideoTrackSourceConstraints& constraints) override;
void MaybeEncodeVideoFrame(const VideoFrame& frame,
int64_t time_when_posted_in_ms);
@@ -239,6 +241,10 @@
encoder_selector_;
VideoStreamEncoderObserver* const encoder_stats_observer_;
+ // The source's constraints.
+ absl::optional<VideoTrackSourceConstraints> source_constraints_
+ RTC_GUARDED_BY(main_queue_);
+
VideoEncoderConfig encoder_config_ RTC_GUARDED_BY(&encoder_queue_);
std::unique_ptr<VideoEncoder> encoder_ RTC_GUARDED_BY(&encoder_queue_)
RTC_PT_GUARDED_BY(&encoder_queue_);
