blob: ec7f76e3339d526d2a26ae4f93ba7fc3ee1df9bf [file] [log] [blame]
Tommiae4d0972020-05-18 06:45:381/*
2 * Copyright 2017 The WebRTC Project Authors. All rights reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "video/video_receive_stream2.h"
12
13#include <algorithm>
Evan Shrubsolea4062722022-05-02 15:46:0714#include <cstddef>
Philipp Hancke80859bc2025-04-10 18:04:5415#include <cstdint>
16#include <cstring>
Evan Shrubsole7cbd8de2022-08-16 08:08:5317#include <deque>
Markus Handell588f9b32021-04-08 17:19:5018#include <limits>
Tommiae4d0972020-05-18 06:45:3819#include <memory>
Florent Castelli8037fc62024-08-29 13:00:4020#include <optional>
Tommiae4d0972020-05-18 06:45:3821#include <utility>
22#include <vector>
23
Evan Shrubsole6dbc1722022-03-22 11:20:1124#include "absl/memory/memory.h"
Philipp Hanckece6aa1f2025-04-28 18:59:2225#include "api/array_view.h"
Danil Chapovalov22333492023-12-07 11:10:4626#include "api/environment/environment.h"
27#include "api/environment/environment_factory.h"
Evan Shrubsolea4062722022-05-02 15:46:0728#include "api/metronome/test/fake_metronome.h"
Philipp Hancke80859bc2025-04-10 18:04:5429#include "api/rtp_packet_info.h"
30#include "api/rtp_packet_infos.h"
Danil Chapovalov9cd4d492021-08-03 12:59:0031#include "api/test/mock_video_decoder.h"
32#include "api/test/mock_video_decoder_factory.h"
Evan Shrubsole44be5792022-04-26 14:24:4133#include "api/test/time_controller.h"
Philipp Hancke80859bc2025-04-10 18:04:5434#include "api/transport/rtp/rtp_source.h"
Evan Shrubsolea4062722022-05-02 15:46:0735#include "api/units/frequency.h"
Evan Shrubsole44be5792022-04-26 14:24:4136#include "api/units/time_delta.h"
Philipp Hancke80859bc2025-04-10 18:04:5437#include "api/units/timestamp.h"
Evan Shrubsole44be5792022-04-26 14:24:4138#include "api/video/recordable_encoded_frame.h"
39#include "api/video/test/video_frame_matchers.h"
Markus Handell588f9b32021-04-08 17:19:5040#include "api/video/video_frame.h"
Philipp Hancke80859bc2025-04-10 18:04:5441#include "api/video/video_frame_type.h"
42#include "api/video/video_rotation.h"
43#include "api/video/video_sink_interface.h"
44#include "api/video/video_timing.h"
Evan Shrubsole1c184772022-04-26 07:47:4945#include "api/video_codecs/sdp_video_format.h"
Tommiae4d0972020-05-18 06:45:3846#include "call/rtp_stream_receiver_controller.h"
Evan Shrubsole1c184772022-04-26 07:47:4947#include "call/video_receive_stream.h"
Tommiae4d0972020-05-18 06:45:3848#include "common_video/test/utilities.h"
Tommi90738dd2021-05-31 15:36:4749#include "media/engine/fake_webrtc_call.h"
Tommiae4d0972020-05-18 06:45:3850#include "modules/pacing/packet_router.h"
51#include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
Philipp Hancke80859bc2025-04-10 18:04:5452#include "modules/video_coding/nack_requester.h"
Evan Shrubsoledcb9c5d2022-06-13 15:39:5353#include "rtc_base/logging.h"
Tommiae4d0972020-05-18 06:45:3854#include "system_wrappers/include/clock.h"
55#include "test/fake_decoder.h"
Evan Shrubsolea0ee64c2022-04-26 08:09:0456#include "test/fake_encoded_frame.h"
Tommiae4d0972020-05-18 06:45:3857#include "test/gmock.h"
58#include "test/gtest.h"
Danil Chapovalov9cd4d492021-08-03 12:59:0059#include "test/mock_transport.h"
philipel27b35a72022-07-05 07:59:5560#include "test/rtcp_packet_parser.h"
Tommiae4d0972020-05-18 06:45:3861#include "test/time_controller/simulated_time_controller.h"
62#include "test/video_decoder_proxy_factory.h"
63#include "video/call_stats2.h"
Philipp Hancke80859bc2025-04-10 18:04:5464#include "video/decode_synchronizer.h"
Tommiae4d0972020-05-18 06:45:3865
66namespace webrtc {
Evan Shrubsole1c184772022-04-26 07:47:4967
Tommiae4d0972020-05-18 06:45:3868namespace {
69
Evan Shrubsole44be5792022-04-26 14:24:4170using test::video_frame_matchers::NtpTimestamp;
71using test::video_frame_matchers::PacketInfos;
72using test::video_frame_matchers::Rotation;
Tommiae4d0972020-05-18 06:45:3873using ::testing::_;
Markus Handell588f9b32021-04-08 17:19:5074using ::testing::AllOf;
Evan Shrubsoledcb9c5d2022-06-13 15:39:5375using ::testing::AnyNumber;
Tommiae4d0972020-05-18 06:45:3876using ::testing::ElementsAreArray;
Evan Shrubsole44be5792022-04-26 14:24:4177using ::testing::Eq;
Markus Handell588f9b32021-04-08 17:19:5078using ::testing::Field;
79using ::testing::InSequence;
Tommiae4d0972020-05-18 06:45:3880using ::testing::Invoke;
81using ::testing::IsEmpty;
Johannes Kronbbf639e2022-06-15 10:27:2382using ::testing::Optional;
Evan Shrubsolea4062722022-05-02 15:46:0783using ::testing::Pointee;
Markus Handell588f9b32021-04-08 17:19:5084using ::testing::Property;
Evan Shrubsole44be5792022-04-26 14:24:4185using ::testing::Return;
Tommiae4d0972020-05-18 06:45:3886using ::testing::SizeIs;
Danil Chapovalovd08930d2021-08-12 11:26:5587using ::testing::WithoutArgs;
Tommiae4d0972020-05-18 06:45:3888
Evan Shrubsole44be5792022-04-26 14:24:4189auto RenderedFrameWith(::testing::Matcher<VideoFrame> m) {
90 return Optional(m);
91}
Evan Shrubsolea4062722022-05-02 15:46:0792auto RenderedFrame() {
93 return RenderedFrameWith(_);
94}
Florent Castelli8037fc62024-08-29 13:00:4095testing::Matcher<std::optional<VideoFrame>> DidNotReceiveFrame() {
96 return Eq(std::nullopt);
Evan Shrubsolea4062722022-05-02 15:46:0797}
Evan Shrubsole44be5792022-04-26 14:24:4198
99constexpr TimeDelta kDefaultTimeOut = TimeDelta::Millis(50);
Evan Shrubsole14ee8032022-04-26 14:01:44100constexpr int kDefaultNumCpuCores = 2;
Tommiae4d0972020-05-18 06:45:38101
Evan Shrubsole44be5792022-04-26 14:24:41102constexpr Timestamp kStartTime = Timestamp::Millis(1'337'000);
Evan Shrubsolea4062722022-05-02 15:46:07103constexpr Frequency k30Fps = Frequency::Hertz(30);
104constexpr TimeDelta k30FpsDelay = 1 / k30Fps;
105constexpr Frequency kRtpTimestampHz = Frequency::KiloHertz(90);
106constexpr uint32_t k30FpsRtpTimestampDelta = kRtpTimestampHz / k30Fps;
107constexpr uint32_t kFirstRtpTimestamp = 90000;
Sergio Garcia Murillob5289d72024-10-24 12:10:53108constexpr uint8_t kH264PayloadType = 99;
109constexpr uint8_t kH265PayloadType = 100;
110constexpr uint8_t kAv1PayloadType = 101;
111constexpr uint32_t kRemoteSsrc = 1111;
112constexpr uint32_t kLocalSsrc = 2222;
Evan Shrubsole44be5792022-04-26 14:24:41113
Evan Shrubsole8f7678f2025-04-01 14:23:55114class FakeVideoRenderer : public VideoSinkInterface<VideoFrame> {
Evan Shrubsole44be5792022-04-26 14:24:41115 public:
Evan Shrubsole7cbd8de2022-08-16 08:08:53116 explicit FakeVideoRenderer(TimeController* time_controller)
117 : time_controller_(time_controller) {}
Evan Shrubsole44be5792022-04-26 14:24:41118 ~FakeVideoRenderer() override = default;
119
Evan Shrubsoledcb9c5d2022-06-13 15:39:53120 void OnFrame(const VideoFrame& frame) override {
121 RTC_LOG(LS_VERBOSE) << "Received frame with timestamp="
Per K0fa90882024-03-13 08:52:41122 << frame.rtp_timestamp();
Evan Shrubsole7cbd8de2022-08-16 08:08:53123 if (!last_frame_.empty()) {
124 RTC_LOG(LS_INFO) << "Already had frame queue with timestamp="
Per K0fa90882024-03-13 08:52:41125 << last_frame_.back().rtp_timestamp();
Evan Shrubsoledcb9c5d2022-06-13 15:39:53126 }
Evan Shrubsole7cbd8de2022-08-16 08:08:53127 last_frame_.push_back(frame);
Evan Shrubsole44be5792022-04-26 14:24:41128 }
129
Evan Shrubsoledcb9c5d2022-06-13 15:39:53130 // If `advance_time`, then the clock will always advance by `timeout`.
Florent Castelli8037fc62024-08-29 13:00:40131 std::optional<VideoFrame> WaitForFrame(TimeDelta timeout,
132 bool advance_time = false) {
Evan Shrubsoledcb9c5d2022-06-13 15:39:53133 auto start = time_controller_->GetClock()->CurrentTime();
Evan Shrubsole7cbd8de2022-08-16 08:08:53134 if (last_frame_.empty()) {
Evan Shrubsolea4062722022-05-02 15:46:07135 time_controller_->AdvanceTime(TimeDelta::Zero());
Evan Shrubsole7cbd8de2022-08-16 08:08:53136 time_controller_->Wait([this] { return !last_frame_.empty(); }, timeout);
Evan Shrubsole44be5792022-04-26 14:24:41137 }
Florent Castelli8037fc62024-08-29 13:00:40138 std::optional<VideoFrame> ret;
Evan Shrubsole7cbd8de2022-08-16 08:08:53139 if (!last_frame_.empty()) {
140 ret = last_frame_.front();
141 last_frame_.pop_front();
142 }
Evan Shrubsoledcb9c5d2022-06-13 15:39:53143 if (advance_time) {
144 time_controller_->AdvanceTime(
145 timeout - (time_controller_->GetClock()->CurrentTime() - start));
146 }
Evan Shrubsole44be5792022-04-26 14:24:41147 return ret;
148 }
149
150 private:
Evan Shrubsole7cbd8de2022-08-16 08:08:53151 std::deque<VideoFrame> last_frame_;
Evan Shrubsolea4062722022-05-02 15:46:07152 TimeController* const time_controller_;
Evan Shrubsole44be5792022-04-26 14:24:41153};
154
Jonas Oreland0deda152022-09-23 10:08:57155MATCHER_P2(MatchResolution, w, h, "") {
Evan Shrubsole44be5792022-04-26 14:24:41156 return arg.resolution().width == w && arg.resolution().height == h;
157}
158
Evan Shrubsoledcb9c5d2022-06-13 15:39:53159MATCHER_P(RtpTimestamp, timestamp, "") {
Per K0fa90882024-03-13 08:52:41160 if (arg.rtp_timestamp() != timestamp) {
Evan Shrubsole7cbd8de2022-08-16 08:08:53161 *result_listener->stream()
Per K0fa90882024-03-13 08:52:41162 << "rtp timestamp was " << arg.rtp_timestamp() << " != " << timestamp;
Evan Shrubsole7cbd8de2022-08-16 08:08:53163 return false;
164 }
165 return true;
Evan Shrubsolea4062722022-05-02 15:46:07166}
167
168// Rtp timestamp for in order frame at 30fps.
169uint32_t RtpTimestampForFrame(int id) {
170 return kFirstRtpTimestamp + id * k30FpsRtpTimestampDelta;
171}
172
173// Receive time for in order frame at 30fps.
174Timestamp ReceiveTimeForFrame(int id) {
175 return kStartTime + id * k30FpsDelay;
176}
177
Tommiae4d0972020-05-18 06:45:38178} // namespace
179
Evan Shrubsolea4062722022-05-02 15:46:07180class VideoReceiveStream2Test : public ::testing::TestWithParam<bool> {
Tommiae4d0972020-05-18 06:45:38181 public:
Evan Shrubsolea4062722022-05-02 15:46:07182 auto DefaultDecodeAction() {
philipel27b35a72022-07-05 07:59:55183 return Invoke(&fake_decoder_, &test::FakeDecoder::Decode);
Evan Shrubsolea4062722022-05-02 15:46:07184 }
185
186 bool UseMetronome() const { return GetParam(); }
187
Tommiae4d0972020-05-18 06:45:38188 VideoReceiveStream2Test()
Evan Shrubsole44be5792022-04-26 14:24:41189 : time_controller_(kStartTime),
Danil Chapovalov22333492023-12-07 11:10:46190 env_(CreateEnvironment(time_controller_.CreateTaskQueueFactory(),
191 time_controller_.GetClock())),
Sergio Garcia Murillob5289d72024-10-24 12:10:53192 config_(&mock_transport_, &mock_decoder_factory_),
Danil Chapovalov22333492023-12-07 11:10:46193 call_stats_(&env_.clock(), time_controller_.GetMainThread()),
Evan Shrubsole7cbd8de2022-08-16 08:08:53194 fake_renderer_(&time_controller_),
Danil Chapovalov8d079be2024-04-08 16:35:40195 fake_call_(env_),
Markus Handellbe400e42022-11-08 11:14:23196 fake_metronome_(TimeDelta::Millis(16)),
Danil Chapovalov22333492023-12-07 11:10:46197 decode_sync_(&env_.clock(),
Evan Shrubsole7cbd8de2022-08-16 08:08:53198 &fake_metronome_,
199 time_controller_.GetMainThread()),
Evan Shrubsolea4062722022-05-02 15:46:07200 h264_decoder_factory_(&mock_decoder_) {
Evan Shrubsole1c184772022-04-26 07:47:49201 // By default, mock decoder factory is backed by VideoDecoderProxyFactory.
Sergio Garcia Murillob5289d72024-10-24 12:10:53202 ON_CALL(mock_decoder_factory_, Create)
Danil Chapovalov08173802024-01-23 12:02:46203 .WillByDefault(Invoke(&h264_decoder_factory_,
204 &test::VideoDecoderProxyFactory::Create));
Evan Shrubsole14ee8032022-04-26 14:01:44205
206 // By default, mock decode will wrap the fake decoder.
Evan Shrubsolea4062722022-05-02 15:46:07207 ON_CALL(mock_decoder_, Configure)
philipel27b35a72022-07-05 07:59:55208 .WillByDefault(Invoke(&fake_decoder_, &test::FakeDecoder::Configure));
Tony Herre55b593f2023-08-29 14:05:49209 ON_CALL(mock_decoder_, Decode(_, _)).WillByDefault(DefaultDecodeAction());
Evan Shrubsolea4062722022-05-02 15:46:07210 ON_CALL(mock_decoder_, RegisterDecodeCompleteCallback)
Evan Shrubsole14ee8032022-04-26 14:01:44211 .WillByDefault(
philipel27b35a72022-07-05 07:59:55212 Invoke(&fake_decoder_,
213 &test::FakeDecoder::RegisterDecodeCompleteCallback));
214 ON_CALL(mock_decoder_, Release)
215 .WillByDefault(Invoke(&fake_decoder_, &test::FakeDecoder::Release));
216 ON_CALL(mock_transport_, SendRtcp)
217 .WillByDefault(
Philipp Hanckece6aa1f2025-04-28 18:59:22218 Invoke([this](ArrayView<const uint8_t> packet, ::testing::Unused) {
219 return rtcp_packet_parser_.Parse(packet);
220 }));
Jonas Oreland8ca06132022-03-14 11:52:48221 }
Evan Shrubsole7cbd8de2022-08-16 08:08:53222
Tommi90738dd2021-05-31 15:36:47223 ~VideoReceiveStream2Test() override {
Evan Shrubsolea4062722022-05-02 15:46:07224 if (video_receive_stream_) {
225 video_receive_stream_->Stop();
Tommi90738dd2021-05-31 15:36:47226 video_receive_stream_->UnregisterFromTransport();
Evan Shrubsolea4062722022-05-02 15:46:07227 }
Evan Shrubsolea4062722022-05-02 15:46:07228 time_controller_.AdvanceTime(TimeDelta::Zero());
Tommi90738dd2021-05-31 15:36:47229 }
Tommiae4d0972020-05-18 06:45:38230
Tommi90738dd2021-05-31 15:36:47231 void SetUp() override {
Sergio Garcia Murillob5289d72024-10-24 12:10:53232 config_.rtp.remote_ssrc = kRemoteSsrc;
233 config_.rtp.local_ssrc = kLocalSsrc;
Tommiae4d0972020-05-18 06:45:38234 config_.renderer = &fake_renderer_;
Tommif6f45432022-05-20 13:21:20235 VideoReceiveStreamInterface::Decoder h264_decoder;
Sergio Garcia Murillob5289d72024-10-24 12:10:53236 h264_decoder.payload_type = kH264PayloadType;
Philipp Hanckebbff58d2024-02-27 11:18:33237 h264_decoder.video_format = SdpVideoFormat::H264();
Tommiae4d0972020-05-18 06:45:38238 h264_decoder.video_format.parameters.insert(
239 {"sprop-parameter-sets", "Z0IACpZTBYmI,aMljiA=="});
Tommif6f45432022-05-20 13:21:20240 VideoReceiveStreamInterface::Decoder h265_decoder;
Sergio Garcia Murillob5289d72024-10-24 12:10:53241 h265_decoder.payload_type = kH265PayloadType;
Evan Shrubsole1c184772022-04-26 07:47:49242 h265_decoder.video_format = SdpVideoFormat("H265");
Sergio Garcia Murillob5289d72024-10-24 12:10:53243 VideoReceiveStreamInterface::Decoder av1_decoder;
244 av1_decoder.payload_type = kAv1PayloadType;
245 av1_decoder.video_format = SdpVideoFormat("AV1");
Evan Shrubsole1c184772022-04-26 07:47:49246
Sergio Garcia Murillob5289d72024-10-24 12:10:53247 config_.decoders = {av1_decoder, h265_decoder, h264_decoder};
Tommiae4d0972020-05-18 06:45:38248
Evan Shrubsole44be5792022-04-26 14:24:41249 RecreateReceiveStream();
Evan Shrubsole14ee8032022-04-26 14:01:44250 }
251
Tommif6f45432022-05-20 13:21:20252 void RecreateReceiveStream(
Florent Castelli8037fc62024-08-29 13:00:40253 std::optional<VideoReceiveStreamInterface::RecordingState> state =
254 std::nullopt) {
Evan Shrubsole14ee8032022-04-26 14:01:44255 if (video_receive_stream_) {
256 video_receive_stream_->UnregisterFromTransport();
257 video_receive_stream_ = nullptr;
258 }
Danil Chapovalov22333492023-12-07 11:10:46259 timing_ = new VCMTiming(&env_.clock(), env_.field_trials());
Evan Shrubsole2b87a422025-05-09 10:35:38260 video_receive_stream_ = std::make_unique<internal::VideoReceiveStream2>(
261 env_, &fake_call_, kDefaultNumCpuCores, &packet_router_, config_.Copy(),
262 &call_stats_, absl::WrapUnique(timing_), &nack_periodic_processor_,
263 UseMetronome() ? &decode_sync_ : nullptr);
Tommi90738dd2021-05-31 15:36:47264 video_receive_stream_->RegisterWithTransport(
265 &rtp_stream_receiver_controller_);
Evan Shrubsole44be5792022-04-26 14:24:41266 if (state)
267 video_receive_stream_->SetAndGetRecordingState(std::move(*state), false);
Tommiae4d0972020-05-18 06:45:38268 }
269
270 protected:
Evan Shrubsole44be5792022-04-26 14:24:41271 GlobalSimulatedTimeController time_controller_;
Danil Chapovalov22333492023-12-07 11:10:46272 Environment env_;
Markus Handell0e62f7a2021-07-20 11:32:02273 NackPeriodicProcessor nack_periodic_processor_;
Sergio Garcia Murillob5289d72024-10-24 12:10:53274 testing::NiceMock<MockVideoDecoderFactory> mock_decoder_factory_;
Tommif6f45432022-05-20 13:21:20275 VideoReceiveStreamInterface::Config config_;
Tommiae4d0972020-05-18 06:45:38276 internal::CallStats call_stats_;
Evan Shrubsolea4062722022-05-02 15:46:07277 testing::NiceMock<MockVideoDecoder> mock_decoder_;
Evan Shrubsole44be5792022-04-26 14:24:41278 FakeVideoRenderer fake_renderer_;
Evan Shrubsolee4b09d72025-03-17 13:25:49279 FakeCall fake_call_;
Tommiae4d0972020-05-18 06:45:38280 MockTransport mock_transport_;
philipel27b35a72022-07-05 07:59:55281 test::RtcpPacketParser rtcp_packet_parser_;
Tommiae4d0972020-05-18 06:45:38282 PacketRouter packet_router_;
283 RtpStreamReceiverController rtp_stream_receiver_controller_;
Evan Shrubsole2b87a422025-05-09 10:35:38284 std::unique_ptr<internal::VideoReceiveStream2> video_receive_stream_;
Tommiae4d0972020-05-18 06:45:38285 VCMTiming* timing_;
Evan Shrubsolea4062722022-05-02 15:46:07286 test::FakeMetronome fake_metronome_;
287 DecodeSynchronizer decode_sync_;
Evan Shrubsole1c184772022-04-26 07:47:49288
289 private:
290 test::VideoDecoderProxyFactory h264_decoder_factory_;
Evan Shrubsole14ee8032022-04-26 14:01:44291 test::FakeDecoder fake_decoder_;
Tommiae4d0972020-05-18 06:45:38292};
293
Evan Shrubsolea4062722022-05-02 15:46:07294TEST_P(VideoReceiveStream2Test, CreateFrameFromH264FmtpSpropAndIdr) {
Tommiae4d0972020-05-18 06:45:38295 constexpr uint8_t idr_nalu[] = {0x05, 0xFF, 0xFF, 0xFF};
296 RtpPacketToSend rtppacket(nullptr);
Danil Chapovalovadf7d6e2025-07-17 15:20:53297 rtppacket.SetPayload(idr_nalu);
Tommiae4d0972020-05-18 06:45:38298 rtppacket.SetMarker(true);
Sergio Garcia Murillob5289d72024-10-24 12:10:53299 rtppacket.SetSsrc(kRemoteSsrc);
300 rtppacket.SetPayloadType(kH264PayloadType);
Tommiae4d0972020-05-18 06:45:38301 rtppacket.SetSequenceNumber(1);
302 rtppacket.SetTimestamp(0);
Evan Shrubsolea4062722022-05-02 15:46:07303 EXPECT_CALL(mock_decoder_, RegisterDecodeCompleteCallback(_));
Tommiae4d0972020-05-18 06:45:38304 video_receive_stream_->Start();
Tony Herre55b593f2023-08-29 14:05:49305 EXPECT_CALL(mock_decoder_, Decode(_, _));
Tommiae4d0972020-05-18 06:45:38306 RtpPacketReceived parsed_packet;
307 ASSERT_TRUE(parsed_packet.Parse(rtppacket.data(), rtppacket.size()));
308 rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
Evan Shrubsolea4062722022-05-02 15:46:07309 EXPECT_CALL(mock_decoder_, Release());
Evan Shrubsole44be5792022-04-26 14:24:41310
311 time_controller_.AdvanceTime(TimeDelta::Zero());
Tommiae4d0972020-05-18 06:45:38312}
313
Evan Shrubsolea4062722022-05-02 15:46:07314TEST_P(VideoReceiveStream2Test, PlayoutDelay) {
Danil Chapovalov7084e1b2023-08-23 11:16:22315 const VideoPlayoutDelay kPlayoutDelay(TimeDelta::Millis(123),
Danil Chapovalov87b7c1a2025-01-28 09:51:08316 TimeDelta::Millis(521));
Evan Shrubsolea0ee64c2022-04-26 08:09:04317 std::unique_ptr<test::FakeEncodedFrame> test_frame =
Danil Chapovalovc146b5f2023-08-16 09:42:52318 test::FakeFrameBuilder()
319 .Id(0)
Danil Chapovalov7084e1b2023-08-23 11:16:22320 .PlayoutDelay(kPlayoutDelay)
Danil Chapovalovc146b5f2023-08-16 09:42:52321 .AsLast()
322 .Build();
Tommiae4d0972020-05-18 06:45:38323
324 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
Evan Shrubsole8f1159b2022-03-22 11:12:17325 auto timings = timing_->GetTimings();
Danil Chapovalov7084e1b2023-08-23 11:16:22326 EXPECT_EQ(kPlayoutDelay.min(), timings.min_playout_delay);
327 EXPECT_EQ(kPlayoutDelay.max(), timings.max_playout_delay);
Tommiae4d0972020-05-18 06:45:38328
329 // Check that the biggest minimum delay is chosen.
330 video_receive_stream_->SetMinimumPlayoutDelay(400);
Evan Shrubsole8f1159b2022-03-22 11:12:17331 timings = timing_->GetTimings();
332 EXPECT_EQ(400, timings.min_playout_delay.ms());
Tommiae4d0972020-05-18 06:45:38333
334 // Check base minimum delay validation.
335 EXPECT_FALSE(video_receive_stream_->SetBaseMinimumPlayoutDelayMs(12345));
336 EXPECT_FALSE(video_receive_stream_->SetBaseMinimumPlayoutDelayMs(-1));
337 EXPECT_TRUE(video_receive_stream_->SetBaseMinimumPlayoutDelayMs(500));
Evan Shrubsole8f1159b2022-03-22 11:12:17338 timings = timing_->GetTimings();
339 EXPECT_EQ(500, timings.min_playout_delay.ms());
Tommiae4d0972020-05-18 06:45:38340
341 // Check that intermidiate values are remembered and the biggest remembered
342 // is chosen.
343 video_receive_stream_->SetBaseMinimumPlayoutDelayMs(0);
Evan Shrubsole8f1159b2022-03-22 11:12:17344 timings = timing_->GetTimings();
345 EXPECT_EQ(400, timings.min_playout_delay.ms());
Tommiae4d0972020-05-18 06:45:38346
347 video_receive_stream_->SetMinimumPlayoutDelay(0);
Evan Shrubsole8f1159b2022-03-22 11:12:17348 timings = timing_->GetTimings();
349 EXPECT_EQ(123, timings.min_playout_delay.ms());
Tommiae4d0972020-05-18 06:45:38350}
351
Danil Chapovalov87b7c1a2025-01-28 09:51:08352TEST_P(VideoReceiveStream2Test, MinPlayoutDelayIsLimitedByMaxPlayoutDelay) {
353 const VideoPlayoutDelay kPlayoutDelay(TimeDelta::Millis(123),
354 TimeDelta::Millis(321));
355 video_receive_stream_->OnCompleteFrame(test::FakeFrameBuilder()
356 .Id(0)
357 .PlayoutDelay(kPlayoutDelay)
358 .AsLast()
359 .Build());
360 EXPECT_EQ(timing_->GetTimings().min_playout_delay, kPlayoutDelay.min());
361
362 // Check that the biggest minimum delay is limited by the max playout delay.
363 video_receive_stream_->SetMinimumPlayoutDelay(400);
364 EXPECT_EQ(timing_->GetTimings().min_playout_delay, kPlayoutDelay.max());
365}
366
Johannes Kronbbf639e2022-06-15 10:27:23367TEST_P(VideoReceiveStream2Test, RenderParametersSetToDefaultValues) {
368 // Default render parameters.
369 const VideoFrame::RenderParameters kDefaultRenderParameters;
Johannes Kron111e9812020-10-26 12:54:40370 // Default with no playout delay set.
Evan Shrubsolea0ee64c2022-04-26 08:09:04371 std::unique_ptr<test::FakeEncodedFrame> test_frame0 =
372 test::FakeFrameBuilder().Id(0).AsLast().Build();
Johannes Kron111e9812020-10-26 12:54:40373 video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
Johannes Kronbbf639e2022-06-15 10:27:23374 EXPECT_EQ(timing_->RenderParameters(), kDefaultRenderParameters);
375}
Johannes Kron111e9812020-10-26 12:54:40376
Johannes Kronbbf639e2022-06-15 10:27:23377TEST_P(VideoReceiveStream2Test, UseLowLatencyRenderingSetFromPlayoutDelay) {
Johannes Kronbbf639e2022-06-15 10:27:23378 std::unique_ptr<test::FakeEncodedFrame> test_frame0 =
Danil Chapovalovc146b5f2023-08-16 09:42:52379 test::FakeFrameBuilder()
380 .Id(0)
381 .PlayoutDelay(VideoPlayoutDelay::Minimal())
382 .AsLast()
383 .Build();
Johannes Kronbbf639e2022-06-15 10:27:23384 video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
385 EXPECT_TRUE(timing_->RenderParameters().use_low_latency_rendering);
386
Evan Shrubsolea0ee64c2022-04-26 08:09:04387 std::unique_ptr<test::FakeEncodedFrame> test_frame1 =
Danil Chapovalovc146b5f2023-08-16 09:42:52388 test::FakeFrameBuilder()
389 .Id(1)
Danil Chapovalov7084e1b2023-08-23 11:16:22390 .PlayoutDelay({TimeDelta::Zero(), TimeDelta::Millis(500)})
Danil Chapovalovc146b5f2023-08-16 09:42:52391 .AsLast()
392 .Build();
Johannes Kron111e9812020-10-26 12:54:40393 video_receive_stream_->OnCompleteFrame(std::move(test_frame1));
Johannes Kronbbf639e2022-06-15 10:27:23394 EXPECT_TRUE(timing_->RenderParameters().use_low_latency_rendering);
Johannes Kron111e9812020-10-26 12:54:40395}
396
Evan Shrubsolea4062722022-05-02 15:46:07397TEST_P(VideoReceiveStream2Test, MaxCompositionDelaySetFromMaxPlayoutDelay) {
Johannes Kronbbf639e2022-06-15 10:27:23398 // The max composition delay is dependent on the number of frames in the
399 // pre-decode queue. It's therefore important to advance the time as the test
400 // runs to get the correct expectations of max_composition_delay_in_frames.
401 video_receive_stream_->Start();
402 // Max composition delay not set if no playout delay is set.
403 std::unique_ptr<test::FakeEncodedFrame> test_frame0 =
404 test::FakeFrameBuilder()
405 .Id(0)
406 .Time(RtpTimestampForFrame(0))
407 .ReceivedTime(ReceiveTimeForFrame(0))
408 .AsLast()
409 .Build();
410 video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
411 EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
Florent Castelli8037fc62024-08-29 13:00:40412 Eq(std::nullopt));
Johannes Kronbbf639e2022-06-15 10:27:23413 time_controller_.AdvanceTime(k30FpsDelay);
Johannes Kronbbf639e2022-06-15 10:27:23414
415 // Max composition delay not set for playout delay 0,0.
416 std::unique_ptr<test::FakeEncodedFrame> test_frame1 =
417 test::FakeFrameBuilder()
418 .Id(1)
419 .Time(RtpTimestampForFrame(1))
420 .ReceivedTime(ReceiveTimeForFrame(1))
Danil Chapovalov7084e1b2023-08-23 11:16:22421 .PlayoutDelay(VideoPlayoutDelay::Minimal())
Johannes Kronbbf639e2022-06-15 10:27:23422 .AsLast()
423 .Build();
Johannes Kronbbf639e2022-06-15 10:27:23424 video_receive_stream_->OnCompleteFrame(std::move(test_frame1));
425 EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
Florent Castelli8037fc62024-08-29 13:00:40426 Eq(std::nullopt));
Johannes Kronbbf639e2022-06-15 10:27:23427 time_controller_.AdvanceTime(k30FpsDelay);
Johannes Kronbbf639e2022-06-15 10:27:23428
429 // Max composition delay not set for playout delay X,Y, where X,Y>0.
430 std::unique_ptr<test::FakeEncodedFrame> test_frame2 =
431 test::FakeFrameBuilder()
432 .Id(2)
433 .Time(RtpTimestampForFrame(2))
434 .ReceivedTime(ReceiveTimeForFrame(2))
Danil Chapovalov7084e1b2023-08-23 11:16:22435 .PlayoutDelay({TimeDelta::Millis(10), TimeDelta::Millis(30)})
Johannes Kronbbf639e2022-06-15 10:27:23436 .AsLast()
437 .Build();
Johannes Kronbbf639e2022-06-15 10:27:23438 video_receive_stream_->OnCompleteFrame(std::move(test_frame2));
439 EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
Florent Castelli8037fc62024-08-29 13:00:40440 Eq(std::nullopt));
Johannes Kronbbf639e2022-06-15 10:27:23441
442 time_controller_.AdvanceTime(k30FpsDelay);
Johannes Kronbbf639e2022-06-15 10:27:23443
Johannes Kron111e9812020-10-26 12:54:40444 // Max composition delay set if playout delay X,Y, where X=0,Y>0.
Johannes Kron111e9812020-10-26 12:54:40445 const int kExpectedMaxCompositionDelayInFrames = 3; // ~50 ms at 60 fps.
Johannes Kronbbf639e2022-06-15 10:27:23446 std::unique_ptr<test::FakeEncodedFrame> test_frame3 =
447 test::FakeFrameBuilder()
448 .Id(3)
449 .Time(RtpTimestampForFrame(3))
450 .ReceivedTime(ReceiveTimeForFrame(3))
Danil Chapovalov7084e1b2023-08-23 11:16:22451 .PlayoutDelay({TimeDelta::Zero(), TimeDelta::Millis(50)})
Johannes Kronbbf639e2022-06-15 10:27:23452 .AsLast()
453 .Build();
Johannes Kronbbf639e2022-06-15 10:27:23454 video_receive_stream_->OnCompleteFrame(std::move(test_frame3));
455 EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
456 Optional(kExpectedMaxCompositionDelayInFrames));
Johannes Kron111e9812020-10-26 12:54:40457}
458
Evan Shrubsolea4062722022-05-02 15:46:07459TEST_P(VideoReceiveStream2Test, LazyDecoderCreation) {
Evan Shrubsole1c184772022-04-26 07:47:49460 constexpr uint8_t idr_nalu[] = {0x05, 0xFF, 0xFF, 0xFF};
461 RtpPacketToSend rtppacket(nullptr);
Danil Chapovalovadf7d6e2025-07-17 15:20:53462 rtppacket.SetPayload(idr_nalu);
Evan Shrubsole1c184772022-04-26 07:47:49463 rtppacket.SetMarker(true);
Sergio Garcia Murillob5289d72024-10-24 12:10:53464 rtppacket.SetSsrc(kRemoteSsrc);
465 rtppacket.SetPayloadType(kH264PayloadType);
Evan Shrubsole1c184772022-04-26 07:47:49466 rtppacket.SetSequenceNumber(1);
467 rtppacket.SetTimestamp(0);
468
Johannes Kronbb591c42022-09-28 10:10:25469 // No decoders are created by default.
Sergio Garcia Murillob5289d72024-10-24 12:10:53470 EXPECT_CALL(mock_decoder_factory_, Create).Times(0);
Evan Shrubsole1c184772022-04-26 07:47:49471 video_receive_stream_->Start();
Erik Språng7aaeb5a2022-08-18 11:18:27472 time_controller_.AdvanceTime(TimeDelta::Zero());
Evan Shrubsole1c184772022-04-26 07:47:49473
474 EXPECT_TRUE(
Sergio Garcia Murillob5289d72024-10-24 12:10:53475 testing::Mock::VerifyAndClearExpectations(&mock_decoder_factory_));
Johannes Kronbb591c42022-09-28 10:10:25476 // Verify that the decoder is created when we receive payload data and tries
477 // to decode a frame.
Sergio Garcia Murillob5289d72024-10-24 12:10:53478 EXPECT_CALL(mock_decoder_factory_,
Danil Chapovalov08173802024-01-23 12:02:46479 Create(_, Field(&SdpVideoFormat::name, Eq("H264"))));
Erik Språng7aaeb5a2022-08-18 11:18:27480 EXPECT_CALL(mock_decoder_, Configure);
481 EXPECT_CALL(mock_decoder_, RegisterDecodeCompleteCallback);
Tony Herre55b593f2023-08-29 14:05:49482 EXPECT_CALL(mock_decoder_, Decode(_, _));
Evan Shrubsole1c184772022-04-26 07:47:49483 RtpPacketReceived parsed_packet;
484 ASSERT_TRUE(parsed_packet.Parse(rtppacket.data(), rtppacket.size()));
485 rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
Erik Språng7aaeb5a2022-08-18 11:18:27486 EXPECT_CALL(mock_decoder_, Release);
Evan Shrubsole1c184772022-04-26 07:47:49487
488 // Make sure the decoder thread had a chance to run.
Erik Språng7aaeb5a2022-08-18 11:18:27489 time_controller_.AdvanceTime(TimeDelta::Zero());
Evan Shrubsole1c184772022-04-26 07:47:49490}
491
Sergio Garcia Murillob5289d72024-10-24 12:10:53492TEST_P(VideoReceiveStream2Test, LazyDecoderCreationCodecSwitch) {
493 constexpr uint8_t idr_nalu[] = {0x05, 0xFF, 0xFF, 0xFF};
494 RtpPacketToSend rtppacket(nullptr);
Danil Chapovalovadf7d6e2025-07-17 15:20:53495 rtppacket.SetPayload(idr_nalu);
Sergio Garcia Murillob5289d72024-10-24 12:10:53496 rtppacket.SetMarker(true);
497 rtppacket.SetSsrc(kRemoteSsrc);
498 rtppacket.SetPayloadType(kH264PayloadType);
499 rtppacket.SetSequenceNumber(1);
500 rtppacket.SetTimestamp(0);
501
502 // No decoders are created by default.
503 EXPECT_CALL(mock_decoder_factory_, Create).Times(0);
504 video_receive_stream_->Start();
505 time_controller_.AdvanceTime(TimeDelta::Zero());
506
507 EXPECT_TRUE(
508 testing::Mock::VerifyAndClearExpectations(&mock_decoder_factory_));
509 // Verify that the decoder is created when we receive payload data and tries
510 // to decode a frame.
511 EXPECT_CALL(mock_decoder_factory_,
512 Create(_, Field(&SdpVideoFormat::name, Eq("H264"))));
513 EXPECT_CALL(mock_decoder_, Configure);
514 EXPECT_CALL(mock_decoder_, RegisterDecodeCompleteCallback);
515 EXPECT_CALL(mock_decoder_, Decode(_, _));
516 RtpPacketReceived parsed_packet;
517 ASSERT_TRUE(parsed_packet.Parse(rtppacket.data(), rtppacket.size()));
518 rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
519 // H264 decoder is released after receiving the AV1 packet.
520 EXPECT_CALL(mock_decoder_, Release).Times(0);
521
522 // Make sure the decoder thread had a chance to run.
523 time_controller_.AdvanceTime(TimeDelta::Millis(100));
524
525 // Switch to AV1.
526 const uint8_t av1_key_obu[] = {0x18, 0x48, 0x01, 0xAA}; // \ OBU
527 RtpPacketToSend av1_rtppacket(nullptr);
Danil Chapovalovadf7d6e2025-07-17 15:20:53528 av1_rtppacket.SetPayload(av1_key_obu);
Sergio Garcia Murillob5289d72024-10-24 12:10:53529 av1_rtppacket.SetMarker(true);
530 av1_rtppacket.SetSsrc(kRemoteSsrc);
531 av1_rtppacket.SetPayloadType(kAv1PayloadType);
532 av1_rtppacket.SetSequenceNumber(2);
533 av1_rtppacket.SetTimestamp(1);
534
535 EXPECT_TRUE(
536 testing::Mock::VerifyAndClearExpectations(&mock_decoder_factory_));
537 // Release the H264 previous decoder.
538 EXPECT_CALL(mock_decoder_, Release);
539 // Verify that the decoder is created when we receive payload data and tries
540 // to decode a frame.
541 EXPECT_CALL(mock_decoder_factory_,
542 Create(_, Field(&SdpVideoFormat::name, Eq("AV1"))));
543 EXPECT_CALL(mock_decoder_, Configure);
544 EXPECT_CALL(mock_decoder_, RegisterDecodeCompleteCallback);
545 EXPECT_CALL(mock_decoder_, Decode(_, _));
546 ASSERT_TRUE(parsed_packet.Parse(av1_rtppacket.data(), av1_rtppacket.size()));
547 rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
548
549 // Make sure the decoder thread had a chance to run.
550 time_controller_.AdvanceTime(TimeDelta::Millis(100));
551
552 // Switch back to H264.
553 rtppacket.SetPayloadType(kH264PayloadType);
554 rtppacket.SetSequenceNumber(3);
555 rtppacket.SetTimestamp(2);
556
557 EXPECT_TRUE(
558 testing::Mock::VerifyAndClearExpectations(&mock_decoder_factory_));
559 // Release the AV1 previous decoder and the new H264 decoder on test end.
560 EXPECT_CALL(mock_decoder_, Release).Times(2);
561 // Verify that the decoder is created when we receive payload data and tries
562 // to decode a frame.
563 EXPECT_CALL(mock_decoder_factory_,
564 Create(_, Field(&SdpVideoFormat::name, Eq("H264"))));
565 EXPECT_CALL(mock_decoder_, Configure);
566 EXPECT_CALL(mock_decoder_, RegisterDecodeCompleteCallback);
567 EXPECT_CALL(mock_decoder_, Decode(_, _));
568 ASSERT_TRUE(parsed_packet.Parse(rtppacket.data(), rtppacket.size()));
569 rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
570
571 // Make sure the decoder thread had a chance to run.
572 time_controller_.AdvanceTime(TimeDelta::Millis(100));
573}
574
Evan Shrubsolea4062722022-05-02 15:46:07575TEST_P(VideoReceiveStream2Test, PassesNtpTime) {
Evan Shrubsolea0ee64c2022-04-26 08:09:04576 const Timestamp kNtpTimestamp = Timestamp::Millis(12345);
577 std::unique_ptr<test::FakeEncodedFrame> test_frame =
578 test::FakeFrameBuilder()
579 .Id(0)
Sergio Garcia Murillob5289d72024-10-24 12:10:53580 .PayloadType(kH264PayloadType)
Evan Shrubsolea0ee64c2022-04-26 08:09:04581 .NtpTime(kNtpTimestamp)
582 .AsLast()
583 .Build();
Tommiae4d0972020-05-18 06:45:38584
585 video_receive_stream_->Start();
586 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
Evan Shrubsolea4062722022-05-02 15:46:07587 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
Evan Shrubsole44be5792022-04-26 14:24:41588 RenderedFrameWith(NtpTimestamp(kNtpTimestamp)));
Tommiae4d0972020-05-18 06:45:38589}
590
Evan Shrubsolea4062722022-05-02 15:46:07591TEST_P(VideoReceiveStream2Test, PassesRotation) {
Evan Shrubsole2b87a422025-05-09 10:35:38592 const VideoRotation kRotation = kVideoRotation_180;
Sergio Garcia Murillob5289d72024-10-24 12:10:53593 std::unique_ptr<test::FakeEncodedFrame> test_frame =
594 test::FakeFrameBuilder()
595 .Id(0)
596 .PayloadType(kH264PayloadType)
597 .Rotation(kRotation)
598 .AsLast()
599 .Build();
Tommiae4d0972020-05-18 06:45:38600
601 video_receive_stream_->Start();
602 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
Evan Shrubsolea4062722022-05-02 15:46:07603 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
Evan Shrubsole44be5792022-04-26 14:24:41604 RenderedFrameWith(Rotation(kRotation)));
Tommiae4d0972020-05-18 06:45:38605}
606
Evan Shrubsolea4062722022-05-02 15:46:07607TEST_P(VideoReceiveStream2Test, PassesPacketInfos) {
Tommiae4d0972020-05-18 06:45:38608 RtpPacketInfos packet_infos = CreatePacketInfos(3);
Evan Shrubsolea0ee64c2022-04-26 08:09:04609 auto test_frame = test::FakeFrameBuilder()
610 .Id(0)
Sergio Garcia Murillob5289d72024-10-24 12:10:53611 .PayloadType(kH264PayloadType)
Evan Shrubsolea0ee64c2022-04-26 08:09:04612 .PacketInfos(packet_infos)
613 .AsLast()
614 .Build();
Tommiae4d0972020-05-18 06:45:38615
616 video_receive_stream_->Start();
617 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
Evan Shrubsolea4062722022-05-02 15:46:07618 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
Evan Shrubsole44be5792022-04-26 14:24:41619 RenderedFrameWith(PacketInfos(ElementsAreArray(packet_infos))));
Tommiae4d0972020-05-18 06:45:38620}
621
Evan Shrubsolea4062722022-05-02 15:46:07622TEST_P(VideoReceiveStream2Test, RenderedFrameUpdatesGetSources) {
Sergio Garcia Murillob5289d72024-10-24 12:10:53623 constexpr uint32_t kSsrc = kRemoteSsrc;
Tommiae4d0972020-05-18 06:45:38624 constexpr uint32_t kCsrc = 9001;
625 constexpr uint32_t kRtpTimestamp = 12345;
626
627 // Prepare one video frame with per-packet information.
Sergio Garcia Murillob5289d72024-10-24 12:10:53628 auto test_frame = test::FakeFrameBuilder()
629 .Id(0)
630 .PayloadType(kH264PayloadType)
631 .AsLast()
632 .Build();
Tommiae4d0972020-05-18 06:45:38633 RtpPacketInfos packet_infos;
634 {
635 RtpPacketInfos::vector_type infos;
636
637 RtpPacketInfo info;
638 info.set_ssrc(kSsrc);
639 info.set_csrcs({kCsrc});
640 info.set_rtp_timestamp(kRtpTimestamp);
641
Danil Chapovalov22333492023-12-07 11:10:46642 info.set_receive_time(env_.clock().CurrentTime() - TimeDelta::Millis(5000));
Tommiae4d0972020-05-18 06:45:38643 infos.push_back(info);
644
Danil Chapovalov22333492023-12-07 11:10:46645 info.set_receive_time(env_.clock().CurrentTime() - TimeDelta::Millis(3000));
Tommiae4d0972020-05-18 06:45:38646 infos.push_back(info);
647
Danil Chapovalov22333492023-12-07 11:10:46648 info.set_receive_time(env_.clock().CurrentTime() - TimeDelta::Millis(2000));
Tommiae4d0972020-05-18 06:45:38649 infos.push_back(info);
650
Danil Chapovalov22333492023-12-07 11:10:46651 info.set_receive_time(env_.clock().CurrentTime() - TimeDelta::Millis(1000));
Tommiae4d0972020-05-18 06:45:38652 infos.push_back(info);
653
654 packet_infos = RtpPacketInfos(std::move(infos));
655 }
656 test_frame->SetPacketInfos(packet_infos);
657
658 // Start receive stream.
659 video_receive_stream_->Start();
660 EXPECT_THAT(video_receive_stream_->GetSources(), IsEmpty());
661
662 // Render one video frame.
Danil Chapovalov22333492023-12-07 11:10:46663 Timestamp timestamp_min = env_.clock().CurrentTime();
Tommiae4d0972020-05-18 06:45:38664 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
Tommiae4d0972020-05-18 06:45:38665 // Verify that the per-packet information is passed to the renderer.
Evan Shrubsolea4062722022-05-02 15:46:07666 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
Evan Shrubsole44be5792022-04-26 14:24:41667 RenderedFrameWith(PacketInfos(ElementsAreArray(packet_infos))));
Danil Chapovalov22333492023-12-07 11:10:46668 Timestamp timestamp_max = env_.clock().CurrentTime();
Tommiae4d0972020-05-18 06:45:38669
Artem Titovab30d722021-07-27 14:22:11670 // Verify that the per-packet information also updates `GetSources()`.
Tommiae4d0972020-05-18 06:45:38671 std::vector<RtpSource> sources = video_receive_stream_->GetSources();
672 ASSERT_THAT(sources, SizeIs(2));
673 {
674 auto it = std::find_if(sources.begin(), sources.end(),
675 [](const RtpSource& source) {
676 return source.source_type() == RtpSourceType::SSRC;
677 });
678 ASSERT_NE(it, sources.end());
679
680 EXPECT_EQ(it->source_id(), kSsrc);
681 EXPECT_EQ(it->source_type(), RtpSourceType::SSRC);
682 EXPECT_EQ(it->rtp_timestamp(), kRtpTimestamp);
Danil Chapovalovf5359712023-08-28 16:32:25683 EXPECT_GE(it->timestamp(), timestamp_min);
684 EXPECT_LE(it->timestamp(), timestamp_max);
Tommiae4d0972020-05-18 06:45:38685 }
686 {
687 auto it = std::find_if(sources.begin(), sources.end(),
688 [](const RtpSource& source) {
689 return source.source_type() == RtpSourceType::CSRC;
690 });
691 ASSERT_NE(it, sources.end());
692
693 EXPECT_EQ(it->source_id(), kCsrc);
694 EXPECT_EQ(it->source_type(), RtpSourceType::CSRC);
695 EXPECT_EQ(it->rtp_timestamp(), kRtpTimestamp);
Danil Chapovalovf5359712023-08-28 16:32:25696 EXPECT_GE(it->timestamp(), timestamp_min);
697 EXPECT_LE(it->timestamp(), timestamp_max);
Tommiae4d0972020-05-18 06:45:38698 }
699}
700
Evan Shrubsolea0ee64c2022-04-26 08:09:04701std::unique_ptr<test::FakeEncodedFrame> MakeFrameWithResolution(
Markus Handell588f9b32021-04-08 17:19:50702 VideoFrameType frame_type,
703 int picture_id,
704 int width,
705 int height) {
Sergio Garcia Murillob5289d72024-10-24 12:10:53706 auto frame = test::FakeFrameBuilder()
707 .Id(picture_id)
708 .PayloadType(kH264PayloadType)
709 .AsLast()
710 .Build();
Tommiae4d0972020-05-18 06:45:38711 frame->SetFrameType(frame_type);
Markus Handell588f9b32021-04-08 17:19:50712 frame->_encodedWidth = width;
713 frame->_encodedHeight = height;
Tommiae4d0972020-05-18 06:45:38714 return frame;
715}
716
Evan Shrubsolea0ee64c2022-04-26 08:09:04717std::unique_ptr<test::FakeEncodedFrame> MakeFrame(VideoFrameType frame_type,
718 int picture_id) {
Markus Handell588f9b32021-04-08 17:19:50719 return MakeFrameWithResolution(frame_type, picture_id, 320, 240);
720}
721
Evan Shrubsolea4062722022-05-02 15:46:07722TEST_P(VideoReceiveStream2Test, PassesFrameWhenEncodedFramesCallbackSet) {
Tommiae4d0972020-05-18 06:45:38723 testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
724 video_receive_stream_->Start();
Tommiae4d0972020-05-18 06:45:38725 EXPECT_CALL(callback, Call);
726 video_receive_stream_->SetAndGetRecordingState(
Tommif6f45432022-05-20 13:21:20727 VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
728 true);
Tommiae4d0972020-05-18 06:45:38729 video_receive_stream_->OnCompleteFrame(
730 MakeFrame(VideoFrameType::kVideoFrameKey, 0));
Evan Shrubsolea4062722022-05-02 15:46:07731 EXPECT_TRUE(fake_renderer_.WaitForFrame(kDefaultTimeOut));
philipel27b35a72022-07-05 07:59:55732
733 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(1));
734
Tommiae4d0972020-05-18 06:45:38735 video_receive_stream_->Stop();
736}
737
Evan Shrubsolea4062722022-05-02 15:46:07738TEST_P(VideoReceiveStream2Test, MovesEncodedFrameDispatchStateWhenReCreating) {
Tommiae4d0972020-05-18 06:45:38739 testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
740 video_receive_stream_->Start();
741 // Expect a key frame request over RTCP.
Tommiae4d0972020-05-18 06:45:38742 video_receive_stream_->SetAndGetRecordingState(
Tommif6f45432022-05-20 13:21:20743 VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
744 true);
Tommiae4d0972020-05-18 06:45:38745 video_receive_stream_->Stop();
Tommif6f45432022-05-20 13:21:20746 VideoReceiveStreamInterface::RecordingState old_state =
Tommiae4d0972020-05-18 06:45:38747 video_receive_stream_->SetAndGetRecordingState(
Tommif6f45432022-05-20 13:21:20748 VideoReceiveStreamInterface::RecordingState(), false);
Evan Shrubsole44be5792022-04-26 14:24:41749 RecreateReceiveStream(std::move(old_state));
philipel27b35a72022-07-05 07:59:55750
751 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(1));
752
Tommiae4d0972020-05-18 06:45:38753 video_receive_stream_->Stop();
754}
755
Evan Shrubsolea4062722022-05-02 15:46:07756TEST_P(VideoReceiveStream2Test, RequestsKeyFramesUntilKeyFrameReceived) {
Evan Shrubsole44be5792022-04-26 14:24:41757 // Recreate receive stream with shorter delay to test rtx.
758 TimeDelta rtx_delay = TimeDelta::Millis(50);
759 config_.rtp.nack.rtp_history_ms = rtx_delay.ms();
760 auto tick = rtx_delay / 2;
761 RecreateReceiveStream();
762 video_receive_stream_->Start();
Markus Handell588f9b32021-04-08 17:19:50763
Evan Shrubsole44be5792022-04-26 14:24:41764 video_receive_stream_->GenerateKeyFrame();
765 video_receive_stream_->OnCompleteFrame(
766 MakeFrame(VideoFrameType::kVideoFrameDelta, 0));
Evan Shrubsolea4062722022-05-02 15:46:07767 fake_renderer_.WaitForFrame(kDefaultTimeOut);
Tommiae4d0972020-05-18 06:45:38768 time_controller_.AdvanceTime(tick);
Evan Shrubsole44be5792022-04-26 14:24:41769 video_receive_stream_->OnCompleteFrame(
770 MakeFrame(VideoFrameType::kVideoFrameDelta, 1));
Evan Shrubsolea4062722022-05-02 15:46:07771 fake_renderer_.WaitForFrame(kDefaultTimeOut);
Evan Shrubsole44be5792022-04-26 14:24:41772 time_controller_.AdvanceTime(TimeDelta::Zero());
Tommiae4d0972020-05-18 06:45:38773 testing::Mock::VerifyAndClearExpectations(&mock_transport_);
774
philipel27b35a72022-07-05 07:59:55775 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(1));
776
Philipp Hancke006206d2021-03-24 16:49:02777 // T+keyframetimeout: still no key frame received, expect key frame request
778 // sent again.
Tommiae4d0972020-05-18 06:45:38779 time_controller_.AdvanceTime(tick);
Evan Shrubsole44be5792022-04-26 14:24:41780 video_receive_stream_->OnCompleteFrame(
781 MakeFrame(VideoFrameType::kVideoFrameDelta, 2));
Evan Shrubsolea4062722022-05-02 15:46:07782 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Tommiae4d0972020-05-18 06:45:38783 testing::Mock::VerifyAndClearExpectations(&mock_transport_);
784
philipel27b35a72022-07-05 07:59:55785 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(2));
786
Philipp Hancke006206d2021-03-24 16:49:02787 // T+keyframetimeout: now send a key frame - we should not observe new key
788 // frame requests after this.
Evan Shrubsole44be5792022-04-26 14:24:41789 video_receive_stream_->OnCompleteFrame(
790 MakeFrame(VideoFrameType::kVideoFrameKey, 3));
Evan Shrubsolea4062722022-05-02 15:46:07791 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Tommiae4d0972020-05-18 06:45:38792 time_controller_.AdvanceTime(2 * tick);
Evan Shrubsole44be5792022-04-26 14:24:41793 video_receive_stream_->OnCompleteFrame(
794 MakeFrame(VideoFrameType::kVideoFrameDelta, 4));
Evan Shrubsolea4062722022-05-02 15:46:07795 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
philipel27b35a72022-07-05 07:59:55796
797 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(2));
Tommiae4d0972020-05-18 06:45:38798}
799
Evan Shrubsolea4062722022-05-02 15:46:07800TEST_P(VideoReceiveStream2Test,
Markus Handell588f9b32021-04-08 17:19:50801 DispatchesEncodedFrameSequenceStartingWithKeyframeWithoutResolution) {
Evan Shrubsole44be5792022-04-26 14:24:41802 video_receive_stream_->Start();
Markus Handell588f9b32021-04-08 17:19:50803 testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
Evan Shrubsole44be5792022-04-26 14:24:41804 video_receive_stream_->SetAndGetRecordingState(
Tommif6f45432022-05-20 13:21:20805 VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
Markus Handell588f9b32021-04-08 17:19:50806 /*generate_key_frame=*/false);
807
808 InSequence s;
Jonas Oreland0deda152022-09-23 10:08:57809 EXPECT_CALL(callback,
810 Call(MatchResolution(test::FakeDecoder::kDefaultWidth,
811 test::FakeDecoder::kDefaultHeight)));
Markus Handell588f9b32021-04-08 17:19:50812 EXPECT_CALL(callback, Call);
813
Evan Shrubsole44be5792022-04-26 14:24:41814 video_receive_stream_->OnCompleteFrame(
Markus Handell588f9b32021-04-08 17:19:50815 MakeFrameWithResolution(VideoFrameType::kVideoFrameKey, 0, 0, 0));
Evan Shrubsolea4062722022-05-02 15:46:07816 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Evan Shrubsole44be5792022-04-26 14:24:41817 video_receive_stream_->OnCompleteFrame(
Markus Handell588f9b32021-04-08 17:19:50818 MakeFrameWithResolution(VideoFrameType::kVideoFrameDelta, 1, 0, 0));
Evan Shrubsolea4062722022-05-02 15:46:07819 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Markus Handell588f9b32021-04-08 17:19:50820
Evan Shrubsole44be5792022-04-26 14:24:41821 video_receive_stream_->Stop();
Markus Handell588f9b32021-04-08 17:19:50822}
823
Evan Shrubsolea4062722022-05-02 15:46:07824TEST_P(VideoReceiveStream2Test,
Markus Handell588f9b32021-04-08 17:19:50825 DispatchesEncodedFrameSequenceStartingWithKeyframeWithResolution) {
Evan Shrubsole44be5792022-04-26 14:24:41826 video_receive_stream_->Start();
Markus Handell588f9b32021-04-08 17:19:50827 testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
Evan Shrubsole44be5792022-04-26 14:24:41828 video_receive_stream_->SetAndGetRecordingState(
Tommif6f45432022-05-20 13:21:20829 VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
Markus Handell588f9b32021-04-08 17:19:50830 /*generate_key_frame=*/false);
831
832 InSequence s;
Jonas Oreland0deda152022-09-23 10:08:57833 EXPECT_CALL(callback, Call(MatchResolution(1080u, 720u)));
Markus Handell588f9b32021-04-08 17:19:50834 EXPECT_CALL(callback, Call);
835
Evan Shrubsole44be5792022-04-26 14:24:41836 video_receive_stream_->OnCompleteFrame(
Markus Handell588f9b32021-04-08 17:19:50837 MakeFrameWithResolution(VideoFrameType::kVideoFrameKey, 0, 1080, 720));
Evan Shrubsolea4062722022-05-02 15:46:07838 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Evan Shrubsole44be5792022-04-26 14:24:41839 video_receive_stream_->OnCompleteFrame(
Markus Handell588f9b32021-04-08 17:19:50840 MakeFrameWithResolution(VideoFrameType::kVideoFrameDelta, 1, 0, 0));
Evan Shrubsolea4062722022-05-02 15:46:07841 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Markus Handell588f9b32021-04-08 17:19:50842
Evan Shrubsole44be5792022-04-26 14:24:41843 video_receive_stream_->Stop();
Markus Handell588f9b32021-04-08 17:19:50844}
845
Evan Shrubsolea4062722022-05-02 15:46:07846TEST_P(VideoReceiveStream2Test, DependantFramesAreScheduled) {
847 video_receive_stream_->Start();
848
849 auto key_frame = test::FakeFrameBuilder()
850 .Id(0)
Sergio Garcia Murillob5289d72024-10-24 12:10:53851 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:07852 .Time(kFirstRtpTimestamp)
853 .ReceivedTime(kStartTime)
854 .AsLast()
855 .Build();
856 auto delta_frame = test::FakeFrameBuilder()
857 .Id(1)
Sergio Garcia Murillob5289d72024-10-24 12:10:53858 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:07859 .Time(RtpTimestampForFrame(1))
860 .ReceivedTime(ReceiveTimeForFrame(1))
861 .Refs({0})
862 .AsLast()
863 .Build();
864
865 // Expect frames are decoded in order.
866 InSequence seq;
Tony Herre55b593f2023-08-29 14:05:49867 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp), _));
Evan Shrubsolea4062722022-05-02 15:46:07868 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp +
869 k30FpsRtpTimestampDelta),
Tony Herre55b593f2023-08-29 14:05:49870 _))
Evan Shrubsolea4062722022-05-02 15:46:07871 .Times(1);
872 video_receive_stream_->OnCompleteFrame(std::move(key_frame));
873 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
874
875 time_controller_.AdvanceTime(k30FpsDelay);
876 video_receive_stream_->OnCompleteFrame(std::move(delta_frame));
877 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
878
879 video_receive_stream_->Stop();
880}
881
882TEST_P(VideoReceiveStream2Test, FramesScheduledInOrder) {
883 video_receive_stream_->Start();
884
885 auto key_frame = test::FakeFrameBuilder()
886 .Id(0)
Sergio Garcia Murillob5289d72024-10-24 12:10:53887 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:07888 .Time(kFirstRtpTimestamp)
889 .AsLast()
890 .Build();
891 auto delta_frame1 = test::FakeFrameBuilder()
892 .Id(1)
Sergio Garcia Murillob5289d72024-10-24 12:10:53893 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:07894 .Time(RtpTimestampForFrame(1))
895 .Refs({0})
896 .AsLast()
897 .Build();
898 auto delta_frame2 = test::FakeFrameBuilder()
899 .Id(2)
Sergio Garcia Murillob5289d72024-10-24 12:10:53900 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:07901 .Time(RtpTimestampForFrame(2))
902 .Refs({1})
903 .AsLast()
904 .Build();
905
906 // Expect frames are decoded in order despite delta_frame1 arriving first.
907 InSequence seq;
Tony Herre55b593f2023-08-29 14:05:49908 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp), _))
Evan Shrubsolea4062722022-05-02 15:46:07909 .Times(1);
910 EXPECT_CALL(mock_decoder_,
Tony Herre55b593f2023-08-29 14:05:49911 Decode(test::RtpTimestamp(RtpTimestampForFrame(1)), _))
Evan Shrubsolea4062722022-05-02 15:46:07912 .Times(1);
913 EXPECT_CALL(mock_decoder_,
Tony Herre55b593f2023-08-29 14:05:49914 Decode(test::RtpTimestamp(RtpTimestampForFrame(2)), _))
Evan Shrubsolea4062722022-05-02 15:46:07915 .Times(1);
Danil Chapovalov22333492023-12-07 11:10:46916 key_frame->SetReceivedTime(env_.clock().CurrentTime().ms());
Evan Shrubsolea4062722022-05-02 15:46:07917 video_receive_stream_->OnCompleteFrame(std::move(key_frame));
918 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
919
Danil Chapovalov22333492023-12-07 11:10:46920 delta_frame2->SetReceivedTime(env_.clock().CurrentTime().ms());
Evan Shrubsolea4062722022-05-02 15:46:07921 video_receive_stream_->OnCompleteFrame(std::move(delta_frame2));
922 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), DidNotReceiveFrame());
923 // `delta_frame1` arrives late.
Danil Chapovalov22333492023-12-07 11:10:46924 delta_frame1->SetReceivedTime(env_.clock().CurrentTime().ms());
Evan Shrubsolea4062722022-05-02 15:46:07925 video_receive_stream_->OnCompleteFrame(std::move(delta_frame1));
926 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
927 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay * 2), RenderedFrame());
928 video_receive_stream_->Stop();
929}
930
931TEST_P(VideoReceiveStream2Test, WaitsforAllSpatialLayers) {
932 video_receive_stream_->Start();
933 auto sl0 = test::FakeFrameBuilder()
934 .Id(0)
Sergio Garcia Murillob5289d72024-10-24 12:10:53935 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:07936 .Time(kFirstRtpTimestamp)
937 .ReceivedTime(kStartTime)
938 .Build();
939 auto sl1 = test::FakeFrameBuilder()
940 .Id(1)
Sergio Garcia Murillob5289d72024-10-24 12:10:53941 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:07942 .ReceivedTime(kStartTime)
943 .Time(kFirstRtpTimestamp)
944 .Refs({0})
945 .Build();
946 auto sl2 = test::FakeFrameBuilder()
947 .Id(2)
Sergio Garcia Murillob5289d72024-10-24 12:10:53948 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:07949 .ReceivedTime(kStartTime)
950 .Time(kFirstRtpTimestamp)
951 .Refs({0, 1})
952 .AsLast()
953 .Build();
954
955 // No decodes should be called until `sl2` is received.
Tony Herre55b593f2023-08-29 14:05:49956 EXPECT_CALL(mock_decoder_, Decode(_, _)).Times(0);
Danil Chapovalov22333492023-12-07 11:10:46957 sl0->SetReceivedTime(env_.clock().CurrentTime().ms());
Evan Shrubsolea4062722022-05-02 15:46:07958 video_receive_stream_->OnCompleteFrame(std::move(sl0));
959 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
960 DidNotReceiveFrame());
961 video_receive_stream_->OnCompleteFrame(std::move(sl1));
962 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
963 DidNotReceiveFrame());
964 // When `sl2` arrives decode should happen.
Tony Herre55b593f2023-08-29 14:05:49965 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp), _))
Evan Shrubsolea4062722022-05-02 15:46:07966 .Times(1);
967 video_receive_stream_->OnCompleteFrame(std::move(sl2));
968 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
969 video_receive_stream_->Stop();
970}
971
972TEST_P(VideoReceiveStream2Test, FramesFastForwardOnSystemHalt) {
973 video_receive_stream_->Start();
974
975 // The frame structure looks like this,
976 // F1
977 // /
978 // F0 --> F2
979 //
980 // In this case we will have a system halt simulated. By the time the system
981 // resumes, F1 will be old and so F2 should be decoded.
982 auto key_frame = test::FakeFrameBuilder()
983 .Id(0)
Sergio Garcia Murillob5289d72024-10-24 12:10:53984 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:07985 .Time(kFirstRtpTimestamp)
986 .AsLast()
987 .Build();
988 auto ffwd_frame = test::FakeFrameBuilder()
989 .Id(1)
Sergio Garcia Murillob5289d72024-10-24 12:10:53990 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:07991 .Time(RtpTimestampForFrame(1))
992 .Refs({0})
993 .AsLast()
994 .Build();
995 auto rendered_frame = test::FakeFrameBuilder()
996 .Id(2)
Sergio Garcia Murillob5289d72024-10-24 12:10:53997 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:07998 .Time(RtpTimestampForFrame(2))
999 .Refs({0})
1000 .AsLast()
1001 .Build();
1002 InSequence seq;
Tony Herre55b593f2023-08-29 14:05:491003 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp), _))
philipel27b35a72022-07-05 07:59:551004 .WillOnce(testing::DoAll(Invoke([&] {
Evan Shrubsolea4062722022-05-02 15:46:071005 // System halt will be simulated in the decode.
1006 time_controller_.AdvanceTime(k30FpsDelay * 2);
1007 }),
1008 DefaultDecodeAction()));
1009 EXPECT_CALL(mock_decoder_,
Tony Herre55b593f2023-08-29 14:05:491010 Decode(test::RtpTimestamp(RtpTimestampForFrame(2)), _));
Evan Shrubsolea4062722022-05-02 15:46:071011 video_receive_stream_->OnCompleteFrame(std::move(key_frame));
1012 video_receive_stream_->OnCompleteFrame(std::move(ffwd_frame));
1013 video_receive_stream_->OnCompleteFrame(std::move(rendered_frame));
Evan Shrubsole7cbd8de2022-08-16 08:08:531014 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
1015 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(0))));
1016 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
1017 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(2))));
Evan Shrubsolea4062722022-05-02 15:46:071018
1019 // Check stats show correct dropped frames.
1020 auto stats = video_receive_stream_->GetStats();
1021 EXPECT_EQ(stats.frames_dropped, 1u);
1022
1023 video_receive_stream_->Stop();
1024}
1025
1026TEST_P(VideoReceiveStream2Test, BetterFrameInsertedWhileWaitingToDecodeFrame) {
1027 video_receive_stream_->Start();
1028
1029 auto key_frame = test::FakeFrameBuilder()
1030 .Id(0)
Sergio Garcia Murillob5289d72024-10-24 12:10:531031 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:071032 .Time(kFirstRtpTimestamp)
1033 .ReceivedTime(ReceiveTimeForFrame(0))
1034 .AsLast()
1035 .Build();
1036 auto f1 = test::FakeFrameBuilder()
1037 .Id(1)
Sergio Garcia Murillob5289d72024-10-24 12:10:531038 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:071039 .Time(RtpTimestampForFrame(1))
1040 .ReceivedTime(ReceiveTimeForFrame(1))
1041 .Refs({0})
1042 .AsLast()
1043 .Build();
1044 auto f2 = test::FakeFrameBuilder()
1045 .Id(2)
Sergio Garcia Murillob5289d72024-10-24 12:10:531046 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:071047 .Time(RtpTimestampForFrame(2))
1048 .ReceivedTime(ReceiveTimeForFrame(2))
1049 .Refs({0})
1050 .AsLast()
1051 .Build();
1052
1053 video_receive_stream_->OnCompleteFrame(std::move(key_frame));
1054 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
1055
1056 InSequence seq;
1057 EXPECT_CALL(mock_decoder_,
Tony Herre55b593f2023-08-29 14:05:491058 Decode(test::RtpTimestamp(RtpTimestampForFrame(1)), _))
Evan Shrubsolea4062722022-05-02 15:46:071059 .Times(1);
1060 EXPECT_CALL(mock_decoder_,
Tony Herre55b593f2023-08-29 14:05:491061 Decode(test::RtpTimestamp(RtpTimestampForFrame(2)), _))
Evan Shrubsolea4062722022-05-02 15:46:071062 .Times(1);
1063 // Simulate f1 arriving after f2 but before f2 is decoded.
1064 video_receive_stream_->OnCompleteFrame(std::move(f2));
1065 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), DidNotReceiveFrame());
1066 video_receive_stream_->OnCompleteFrame(std::move(f1));
1067 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
1068 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
1069
1070 video_receive_stream_->Stop();
1071}
1072
1073// Note: This test takes a long time (~10s) to run if the fake metronome is
1074// active. Since the test needs to wait for the timestamp to rollover, it has a
1075// fake delay of around 6.5 hours. Even though time is simulated, this will be
1076// around 1,500,000 metronome tick invocations.
1077TEST_P(VideoReceiveStream2Test, RtpTimestampWrapAround) {
Evan Shrubsole7cbd8de2022-08-16 08:08:531078 EXPECT_CALL(mock_transport_, SendRtcp).Times(AnyNumber());
Evan Shrubsolea4062722022-05-02 15:46:071079 video_receive_stream_->Start();
1080
1081 constexpr uint32_t kBaseRtp = std::numeric_limits<uint32_t>::max() / 2;
1082 video_receive_stream_->OnCompleteFrame(
1083 test::FakeFrameBuilder()
1084 .Id(0)
Sergio Garcia Murillob5289d72024-10-24 12:10:531085 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:071086 .Time(kBaseRtp)
Danil Chapovalov22333492023-12-07 11:10:461087 .ReceivedTime(env_.clock().CurrentTime())
Evan Shrubsolea4062722022-05-02 15:46:071088 .AsLast()
1089 .Build());
1090 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
1091 time_controller_.AdvanceTime(k30FpsDelay);
1092 video_receive_stream_->OnCompleteFrame(
1093 test::FakeFrameBuilder()
1094 .Id(1)
Sergio Garcia Murillob5289d72024-10-24 12:10:531095 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:071096 .Time(kBaseRtp + k30FpsRtpTimestampDelta)
Danil Chapovalov22333492023-12-07 11:10:461097 .ReceivedTime(env_.clock().CurrentTime())
Evan Shrubsolea4062722022-05-02 15:46:071098 .AsLast()
1099 .Build());
1100 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
1101
1102 // Pause stream so that RTP timestamp wraps around.
1103 constexpr uint32_t kLastRtp = kBaseRtp + k30FpsRtpTimestampDelta;
1104 constexpr uint32_t kWrapAroundRtp =
1105 kLastRtp + std::numeric_limits<uint32_t>::max() / 2 + 1;
1106 // Pause for corresponding delay such that RTP timestamp would increase this
1107 // much at 30fps.
1108 constexpr TimeDelta kWrapAroundDelay =
1109 (std::numeric_limits<uint32_t>::max() / 2 + 1) / kRtpTimestampHz;
1110
1111 time_controller_.AdvanceTime(kWrapAroundDelay);
1112 video_receive_stream_->OnCompleteFrame(
1113 test::FakeFrameBuilder()
1114 .Id(2)
Sergio Garcia Murillob5289d72024-10-24 12:10:531115 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:071116 .Time(kWrapAroundRtp)
Danil Chapovalov22333492023-12-07 11:10:461117 .ReceivedTime(env_.clock().CurrentTime())
Evan Shrubsolea4062722022-05-02 15:46:071118 .AsLast()
1119 .Build());
Tony Herre55b593f2023-08-29 14:05:491120 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kWrapAroundRtp), _))
Evan Shrubsolea4062722022-05-02 15:46:071121 .Times(1);
philipel2f3b75d2024-03-13 12:02:151122 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Seconds(1)),
1123 RenderedFrame());
Evan Shrubsolea4062722022-05-02 15:46:071124
1125 video_receive_stream_->Stop();
1126}
1127
Evan Shrubsoledcb9c5d2022-06-13 15:39:531128// If a frame was lost causing the stream to become temporarily non-decodable
1129// and the sender reduces their framerate during this time, the video stream
1130// should start decoding at the new framerate. However, if the connection is
1131// poor, a keyframe will take a long time to send. If the timing of the incoming
1132// frames was not kept up to date with the new framerate while the stream was
1133// decodable, this late frame will have a large delay as the rtp timestamp of
1134// this keyframe will look like the frame arrived early if the frame-rate was
1135// not updated.
1136TEST_P(VideoReceiveStream2Test, PoorConnectionWithFpsChangeDuringLostFrame) {
1137 video_receive_stream_->Start();
1138
1139 constexpr Frequency k15Fps = Frequency::Hertz(15);
1140 constexpr TimeDelta k15FpsDelay = 1 / k15Fps;
1141 constexpr uint32_t k15FpsRtpTimestampDelta = kRtpTimestampHz / k15Fps;
1142
1143 // Initial keyframe and frames at 30fps.
1144 video_receive_stream_->OnCompleteFrame(
1145 test::FakeFrameBuilder()
1146 .Id(0)
Sergio Garcia Murillob5289d72024-10-24 12:10:531147 .PayloadType(kH264PayloadType)
Evan Shrubsoledcb9c5d2022-06-13 15:39:531148 .Time(RtpTimestampForFrame(0))
1149 .ReceivedTime(ReceiveTimeForFrame(0))
1150 .AsLast()
1151 .Build());
1152 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1153 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(0))));
1154
1155 video_receive_stream_->OnCompleteFrame(
1156 test::FakeFrameBuilder()
1157 .Id(1)
Sergio Garcia Murillob5289d72024-10-24 12:10:531158 .PayloadType(kH264PayloadType)
Evan Shrubsoledcb9c5d2022-06-13 15:39:531159 .Time(RtpTimestampForFrame(1))
1160 .ReceivedTime(ReceiveTimeForFrame(1))
1161 .Refs({0})
1162 .AsLast()
1163 .Build());
1164 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1165 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(1))));
1166
1167 // Simulate lost frame 2, followed by 2 second of frames at 30fps, followed by
1168 // 2 second of frames at 15 fps, and then a keyframe.
1169 time_controller_.AdvanceTime(k30FpsDelay);
1170
Danil Chapovalov22333492023-12-07 11:10:461171 Timestamp send_30fps_end_time =
1172 env_.clock().CurrentTime() + TimeDelta::Seconds(2);
Evan Shrubsoledcb9c5d2022-06-13 15:39:531173 int id = 3;
1174 EXPECT_CALL(mock_transport_, SendRtcp).Times(AnyNumber());
Danil Chapovalov22333492023-12-07 11:10:461175 while (env_.clock().CurrentTime() < send_30fps_end_time) {
Evan Shrubsoledcb9c5d2022-06-13 15:39:531176 ++id;
1177 video_receive_stream_->OnCompleteFrame(
1178 test::FakeFrameBuilder()
1179 .Id(id)
Sergio Garcia Murillob5289d72024-10-24 12:10:531180 .PayloadType(kH264PayloadType)
Evan Shrubsoledcb9c5d2022-06-13 15:39:531181 .Time(RtpTimestampForFrame(id))
1182 .ReceivedTime(ReceiveTimeForFrame(id))
1183 .Refs({id - 1})
1184 .AsLast()
1185 .Build());
1186 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
Florent Castelli8037fc62024-08-29 13:00:401187 Eq(std::nullopt));
Evan Shrubsoledcb9c5d2022-06-13 15:39:531188 }
1189 uint32_t current_rtp = RtpTimestampForFrame(id);
Danil Chapovalov22333492023-12-07 11:10:461190 Timestamp send_15fps_end_time =
1191 env_.clock().CurrentTime() + TimeDelta::Seconds(2);
1192 while (env_.clock().CurrentTime() < send_15fps_end_time) {
Evan Shrubsoledcb9c5d2022-06-13 15:39:531193 ++id;
1194 current_rtp += k15FpsRtpTimestampDelta;
1195 video_receive_stream_->OnCompleteFrame(
1196 test::FakeFrameBuilder()
1197 .Id(id)
Sergio Garcia Murillob5289d72024-10-24 12:10:531198 .PayloadType(kH264PayloadType)
Evan Shrubsoledcb9c5d2022-06-13 15:39:531199 .Time(current_rtp)
Danil Chapovalov22333492023-12-07 11:10:461200 .ReceivedTime(env_.clock().CurrentTime())
Evan Shrubsoledcb9c5d2022-06-13 15:39:531201 .Refs({id - 1})
1202 .AsLast()
1203 .Build());
1204 EXPECT_THAT(fake_renderer_.WaitForFrame(k15FpsDelay, /*advance_time=*/true),
Florent Castelli8037fc62024-08-29 13:00:401205 Eq(std::nullopt));
Evan Shrubsoledcb9c5d2022-06-13 15:39:531206 }
1207
1208 ++id;
1209 current_rtp += k15FpsRtpTimestampDelta;
1210 // Insert keyframe which will recover the stream. However, on a poor
1211 // connection the keyframe will take significant time to send.
1212 constexpr TimeDelta kKeyframeDelay = TimeDelta::Millis(200);
1213 video_receive_stream_->OnCompleteFrame(
1214 test::FakeFrameBuilder()
1215 .Id(id)
Sergio Garcia Murillob5289d72024-10-24 12:10:531216 .PayloadType(kH264PayloadType)
Evan Shrubsoledcb9c5d2022-06-13 15:39:531217 .Time(current_rtp)
Danil Chapovalov22333492023-12-07 11:10:461218 .ReceivedTime(env_.clock().CurrentTime() + kKeyframeDelay)
Evan Shrubsoledcb9c5d2022-06-13 15:39:531219 .AsLast()
1220 .Build());
1221 // If the framerate was not updated to be 15fps from the frames that arrived
1222 // previously, this will fail, as the delay will be longer.
1223 EXPECT_THAT(fake_renderer_.WaitForFrame(k15FpsDelay, /*advance_time=*/true),
1224 RenderedFrameWith(RtpTimestamp(current_rtp)));
1225
1226 video_receive_stream_->Stop();
1227}
1228
Evan Shrubsole4d3ba772022-06-22 14:32:361229TEST_P(VideoReceiveStream2Test, StreamShouldNotTimeoutWhileWaitingForFrame) {
1230 // Disable smoothing since this makes it hard to test frame timing.
1231 config_.enable_prerenderer_smoothing = false;
1232 RecreateReceiveStream();
1233
1234 video_receive_stream_->Start();
1235 EXPECT_CALL(mock_transport_, SendRtcp).Times(AnyNumber());
1236
1237 video_receive_stream_->OnCompleteFrame(
1238 test::FakeFrameBuilder()
1239 .Id(0)
Sergio Garcia Murillob5289d72024-10-24 12:10:531240 .PayloadType(kH264PayloadType)
Evan Shrubsole4d3ba772022-06-22 14:32:361241 .Time(RtpTimestampForFrame(0))
1242 .ReceivedTime(ReceiveTimeForFrame(0))
1243 .AsLast()
1244 .Build());
1245 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1246 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(0))));
1247
1248 for (int id = 1; id < 30; ++id) {
1249 video_receive_stream_->OnCompleteFrame(
1250 test::FakeFrameBuilder()
1251 .Id(id)
Sergio Garcia Murillob5289d72024-10-24 12:10:531252 .PayloadType(kH264PayloadType)
Evan Shrubsole4d3ba772022-06-22 14:32:361253 .Time(RtpTimestampForFrame(id))
1254 .ReceivedTime(ReceiveTimeForFrame(id))
1255 .Refs({0})
1256 .AsLast()
1257 .Build());
1258 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1259 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(id))));
1260 }
1261
1262 // Simulate a pause in the stream, followed by a decodable frame that is ready
1263 // long in the future. The stream should not timeout in this case, but rather
1264 // decode the frame just before the timeout.
1265 time_controller_.AdvanceTime(TimeDelta::Millis(2900));
1266 uint32_t late_decode_rtp = kFirstRtpTimestamp + 200 * k30FpsRtpTimestampDelta;
1267 video_receive_stream_->OnCompleteFrame(
1268 test::FakeFrameBuilder()
1269 .Id(121)
Sergio Garcia Murillob5289d72024-10-24 12:10:531270 .PayloadType(kH264PayloadType)
Evan Shrubsole4d3ba772022-06-22 14:32:361271 .Time(late_decode_rtp)
Danil Chapovalov22333492023-12-07 11:10:461272 .ReceivedTime(env_.clock().CurrentTime())
Evan Shrubsole4d3ba772022-06-22 14:32:361273 .AsLast()
1274 .Build());
1275 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Millis(100),
1276 /*advance_time=*/true),
1277 RenderedFrameWith(RtpTimestamp(late_decode_rtp)));
1278
1279 video_receive_stream_->Stop();
1280}
1281
Evan Shrubsolea4062722022-05-02 15:46:071282INSTANTIATE_TEST_SUITE_P(VideoReceiveStream2Test,
1283 VideoReceiveStream2Test,
1284 testing::Bool(),
1285 [](const auto& test_param_info) {
Evan Shrubsoledcb9c5d2022-06-13 15:39:531286 return (test_param_info.param
1287 ? "ScheduleDecodesWithMetronome"
1288 : "ScheduleDecodesWithPostTask");
Evan Shrubsolea4062722022-05-02 15:46:071289 });
1290
Tommiae4d0972020-05-18 06:45:381291} // namespace webrtc