blob: ef89290666828ac4d99ce1da899afa38b4978a8b [file] [log] [blame]
Tommiae4d0972020-05-18 06:45:381/*
2 * Copyright 2017 The WebRTC Project Authors. All rights reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "video/video_receive_stream2.h"
12
13#include <algorithm>
Evan Shrubsolea4062722022-05-02 15:46:0714#include <cstddef>
Evan Shrubsole7cbd8de2022-08-16 08:08:5315#include <deque>
Markus Handell588f9b32021-04-08 17:19:5016#include <limits>
Tommiae4d0972020-05-18 06:45:3817#include <memory>
Florent Castelli8037fc62024-08-29 13:00:4018#include <optional>
Evan Shrubsole7cbd8de2022-08-16 08:08:5319#include <queue>
Evan Shrubsole9a999052021-12-12 14:27:0020#include <tuple>
Tommiae4d0972020-05-18 06:45:3821#include <utility>
22#include <vector>
23
Evan Shrubsole6dbc1722022-03-22 11:20:1124#include "absl/memory/memory.h"
Danil Chapovalov22333492023-12-07 11:10:4625#include "api/environment/environment.h"
26#include "api/environment/environment_factory.h"
Evan Shrubsolea4062722022-05-02 15:46:0727#include "api/metronome/test/fake_metronome.h"
Danil Chapovalov9cd4d492021-08-03 12:59:0028#include "api/test/mock_video_decoder.h"
29#include "api/test/mock_video_decoder_factory.h"
Evan Shrubsole44be5792022-04-26 14:24:4130#include "api/test/time_controller.h"
Evan Shrubsolea4062722022-05-02 15:46:0731#include "api/units/frequency.h"
Evan Shrubsole44be5792022-04-26 14:24:4132#include "api/units/time_delta.h"
Evan Shrubsolea4062722022-05-02 15:46:0733#include "api/video/encoded_image.h"
Evan Shrubsole44be5792022-04-26 14:24:4134#include "api/video/recordable_encoded_frame.h"
35#include "api/video/test/video_frame_matchers.h"
Markus Handell588f9b32021-04-08 17:19:5036#include "api/video/video_frame.h"
Evan Shrubsole1c184772022-04-26 07:47:4937#include "api/video_codecs/sdp_video_format.h"
Tommiae4d0972020-05-18 06:45:3838#include "api/video_codecs/video_decoder.h"
39#include "call/rtp_stream_receiver_controller.h"
Evan Shrubsole1c184772022-04-26 07:47:4940#include "call/video_receive_stream.h"
Tommiae4d0972020-05-18 06:45:3841#include "common_video/test/utilities.h"
Tommi90738dd2021-05-31 15:36:4742#include "media/engine/fake_webrtc_call.h"
Tommiae4d0972020-05-18 06:45:3843#include "modules/pacing/packet_router.h"
44#include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
Tommiae4d0972020-05-18 06:45:3845#include "modules/video_coding/encoded_frame.h"
Evan Shrubsoledcb9c5d2022-06-13 15:39:5346#include "rtc_base/logging.h"
Tommiae4d0972020-05-18 06:45:3847#include "system_wrappers/include/clock.h"
48#include "test/fake_decoder.h"
Evan Shrubsolea0ee64c2022-04-26 08:09:0449#include "test/fake_encoded_frame.h"
Tommiae4d0972020-05-18 06:45:3850#include "test/gmock.h"
51#include "test/gtest.h"
Danil Chapovalov9cd4d492021-08-03 12:59:0052#include "test/mock_transport.h"
philipel27b35a72022-07-05 07:59:5553#include "test/rtcp_packet_parser.h"
Tommiae4d0972020-05-18 06:45:3854#include "test/time_controller/simulated_time_controller.h"
55#include "test/video_decoder_proxy_factory.h"
56#include "video/call_stats2.h"
57
58namespace webrtc {
Evan Shrubsole1c184772022-04-26 07:47:4959
Tommiae4d0972020-05-18 06:45:3860namespace {
61
Evan Shrubsole44be5792022-04-26 14:24:4162using test::video_frame_matchers::NtpTimestamp;
63using test::video_frame_matchers::PacketInfos;
64using test::video_frame_matchers::Rotation;
Tommiae4d0972020-05-18 06:45:3865using ::testing::_;
Markus Handell588f9b32021-04-08 17:19:5066using ::testing::AllOf;
Evan Shrubsoledcb9c5d2022-06-13 15:39:5367using ::testing::AnyNumber;
Tommiae4d0972020-05-18 06:45:3868using ::testing::ElementsAreArray;
Evan Shrubsole44be5792022-04-26 14:24:4169using ::testing::Eq;
Markus Handell588f9b32021-04-08 17:19:5070using ::testing::Field;
71using ::testing::InSequence;
Tommiae4d0972020-05-18 06:45:3872using ::testing::Invoke;
73using ::testing::IsEmpty;
Johannes Kronbbf639e2022-06-15 10:27:2374using ::testing::Optional;
Evan Shrubsolea4062722022-05-02 15:46:0775using ::testing::Pointee;
Markus Handell588f9b32021-04-08 17:19:5076using ::testing::Property;
Evan Shrubsole44be5792022-04-26 14:24:4177using ::testing::Return;
Tommiae4d0972020-05-18 06:45:3878using ::testing::SizeIs;
Danil Chapovalovd08930d2021-08-12 11:26:5579using ::testing::WithoutArgs;
Tommiae4d0972020-05-18 06:45:3880
Evan Shrubsole44be5792022-04-26 14:24:4181auto RenderedFrameWith(::testing::Matcher<VideoFrame> m) {
82 return Optional(m);
83}
Evan Shrubsolea4062722022-05-02 15:46:0784auto RenderedFrame() {
85 return RenderedFrameWith(_);
86}
Florent Castelli8037fc62024-08-29 13:00:4087testing::Matcher<std::optional<VideoFrame>> DidNotReceiveFrame() {
88 return Eq(std::nullopt);
Evan Shrubsolea4062722022-05-02 15:46:0789}
Evan Shrubsole44be5792022-04-26 14:24:4190
91constexpr TimeDelta kDefaultTimeOut = TimeDelta::Millis(50);
Evan Shrubsole14ee8032022-04-26 14:01:4492constexpr int kDefaultNumCpuCores = 2;
Tommiae4d0972020-05-18 06:45:3893
Evan Shrubsole44be5792022-04-26 14:24:4194constexpr Timestamp kStartTime = Timestamp::Millis(1'337'000);
Evan Shrubsolea4062722022-05-02 15:46:0795constexpr Frequency k30Fps = Frequency::Hertz(30);
96constexpr TimeDelta k30FpsDelay = 1 / k30Fps;
97constexpr Frequency kRtpTimestampHz = Frequency::KiloHertz(90);
98constexpr uint32_t k30FpsRtpTimestampDelta = kRtpTimestampHz / k30Fps;
99constexpr uint32_t kFirstRtpTimestamp = 90000;
Evan Shrubsole44be5792022-04-26 14:24:41100
Evan Shrubsoledcb9c5d2022-06-13 15:39:53101class FakeVideoRenderer : public rtc::VideoSinkInterface<VideoFrame> {
Evan Shrubsole44be5792022-04-26 14:24:41102 public:
Evan Shrubsole7cbd8de2022-08-16 08:08:53103 explicit FakeVideoRenderer(TimeController* time_controller)
104 : time_controller_(time_controller) {}
Evan Shrubsole44be5792022-04-26 14:24:41105 ~FakeVideoRenderer() override = default;
106
Evan Shrubsoledcb9c5d2022-06-13 15:39:53107 void OnFrame(const VideoFrame& frame) override {
108 RTC_LOG(LS_VERBOSE) << "Received frame with timestamp="
Per K0fa90882024-03-13 08:52:41109 << frame.rtp_timestamp();
Evan Shrubsole7cbd8de2022-08-16 08:08:53110 if (!last_frame_.empty()) {
111 RTC_LOG(LS_INFO) << "Already had frame queue with timestamp="
Per K0fa90882024-03-13 08:52:41112 << last_frame_.back().rtp_timestamp();
Evan Shrubsoledcb9c5d2022-06-13 15:39:53113 }
Evan Shrubsole7cbd8de2022-08-16 08:08:53114 last_frame_.push_back(frame);
Evan Shrubsole44be5792022-04-26 14:24:41115 }
116
Evan Shrubsoledcb9c5d2022-06-13 15:39:53117 // If `advance_time`, then the clock will always advance by `timeout`.
Florent Castelli8037fc62024-08-29 13:00:40118 std::optional<VideoFrame> WaitForFrame(TimeDelta timeout,
119 bool advance_time = false) {
Evan Shrubsoledcb9c5d2022-06-13 15:39:53120 auto start = time_controller_->GetClock()->CurrentTime();
Evan Shrubsole7cbd8de2022-08-16 08:08:53121 if (last_frame_.empty()) {
Evan Shrubsolea4062722022-05-02 15:46:07122 time_controller_->AdvanceTime(TimeDelta::Zero());
Evan Shrubsole7cbd8de2022-08-16 08:08:53123 time_controller_->Wait([this] { return !last_frame_.empty(); }, timeout);
Evan Shrubsole44be5792022-04-26 14:24:41124 }
Florent Castelli8037fc62024-08-29 13:00:40125 std::optional<VideoFrame> ret;
Evan Shrubsole7cbd8de2022-08-16 08:08:53126 if (!last_frame_.empty()) {
127 ret = last_frame_.front();
128 last_frame_.pop_front();
129 }
Evan Shrubsoledcb9c5d2022-06-13 15:39:53130 if (advance_time) {
131 time_controller_->AdvanceTime(
132 timeout - (time_controller_->GetClock()->CurrentTime() - start));
133 }
Evan Shrubsole44be5792022-04-26 14:24:41134 return ret;
135 }
136
137 private:
Evan Shrubsole7cbd8de2022-08-16 08:08:53138 std::deque<VideoFrame> last_frame_;
Evan Shrubsolea4062722022-05-02 15:46:07139 TimeController* const time_controller_;
Evan Shrubsole44be5792022-04-26 14:24:41140};
141
Jonas Oreland0deda152022-09-23 10:08:57142MATCHER_P2(MatchResolution, w, h, "") {
Evan Shrubsole44be5792022-04-26 14:24:41143 return arg.resolution().width == w && arg.resolution().height == h;
144}
145
Evan Shrubsoledcb9c5d2022-06-13 15:39:53146MATCHER_P(RtpTimestamp, timestamp, "") {
Per K0fa90882024-03-13 08:52:41147 if (arg.rtp_timestamp() != timestamp) {
Evan Shrubsole7cbd8de2022-08-16 08:08:53148 *result_listener->stream()
Per K0fa90882024-03-13 08:52:41149 << "rtp timestamp was " << arg.rtp_timestamp() << " != " << timestamp;
Evan Shrubsole7cbd8de2022-08-16 08:08:53150 return false;
151 }
152 return true;
Evan Shrubsolea4062722022-05-02 15:46:07153}
154
155// Rtp timestamp for in order frame at 30fps.
156uint32_t RtpTimestampForFrame(int id) {
157 return kFirstRtpTimestamp + id * k30FpsRtpTimestampDelta;
158}
159
160// Receive time for in order frame at 30fps.
161Timestamp ReceiveTimeForFrame(int id) {
162 return kStartTime + id * k30FpsDelay;
163}
164
Tommiae4d0972020-05-18 06:45:38165} // namespace
166
Evan Shrubsolea4062722022-05-02 15:46:07167class VideoReceiveStream2Test : public ::testing::TestWithParam<bool> {
Tommiae4d0972020-05-18 06:45:38168 public:
Evan Shrubsolea4062722022-05-02 15:46:07169 auto DefaultDecodeAction() {
philipel27b35a72022-07-05 07:59:55170 return Invoke(&fake_decoder_, &test::FakeDecoder::Decode);
Evan Shrubsolea4062722022-05-02 15:46:07171 }
172
173 bool UseMetronome() const { return GetParam(); }
174
Tommiae4d0972020-05-18 06:45:38175 VideoReceiveStream2Test()
Evan Shrubsole44be5792022-04-26 14:24:41176 : time_controller_(kStartTime),
Danil Chapovalov22333492023-12-07 11:10:46177 env_(CreateEnvironment(time_controller_.CreateTaskQueueFactory(),
178 time_controller_.GetClock())),
Evan Shrubsole1c184772022-04-26 07:47:49179 config_(&mock_transport_, &mock_h264_decoder_factory_),
Danil Chapovalov22333492023-12-07 11:10:46180 call_stats_(&env_.clock(), time_controller_.GetMainThread()),
Evan Shrubsole7cbd8de2022-08-16 08:08:53181 fake_renderer_(&time_controller_),
Danil Chapovalov8d079be2024-04-08 16:35:40182 fake_call_(env_),
Markus Handellbe400e42022-11-08 11:14:23183 fake_metronome_(TimeDelta::Millis(16)),
Danil Chapovalov22333492023-12-07 11:10:46184 decode_sync_(&env_.clock(),
Evan Shrubsole7cbd8de2022-08-16 08:08:53185 &fake_metronome_,
186 time_controller_.GetMainThread()),
Evan Shrubsolea4062722022-05-02 15:46:07187 h264_decoder_factory_(&mock_decoder_) {
Evan Shrubsole1c184772022-04-26 07:47:49188 // By default, mock decoder factory is backed by VideoDecoderProxyFactory.
Danil Chapovalov08173802024-01-23 12:02:46189 ON_CALL(mock_h264_decoder_factory_, Create)
190 .WillByDefault(Invoke(&h264_decoder_factory_,
191 &test::VideoDecoderProxyFactory::Create));
Evan Shrubsole14ee8032022-04-26 14:01:44192
193 // By default, mock decode will wrap the fake decoder.
Evan Shrubsolea4062722022-05-02 15:46:07194 ON_CALL(mock_decoder_, Configure)
philipel27b35a72022-07-05 07:59:55195 .WillByDefault(Invoke(&fake_decoder_, &test::FakeDecoder::Configure));
Tony Herre55b593f2023-08-29 14:05:49196 ON_CALL(mock_decoder_, Decode(_, _)).WillByDefault(DefaultDecodeAction());
Evan Shrubsolea4062722022-05-02 15:46:07197 ON_CALL(mock_decoder_, RegisterDecodeCompleteCallback)
Evan Shrubsole14ee8032022-04-26 14:01:44198 .WillByDefault(
philipel27b35a72022-07-05 07:59:55199 Invoke(&fake_decoder_,
200 &test::FakeDecoder::RegisterDecodeCompleteCallback));
201 ON_CALL(mock_decoder_, Release)
202 .WillByDefault(Invoke(&fake_decoder_, &test::FakeDecoder::Release));
203 ON_CALL(mock_transport_, SendRtcp)
204 .WillByDefault(
205 Invoke(&rtcp_packet_parser_, &test::RtcpPacketParser::Parse));
Jonas Oreland8ca06132022-03-14 11:52:48206 }
Evan Shrubsole7cbd8de2022-08-16 08:08:53207
Tommi90738dd2021-05-31 15:36:47208 ~VideoReceiveStream2Test() override {
Evan Shrubsolea4062722022-05-02 15:46:07209 if (video_receive_stream_) {
210 video_receive_stream_->Stop();
Tommi90738dd2021-05-31 15:36:47211 video_receive_stream_->UnregisterFromTransport();
Evan Shrubsolea4062722022-05-02 15:46:07212 }
Evan Shrubsolea4062722022-05-02 15:46:07213 time_controller_.AdvanceTime(TimeDelta::Zero());
Tommi90738dd2021-05-31 15:36:47214 }
Tommiae4d0972020-05-18 06:45:38215
Tommi90738dd2021-05-31 15:36:47216 void SetUp() override {
Tommiae4d0972020-05-18 06:45:38217 config_.rtp.remote_ssrc = 1111;
218 config_.rtp.local_ssrc = 2222;
219 config_.renderer = &fake_renderer_;
Tommif6f45432022-05-20 13:21:20220 VideoReceiveStreamInterface::Decoder h264_decoder;
Tommiae4d0972020-05-18 06:45:38221 h264_decoder.payload_type = 99;
Philipp Hanckebbff58d2024-02-27 11:18:33222 h264_decoder.video_format = SdpVideoFormat::H264();
Tommiae4d0972020-05-18 06:45:38223 h264_decoder.video_format.parameters.insert(
224 {"sprop-parameter-sets", "Z0IACpZTBYmI,aMljiA=="});
Tommif6f45432022-05-20 13:21:20225 VideoReceiveStreamInterface::Decoder h265_decoder;
Evan Shrubsole1c184772022-04-26 07:47:49226 h265_decoder.payload_type = 100;
227 h265_decoder.video_format = SdpVideoFormat("H265");
228
229 config_.decoders = {h265_decoder, h264_decoder};
Tommiae4d0972020-05-18 06:45:38230
Evan Shrubsole44be5792022-04-26 14:24:41231 RecreateReceiveStream();
Evan Shrubsole14ee8032022-04-26 14:01:44232 }
233
Tommif6f45432022-05-20 13:21:20234 void RecreateReceiveStream(
Florent Castelli8037fc62024-08-29 13:00:40235 std::optional<VideoReceiveStreamInterface::RecordingState> state =
236 std::nullopt) {
Evan Shrubsole14ee8032022-04-26 14:01:44237 if (video_receive_stream_) {
238 video_receive_stream_->UnregisterFromTransport();
239 video_receive_stream_ = nullptr;
240 }
Danil Chapovalov22333492023-12-07 11:10:46241 timing_ = new VCMTiming(&env_.clock(), env_.field_trials());
Tommiae4d0972020-05-18 06:45:38242 video_receive_stream_ =
243 std::make_unique<webrtc::internal::VideoReceiveStream2>(
Danil Chapovalov22333492023-12-07 11:10:46244 env_, &fake_call_, kDefaultNumCpuCores, &packet_router_,
245 config_.Copy(), &call_stats_, absl::WrapUnique(timing_),
246 &nack_periodic_processor_,
247 UseMetronome() ? &decode_sync_ : nullptr);
Tommi90738dd2021-05-31 15:36:47248 video_receive_stream_->RegisterWithTransport(
249 &rtp_stream_receiver_controller_);
Evan Shrubsole44be5792022-04-26 14:24:41250 if (state)
251 video_receive_stream_->SetAndGetRecordingState(std::move(*state), false);
Tommiae4d0972020-05-18 06:45:38252 }
253
254 protected:
Evan Shrubsole44be5792022-04-26 14:24:41255 GlobalSimulatedTimeController time_controller_;
Danil Chapovalov22333492023-12-07 11:10:46256 Environment env_;
Markus Handell0e62f7a2021-07-20 11:32:02257 NackPeriodicProcessor nack_periodic_processor_;
Evan Shrubsole1c184772022-04-26 07:47:49258 testing::NiceMock<MockVideoDecoderFactory> mock_h264_decoder_factory_;
Tommif6f45432022-05-20 13:21:20259 VideoReceiveStreamInterface::Config config_;
Tommiae4d0972020-05-18 06:45:38260 internal::CallStats call_stats_;
Evan Shrubsolea4062722022-05-02 15:46:07261 testing::NiceMock<MockVideoDecoder> mock_decoder_;
Evan Shrubsole44be5792022-04-26 14:24:41262 FakeVideoRenderer fake_renderer_;
Tommi90738dd2021-05-31 15:36:47263 cricket::FakeCall fake_call_;
Tommiae4d0972020-05-18 06:45:38264 MockTransport mock_transport_;
philipel27b35a72022-07-05 07:59:55265 test::RtcpPacketParser rtcp_packet_parser_;
Tommiae4d0972020-05-18 06:45:38266 PacketRouter packet_router_;
267 RtpStreamReceiverController rtp_stream_receiver_controller_;
268 std::unique_ptr<webrtc::internal::VideoReceiveStream2> video_receive_stream_;
Tommiae4d0972020-05-18 06:45:38269 VCMTiming* timing_;
Evan Shrubsolea4062722022-05-02 15:46:07270 test::FakeMetronome fake_metronome_;
271 DecodeSynchronizer decode_sync_;
Evan Shrubsole1c184772022-04-26 07:47:49272
273 private:
274 test::VideoDecoderProxyFactory h264_decoder_factory_;
Evan Shrubsole14ee8032022-04-26 14:01:44275 test::FakeDecoder fake_decoder_;
Tommiae4d0972020-05-18 06:45:38276};
277
Evan Shrubsolea4062722022-05-02 15:46:07278TEST_P(VideoReceiveStream2Test, CreateFrameFromH264FmtpSpropAndIdr) {
Tommiae4d0972020-05-18 06:45:38279 constexpr uint8_t idr_nalu[] = {0x05, 0xFF, 0xFF, 0xFF};
280 RtpPacketToSend rtppacket(nullptr);
281 uint8_t* payload = rtppacket.AllocatePayload(sizeof(idr_nalu));
282 memcpy(payload, idr_nalu, sizeof(idr_nalu));
283 rtppacket.SetMarker(true);
284 rtppacket.SetSsrc(1111);
285 rtppacket.SetPayloadType(99);
286 rtppacket.SetSequenceNumber(1);
287 rtppacket.SetTimestamp(0);
Evan Shrubsolea4062722022-05-02 15:46:07288 EXPECT_CALL(mock_decoder_, RegisterDecodeCompleteCallback(_));
Tommiae4d0972020-05-18 06:45:38289 video_receive_stream_->Start();
Tony Herre55b593f2023-08-29 14:05:49290 EXPECT_CALL(mock_decoder_, Decode(_, _));
Tommiae4d0972020-05-18 06:45:38291 RtpPacketReceived parsed_packet;
292 ASSERT_TRUE(parsed_packet.Parse(rtppacket.data(), rtppacket.size()));
293 rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
Evan Shrubsolea4062722022-05-02 15:46:07294 EXPECT_CALL(mock_decoder_, Release());
Evan Shrubsole44be5792022-04-26 14:24:41295
296 time_controller_.AdvanceTime(TimeDelta::Zero());
Tommiae4d0972020-05-18 06:45:38297}
298
Evan Shrubsolea4062722022-05-02 15:46:07299TEST_P(VideoReceiveStream2Test, PlayoutDelay) {
Danil Chapovalov7084e1b2023-08-23 11:16:22300 const VideoPlayoutDelay kPlayoutDelay(TimeDelta::Millis(123),
301 TimeDelta::Millis(321));
Evan Shrubsolea0ee64c2022-04-26 08:09:04302 std::unique_ptr<test::FakeEncodedFrame> test_frame =
Danil Chapovalovc146b5f2023-08-16 09:42:52303 test::FakeFrameBuilder()
304 .Id(0)
Danil Chapovalov7084e1b2023-08-23 11:16:22305 .PlayoutDelay(kPlayoutDelay)
Danil Chapovalovc146b5f2023-08-16 09:42:52306 .AsLast()
307 .Build();
Tommiae4d0972020-05-18 06:45:38308
309 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
Evan Shrubsole8f1159b2022-03-22 11:12:17310 auto timings = timing_->GetTimings();
Danil Chapovalov7084e1b2023-08-23 11:16:22311 EXPECT_EQ(kPlayoutDelay.min(), timings.min_playout_delay);
312 EXPECT_EQ(kPlayoutDelay.max(), timings.max_playout_delay);
Tommiae4d0972020-05-18 06:45:38313
314 // Check that the biggest minimum delay is chosen.
315 video_receive_stream_->SetMinimumPlayoutDelay(400);
Evan Shrubsole8f1159b2022-03-22 11:12:17316 timings = timing_->GetTimings();
317 EXPECT_EQ(400, timings.min_playout_delay.ms());
Tommiae4d0972020-05-18 06:45:38318
319 // Check base minimum delay validation.
320 EXPECT_FALSE(video_receive_stream_->SetBaseMinimumPlayoutDelayMs(12345));
321 EXPECT_FALSE(video_receive_stream_->SetBaseMinimumPlayoutDelayMs(-1));
322 EXPECT_TRUE(video_receive_stream_->SetBaseMinimumPlayoutDelayMs(500));
Evan Shrubsole8f1159b2022-03-22 11:12:17323 timings = timing_->GetTimings();
324 EXPECT_EQ(500, timings.min_playout_delay.ms());
Tommiae4d0972020-05-18 06:45:38325
326 // Check that intermidiate values are remembered and the biggest remembered
327 // is chosen.
328 video_receive_stream_->SetBaseMinimumPlayoutDelayMs(0);
Evan Shrubsole8f1159b2022-03-22 11:12:17329 timings = timing_->GetTimings();
330 EXPECT_EQ(400, timings.min_playout_delay.ms());
Tommiae4d0972020-05-18 06:45:38331
332 video_receive_stream_->SetMinimumPlayoutDelay(0);
Evan Shrubsole8f1159b2022-03-22 11:12:17333 timings = timing_->GetTimings();
334 EXPECT_EQ(123, timings.min_playout_delay.ms());
Tommiae4d0972020-05-18 06:45:38335}
336
Johannes Kronbbf639e2022-06-15 10:27:23337TEST_P(VideoReceiveStream2Test, RenderParametersSetToDefaultValues) {
338 // Default render parameters.
339 const VideoFrame::RenderParameters kDefaultRenderParameters;
Johannes Kron111e9812020-10-26 12:54:40340 // Default with no playout delay set.
Evan Shrubsolea0ee64c2022-04-26 08:09:04341 std::unique_ptr<test::FakeEncodedFrame> test_frame0 =
342 test::FakeFrameBuilder().Id(0).AsLast().Build();
Johannes Kron111e9812020-10-26 12:54:40343 video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
Johannes Kronbbf639e2022-06-15 10:27:23344 EXPECT_EQ(timing_->RenderParameters(), kDefaultRenderParameters);
345}
Johannes Kron111e9812020-10-26 12:54:40346
Johannes Kronbbf639e2022-06-15 10:27:23347TEST_P(VideoReceiveStream2Test, UseLowLatencyRenderingSetFromPlayoutDelay) {
Johannes Kronbbf639e2022-06-15 10:27:23348 std::unique_ptr<test::FakeEncodedFrame> test_frame0 =
Danil Chapovalovc146b5f2023-08-16 09:42:52349 test::FakeFrameBuilder()
350 .Id(0)
351 .PlayoutDelay(VideoPlayoutDelay::Minimal())
352 .AsLast()
353 .Build();
Johannes Kronbbf639e2022-06-15 10:27:23354 video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
355 EXPECT_TRUE(timing_->RenderParameters().use_low_latency_rendering);
356
Evan Shrubsolea0ee64c2022-04-26 08:09:04357 std::unique_ptr<test::FakeEncodedFrame> test_frame1 =
Danil Chapovalovc146b5f2023-08-16 09:42:52358 test::FakeFrameBuilder()
359 .Id(1)
Danil Chapovalov7084e1b2023-08-23 11:16:22360 .PlayoutDelay({TimeDelta::Zero(), TimeDelta::Millis(500)})
Danil Chapovalovc146b5f2023-08-16 09:42:52361 .AsLast()
362 .Build();
Johannes Kron111e9812020-10-26 12:54:40363 video_receive_stream_->OnCompleteFrame(std::move(test_frame1));
Johannes Kronbbf639e2022-06-15 10:27:23364 EXPECT_TRUE(timing_->RenderParameters().use_low_latency_rendering);
Johannes Kron111e9812020-10-26 12:54:40365}
366
Evan Shrubsolea4062722022-05-02 15:46:07367TEST_P(VideoReceiveStream2Test, MaxCompositionDelaySetFromMaxPlayoutDelay) {
Johannes Kronbbf639e2022-06-15 10:27:23368 // The max composition delay is dependent on the number of frames in the
369 // pre-decode queue. It's therefore important to advance the time as the test
370 // runs to get the correct expectations of max_composition_delay_in_frames.
371 video_receive_stream_->Start();
372 // Max composition delay not set if no playout delay is set.
373 std::unique_ptr<test::FakeEncodedFrame> test_frame0 =
374 test::FakeFrameBuilder()
375 .Id(0)
376 .Time(RtpTimestampForFrame(0))
377 .ReceivedTime(ReceiveTimeForFrame(0))
378 .AsLast()
379 .Build();
380 video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
381 EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
Florent Castelli8037fc62024-08-29 13:00:40382 Eq(std::nullopt));
Johannes Kronbbf639e2022-06-15 10:27:23383 time_controller_.AdvanceTime(k30FpsDelay);
Johannes Kronbbf639e2022-06-15 10:27:23384
385 // Max composition delay not set for playout delay 0,0.
386 std::unique_ptr<test::FakeEncodedFrame> test_frame1 =
387 test::FakeFrameBuilder()
388 .Id(1)
389 .Time(RtpTimestampForFrame(1))
390 .ReceivedTime(ReceiveTimeForFrame(1))
Danil Chapovalov7084e1b2023-08-23 11:16:22391 .PlayoutDelay(VideoPlayoutDelay::Minimal())
Johannes Kronbbf639e2022-06-15 10:27:23392 .AsLast()
393 .Build();
Johannes Kronbbf639e2022-06-15 10:27:23394 video_receive_stream_->OnCompleteFrame(std::move(test_frame1));
395 EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
Florent Castelli8037fc62024-08-29 13:00:40396 Eq(std::nullopt));
Johannes Kronbbf639e2022-06-15 10:27:23397 time_controller_.AdvanceTime(k30FpsDelay);
Johannes Kronbbf639e2022-06-15 10:27:23398
399 // Max composition delay not set for playout delay X,Y, where X,Y>0.
400 std::unique_ptr<test::FakeEncodedFrame> test_frame2 =
401 test::FakeFrameBuilder()
402 .Id(2)
403 .Time(RtpTimestampForFrame(2))
404 .ReceivedTime(ReceiveTimeForFrame(2))
Danil Chapovalov7084e1b2023-08-23 11:16:22405 .PlayoutDelay({TimeDelta::Millis(10), TimeDelta::Millis(30)})
Johannes Kronbbf639e2022-06-15 10:27:23406 .AsLast()
407 .Build();
Johannes Kronbbf639e2022-06-15 10:27:23408 video_receive_stream_->OnCompleteFrame(std::move(test_frame2));
409 EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
Florent Castelli8037fc62024-08-29 13:00:40410 Eq(std::nullopt));
Johannes Kronbbf639e2022-06-15 10:27:23411
412 time_controller_.AdvanceTime(k30FpsDelay);
Johannes Kronbbf639e2022-06-15 10:27:23413
Johannes Kron111e9812020-10-26 12:54:40414 // Max composition delay set if playout delay X,Y, where X=0,Y>0.
Johannes Kron111e9812020-10-26 12:54:40415 const int kExpectedMaxCompositionDelayInFrames = 3; // ~50 ms at 60 fps.
Johannes Kronbbf639e2022-06-15 10:27:23416 std::unique_ptr<test::FakeEncodedFrame> test_frame3 =
417 test::FakeFrameBuilder()
418 .Id(3)
419 .Time(RtpTimestampForFrame(3))
420 .ReceivedTime(ReceiveTimeForFrame(3))
Danil Chapovalov7084e1b2023-08-23 11:16:22421 .PlayoutDelay({TimeDelta::Zero(), TimeDelta::Millis(50)})
Johannes Kronbbf639e2022-06-15 10:27:23422 .AsLast()
423 .Build();
Johannes Kronbbf639e2022-06-15 10:27:23424 video_receive_stream_->OnCompleteFrame(std::move(test_frame3));
425 EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
426 Optional(kExpectedMaxCompositionDelayInFrames));
Johannes Kron111e9812020-10-26 12:54:40427}
428
Evan Shrubsolea4062722022-05-02 15:46:07429TEST_P(VideoReceiveStream2Test, LazyDecoderCreation) {
Evan Shrubsole1c184772022-04-26 07:47:49430 constexpr uint8_t idr_nalu[] = {0x05, 0xFF, 0xFF, 0xFF};
431 RtpPacketToSend rtppacket(nullptr);
432 uint8_t* payload = rtppacket.AllocatePayload(sizeof(idr_nalu));
433 memcpy(payload, idr_nalu, sizeof(idr_nalu));
434 rtppacket.SetMarker(true);
435 rtppacket.SetSsrc(1111);
436 // H265 payload type.
437 rtppacket.SetPayloadType(99);
438 rtppacket.SetSequenceNumber(1);
439 rtppacket.SetTimestamp(0);
440
Johannes Kronbb591c42022-09-28 10:10:25441 // No decoders are created by default.
Danil Chapovalov08173802024-01-23 12:02:46442 EXPECT_CALL(mock_h264_decoder_factory_, Create).Times(0);
Evan Shrubsole1c184772022-04-26 07:47:49443 video_receive_stream_->Start();
Erik Språng7aaeb5a2022-08-18 11:18:27444 time_controller_.AdvanceTime(TimeDelta::Zero());
Evan Shrubsole1c184772022-04-26 07:47:49445
446 EXPECT_TRUE(
447 testing::Mock::VerifyAndClearExpectations(&mock_h264_decoder_factory_));
Johannes Kronbb591c42022-09-28 10:10:25448 // Verify that the decoder is created when we receive payload data and tries
449 // to decode a frame.
Danil Chapovalov08173802024-01-23 12:02:46450 EXPECT_CALL(mock_h264_decoder_factory_,
451 Create(_, Field(&SdpVideoFormat::name, Eq("H264"))));
Erik Språng7aaeb5a2022-08-18 11:18:27452 EXPECT_CALL(mock_decoder_, Configure);
453 EXPECT_CALL(mock_decoder_, RegisterDecodeCompleteCallback);
Tony Herre55b593f2023-08-29 14:05:49454 EXPECT_CALL(mock_decoder_, Decode(_, _));
Evan Shrubsole1c184772022-04-26 07:47:49455 RtpPacketReceived parsed_packet;
456 ASSERT_TRUE(parsed_packet.Parse(rtppacket.data(), rtppacket.size()));
457 rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
Erik Språng7aaeb5a2022-08-18 11:18:27458 EXPECT_CALL(mock_decoder_, Release);
Evan Shrubsole1c184772022-04-26 07:47:49459
460 // Make sure the decoder thread had a chance to run.
Erik Språng7aaeb5a2022-08-18 11:18:27461 time_controller_.AdvanceTime(TimeDelta::Zero());
Evan Shrubsole1c184772022-04-26 07:47:49462}
463
Evan Shrubsolea4062722022-05-02 15:46:07464TEST_P(VideoReceiveStream2Test, PassesNtpTime) {
Evan Shrubsolea0ee64c2022-04-26 08:09:04465 const Timestamp kNtpTimestamp = Timestamp::Millis(12345);
466 std::unique_ptr<test::FakeEncodedFrame> test_frame =
467 test::FakeFrameBuilder()
468 .Id(0)
469 .PayloadType(99)
470 .NtpTime(kNtpTimestamp)
471 .AsLast()
472 .Build();
Tommiae4d0972020-05-18 06:45:38473
474 video_receive_stream_->Start();
475 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
Evan Shrubsolea4062722022-05-02 15:46:07476 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
Evan Shrubsole44be5792022-04-26 14:24:41477 RenderedFrameWith(NtpTimestamp(kNtpTimestamp)));
Tommiae4d0972020-05-18 06:45:38478}
479
Evan Shrubsolea4062722022-05-02 15:46:07480TEST_P(VideoReceiveStream2Test, PassesRotation) {
Tommiae4d0972020-05-18 06:45:38481 const webrtc::VideoRotation kRotation = webrtc::kVideoRotation_180;
Evan Shrubsolea0ee64c2022-04-26 08:09:04482 std::unique_ptr<test::FakeEncodedFrame> test_frame = test::FakeFrameBuilder()
483 .Id(0)
484 .PayloadType(99)
485 .Rotation(kRotation)
486 .AsLast()
487 .Build();
Tommiae4d0972020-05-18 06:45:38488
489 video_receive_stream_->Start();
490 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
Evan Shrubsolea4062722022-05-02 15:46:07491 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
Evan Shrubsole44be5792022-04-26 14:24:41492 RenderedFrameWith(Rotation(kRotation)));
Tommiae4d0972020-05-18 06:45:38493}
494
Evan Shrubsolea4062722022-05-02 15:46:07495TEST_P(VideoReceiveStream2Test, PassesPacketInfos) {
Tommiae4d0972020-05-18 06:45:38496 RtpPacketInfos packet_infos = CreatePacketInfos(3);
Evan Shrubsolea0ee64c2022-04-26 08:09:04497 auto test_frame = test::FakeFrameBuilder()
498 .Id(0)
499 .PayloadType(99)
500 .PacketInfos(packet_infos)
501 .AsLast()
502 .Build();
Tommiae4d0972020-05-18 06:45:38503
504 video_receive_stream_->Start();
505 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
Evan Shrubsolea4062722022-05-02 15:46:07506 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
Evan Shrubsole44be5792022-04-26 14:24:41507 RenderedFrameWith(PacketInfos(ElementsAreArray(packet_infos))));
Tommiae4d0972020-05-18 06:45:38508}
509
Evan Shrubsolea4062722022-05-02 15:46:07510TEST_P(VideoReceiveStream2Test, RenderedFrameUpdatesGetSources) {
Tommiae4d0972020-05-18 06:45:38511 constexpr uint32_t kSsrc = 1111;
512 constexpr uint32_t kCsrc = 9001;
513 constexpr uint32_t kRtpTimestamp = 12345;
514
515 // Prepare one video frame with per-packet information.
Evan Shrubsolea0ee64c2022-04-26 08:09:04516 auto test_frame =
517 test::FakeFrameBuilder().Id(0).PayloadType(99).AsLast().Build();
Tommiae4d0972020-05-18 06:45:38518 RtpPacketInfos packet_infos;
519 {
520 RtpPacketInfos::vector_type infos;
521
522 RtpPacketInfo info;
523 info.set_ssrc(kSsrc);
524 info.set_csrcs({kCsrc});
525 info.set_rtp_timestamp(kRtpTimestamp);
526
Danil Chapovalov22333492023-12-07 11:10:46527 info.set_receive_time(env_.clock().CurrentTime() - TimeDelta::Millis(5000));
Tommiae4d0972020-05-18 06:45:38528 infos.push_back(info);
529
Danil Chapovalov22333492023-12-07 11:10:46530 info.set_receive_time(env_.clock().CurrentTime() - TimeDelta::Millis(3000));
Tommiae4d0972020-05-18 06:45:38531 infos.push_back(info);
532
Danil Chapovalov22333492023-12-07 11:10:46533 info.set_receive_time(env_.clock().CurrentTime() - TimeDelta::Millis(2000));
Tommiae4d0972020-05-18 06:45:38534 infos.push_back(info);
535
Danil Chapovalov22333492023-12-07 11:10:46536 info.set_receive_time(env_.clock().CurrentTime() - TimeDelta::Millis(1000));
Tommiae4d0972020-05-18 06:45:38537 infos.push_back(info);
538
539 packet_infos = RtpPacketInfos(std::move(infos));
540 }
541 test_frame->SetPacketInfos(packet_infos);
542
543 // Start receive stream.
544 video_receive_stream_->Start();
545 EXPECT_THAT(video_receive_stream_->GetSources(), IsEmpty());
546
547 // Render one video frame.
Danil Chapovalov22333492023-12-07 11:10:46548 Timestamp timestamp_min = env_.clock().CurrentTime();
Tommiae4d0972020-05-18 06:45:38549 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
Tommiae4d0972020-05-18 06:45:38550 // Verify that the per-packet information is passed to the renderer.
Evan Shrubsolea4062722022-05-02 15:46:07551 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
Evan Shrubsole44be5792022-04-26 14:24:41552 RenderedFrameWith(PacketInfos(ElementsAreArray(packet_infos))));
Danil Chapovalov22333492023-12-07 11:10:46553 Timestamp timestamp_max = env_.clock().CurrentTime();
Tommiae4d0972020-05-18 06:45:38554
Artem Titovab30d722021-07-27 14:22:11555 // Verify that the per-packet information also updates `GetSources()`.
Tommiae4d0972020-05-18 06:45:38556 std::vector<RtpSource> sources = video_receive_stream_->GetSources();
557 ASSERT_THAT(sources, SizeIs(2));
558 {
559 auto it = std::find_if(sources.begin(), sources.end(),
560 [](const RtpSource& source) {
561 return source.source_type() == RtpSourceType::SSRC;
562 });
563 ASSERT_NE(it, sources.end());
564
565 EXPECT_EQ(it->source_id(), kSsrc);
566 EXPECT_EQ(it->source_type(), RtpSourceType::SSRC);
567 EXPECT_EQ(it->rtp_timestamp(), kRtpTimestamp);
Danil Chapovalovf5359712023-08-28 16:32:25568 EXPECT_GE(it->timestamp(), timestamp_min);
569 EXPECT_LE(it->timestamp(), timestamp_max);
Tommiae4d0972020-05-18 06:45:38570 }
571 {
572 auto it = std::find_if(sources.begin(), sources.end(),
573 [](const RtpSource& source) {
574 return source.source_type() == RtpSourceType::CSRC;
575 });
576 ASSERT_NE(it, sources.end());
577
578 EXPECT_EQ(it->source_id(), kCsrc);
579 EXPECT_EQ(it->source_type(), RtpSourceType::CSRC);
580 EXPECT_EQ(it->rtp_timestamp(), kRtpTimestamp);
Danil Chapovalovf5359712023-08-28 16:32:25581 EXPECT_GE(it->timestamp(), timestamp_min);
582 EXPECT_LE(it->timestamp(), timestamp_max);
Tommiae4d0972020-05-18 06:45:38583 }
584}
585
Evan Shrubsolea0ee64c2022-04-26 08:09:04586std::unique_ptr<test::FakeEncodedFrame> MakeFrameWithResolution(
Markus Handell588f9b32021-04-08 17:19:50587 VideoFrameType frame_type,
588 int picture_id,
589 int width,
590 int height) {
Evan Shrubsolea0ee64c2022-04-26 08:09:04591 auto frame =
592 test::FakeFrameBuilder().Id(picture_id).PayloadType(99).AsLast().Build();
Tommiae4d0972020-05-18 06:45:38593 frame->SetFrameType(frame_type);
Markus Handell588f9b32021-04-08 17:19:50594 frame->_encodedWidth = width;
595 frame->_encodedHeight = height;
Tommiae4d0972020-05-18 06:45:38596 return frame;
597}
598
Evan Shrubsolea0ee64c2022-04-26 08:09:04599std::unique_ptr<test::FakeEncodedFrame> MakeFrame(VideoFrameType frame_type,
600 int picture_id) {
Markus Handell588f9b32021-04-08 17:19:50601 return MakeFrameWithResolution(frame_type, picture_id, 320, 240);
602}
603
Evan Shrubsolea4062722022-05-02 15:46:07604TEST_P(VideoReceiveStream2Test, PassesFrameWhenEncodedFramesCallbackSet) {
Tommiae4d0972020-05-18 06:45:38605 testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
606 video_receive_stream_->Start();
Tommiae4d0972020-05-18 06:45:38607 EXPECT_CALL(callback, Call);
608 video_receive_stream_->SetAndGetRecordingState(
Tommif6f45432022-05-20 13:21:20609 VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
610 true);
Tommiae4d0972020-05-18 06:45:38611 video_receive_stream_->OnCompleteFrame(
612 MakeFrame(VideoFrameType::kVideoFrameKey, 0));
Evan Shrubsolea4062722022-05-02 15:46:07613 EXPECT_TRUE(fake_renderer_.WaitForFrame(kDefaultTimeOut));
philipel27b35a72022-07-05 07:59:55614
615 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(1));
616
Tommiae4d0972020-05-18 06:45:38617 video_receive_stream_->Stop();
618}
619
Evan Shrubsolea4062722022-05-02 15:46:07620TEST_P(VideoReceiveStream2Test, MovesEncodedFrameDispatchStateWhenReCreating) {
Tommiae4d0972020-05-18 06:45:38621 testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
622 video_receive_stream_->Start();
623 // Expect a key frame request over RTCP.
Tommiae4d0972020-05-18 06:45:38624 video_receive_stream_->SetAndGetRecordingState(
Tommif6f45432022-05-20 13:21:20625 VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
626 true);
Tommiae4d0972020-05-18 06:45:38627 video_receive_stream_->Stop();
Tommif6f45432022-05-20 13:21:20628 VideoReceiveStreamInterface::RecordingState old_state =
Tommiae4d0972020-05-18 06:45:38629 video_receive_stream_->SetAndGetRecordingState(
Tommif6f45432022-05-20 13:21:20630 VideoReceiveStreamInterface::RecordingState(), false);
Evan Shrubsole44be5792022-04-26 14:24:41631 RecreateReceiveStream(std::move(old_state));
philipel27b35a72022-07-05 07:59:55632
633 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(1));
634
Tommiae4d0972020-05-18 06:45:38635 video_receive_stream_->Stop();
636}
637
Evan Shrubsolea4062722022-05-02 15:46:07638TEST_P(VideoReceiveStream2Test, RequestsKeyFramesUntilKeyFrameReceived) {
Evan Shrubsole44be5792022-04-26 14:24:41639 // Recreate receive stream with shorter delay to test rtx.
640 TimeDelta rtx_delay = TimeDelta::Millis(50);
641 config_.rtp.nack.rtp_history_ms = rtx_delay.ms();
642 auto tick = rtx_delay / 2;
643 RecreateReceiveStream();
644 video_receive_stream_->Start();
Markus Handell588f9b32021-04-08 17:19:50645
Evan Shrubsole44be5792022-04-26 14:24:41646 video_receive_stream_->GenerateKeyFrame();
647 video_receive_stream_->OnCompleteFrame(
648 MakeFrame(VideoFrameType::kVideoFrameDelta, 0));
Evan Shrubsolea4062722022-05-02 15:46:07649 fake_renderer_.WaitForFrame(kDefaultTimeOut);
Tommiae4d0972020-05-18 06:45:38650 time_controller_.AdvanceTime(tick);
Evan Shrubsole44be5792022-04-26 14:24:41651 video_receive_stream_->OnCompleteFrame(
652 MakeFrame(VideoFrameType::kVideoFrameDelta, 1));
Evan Shrubsolea4062722022-05-02 15:46:07653 fake_renderer_.WaitForFrame(kDefaultTimeOut);
Evan Shrubsole44be5792022-04-26 14:24:41654 time_controller_.AdvanceTime(TimeDelta::Zero());
Tommiae4d0972020-05-18 06:45:38655 testing::Mock::VerifyAndClearExpectations(&mock_transport_);
656
philipel27b35a72022-07-05 07:59:55657 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(1));
658
Philipp Hancke006206d2021-03-24 16:49:02659 // T+keyframetimeout: still no key frame received, expect key frame request
660 // sent again.
Tommiae4d0972020-05-18 06:45:38661 time_controller_.AdvanceTime(tick);
Evan Shrubsole44be5792022-04-26 14:24:41662 video_receive_stream_->OnCompleteFrame(
663 MakeFrame(VideoFrameType::kVideoFrameDelta, 2));
Evan Shrubsolea4062722022-05-02 15:46:07664 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Tommiae4d0972020-05-18 06:45:38665 testing::Mock::VerifyAndClearExpectations(&mock_transport_);
666
philipel27b35a72022-07-05 07:59:55667 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(2));
668
Philipp Hancke006206d2021-03-24 16:49:02669 // T+keyframetimeout: now send a key frame - we should not observe new key
670 // frame requests after this.
Evan Shrubsole44be5792022-04-26 14:24:41671 video_receive_stream_->OnCompleteFrame(
672 MakeFrame(VideoFrameType::kVideoFrameKey, 3));
Evan Shrubsolea4062722022-05-02 15:46:07673 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Tommiae4d0972020-05-18 06:45:38674 time_controller_.AdvanceTime(2 * tick);
Evan Shrubsole44be5792022-04-26 14:24:41675 video_receive_stream_->OnCompleteFrame(
676 MakeFrame(VideoFrameType::kVideoFrameDelta, 4));
Evan Shrubsolea4062722022-05-02 15:46:07677 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
philipel27b35a72022-07-05 07:59:55678
679 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(2));
Tommiae4d0972020-05-18 06:45:38680}
681
Evan Shrubsolea4062722022-05-02 15:46:07682TEST_P(VideoReceiveStream2Test,
Markus Handell588f9b32021-04-08 17:19:50683 DispatchesEncodedFrameSequenceStartingWithKeyframeWithoutResolution) {
Evan Shrubsole44be5792022-04-26 14:24:41684 video_receive_stream_->Start();
Markus Handell588f9b32021-04-08 17:19:50685 testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
Evan Shrubsole44be5792022-04-26 14:24:41686 video_receive_stream_->SetAndGetRecordingState(
Tommif6f45432022-05-20 13:21:20687 VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
Markus Handell588f9b32021-04-08 17:19:50688 /*generate_key_frame=*/false);
689
690 InSequence s;
Jonas Oreland0deda152022-09-23 10:08:57691 EXPECT_CALL(callback,
692 Call(MatchResolution(test::FakeDecoder::kDefaultWidth,
693 test::FakeDecoder::kDefaultHeight)));
Markus Handell588f9b32021-04-08 17:19:50694 EXPECT_CALL(callback, Call);
695
Evan Shrubsole44be5792022-04-26 14:24:41696 video_receive_stream_->OnCompleteFrame(
Markus Handell588f9b32021-04-08 17:19:50697 MakeFrameWithResolution(VideoFrameType::kVideoFrameKey, 0, 0, 0));
Evan Shrubsolea4062722022-05-02 15:46:07698 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Evan Shrubsole44be5792022-04-26 14:24:41699 video_receive_stream_->OnCompleteFrame(
Markus Handell588f9b32021-04-08 17:19:50700 MakeFrameWithResolution(VideoFrameType::kVideoFrameDelta, 1, 0, 0));
Evan Shrubsolea4062722022-05-02 15:46:07701 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Markus Handell588f9b32021-04-08 17:19:50702
Evan Shrubsole44be5792022-04-26 14:24:41703 video_receive_stream_->Stop();
Markus Handell588f9b32021-04-08 17:19:50704}
705
Evan Shrubsolea4062722022-05-02 15:46:07706TEST_P(VideoReceiveStream2Test,
Markus Handell588f9b32021-04-08 17:19:50707 DispatchesEncodedFrameSequenceStartingWithKeyframeWithResolution) {
Evan Shrubsole44be5792022-04-26 14:24:41708 video_receive_stream_->Start();
Markus Handell588f9b32021-04-08 17:19:50709 testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
Evan Shrubsole44be5792022-04-26 14:24:41710 video_receive_stream_->SetAndGetRecordingState(
Tommif6f45432022-05-20 13:21:20711 VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
Markus Handell588f9b32021-04-08 17:19:50712 /*generate_key_frame=*/false);
713
714 InSequence s;
Jonas Oreland0deda152022-09-23 10:08:57715 EXPECT_CALL(callback, Call(MatchResolution(1080u, 720u)));
Markus Handell588f9b32021-04-08 17:19:50716 EXPECT_CALL(callback, Call);
717
Evan Shrubsole44be5792022-04-26 14:24:41718 video_receive_stream_->OnCompleteFrame(
Markus Handell588f9b32021-04-08 17:19:50719 MakeFrameWithResolution(VideoFrameType::kVideoFrameKey, 0, 1080, 720));
Evan Shrubsolea4062722022-05-02 15:46:07720 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Evan Shrubsole44be5792022-04-26 14:24:41721 video_receive_stream_->OnCompleteFrame(
Markus Handell588f9b32021-04-08 17:19:50722 MakeFrameWithResolution(VideoFrameType::kVideoFrameDelta, 1, 0, 0));
Evan Shrubsolea4062722022-05-02 15:46:07723 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Markus Handell588f9b32021-04-08 17:19:50724
Evan Shrubsole44be5792022-04-26 14:24:41725 video_receive_stream_->Stop();
Markus Handell588f9b32021-04-08 17:19:50726}
727
Evan Shrubsolea4062722022-05-02 15:46:07728TEST_P(VideoReceiveStream2Test, DependantFramesAreScheduled) {
729 video_receive_stream_->Start();
730
731 auto key_frame = test::FakeFrameBuilder()
732 .Id(0)
733 .PayloadType(99)
734 .Time(kFirstRtpTimestamp)
735 .ReceivedTime(kStartTime)
736 .AsLast()
737 .Build();
738 auto delta_frame = test::FakeFrameBuilder()
739 .Id(1)
740 .PayloadType(99)
741 .Time(RtpTimestampForFrame(1))
742 .ReceivedTime(ReceiveTimeForFrame(1))
743 .Refs({0})
744 .AsLast()
745 .Build();
746
747 // Expect frames are decoded in order.
748 InSequence seq;
Tony Herre55b593f2023-08-29 14:05:49749 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp), _));
Evan Shrubsolea4062722022-05-02 15:46:07750 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp +
751 k30FpsRtpTimestampDelta),
Tony Herre55b593f2023-08-29 14:05:49752 _))
Evan Shrubsolea4062722022-05-02 15:46:07753 .Times(1);
754 video_receive_stream_->OnCompleteFrame(std::move(key_frame));
755 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
756
757 time_controller_.AdvanceTime(k30FpsDelay);
758 video_receive_stream_->OnCompleteFrame(std::move(delta_frame));
759 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
760
761 video_receive_stream_->Stop();
762}
763
764TEST_P(VideoReceiveStream2Test, FramesScheduledInOrder) {
765 video_receive_stream_->Start();
766
767 auto key_frame = test::FakeFrameBuilder()
768 .Id(0)
769 .PayloadType(99)
770 .Time(kFirstRtpTimestamp)
771 .AsLast()
772 .Build();
773 auto delta_frame1 = test::FakeFrameBuilder()
774 .Id(1)
775 .PayloadType(99)
776 .Time(RtpTimestampForFrame(1))
777 .Refs({0})
778 .AsLast()
779 .Build();
780 auto delta_frame2 = test::FakeFrameBuilder()
781 .Id(2)
782 .PayloadType(99)
783 .Time(RtpTimestampForFrame(2))
784 .Refs({1})
785 .AsLast()
786 .Build();
787
788 // Expect frames are decoded in order despite delta_frame1 arriving first.
789 InSequence seq;
Tony Herre55b593f2023-08-29 14:05:49790 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp), _))
Evan Shrubsolea4062722022-05-02 15:46:07791 .Times(1);
792 EXPECT_CALL(mock_decoder_,
Tony Herre55b593f2023-08-29 14:05:49793 Decode(test::RtpTimestamp(RtpTimestampForFrame(1)), _))
Evan Shrubsolea4062722022-05-02 15:46:07794 .Times(1);
795 EXPECT_CALL(mock_decoder_,
Tony Herre55b593f2023-08-29 14:05:49796 Decode(test::RtpTimestamp(RtpTimestampForFrame(2)), _))
Evan Shrubsolea4062722022-05-02 15:46:07797 .Times(1);
Danil Chapovalov22333492023-12-07 11:10:46798 key_frame->SetReceivedTime(env_.clock().CurrentTime().ms());
Evan Shrubsolea4062722022-05-02 15:46:07799 video_receive_stream_->OnCompleteFrame(std::move(key_frame));
800 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
801
Danil Chapovalov22333492023-12-07 11:10:46802 delta_frame2->SetReceivedTime(env_.clock().CurrentTime().ms());
Evan Shrubsolea4062722022-05-02 15:46:07803 video_receive_stream_->OnCompleteFrame(std::move(delta_frame2));
804 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), DidNotReceiveFrame());
805 // `delta_frame1` arrives late.
Danil Chapovalov22333492023-12-07 11:10:46806 delta_frame1->SetReceivedTime(env_.clock().CurrentTime().ms());
Evan Shrubsolea4062722022-05-02 15:46:07807 video_receive_stream_->OnCompleteFrame(std::move(delta_frame1));
808 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
809 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay * 2), RenderedFrame());
810 video_receive_stream_->Stop();
811}
812
813TEST_P(VideoReceiveStream2Test, WaitsforAllSpatialLayers) {
814 video_receive_stream_->Start();
815 auto sl0 = test::FakeFrameBuilder()
816 .Id(0)
817 .PayloadType(99)
818 .Time(kFirstRtpTimestamp)
819 .ReceivedTime(kStartTime)
820 .Build();
821 auto sl1 = test::FakeFrameBuilder()
822 .Id(1)
823 .PayloadType(99)
824 .ReceivedTime(kStartTime)
825 .Time(kFirstRtpTimestamp)
826 .Refs({0})
827 .Build();
828 auto sl2 = test::FakeFrameBuilder()
829 .Id(2)
830 .PayloadType(99)
831 .ReceivedTime(kStartTime)
832 .Time(kFirstRtpTimestamp)
833 .Refs({0, 1})
834 .AsLast()
835 .Build();
836
837 // No decodes should be called until `sl2` is received.
Tony Herre55b593f2023-08-29 14:05:49838 EXPECT_CALL(mock_decoder_, Decode(_, _)).Times(0);
Danil Chapovalov22333492023-12-07 11:10:46839 sl0->SetReceivedTime(env_.clock().CurrentTime().ms());
Evan Shrubsolea4062722022-05-02 15:46:07840 video_receive_stream_->OnCompleteFrame(std::move(sl0));
841 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
842 DidNotReceiveFrame());
843 video_receive_stream_->OnCompleteFrame(std::move(sl1));
844 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
845 DidNotReceiveFrame());
846 // When `sl2` arrives decode should happen.
Tony Herre55b593f2023-08-29 14:05:49847 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp), _))
Evan Shrubsolea4062722022-05-02 15:46:07848 .Times(1);
849 video_receive_stream_->OnCompleteFrame(std::move(sl2));
850 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
851 video_receive_stream_->Stop();
852}
853
854TEST_P(VideoReceiveStream2Test, FramesFastForwardOnSystemHalt) {
855 video_receive_stream_->Start();
856
857 // The frame structure looks like this,
858 // F1
859 // /
860 // F0 --> F2
861 //
862 // In this case we will have a system halt simulated. By the time the system
863 // resumes, F1 will be old and so F2 should be decoded.
864 auto key_frame = test::FakeFrameBuilder()
865 .Id(0)
866 .PayloadType(99)
867 .Time(kFirstRtpTimestamp)
868 .AsLast()
869 .Build();
870 auto ffwd_frame = test::FakeFrameBuilder()
871 .Id(1)
872 .PayloadType(99)
873 .Time(RtpTimestampForFrame(1))
874 .Refs({0})
875 .AsLast()
876 .Build();
877 auto rendered_frame = test::FakeFrameBuilder()
878 .Id(2)
879 .PayloadType(99)
880 .Time(RtpTimestampForFrame(2))
881 .Refs({0})
882 .AsLast()
883 .Build();
884 InSequence seq;
Tony Herre55b593f2023-08-29 14:05:49885 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp), _))
philipel27b35a72022-07-05 07:59:55886 .WillOnce(testing::DoAll(Invoke([&] {
Evan Shrubsolea4062722022-05-02 15:46:07887 // System halt will be simulated in the decode.
888 time_controller_.AdvanceTime(k30FpsDelay * 2);
889 }),
890 DefaultDecodeAction()));
891 EXPECT_CALL(mock_decoder_,
Tony Herre55b593f2023-08-29 14:05:49892 Decode(test::RtpTimestamp(RtpTimestampForFrame(2)), _));
Evan Shrubsolea4062722022-05-02 15:46:07893 video_receive_stream_->OnCompleteFrame(std::move(key_frame));
894 video_receive_stream_->OnCompleteFrame(std::move(ffwd_frame));
895 video_receive_stream_->OnCompleteFrame(std::move(rendered_frame));
Evan Shrubsole7cbd8de2022-08-16 08:08:53896 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
897 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(0))));
898 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
899 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(2))));
Evan Shrubsolea4062722022-05-02 15:46:07900
901 // Check stats show correct dropped frames.
902 auto stats = video_receive_stream_->GetStats();
903 EXPECT_EQ(stats.frames_dropped, 1u);
904
905 video_receive_stream_->Stop();
906}
907
908TEST_P(VideoReceiveStream2Test, BetterFrameInsertedWhileWaitingToDecodeFrame) {
909 video_receive_stream_->Start();
910
911 auto key_frame = test::FakeFrameBuilder()
912 .Id(0)
913 .PayloadType(99)
914 .Time(kFirstRtpTimestamp)
915 .ReceivedTime(ReceiveTimeForFrame(0))
916 .AsLast()
917 .Build();
918 auto f1 = test::FakeFrameBuilder()
919 .Id(1)
920 .PayloadType(99)
921 .Time(RtpTimestampForFrame(1))
922 .ReceivedTime(ReceiveTimeForFrame(1))
923 .Refs({0})
924 .AsLast()
925 .Build();
926 auto f2 = test::FakeFrameBuilder()
927 .Id(2)
928 .PayloadType(99)
929 .Time(RtpTimestampForFrame(2))
930 .ReceivedTime(ReceiveTimeForFrame(2))
931 .Refs({0})
932 .AsLast()
933 .Build();
934
935 video_receive_stream_->OnCompleteFrame(std::move(key_frame));
936 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
937
938 InSequence seq;
939 EXPECT_CALL(mock_decoder_,
Tony Herre55b593f2023-08-29 14:05:49940 Decode(test::RtpTimestamp(RtpTimestampForFrame(1)), _))
Evan Shrubsolea4062722022-05-02 15:46:07941 .Times(1);
942 EXPECT_CALL(mock_decoder_,
Tony Herre55b593f2023-08-29 14:05:49943 Decode(test::RtpTimestamp(RtpTimestampForFrame(2)), _))
Evan Shrubsolea4062722022-05-02 15:46:07944 .Times(1);
945 // Simulate f1 arriving after f2 but before f2 is decoded.
946 video_receive_stream_->OnCompleteFrame(std::move(f2));
947 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), DidNotReceiveFrame());
948 video_receive_stream_->OnCompleteFrame(std::move(f1));
949 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
950 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
951
952 video_receive_stream_->Stop();
953}
954
955// Note: This test takes a long time (~10s) to run if the fake metronome is
956// active. Since the test needs to wait for the timestamp to rollover, it has a
957// fake delay of around 6.5 hours. Even though time is simulated, this will be
958// around 1,500,000 metronome tick invocations.
959TEST_P(VideoReceiveStream2Test, RtpTimestampWrapAround) {
Evan Shrubsole7cbd8de2022-08-16 08:08:53960 EXPECT_CALL(mock_transport_, SendRtcp).Times(AnyNumber());
Evan Shrubsolea4062722022-05-02 15:46:07961 video_receive_stream_->Start();
962
963 constexpr uint32_t kBaseRtp = std::numeric_limits<uint32_t>::max() / 2;
964 video_receive_stream_->OnCompleteFrame(
965 test::FakeFrameBuilder()
966 .Id(0)
967 .PayloadType(99)
968 .Time(kBaseRtp)
Danil Chapovalov22333492023-12-07 11:10:46969 .ReceivedTime(env_.clock().CurrentTime())
Evan Shrubsolea4062722022-05-02 15:46:07970 .AsLast()
971 .Build());
972 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
973 time_controller_.AdvanceTime(k30FpsDelay);
974 video_receive_stream_->OnCompleteFrame(
975 test::FakeFrameBuilder()
976 .Id(1)
977 .PayloadType(99)
978 .Time(kBaseRtp + k30FpsRtpTimestampDelta)
Danil Chapovalov22333492023-12-07 11:10:46979 .ReceivedTime(env_.clock().CurrentTime())
Evan Shrubsolea4062722022-05-02 15:46:07980 .AsLast()
981 .Build());
982 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
983
984 // Pause stream so that RTP timestamp wraps around.
985 constexpr uint32_t kLastRtp = kBaseRtp + k30FpsRtpTimestampDelta;
986 constexpr uint32_t kWrapAroundRtp =
987 kLastRtp + std::numeric_limits<uint32_t>::max() / 2 + 1;
988 // Pause for corresponding delay such that RTP timestamp would increase this
989 // much at 30fps.
990 constexpr TimeDelta kWrapAroundDelay =
991 (std::numeric_limits<uint32_t>::max() / 2 + 1) / kRtpTimestampHz;
992
993 time_controller_.AdvanceTime(kWrapAroundDelay);
994 video_receive_stream_->OnCompleteFrame(
995 test::FakeFrameBuilder()
996 .Id(2)
997 .PayloadType(99)
998 .Time(kWrapAroundRtp)
Danil Chapovalov22333492023-12-07 11:10:46999 .ReceivedTime(env_.clock().CurrentTime())
Evan Shrubsolea4062722022-05-02 15:46:071000 .AsLast()
1001 .Build());
Tony Herre55b593f2023-08-29 14:05:491002 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kWrapAroundRtp), _))
Evan Shrubsolea4062722022-05-02 15:46:071003 .Times(1);
philipel2f3b75d2024-03-13 12:02:151004 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Seconds(1)),
1005 RenderedFrame());
Evan Shrubsolea4062722022-05-02 15:46:071006
1007 video_receive_stream_->Stop();
1008}
1009
Evan Shrubsoledcb9c5d2022-06-13 15:39:531010// If a frame was lost causing the stream to become temporarily non-decodable
1011// and the sender reduces their framerate during this time, the video stream
1012// should start decoding at the new framerate. However, if the connection is
1013// poor, a keyframe will take a long time to send. If the timing of the incoming
1014// frames was not kept up to date with the new framerate while the stream was
1015// decodable, this late frame will have a large delay as the rtp timestamp of
1016// this keyframe will look like the frame arrived early if the frame-rate was
1017// not updated.
1018TEST_P(VideoReceiveStream2Test, PoorConnectionWithFpsChangeDuringLostFrame) {
1019 video_receive_stream_->Start();
1020
1021 constexpr Frequency k15Fps = Frequency::Hertz(15);
1022 constexpr TimeDelta k15FpsDelay = 1 / k15Fps;
1023 constexpr uint32_t k15FpsRtpTimestampDelta = kRtpTimestampHz / k15Fps;
1024
1025 // Initial keyframe and frames at 30fps.
1026 video_receive_stream_->OnCompleteFrame(
1027 test::FakeFrameBuilder()
1028 .Id(0)
1029 .PayloadType(99)
1030 .Time(RtpTimestampForFrame(0))
1031 .ReceivedTime(ReceiveTimeForFrame(0))
1032 .AsLast()
1033 .Build());
1034 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1035 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(0))));
1036
1037 video_receive_stream_->OnCompleteFrame(
1038 test::FakeFrameBuilder()
1039 .Id(1)
1040 .PayloadType(99)
1041 .Time(RtpTimestampForFrame(1))
1042 .ReceivedTime(ReceiveTimeForFrame(1))
1043 .Refs({0})
1044 .AsLast()
1045 .Build());
1046 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1047 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(1))));
1048
1049 // Simulate lost frame 2, followed by 2 second of frames at 30fps, followed by
1050 // 2 second of frames at 15 fps, and then a keyframe.
1051 time_controller_.AdvanceTime(k30FpsDelay);
1052
Danil Chapovalov22333492023-12-07 11:10:461053 Timestamp send_30fps_end_time =
1054 env_.clock().CurrentTime() + TimeDelta::Seconds(2);
Evan Shrubsoledcb9c5d2022-06-13 15:39:531055 int id = 3;
1056 EXPECT_CALL(mock_transport_, SendRtcp).Times(AnyNumber());
Danil Chapovalov22333492023-12-07 11:10:461057 while (env_.clock().CurrentTime() < send_30fps_end_time) {
Evan Shrubsoledcb9c5d2022-06-13 15:39:531058 ++id;
1059 video_receive_stream_->OnCompleteFrame(
1060 test::FakeFrameBuilder()
1061 .Id(id)
1062 .PayloadType(99)
1063 .Time(RtpTimestampForFrame(id))
1064 .ReceivedTime(ReceiveTimeForFrame(id))
1065 .Refs({id - 1})
1066 .AsLast()
1067 .Build());
1068 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
Florent Castelli8037fc62024-08-29 13:00:401069 Eq(std::nullopt));
Evan Shrubsoledcb9c5d2022-06-13 15:39:531070 }
1071 uint32_t current_rtp = RtpTimestampForFrame(id);
Danil Chapovalov22333492023-12-07 11:10:461072 Timestamp send_15fps_end_time =
1073 env_.clock().CurrentTime() + TimeDelta::Seconds(2);
1074 while (env_.clock().CurrentTime() < send_15fps_end_time) {
Evan Shrubsoledcb9c5d2022-06-13 15:39:531075 ++id;
1076 current_rtp += k15FpsRtpTimestampDelta;
1077 video_receive_stream_->OnCompleteFrame(
1078 test::FakeFrameBuilder()
1079 .Id(id)
1080 .PayloadType(99)
1081 .Time(current_rtp)
Danil Chapovalov22333492023-12-07 11:10:461082 .ReceivedTime(env_.clock().CurrentTime())
Evan Shrubsoledcb9c5d2022-06-13 15:39:531083 .Refs({id - 1})
1084 .AsLast()
1085 .Build());
1086 EXPECT_THAT(fake_renderer_.WaitForFrame(k15FpsDelay, /*advance_time=*/true),
Florent Castelli8037fc62024-08-29 13:00:401087 Eq(std::nullopt));
Evan Shrubsoledcb9c5d2022-06-13 15:39:531088 }
1089
1090 ++id;
1091 current_rtp += k15FpsRtpTimestampDelta;
1092 // Insert keyframe which will recover the stream. However, on a poor
1093 // connection the keyframe will take significant time to send.
1094 constexpr TimeDelta kKeyframeDelay = TimeDelta::Millis(200);
1095 video_receive_stream_->OnCompleteFrame(
1096 test::FakeFrameBuilder()
1097 .Id(id)
1098 .PayloadType(99)
1099 .Time(current_rtp)
Danil Chapovalov22333492023-12-07 11:10:461100 .ReceivedTime(env_.clock().CurrentTime() + kKeyframeDelay)
Evan Shrubsoledcb9c5d2022-06-13 15:39:531101 .AsLast()
1102 .Build());
1103 // If the framerate was not updated to be 15fps from the frames that arrived
1104 // previously, this will fail, as the delay will be longer.
1105 EXPECT_THAT(fake_renderer_.WaitForFrame(k15FpsDelay, /*advance_time=*/true),
1106 RenderedFrameWith(RtpTimestamp(current_rtp)));
1107
1108 video_receive_stream_->Stop();
1109}
1110
Evan Shrubsole4d3ba772022-06-22 14:32:361111TEST_P(VideoReceiveStream2Test, StreamShouldNotTimeoutWhileWaitingForFrame) {
1112 // Disable smoothing since this makes it hard to test frame timing.
1113 config_.enable_prerenderer_smoothing = false;
1114 RecreateReceiveStream();
1115
1116 video_receive_stream_->Start();
1117 EXPECT_CALL(mock_transport_, SendRtcp).Times(AnyNumber());
1118
1119 video_receive_stream_->OnCompleteFrame(
1120 test::FakeFrameBuilder()
1121 .Id(0)
1122 .PayloadType(99)
1123 .Time(RtpTimestampForFrame(0))
1124 .ReceivedTime(ReceiveTimeForFrame(0))
1125 .AsLast()
1126 .Build());
1127 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1128 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(0))));
1129
1130 for (int id = 1; id < 30; ++id) {
1131 video_receive_stream_->OnCompleteFrame(
1132 test::FakeFrameBuilder()
1133 .Id(id)
1134 .PayloadType(99)
1135 .Time(RtpTimestampForFrame(id))
1136 .ReceivedTime(ReceiveTimeForFrame(id))
1137 .Refs({0})
1138 .AsLast()
1139 .Build());
1140 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1141 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(id))));
1142 }
1143
1144 // Simulate a pause in the stream, followed by a decodable frame that is ready
1145 // long in the future. The stream should not timeout in this case, but rather
1146 // decode the frame just before the timeout.
1147 time_controller_.AdvanceTime(TimeDelta::Millis(2900));
1148 uint32_t late_decode_rtp = kFirstRtpTimestamp + 200 * k30FpsRtpTimestampDelta;
1149 video_receive_stream_->OnCompleteFrame(
1150 test::FakeFrameBuilder()
1151 .Id(121)
1152 .PayloadType(99)
1153 .Time(late_decode_rtp)
Danil Chapovalov22333492023-12-07 11:10:461154 .ReceivedTime(env_.clock().CurrentTime())
Evan Shrubsole4d3ba772022-06-22 14:32:361155 .AsLast()
1156 .Build());
1157 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Millis(100),
1158 /*advance_time=*/true),
1159 RenderedFrameWith(RtpTimestamp(late_decode_rtp)));
1160
1161 video_receive_stream_->Stop();
1162}
1163
Evan Shrubsolea4062722022-05-02 15:46:071164INSTANTIATE_TEST_SUITE_P(VideoReceiveStream2Test,
1165 VideoReceiveStream2Test,
1166 testing::Bool(),
1167 [](const auto& test_param_info) {
Evan Shrubsoledcb9c5d2022-06-13 15:39:531168 return (test_param_info.param
1169 ? "ScheduleDecodesWithMetronome"
1170 : "ScheduleDecodesWithPostTask");
Evan Shrubsolea4062722022-05-02 15:46:071171 });
1172
Tommiae4d0972020-05-18 06:45:381173} // namespace webrtc