blob: 14e5f96c13b29644df73e68e2d304b976ac89f89 [file] [log] [blame]
Tommiae4d0972020-05-18 06:45:381/*
2 * Copyright 2017 The WebRTC Project Authors. All rights reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "video/video_receive_stream2.h"
12
13#include <algorithm>
Philipp Hancke80859bc2025-04-10 18:04:5414#include <cstdint>
Evan Shrubsole7cbd8de2022-08-16 08:08:5315#include <deque>
Markus Handell588f9b32021-04-08 17:19:5016#include <limits>
Tommiae4d0972020-05-18 06:45:3817#include <memory>
Florent Castelli8037fc62024-08-29 13:00:4018#include <optional>
Tommiae4d0972020-05-18 06:45:3819#include <utility>
20#include <vector>
21
Evan Shrubsole6dbc1722022-03-22 11:20:1122#include "absl/memory/memory.h"
Philipp Hanckece6aa1f2025-04-28 18:59:2223#include "api/array_view.h"
Danil Chapovalov22333492023-12-07 11:10:4624#include "api/environment/environment.h"
25#include "api/environment/environment_factory.h"
Evan Shrubsolea4062722022-05-02 15:46:0726#include "api/metronome/test/fake_metronome.h"
Philipp Hancke80859bc2025-04-10 18:04:5427#include "api/rtp_packet_info.h"
28#include "api/rtp_packet_infos.h"
Danil Chapovalov9cd4d492021-08-03 12:59:0029#include "api/test/mock_video_decoder.h"
30#include "api/test/mock_video_decoder_factory.h"
Evan Shrubsole44be5792022-04-26 14:24:4131#include "api/test/time_controller.h"
Philipp Hancke80859bc2025-04-10 18:04:5432#include "api/transport/rtp/rtp_source.h"
Evan Shrubsolea4062722022-05-02 15:46:0733#include "api/units/frequency.h"
Evan Shrubsole44be5792022-04-26 14:24:4134#include "api/units/time_delta.h"
Philipp Hancke80859bc2025-04-10 18:04:5435#include "api/units/timestamp.h"
Evan Shrubsole44be5792022-04-26 14:24:4136#include "api/video/recordable_encoded_frame.h"
37#include "api/video/test/video_frame_matchers.h"
Markus Handell588f9b32021-04-08 17:19:5038#include "api/video/video_frame.h"
Philipp Hancke80859bc2025-04-10 18:04:5439#include "api/video/video_frame_type.h"
40#include "api/video/video_rotation.h"
41#include "api/video/video_sink_interface.h"
42#include "api/video/video_timing.h"
Evan Shrubsole1c184772022-04-26 07:47:4943#include "api/video_codecs/sdp_video_format.h"
Tommiae4d0972020-05-18 06:45:3844#include "call/rtp_stream_receiver_controller.h"
Evan Shrubsole1c184772022-04-26 07:47:4945#include "call/video_receive_stream.h"
Tommiae4d0972020-05-18 06:45:3846#include "common_video/test/utilities.h"
Tommi90738dd2021-05-31 15:36:4747#include "media/engine/fake_webrtc_call.h"
Tommiae4d0972020-05-18 06:45:3848#include "modules/pacing/packet_router.h"
49#include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
Philipp Hancke80859bc2025-04-10 18:04:5450#include "modules/video_coding/nack_requester.h"
Evan Shrubsoledcb9c5d2022-06-13 15:39:5351#include "rtc_base/logging.h"
Tommiae4d0972020-05-18 06:45:3852#include "system_wrappers/include/clock.h"
53#include "test/fake_decoder.h"
Evan Shrubsolea0ee64c2022-04-26 08:09:0454#include "test/fake_encoded_frame.h"
Tommiae4d0972020-05-18 06:45:3855#include "test/gmock.h"
56#include "test/gtest.h"
Danil Chapovalov9cd4d492021-08-03 12:59:0057#include "test/mock_transport.h"
philipel27b35a72022-07-05 07:59:5558#include "test/rtcp_packet_parser.h"
Tommiae4d0972020-05-18 06:45:3859#include "test/time_controller/simulated_time_controller.h"
60#include "test/video_decoder_proxy_factory.h"
61#include "video/call_stats2.h"
Philipp Hancke80859bc2025-04-10 18:04:5462#include "video/decode_synchronizer.h"
Tommiae4d0972020-05-18 06:45:3863
64namespace webrtc {
Evan Shrubsole1c184772022-04-26 07:47:4965
Tommiae4d0972020-05-18 06:45:3866namespace {
67
Evan Shrubsole44be5792022-04-26 14:24:4168using test::video_frame_matchers::NtpTimestamp;
69using test::video_frame_matchers::PacketInfos;
70using test::video_frame_matchers::Rotation;
Tommiae4d0972020-05-18 06:45:3871using ::testing::_;
Markus Handell588f9b32021-04-08 17:19:5072using ::testing::AllOf;
Evan Shrubsoledcb9c5d2022-06-13 15:39:5373using ::testing::AnyNumber;
Tommiae4d0972020-05-18 06:45:3874using ::testing::ElementsAreArray;
Evan Shrubsole44be5792022-04-26 14:24:4175using ::testing::Eq;
Markus Handell588f9b32021-04-08 17:19:5076using ::testing::Field;
77using ::testing::InSequence;
Tommiae4d0972020-05-18 06:45:3878using ::testing::Invoke;
79using ::testing::IsEmpty;
Johannes Kronbbf639e2022-06-15 10:27:2380using ::testing::Optional;
Evan Shrubsolea4062722022-05-02 15:46:0781using ::testing::Pointee;
Markus Handell588f9b32021-04-08 17:19:5082using ::testing::Property;
Evan Shrubsole44be5792022-04-26 14:24:4183using ::testing::Return;
Tommiae4d0972020-05-18 06:45:3884using ::testing::SizeIs;
Danil Chapovalovd08930d2021-08-12 11:26:5585using ::testing::WithoutArgs;
Tommiae4d0972020-05-18 06:45:3886
Evan Shrubsole44be5792022-04-26 14:24:4187auto RenderedFrameWith(::testing::Matcher<VideoFrame> m) {
88 return Optional(m);
89}
Evan Shrubsolea4062722022-05-02 15:46:0790auto RenderedFrame() {
91 return RenderedFrameWith(_);
92}
Florent Castelli8037fc62024-08-29 13:00:4093testing::Matcher<std::optional<VideoFrame>> DidNotReceiveFrame() {
94 return Eq(std::nullopt);
Evan Shrubsolea4062722022-05-02 15:46:0795}
Evan Shrubsole44be5792022-04-26 14:24:4196
97constexpr TimeDelta kDefaultTimeOut = TimeDelta::Millis(50);
Evan Shrubsole14ee8032022-04-26 14:01:4498constexpr int kDefaultNumCpuCores = 2;
Tommiae4d0972020-05-18 06:45:3899
Evan Shrubsole44be5792022-04-26 14:24:41100constexpr Timestamp kStartTime = Timestamp::Millis(1'337'000);
Evan Shrubsolea4062722022-05-02 15:46:07101constexpr Frequency k30Fps = Frequency::Hertz(30);
102constexpr TimeDelta k30FpsDelay = 1 / k30Fps;
103constexpr Frequency kRtpTimestampHz = Frequency::KiloHertz(90);
104constexpr uint32_t k30FpsRtpTimestampDelta = kRtpTimestampHz / k30Fps;
105constexpr uint32_t kFirstRtpTimestamp = 90000;
Sergio Garcia Murillob5289d72024-10-24 12:10:53106constexpr uint8_t kH264PayloadType = 99;
107constexpr uint8_t kH265PayloadType = 100;
108constexpr uint8_t kAv1PayloadType = 101;
109constexpr uint32_t kRemoteSsrc = 1111;
110constexpr uint32_t kLocalSsrc = 2222;
Evan Shrubsole44be5792022-04-26 14:24:41111
Evan Shrubsole8f7678f2025-04-01 14:23:55112class FakeVideoRenderer : public VideoSinkInterface<VideoFrame> {
Evan Shrubsole44be5792022-04-26 14:24:41113 public:
Evan Shrubsole7cbd8de2022-08-16 08:08:53114 explicit FakeVideoRenderer(TimeController* time_controller)
115 : time_controller_(time_controller) {}
Evan Shrubsole44be5792022-04-26 14:24:41116 ~FakeVideoRenderer() override = default;
117
Evan Shrubsoledcb9c5d2022-06-13 15:39:53118 void OnFrame(const VideoFrame& frame) override {
119 RTC_LOG(LS_VERBOSE) << "Received frame with timestamp="
Per K0fa90882024-03-13 08:52:41120 << frame.rtp_timestamp();
Evan Shrubsole7cbd8de2022-08-16 08:08:53121 if (!last_frame_.empty()) {
122 RTC_LOG(LS_INFO) << "Already had frame queue with timestamp="
Per K0fa90882024-03-13 08:52:41123 << last_frame_.back().rtp_timestamp();
Evan Shrubsoledcb9c5d2022-06-13 15:39:53124 }
Evan Shrubsole7cbd8de2022-08-16 08:08:53125 last_frame_.push_back(frame);
Evan Shrubsole44be5792022-04-26 14:24:41126 }
127
Evan Shrubsoledcb9c5d2022-06-13 15:39:53128 // If `advance_time`, then the clock will always advance by `timeout`.
Florent Castelli8037fc62024-08-29 13:00:40129 std::optional<VideoFrame> WaitForFrame(TimeDelta timeout,
130 bool advance_time = false) {
Evan Shrubsoledcb9c5d2022-06-13 15:39:53131 auto start = time_controller_->GetClock()->CurrentTime();
Evan Shrubsole7cbd8de2022-08-16 08:08:53132 if (last_frame_.empty()) {
Evan Shrubsolea4062722022-05-02 15:46:07133 time_controller_->AdvanceTime(TimeDelta::Zero());
Evan Shrubsole7cbd8de2022-08-16 08:08:53134 time_controller_->Wait([this] { return !last_frame_.empty(); }, timeout);
Evan Shrubsole44be5792022-04-26 14:24:41135 }
Florent Castelli8037fc62024-08-29 13:00:40136 std::optional<VideoFrame> ret;
Evan Shrubsole7cbd8de2022-08-16 08:08:53137 if (!last_frame_.empty()) {
138 ret = last_frame_.front();
139 last_frame_.pop_front();
140 }
Evan Shrubsoledcb9c5d2022-06-13 15:39:53141 if (advance_time) {
142 time_controller_->AdvanceTime(
143 timeout - (time_controller_->GetClock()->CurrentTime() - start));
144 }
Evan Shrubsole44be5792022-04-26 14:24:41145 return ret;
146 }
147
148 private:
Evan Shrubsole7cbd8de2022-08-16 08:08:53149 std::deque<VideoFrame> last_frame_;
Evan Shrubsolea4062722022-05-02 15:46:07150 TimeController* const time_controller_;
Evan Shrubsole44be5792022-04-26 14:24:41151};
152
Jonas Oreland0deda152022-09-23 10:08:57153MATCHER_P2(MatchResolution, w, h, "") {
Evan Shrubsole44be5792022-04-26 14:24:41154 return arg.resolution().width == w && arg.resolution().height == h;
155}
156
Evan Shrubsoledcb9c5d2022-06-13 15:39:53157MATCHER_P(RtpTimestamp, timestamp, "") {
Per K0fa90882024-03-13 08:52:41158 if (arg.rtp_timestamp() != timestamp) {
Evan Shrubsole7cbd8de2022-08-16 08:08:53159 *result_listener->stream()
Per K0fa90882024-03-13 08:52:41160 << "rtp timestamp was " << arg.rtp_timestamp() << " != " << timestamp;
Evan Shrubsole7cbd8de2022-08-16 08:08:53161 return false;
162 }
163 return true;
Evan Shrubsolea4062722022-05-02 15:46:07164}
165
166// Rtp timestamp for in order frame at 30fps.
167uint32_t RtpTimestampForFrame(int id) {
168 return kFirstRtpTimestamp + id * k30FpsRtpTimestampDelta;
169}
170
171// Receive time for in order frame at 30fps.
172Timestamp ReceiveTimeForFrame(int id) {
173 return kStartTime + id * k30FpsDelay;
174}
175
Tommiae4d0972020-05-18 06:45:38176} // namespace
177
Evan Shrubsolea4062722022-05-02 15:46:07178class VideoReceiveStream2Test : public ::testing::TestWithParam<bool> {
Tommiae4d0972020-05-18 06:45:38179 public:
Evan Shrubsolea4062722022-05-02 15:46:07180 auto DefaultDecodeAction() {
philipel27b35a72022-07-05 07:59:55181 return Invoke(&fake_decoder_, &test::FakeDecoder::Decode);
Evan Shrubsolea4062722022-05-02 15:46:07182 }
183
184 bool UseMetronome() const { return GetParam(); }
185
Tommiae4d0972020-05-18 06:45:38186 VideoReceiveStream2Test()
Evan Shrubsole44be5792022-04-26 14:24:41187 : time_controller_(kStartTime),
Danil Chapovalov22333492023-12-07 11:10:46188 env_(CreateEnvironment(time_controller_.CreateTaskQueueFactory(),
189 time_controller_.GetClock())),
Sergio Garcia Murillob5289d72024-10-24 12:10:53190 config_(&mock_transport_, &mock_decoder_factory_),
Danil Chapovalov22333492023-12-07 11:10:46191 call_stats_(&env_.clock(), time_controller_.GetMainThread()),
Evan Shrubsole7cbd8de2022-08-16 08:08:53192 fake_renderer_(&time_controller_),
Danil Chapovalov8d079be2024-04-08 16:35:40193 fake_call_(env_),
Markus Handellbe400e42022-11-08 11:14:23194 fake_metronome_(TimeDelta::Millis(16)),
Danil Chapovalov22333492023-12-07 11:10:46195 decode_sync_(&env_.clock(),
Evan Shrubsole7cbd8de2022-08-16 08:08:53196 &fake_metronome_,
197 time_controller_.GetMainThread()),
Evan Shrubsolea4062722022-05-02 15:46:07198 h264_decoder_factory_(&mock_decoder_) {
Evan Shrubsole1c184772022-04-26 07:47:49199 // By default, mock decoder factory is backed by VideoDecoderProxyFactory.
Sergio Garcia Murillob5289d72024-10-24 12:10:53200 ON_CALL(mock_decoder_factory_, Create)
Danil Chapovalov08173802024-01-23 12:02:46201 .WillByDefault(Invoke(&h264_decoder_factory_,
202 &test::VideoDecoderProxyFactory::Create));
Evan Shrubsole14ee8032022-04-26 14:01:44203
204 // By default, mock decode will wrap the fake decoder.
Evan Shrubsolea4062722022-05-02 15:46:07205 ON_CALL(mock_decoder_, Configure)
philipel27b35a72022-07-05 07:59:55206 .WillByDefault(Invoke(&fake_decoder_, &test::FakeDecoder::Configure));
Tony Herre55b593f2023-08-29 14:05:49207 ON_CALL(mock_decoder_, Decode(_, _)).WillByDefault(DefaultDecodeAction());
Evan Shrubsolea4062722022-05-02 15:46:07208 ON_CALL(mock_decoder_, RegisterDecodeCompleteCallback)
Evan Shrubsole14ee8032022-04-26 14:01:44209 .WillByDefault(
philipel27b35a72022-07-05 07:59:55210 Invoke(&fake_decoder_,
211 &test::FakeDecoder::RegisterDecodeCompleteCallback));
212 ON_CALL(mock_decoder_, Release)
213 .WillByDefault(Invoke(&fake_decoder_, &test::FakeDecoder::Release));
214 ON_CALL(mock_transport_, SendRtcp)
215 .WillByDefault(
Ilya Nikolaevskiy47ad4b62025-10-17 10:13:55216 [this](ArrayView<const uint8_t> packet, ::testing::Unused) {
Philipp Hanckece6aa1f2025-04-28 18:59:22217 return rtcp_packet_parser_.Parse(packet);
Ilya Nikolaevskiy47ad4b62025-10-17 10:13:55218 });
Jonas Oreland8ca06132022-03-14 11:52:48219 }
Evan Shrubsole7cbd8de2022-08-16 08:08:53220
Tommi90738dd2021-05-31 15:36:47221 ~VideoReceiveStream2Test() override {
Evan Shrubsolea4062722022-05-02 15:46:07222 if (video_receive_stream_) {
223 video_receive_stream_->Stop();
Tommi90738dd2021-05-31 15:36:47224 video_receive_stream_->UnregisterFromTransport();
Evan Shrubsolea4062722022-05-02 15:46:07225 }
Evan Shrubsolea4062722022-05-02 15:46:07226 time_controller_.AdvanceTime(TimeDelta::Zero());
Tommi90738dd2021-05-31 15:36:47227 }
Tommiae4d0972020-05-18 06:45:38228
Tommi90738dd2021-05-31 15:36:47229 void SetUp() override {
Sergio Garcia Murillob5289d72024-10-24 12:10:53230 config_.rtp.remote_ssrc = kRemoteSsrc;
231 config_.rtp.local_ssrc = kLocalSsrc;
Tommiae4d0972020-05-18 06:45:38232 config_.renderer = &fake_renderer_;
Tommif6f45432022-05-20 13:21:20233 VideoReceiveStreamInterface::Decoder h264_decoder;
Sergio Garcia Murillob5289d72024-10-24 12:10:53234 h264_decoder.payload_type = kH264PayloadType;
Philipp Hanckebbff58d2024-02-27 11:18:33235 h264_decoder.video_format = SdpVideoFormat::H264();
Tommiae4d0972020-05-18 06:45:38236 h264_decoder.video_format.parameters.insert(
237 {"sprop-parameter-sets", "Z0IACpZTBYmI,aMljiA=="});
Tommif6f45432022-05-20 13:21:20238 VideoReceiveStreamInterface::Decoder h265_decoder;
Sergio Garcia Murillob5289d72024-10-24 12:10:53239 h265_decoder.payload_type = kH265PayloadType;
Evan Shrubsole1c184772022-04-26 07:47:49240 h265_decoder.video_format = SdpVideoFormat("H265");
Sergio Garcia Murillob5289d72024-10-24 12:10:53241 VideoReceiveStreamInterface::Decoder av1_decoder;
242 av1_decoder.payload_type = kAv1PayloadType;
243 av1_decoder.video_format = SdpVideoFormat("AV1");
Evan Shrubsole1c184772022-04-26 07:47:49244
Sergio Garcia Murillob5289d72024-10-24 12:10:53245 config_.decoders = {av1_decoder, h265_decoder, h264_decoder};
Tommiae4d0972020-05-18 06:45:38246
Evan Shrubsole44be5792022-04-26 14:24:41247 RecreateReceiveStream();
Evan Shrubsole14ee8032022-04-26 14:01:44248 }
249
Tommif6f45432022-05-20 13:21:20250 void RecreateReceiveStream(
Florent Castelli8037fc62024-08-29 13:00:40251 std::optional<VideoReceiveStreamInterface::RecordingState> state =
252 std::nullopt) {
Evan Shrubsole14ee8032022-04-26 14:01:44253 if (video_receive_stream_) {
254 video_receive_stream_->UnregisterFromTransport();
255 video_receive_stream_ = nullptr;
256 }
Danil Chapovalov22333492023-12-07 11:10:46257 timing_ = new VCMTiming(&env_.clock(), env_.field_trials());
Evan Shrubsole2b87a422025-05-09 10:35:38258 video_receive_stream_ = std::make_unique<internal::VideoReceiveStream2>(
259 env_, &fake_call_, kDefaultNumCpuCores, &packet_router_, config_.Copy(),
260 &call_stats_, absl::WrapUnique(timing_), &nack_periodic_processor_,
261 UseMetronome() ? &decode_sync_ : nullptr);
Tommi90738dd2021-05-31 15:36:47262 video_receive_stream_->RegisterWithTransport(
263 &rtp_stream_receiver_controller_);
Evan Shrubsole44be5792022-04-26 14:24:41264 if (state)
265 video_receive_stream_->SetAndGetRecordingState(std::move(*state), false);
Tommiae4d0972020-05-18 06:45:38266 }
267
268 protected:
Evan Shrubsole44be5792022-04-26 14:24:41269 GlobalSimulatedTimeController time_controller_;
Danil Chapovalov22333492023-12-07 11:10:46270 Environment env_;
Markus Handell0e62f7a2021-07-20 11:32:02271 NackPeriodicProcessor nack_periodic_processor_;
Sergio Garcia Murillob5289d72024-10-24 12:10:53272 testing::NiceMock<MockVideoDecoderFactory> mock_decoder_factory_;
Tommif6f45432022-05-20 13:21:20273 VideoReceiveStreamInterface::Config config_;
Tommiae4d0972020-05-18 06:45:38274 internal::CallStats call_stats_;
Evan Shrubsolea4062722022-05-02 15:46:07275 testing::NiceMock<MockVideoDecoder> mock_decoder_;
Evan Shrubsole44be5792022-04-26 14:24:41276 FakeVideoRenderer fake_renderer_;
Evan Shrubsolee4b09d72025-03-17 13:25:49277 FakeCall fake_call_;
Tommiae4d0972020-05-18 06:45:38278 MockTransport mock_transport_;
philipel27b35a72022-07-05 07:59:55279 test::RtcpPacketParser rtcp_packet_parser_;
Tommiae4d0972020-05-18 06:45:38280 PacketRouter packet_router_;
281 RtpStreamReceiverController rtp_stream_receiver_controller_;
Evan Shrubsole2b87a422025-05-09 10:35:38282 std::unique_ptr<internal::VideoReceiveStream2> video_receive_stream_;
Tommiae4d0972020-05-18 06:45:38283 VCMTiming* timing_;
Evan Shrubsolea4062722022-05-02 15:46:07284 test::FakeMetronome fake_metronome_;
285 DecodeSynchronizer decode_sync_;
Evan Shrubsole1c184772022-04-26 07:47:49286
287 private:
288 test::VideoDecoderProxyFactory h264_decoder_factory_;
Evan Shrubsole14ee8032022-04-26 14:01:44289 test::FakeDecoder fake_decoder_;
Tommiae4d0972020-05-18 06:45:38290};
291
Evan Shrubsolea4062722022-05-02 15:46:07292TEST_P(VideoReceiveStream2Test, CreateFrameFromH264FmtpSpropAndIdr) {
Tommiae4d0972020-05-18 06:45:38293 constexpr uint8_t idr_nalu[] = {0x05, 0xFF, 0xFF, 0xFF};
294 RtpPacketToSend rtppacket(nullptr);
Danil Chapovalovadf7d6e2025-07-17 15:20:53295 rtppacket.SetPayload(idr_nalu);
Tommiae4d0972020-05-18 06:45:38296 rtppacket.SetMarker(true);
Sergio Garcia Murillob5289d72024-10-24 12:10:53297 rtppacket.SetSsrc(kRemoteSsrc);
298 rtppacket.SetPayloadType(kH264PayloadType);
Tommiae4d0972020-05-18 06:45:38299 rtppacket.SetSequenceNumber(1);
300 rtppacket.SetTimestamp(0);
Evan Shrubsolea4062722022-05-02 15:46:07301 EXPECT_CALL(mock_decoder_, RegisterDecodeCompleteCallback(_));
Tommiae4d0972020-05-18 06:45:38302 video_receive_stream_->Start();
Tony Herre55b593f2023-08-29 14:05:49303 EXPECT_CALL(mock_decoder_, Decode(_, _));
Tommiae4d0972020-05-18 06:45:38304 RtpPacketReceived parsed_packet;
305 ASSERT_TRUE(parsed_packet.Parse(rtppacket.data(), rtppacket.size()));
306 rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
Evan Shrubsolea4062722022-05-02 15:46:07307 EXPECT_CALL(mock_decoder_, Release());
Evan Shrubsole44be5792022-04-26 14:24:41308
309 time_controller_.AdvanceTime(TimeDelta::Zero());
Tommiae4d0972020-05-18 06:45:38310}
311
Evan Shrubsolea4062722022-05-02 15:46:07312TEST_P(VideoReceiveStream2Test, PlayoutDelay) {
Danil Chapovalov7084e1b2023-08-23 11:16:22313 const VideoPlayoutDelay kPlayoutDelay(TimeDelta::Millis(123),
Danil Chapovalov87b7c1a2025-01-28 09:51:08314 TimeDelta::Millis(521));
Evan Shrubsolea0ee64c2022-04-26 08:09:04315 std::unique_ptr<test::FakeEncodedFrame> test_frame =
Danil Chapovalovc146b5f2023-08-16 09:42:52316 test::FakeFrameBuilder()
317 .Id(0)
Danil Chapovalov7084e1b2023-08-23 11:16:22318 .PlayoutDelay(kPlayoutDelay)
Danil Chapovalovc146b5f2023-08-16 09:42:52319 .AsLast()
320 .Build();
Tommiae4d0972020-05-18 06:45:38321
322 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
Evan Shrubsole8f1159b2022-03-22 11:12:17323 auto timings = timing_->GetTimings();
Danil Chapovalov7084e1b2023-08-23 11:16:22324 EXPECT_EQ(kPlayoutDelay.min(), timings.min_playout_delay);
325 EXPECT_EQ(kPlayoutDelay.max(), timings.max_playout_delay);
Tommiae4d0972020-05-18 06:45:38326
327 // Check that the biggest minimum delay is chosen.
Danil Chapovalovd1217342025-08-08 17:04:51328 video_receive_stream_->SetMinimumPlayoutDelay(TimeDelta::Millis(400));
Evan Shrubsole8f1159b2022-03-22 11:12:17329 timings = timing_->GetTimings();
330 EXPECT_EQ(400, timings.min_playout_delay.ms());
Tommiae4d0972020-05-18 06:45:38331
332 // Check base minimum delay validation.
333 EXPECT_FALSE(video_receive_stream_->SetBaseMinimumPlayoutDelayMs(12345));
334 EXPECT_FALSE(video_receive_stream_->SetBaseMinimumPlayoutDelayMs(-1));
335 EXPECT_TRUE(video_receive_stream_->SetBaseMinimumPlayoutDelayMs(500));
Evan Shrubsole8f1159b2022-03-22 11:12:17336 timings = timing_->GetTimings();
337 EXPECT_EQ(500, timings.min_playout_delay.ms());
Tommiae4d0972020-05-18 06:45:38338
339 // Check that intermidiate values are remembered and the biggest remembered
340 // is chosen.
341 video_receive_stream_->SetBaseMinimumPlayoutDelayMs(0);
Evan Shrubsole8f1159b2022-03-22 11:12:17342 timings = timing_->GetTimings();
343 EXPECT_EQ(400, timings.min_playout_delay.ms());
Tommiae4d0972020-05-18 06:45:38344
Danil Chapovalovd1217342025-08-08 17:04:51345 video_receive_stream_->SetMinimumPlayoutDelay(TimeDelta::Zero());
Evan Shrubsole8f1159b2022-03-22 11:12:17346 timings = timing_->GetTimings();
347 EXPECT_EQ(123, timings.min_playout_delay.ms());
Tommiae4d0972020-05-18 06:45:38348}
349
Danil Chapovalov87b7c1a2025-01-28 09:51:08350TEST_P(VideoReceiveStream2Test, MinPlayoutDelayIsLimitedByMaxPlayoutDelay) {
351 const VideoPlayoutDelay kPlayoutDelay(TimeDelta::Millis(123),
352 TimeDelta::Millis(321));
353 video_receive_stream_->OnCompleteFrame(test::FakeFrameBuilder()
354 .Id(0)
355 .PlayoutDelay(kPlayoutDelay)
356 .AsLast()
357 .Build());
358 EXPECT_EQ(timing_->GetTimings().min_playout_delay, kPlayoutDelay.min());
359
360 // Check that the biggest minimum delay is limited by the max playout delay.
Danil Chapovalovd1217342025-08-08 17:04:51361 video_receive_stream_->SetMinimumPlayoutDelay(TimeDelta::Millis(400));
Danil Chapovalov87b7c1a2025-01-28 09:51:08362 EXPECT_EQ(timing_->GetTimings().min_playout_delay, kPlayoutDelay.max());
363}
364
Johannes Kronbbf639e2022-06-15 10:27:23365TEST_P(VideoReceiveStream2Test, RenderParametersSetToDefaultValues) {
366 // Default render parameters.
367 const VideoFrame::RenderParameters kDefaultRenderParameters;
Johannes Kron111e9812020-10-26 12:54:40368 // Default with no playout delay set.
Evan Shrubsolea0ee64c2022-04-26 08:09:04369 std::unique_ptr<test::FakeEncodedFrame> test_frame0 =
370 test::FakeFrameBuilder().Id(0).AsLast().Build();
Johannes Kron111e9812020-10-26 12:54:40371 video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
Johannes Kronbbf639e2022-06-15 10:27:23372 EXPECT_EQ(timing_->RenderParameters(), kDefaultRenderParameters);
373}
Johannes Kron111e9812020-10-26 12:54:40374
Johannes Kronbbf639e2022-06-15 10:27:23375TEST_P(VideoReceiveStream2Test, UseLowLatencyRenderingSetFromPlayoutDelay) {
Johannes Kronbbf639e2022-06-15 10:27:23376 std::unique_ptr<test::FakeEncodedFrame> test_frame0 =
Danil Chapovalovc146b5f2023-08-16 09:42:52377 test::FakeFrameBuilder()
378 .Id(0)
379 .PlayoutDelay(VideoPlayoutDelay::Minimal())
380 .AsLast()
381 .Build();
Johannes Kronbbf639e2022-06-15 10:27:23382 video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
383 EXPECT_TRUE(timing_->RenderParameters().use_low_latency_rendering);
384
Evan Shrubsolea0ee64c2022-04-26 08:09:04385 std::unique_ptr<test::FakeEncodedFrame> test_frame1 =
Danil Chapovalovc146b5f2023-08-16 09:42:52386 test::FakeFrameBuilder()
387 .Id(1)
Danil Chapovalov7084e1b2023-08-23 11:16:22388 .PlayoutDelay({TimeDelta::Zero(), TimeDelta::Millis(500)})
Danil Chapovalovc146b5f2023-08-16 09:42:52389 .AsLast()
390 .Build();
Johannes Kron111e9812020-10-26 12:54:40391 video_receive_stream_->OnCompleteFrame(std::move(test_frame1));
Johannes Kronbbf639e2022-06-15 10:27:23392 EXPECT_TRUE(timing_->RenderParameters().use_low_latency_rendering);
Johannes Kron111e9812020-10-26 12:54:40393}
394
Evan Shrubsolea4062722022-05-02 15:46:07395TEST_P(VideoReceiveStream2Test, MaxCompositionDelaySetFromMaxPlayoutDelay) {
Johannes Kronbbf639e2022-06-15 10:27:23396 // The max composition delay is dependent on the number of frames in the
397 // pre-decode queue. It's therefore important to advance the time as the test
398 // runs to get the correct expectations of max_composition_delay_in_frames.
399 video_receive_stream_->Start();
400 // Max composition delay not set if no playout delay is set.
401 std::unique_ptr<test::FakeEncodedFrame> test_frame0 =
402 test::FakeFrameBuilder()
403 .Id(0)
404 .Time(RtpTimestampForFrame(0))
405 .ReceivedTime(ReceiveTimeForFrame(0))
406 .AsLast()
407 .Build();
408 video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
409 EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
Florent Castelli8037fc62024-08-29 13:00:40410 Eq(std::nullopt));
Johannes Kronbbf639e2022-06-15 10:27:23411 time_controller_.AdvanceTime(k30FpsDelay);
Johannes Kronbbf639e2022-06-15 10:27:23412
413 // Max composition delay not set for playout delay 0,0.
414 std::unique_ptr<test::FakeEncodedFrame> test_frame1 =
415 test::FakeFrameBuilder()
416 .Id(1)
417 .Time(RtpTimestampForFrame(1))
418 .ReceivedTime(ReceiveTimeForFrame(1))
Danil Chapovalov7084e1b2023-08-23 11:16:22419 .PlayoutDelay(VideoPlayoutDelay::Minimal())
Johannes Kronbbf639e2022-06-15 10:27:23420 .AsLast()
421 .Build();
Johannes Kronbbf639e2022-06-15 10:27:23422 video_receive_stream_->OnCompleteFrame(std::move(test_frame1));
423 EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
Florent Castelli8037fc62024-08-29 13:00:40424 Eq(std::nullopt));
Johannes Kronbbf639e2022-06-15 10:27:23425 time_controller_.AdvanceTime(k30FpsDelay);
Johannes Kronbbf639e2022-06-15 10:27:23426
427 // Max composition delay not set for playout delay X,Y, where X,Y>0.
428 std::unique_ptr<test::FakeEncodedFrame> test_frame2 =
429 test::FakeFrameBuilder()
430 .Id(2)
431 .Time(RtpTimestampForFrame(2))
432 .ReceivedTime(ReceiveTimeForFrame(2))
Danil Chapovalov7084e1b2023-08-23 11:16:22433 .PlayoutDelay({TimeDelta::Millis(10), TimeDelta::Millis(30)})
Johannes Kronbbf639e2022-06-15 10:27:23434 .AsLast()
435 .Build();
Johannes Kronbbf639e2022-06-15 10:27:23436 video_receive_stream_->OnCompleteFrame(std::move(test_frame2));
437 EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
Florent Castelli8037fc62024-08-29 13:00:40438 Eq(std::nullopt));
Johannes Kronbbf639e2022-06-15 10:27:23439
440 time_controller_.AdvanceTime(k30FpsDelay);
Johannes Kronbbf639e2022-06-15 10:27:23441
Johannes Kron111e9812020-10-26 12:54:40442 // Max composition delay set if playout delay X,Y, where X=0,Y>0.
Johannes Kron111e9812020-10-26 12:54:40443 const int kExpectedMaxCompositionDelayInFrames = 3; // ~50 ms at 60 fps.
Johannes Kronbbf639e2022-06-15 10:27:23444 std::unique_ptr<test::FakeEncodedFrame> test_frame3 =
445 test::FakeFrameBuilder()
446 .Id(3)
447 .Time(RtpTimestampForFrame(3))
448 .ReceivedTime(ReceiveTimeForFrame(3))
Danil Chapovalov7084e1b2023-08-23 11:16:22449 .PlayoutDelay({TimeDelta::Zero(), TimeDelta::Millis(50)})
Johannes Kronbbf639e2022-06-15 10:27:23450 .AsLast()
451 .Build();
Johannes Kronbbf639e2022-06-15 10:27:23452 video_receive_stream_->OnCompleteFrame(std::move(test_frame3));
453 EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
454 Optional(kExpectedMaxCompositionDelayInFrames));
Johannes Kron111e9812020-10-26 12:54:40455}
456
Evan Shrubsolea4062722022-05-02 15:46:07457TEST_P(VideoReceiveStream2Test, LazyDecoderCreation) {
Evan Shrubsole1c184772022-04-26 07:47:49458 constexpr uint8_t idr_nalu[] = {0x05, 0xFF, 0xFF, 0xFF};
459 RtpPacketToSend rtppacket(nullptr);
Danil Chapovalovadf7d6e2025-07-17 15:20:53460 rtppacket.SetPayload(idr_nalu);
Evan Shrubsole1c184772022-04-26 07:47:49461 rtppacket.SetMarker(true);
Sergio Garcia Murillob5289d72024-10-24 12:10:53462 rtppacket.SetSsrc(kRemoteSsrc);
463 rtppacket.SetPayloadType(kH264PayloadType);
Evan Shrubsole1c184772022-04-26 07:47:49464 rtppacket.SetSequenceNumber(1);
465 rtppacket.SetTimestamp(0);
466
Johannes Kronbb591c42022-09-28 10:10:25467 // No decoders are created by default.
Sergio Garcia Murillob5289d72024-10-24 12:10:53468 EXPECT_CALL(mock_decoder_factory_, Create).Times(0);
Evan Shrubsole1c184772022-04-26 07:47:49469 video_receive_stream_->Start();
Erik Språng7aaeb5a2022-08-18 11:18:27470 time_controller_.AdvanceTime(TimeDelta::Zero());
Evan Shrubsole1c184772022-04-26 07:47:49471
472 EXPECT_TRUE(
Sergio Garcia Murillob5289d72024-10-24 12:10:53473 testing::Mock::VerifyAndClearExpectations(&mock_decoder_factory_));
Johannes Kronbb591c42022-09-28 10:10:25474 // Verify that the decoder is created when we receive payload data and tries
475 // to decode a frame.
Sergio Garcia Murillob5289d72024-10-24 12:10:53476 EXPECT_CALL(mock_decoder_factory_,
Danil Chapovalov08173802024-01-23 12:02:46477 Create(_, Field(&SdpVideoFormat::name, Eq("H264"))));
Erik Språng7aaeb5a2022-08-18 11:18:27478 EXPECT_CALL(mock_decoder_, Configure);
479 EXPECT_CALL(mock_decoder_, RegisterDecodeCompleteCallback);
Tony Herre55b593f2023-08-29 14:05:49480 EXPECT_CALL(mock_decoder_, Decode(_, _));
Evan Shrubsole1c184772022-04-26 07:47:49481 RtpPacketReceived parsed_packet;
482 ASSERT_TRUE(parsed_packet.Parse(rtppacket.data(), rtppacket.size()));
483 rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
Erik Språng7aaeb5a2022-08-18 11:18:27484 EXPECT_CALL(mock_decoder_, Release);
Evan Shrubsole1c184772022-04-26 07:47:49485
486 // Make sure the decoder thread had a chance to run.
Erik Språng7aaeb5a2022-08-18 11:18:27487 time_controller_.AdvanceTime(TimeDelta::Zero());
Evan Shrubsole1c184772022-04-26 07:47:49488}
489
Sergio Garcia Murillob5289d72024-10-24 12:10:53490TEST_P(VideoReceiveStream2Test, LazyDecoderCreationCodecSwitch) {
491 constexpr uint8_t idr_nalu[] = {0x05, 0xFF, 0xFF, 0xFF};
492 RtpPacketToSend rtppacket(nullptr);
Danil Chapovalovadf7d6e2025-07-17 15:20:53493 rtppacket.SetPayload(idr_nalu);
Sergio Garcia Murillob5289d72024-10-24 12:10:53494 rtppacket.SetMarker(true);
495 rtppacket.SetSsrc(kRemoteSsrc);
496 rtppacket.SetPayloadType(kH264PayloadType);
497 rtppacket.SetSequenceNumber(1);
498 rtppacket.SetTimestamp(0);
499
500 // No decoders are created by default.
501 EXPECT_CALL(mock_decoder_factory_, Create).Times(0);
502 video_receive_stream_->Start();
503 time_controller_.AdvanceTime(TimeDelta::Zero());
504
505 EXPECT_TRUE(
506 testing::Mock::VerifyAndClearExpectations(&mock_decoder_factory_));
507 // Verify that the decoder is created when we receive payload data and tries
508 // to decode a frame.
509 EXPECT_CALL(mock_decoder_factory_,
510 Create(_, Field(&SdpVideoFormat::name, Eq("H264"))));
511 EXPECT_CALL(mock_decoder_, Configure);
512 EXPECT_CALL(mock_decoder_, RegisterDecodeCompleteCallback);
513 EXPECT_CALL(mock_decoder_, Decode(_, _));
514 RtpPacketReceived parsed_packet;
515 ASSERT_TRUE(parsed_packet.Parse(rtppacket.data(), rtppacket.size()));
516 rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
517 // H264 decoder is released after receiving the AV1 packet.
518 EXPECT_CALL(mock_decoder_, Release).Times(0);
519
520 // Make sure the decoder thread had a chance to run.
521 time_controller_.AdvanceTime(TimeDelta::Millis(100));
522
523 // Switch to AV1.
524 const uint8_t av1_key_obu[] = {0x18, 0x48, 0x01, 0xAA}; // \ OBU
525 RtpPacketToSend av1_rtppacket(nullptr);
Danil Chapovalovadf7d6e2025-07-17 15:20:53526 av1_rtppacket.SetPayload(av1_key_obu);
Sergio Garcia Murillob5289d72024-10-24 12:10:53527 av1_rtppacket.SetMarker(true);
528 av1_rtppacket.SetSsrc(kRemoteSsrc);
529 av1_rtppacket.SetPayloadType(kAv1PayloadType);
530 av1_rtppacket.SetSequenceNumber(2);
531 av1_rtppacket.SetTimestamp(1);
532
533 EXPECT_TRUE(
534 testing::Mock::VerifyAndClearExpectations(&mock_decoder_factory_));
535 // Release the H264 previous decoder.
536 EXPECT_CALL(mock_decoder_, Release);
537 // Verify that the decoder is created when we receive payload data and tries
538 // to decode a frame.
539 EXPECT_CALL(mock_decoder_factory_,
540 Create(_, Field(&SdpVideoFormat::name, Eq("AV1"))));
541 EXPECT_CALL(mock_decoder_, Configure);
542 EXPECT_CALL(mock_decoder_, RegisterDecodeCompleteCallback);
543 EXPECT_CALL(mock_decoder_, Decode(_, _));
544 ASSERT_TRUE(parsed_packet.Parse(av1_rtppacket.data(), av1_rtppacket.size()));
545 rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
546
547 // Make sure the decoder thread had a chance to run.
548 time_controller_.AdvanceTime(TimeDelta::Millis(100));
549
550 // Switch back to H264.
551 rtppacket.SetPayloadType(kH264PayloadType);
552 rtppacket.SetSequenceNumber(3);
553 rtppacket.SetTimestamp(2);
554
555 EXPECT_TRUE(
556 testing::Mock::VerifyAndClearExpectations(&mock_decoder_factory_));
557 // Release the AV1 previous decoder and the new H264 decoder on test end.
558 EXPECT_CALL(mock_decoder_, Release).Times(2);
559 // Verify that the decoder is created when we receive payload data and tries
560 // to decode a frame.
561 EXPECT_CALL(mock_decoder_factory_,
562 Create(_, Field(&SdpVideoFormat::name, Eq("H264"))));
563 EXPECT_CALL(mock_decoder_, Configure);
564 EXPECT_CALL(mock_decoder_, RegisterDecodeCompleteCallback);
565 EXPECT_CALL(mock_decoder_, Decode(_, _));
566 ASSERT_TRUE(parsed_packet.Parse(rtppacket.data(), rtppacket.size()));
567 rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
568
569 // Make sure the decoder thread had a chance to run.
570 time_controller_.AdvanceTime(TimeDelta::Millis(100));
571}
572
Evan Shrubsolea4062722022-05-02 15:46:07573TEST_P(VideoReceiveStream2Test, PassesNtpTime) {
Evan Shrubsolea0ee64c2022-04-26 08:09:04574 const Timestamp kNtpTimestamp = Timestamp::Millis(12345);
575 std::unique_ptr<test::FakeEncodedFrame> test_frame =
576 test::FakeFrameBuilder()
577 .Id(0)
Sergio Garcia Murillob5289d72024-10-24 12:10:53578 .PayloadType(kH264PayloadType)
Evan Shrubsolea0ee64c2022-04-26 08:09:04579 .NtpTime(kNtpTimestamp)
580 .AsLast()
581 .Build();
Tommiae4d0972020-05-18 06:45:38582
583 video_receive_stream_->Start();
584 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
Evan Shrubsolea4062722022-05-02 15:46:07585 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
Evan Shrubsole44be5792022-04-26 14:24:41586 RenderedFrameWith(NtpTimestamp(kNtpTimestamp)));
Tommiae4d0972020-05-18 06:45:38587}
588
Evan Shrubsolea4062722022-05-02 15:46:07589TEST_P(VideoReceiveStream2Test, PassesRotation) {
Evan Shrubsole2b87a422025-05-09 10:35:38590 const VideoRotation kRotation = kVideoRotation_180;
Sergio Garcia Murillob5289d72024-10-24 12:10:53591 std::unique_ptr<test::FakeEncodedFrame> test_frame =
592 test::FakeFrameBuilder()
593 .Id(0)
594 .PayloadType(kH264PayloadType)
595 .Rotation(kRotation)
596 .AsLast()
597 .Build();
Tommiae4d0972020-05-18 06:45:38598
599 video_receive_stream_->Start();
600 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
Evan Shrubsolea4062722022-05-02 15:46:07601 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
Evan Shrubsole44be5792022-04-26 14:24:41602 RenderedFrameWith(Rotation(kRotation)));
Tommiae4d0972020-05-18 06:45:38603}
604
Evan Shrubsolea4062722022-05-02 15:46:07605TEST_P(VideoReceiveStream2Test, PassesPacketInfos) {
Tommiae4d0972020-05-18 06:45:38606 RtpPacketInfos packet_infos = CreatePacketInfos(3);
Evan Shrubsolea0ee64c2022-04-26 08:09:04607 auto test_frame = test::FakeFrameBuilder()
608 .Id(0)
Sergio Garcia Murillob5289d72024-10-24 12:10:53609 .PayloadType(kH264PayloadType)
Evan Shrubsolea0ee64c2022-04-26 08:09:04610 .PacketInfos(packet_infos)
611 .AsLast()
612 .Build();
Tommiae4d0972020-05-18 06:45:38613
614 video_receive_stream_->Start();
615 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
Evan Shrubsolea4062722022-05-02 15:46:07616 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
Evan Shrubsole44be5792022-04-26 14:24:41617 RenderedFrameWith(PacketInfos(ElementsAreArray(packet_infos))));
Tommiae4d0972020-05-18 06:45:38618}
619
Evan Shrubsolea4062722022-05-02 15:46:07620TEST_P(VideoReceiveStream2Test, RenderedFrameUpdatesGetSources) {
Sergio Garcia Murillob5289d72024-10-24 12:10:53621 constexpr uint32_t kSsrc = kRemoteSsrc;
Tommiae4d0972020-05-18 06:45:38622 constexpr uint32_t kCsrc = 9001;
623 constexpr uint32_t kRtpTimestamp = 12345;
624
625 // Prepare one video frame with per-packet information.
Sergio Garcia Murillob5289d72024-10-24 12:10:53626 auto test_frame = test::FakeFrameBuilder()
627 .Id(0)
628 .PayloadType(kH264PayloadType)
629 .AsLast()
630 .Build();
Tommiae4d0972020-05-18 06:45:38631 RtpPacketInfos packet_infos;
632 {
633 RtpPacketInfos::vector_type infos;
634
635 RtpPacketInfo info;
636 info.set_ssrc(kSsrc);
637 info.set_csrcs({kCsrc});
638 info.set_rtp_timestamp(kRtpTimestamp);
639
Danil Chapovalov22333492023-12-07 11:10:46640 info.set_receive_time(env_.clock().CurrentTime() - TimeDelta::Millis(5000));
Tommiae4d0972020-05-18 06:45:38641 infos.push_back(info);
642
Danil Chapovalov22333492023-12-07 11:10:46643 info.set_receive_time(env_.clock().CurrentTime() - TimeDelta::Millis(3000));
Tommiae4d0972020-05-18 06:45:38644 infos.push_back(info);
645
Danil Chapovalov22333492023-12-07 11:10:46646 info.set_receive_time(env_.clock().CurrentTime() - TimeDelta::Millis(2000));
Tommiae4d0972020-05-18 06:45:38647 infos.push_back(info);
648
Danil Chapovalov22333492023-12-07 11:10:46649 info.set_receive_time(env_.clock().CurrentTime() - TimeDelta::Millis(1000));
Tommiae4d0972020-05-18 06:45:38650 infos.push_back(info);
651
652 packet_infos = RtpPacketInfos(std::move(infos));
653 }
654 test_frame->SetPacketInfos(packet_infos);
655
656 // Start receive stream.
657 video_receive_stream_->Start();
658 EXPECT_THAT(video_receive_stream_->GetSources(), IsEmpty());
659
660 // Render one video frame.
Danil Chapovalov22333492023-12-07 11:10:46661 Timestamp timestamp_min = env_.clock().CurrentTime();
Tommiae4d0972020-05-18 06:45:38662 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
Tommiae4d0972020-05-18 06:45:38663 // Verify that the per-packet information is passed to the renderer.
Evan Shrubsolea4062722022-05-02 15:46:07664 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
Evan Shrubsole44be5792022-04-26 14:24:41665 RenderedFrameWith(PacketInfos(ElementsAreArray(packet_infos))));
Danil Chapovalov22333492023-12-07 11:10:46666 Timestamp timestamp_max = env_.clock().CurrentTime();
Tommiae4d0972020-05-18 06:45:38667
Artem Titovab30d722021-07-27 14:22:11668 // Verify that the per-packet information also updates `GetSources()`.
Tommiae4d0972020-05-18 06:45:38669 std::vector<RtpSource> sources = video_receive_stream_->GetSources();
670 ASSERT_THAT(sources, SizeIs(2));
671 {
672 auto it = std::find_if(sources.begin(), sources.end(),
673 [](const RtpSource& source) {
674 return source.source_type() == RtpSourceType::SSRC;
675 });
676 ASSERT_NE(it, sources.end());
677
678 EXPECT_EQ(it->source_id(), kSsrc);
679 EXPECT_EQ(it->source_type(), RtpSourceType::SSRC);
680 EXPECT_EQ(it->rtp_timestamp(), kRtpTimestamp);
Danil Chapovalovf5359712023-08-28 16:32:25681 EXPECT_GE(it->timestamp(), timestamp_min);
682 EXPECT_LE(it->timestamp(), timestamp_max);
Tommiae4d0972020-05-18 06:45:38683 }
684 {
685 auto it = std::find_if(sources.begin(), sources.end(),
686 [](const RtpSource& source) {
687 return source.source_type() == RtpSourceType::CSRC;
688 });
689 ASSERT_NE(it, sources.end());
690
691 EXPECT_EQ(it->source_id(), kCsrc);
692 EXPECT_EQ(it->source_type(), RtpSourceType::CSRC);
693 EXPECT_EQ(it->rtp_timestamp(), kRtpTimestamp);
Danil Chapovalovf5359712023-08-28 16:32:25694 EXPECT_GE(it->timestamp(), timestamp_min);
695 EXPECT_LE(it->timestamp(), timestamp_max);
Tommiae4d0972020-05-18 06:45:38696 }
697}
698
Evan Shrubsolea0ee64c2022-04-26 08:09:04699std::unique_ptr<test::FakeEncodedFrame> MakeFrameWithResolution(
Markus Handell588f9b32021-04-08 17:19:50700 VideoFrameType frame_type,
701 int picture_id,
702 int width,
703 int height) {
Sergio Garcia Murillob5289d72024-10-24 12:10:53704 auto frame = test::FakeFrameBuilder()
705 .Id(picture_id)
706 .PayloadType(kH264PayloadType)
707 .AsLast()
708 .Build();
Tommiae4d0972020-05-18 06:45:38709 frame->SetFrameType(frame_type);
Markus Handell588f9b32021-04-08 17:19:50710 frame->_encodedWidth = width;
711 frame->_encodedHeight = height;
Tommiae4d0972020-05-18 06:45:38712 return frame;
713}
714
Evan Shrubsolea0ee64c2022-04-26 08:09:04715std::unique_ptr<test::FakeEncodedFrame> MakeFrame(VideoFrameType frame_type,
716 int picture_id) {
Markus Handell588f9b32021-04-08 17:19:50717 return MakeFrameWithResolution(frame_type, picture_id, 320, 240);
718}
719
Evan Shrubsolea4062722022-05-02 15:46:07720TEST_P(VideoReceiveStream2Test, PassesFrameWhenEncodedFramesCallbackSet) {
Tommiae4d0972020-05-18 06:45:38721 testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
722 video_receive_stream_->Start();
Tommiae4d0972020-05-18 06:45:38723 EXPECT_CALL(callback, Call);
724 video_receive_stream_->SetAndGetRecordingState(
Tommif6f45432022-05-20 13:21:20725 VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
726 true);
Tommiae4d0972020-05-18 06:45:38727 video_receive_stream_->OnCompleteFrame(
728 MakeFrame(VideoFrameType::kVideoFrameKey, 0));
Evan Shrubsolea4062722022-05-02 15:46:07729 EXPECT_TRUE(fake_renderer_.WaitForFrame(kDefaultTimeOut));
philipel27b35a72022-07-05 07:59:55730
731 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(1));
732
Tommiae4d0972020-05-18 06:45:38733 video_receive_stream_->Stop();
734}
735
Evan Shrubsolea4062722022-05-02 15:46:07736TEST_P(VideoReceiveStream2Test, MovesEncodedFrameDispatchStateWhenReCreating) {
Tommiae4d0972020-05-18 06:45:38737 testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
738 video_receive_stream_->Start();
739 // Expect a key frame request over RTCP.
Tommiae4d0972020-05-18 06:45:38740 video_receive_stream_->SetAndGetRecordingState(
Tommif6f45432022-05-20 13:21:20741 VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
742 true);
Tommiae4d0972020-05-18 06:45:38743 video_receive_stream_->Stop();
Tommif6f45432022-05-20 13:21:20744 VideoReceiveStreamInterface::RecordingState old_state =
Tommiae4d0972020-05-18 06:45:38745 video_receive_stream_->SetAndGetRecordingState(
Tommif6f45432022-05-20 13:21:20746 VideoReceiveStreamInterface::RecordingState(), false);
Evan Shrubsole44be5792022-04-26 14:24:41747 RecreateReceiveStream(std::move(old_state));
philipel27b35a72022-07-05 07:59:55748
749 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(1));
750
Tommiae4d0972020-05-18 06:45:38751 video_receive_stream_->Stop();
752}
753
Evan Shrubsolea4062722022-05-02 15:46:07754TEST_P(VideoReceiveStream2Test, RequestsKeyFramesUntilKeyFrameReceived) {
Evan Shrubsole44be5792022-04-26 14:24:41755 // Recreate receive stream with shorter delay to test rtx.
756 TimeDelta rtx_delay = TimeDelta::Millis(50);
757 config_.rtp.nack.rtp_history_ms = rtx_delay.ms();
758 auto tick = rtx_delay / 2;
759 RecreateReceiveStream();
760 video_receive_stream_->Start();
Markus Handell588f9b32021-04-08 17:19:50761
Evan Shrubsole44be5792022-04-26 14:24:41762 video_receive_stream_->GenerateKeyFrame();
763 video_receive_stream_->OnCompleteFrame(
764 MakeFrame(VideoFrameType::kVideoFrameDelta, 0));
Evan Shrubsolea4062722022-05-02 15:46:07765 fake_renderer_.WaitForFrame(kDefaultTimeOut);
Tommiae4d0972020-05-18 06:45:38766 time_controller_.AdvanceTime(tick);
Evan Shrubsole44be5792022-04-26 14:24:41767 video_receive_stream_->OnCompleteFrame(
768 MakeFrame(VideoFrameType::kVideoFrameDelta, 1));
Evan Shrubsolea4062722022-05-02 15:46:07769 fake_renderer_.WaitForFrame(kDefaultTimeOut);
Evan Shrubsole44be5792022-04-26 14:24:41770 time_controller_.AdvanceTime(TimeDelta::Zero());
Tommiae4d0972020-05-18 06:45:38771 testing::Mock::VerifyAndClearExpectations(&mock_transport_);
772
philipel27b35a72022-07-05 07:59:55773 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(1));
774
Philipp Hancke006206d2021-03-24 16:49:02775 // T+keyframetimeout: still no key frame received, expect key frame request
776 // sent again.
Tommiae4d0972020-05-18 06:45:38777 time_controller_.AdvanceTime(tick);
Evan Shrubsole44be5792022-04-26 14:24:41778 video_receive_stream_->OnCompleteFrame(
779 MakeFrame(VideoFrameType::kVideoFrameDelta, 2));
Evan Shrubsolea4062722022-05-02 15:46:07780 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Tommiae4d0972020-05-18 06:45:38781 testing::Mock::VerifyAndClearExpectations(&mock_transport_);
782
philipel27b35a72022-07-05 07:59:55783 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(2));
784
Philipp Hancke006206d2021-03-24 16:49:02785 // T+keyframetimeout: now send a key frame - we should not observe new key
786 // frame requests after this.
Evan Shrubsole44be5792022-04-26 14:24:41787 video_receive_stream_->OnCompleteFrame(
788 MakeFrame(VideoFrameType::kVideoFrameKey, 3));
Evan Shrubsolea4062722022-05-02 15:46:07789 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Tommiae4d0972020-05-18 06:45:38790 time_controller_.AdvanceTime(2 * tick);
Evan Shrubsole44be5792022-04-26 14:24:41791 video_receive_stream_->OnCompleteFrame(
792 MakeFrame(VideoFrameType::kVideoFrameDelta, 4));
Evan Shrubsolea4062722022-05-02 15:46:07793 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
philipel27b35a72022-07-05 07:59:55794
795 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(2));
Tommiae4d0972020-05-18 06:45:38796}
797
Evan Shrubsolea4062722022-05-02 15:46:07798TEST_P(VideoReceiveStream2Test,
Markus Handell588f9b32021-04-08 17:19:50799 DispatchesEncodedFrameSequenceStartingWithKeyframeWithoutResolution) {
Evan Shrubsole44be5792022-04-26 14:24:41800 video_receive_stream_->Start();
Markus Handell588f9b32021-04-08 17:19:50801 testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
Evan Shrubsole44be5792022-04-26 14:24:41802 video_receive_stream_->SetAndGetRecordingState(
Tommif6f45432022-05-20 13:21:20803 VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
Markus Handell588f9b32021-04-08 17:19:50804 /*generate_key_frame=*/false);
805
806 InSequence s;
Jonas Oreland0deda152022-09-23 10:08:57807 EXPECT_CALL(callback,
808 Call(MatchResolution(test::FakeDecoder::kDefaultWidth,
809 test::FakeDecoder::kDefaultHeight)));
Markus Handell588f9b32021-04-08 17:19:50810 EXPECT_CALL(callback, Call);
811
Evan Shrubsole44be5792022-04-26 14:24:41812 video_receive_stream_->OnCompleteFrame(
Markus Handell588f9b32021-04-08 17:19:50813 MakeFrameWithResolution(VideoFrameType::kVideoFrameKey, 0, 0, 0));
Evan Shrubsolea4062722022-05-02 15:46:07814 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Evan Shrubsole44be5792022-04-26 14:24:41815 video_receive_stream_->OnCompleteFrame(
Markus Handell588f9b32021-04-08 17:19:50816 MakeFrameWithResolution(VideoFrameType::kVideoFrameDelta, 1, 0, 0));
Evan Shrubsolea4062722022-05-02 15:46:07817 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Markus Handell588f9b32021-04-08 17:19:50818
Evan Shrubsole44be5792022-04-26 14:24:41819 video_receive_stream_->Stop();
Markus Handell588f9b32021-04-08 17:19:50820}
821
Evan Shrubsolea4062722022-05-02 15:46:07822TEST_P(VideoReceiveStream2Test,
Markus Handell588f9b32021-04-08 17:19:50823 DispatchesEncodedFrameSequenceStartingWithKeyframeWithResolution) {
Evan Shrubsole44be5792022-04-26 14:24:41824 video_receive_stream_->Start();
Markus Handell588f9b32021-04-08 17:19:50825 testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
Evan Shrubsole44be5792022-04-26 14:24:41826 video_receive_stream_->SetAndGetRecordingState(
Tommif6f45432022-05-20 13:21:20827 VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
Markus Handell588f9b32021-04-08 17:19:50828 /*generate_key_frame=*/false);
829
830 InSequence s;
Jonas Oreland0deda152022-09-23 10:08:57831 EXPECT_CALL(callback, Call(MatchResolution(1080u, 720u)));
Markus Handell588f9b32021-04-08 17:19:50832 EXPECT_CALL(callback, Call);
833
Evan Shrubsole44be5792022-04-26 14:24:41834 video_receive_stream_->OnCompleteFrame(
Markus Handell588f9b32021-04-08 17:19:50835 MakeFrameWithResolution(VideoFrameType::kVideoFrameKey, 0, 1080, 720));
Evan Shrubsolea4062722022-05-02 15:46:07836 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Evan Shrubsole44be5792022-04-26 14:24:41837 video_receive_stream_->OnCompleteFrame(
Markus Handell588f9b32021-04-08 17:19:50838 MakeFrameWithResolution(VideoFrameType::kVideoFrameDelta, 1, 0, 0));
Evan Shrubsolea4062722022-05-02 15:46:07839 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Markus Handell588f9b32021-04-08 17:19:50840
Evan Shrubsole44be5792022-04-26 14:24:41841 video_receive_stream_->Stop();
Markus Handell588f9b32021-04-08 17:19:50842}
843
Evan Shrubsolea4062722022-05-02 15:46:07844TEST_P(VideoReceiveStream2Test, DependantFramesAreScheduled) {
845 video_receive_stream_->Start();
846
847 auto key_frame = test::FakeFrameBuilder()
848 .Id(0)
Sergio Garcia Murillob5289d72024-10-24 12:10:53849 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:07850 .Time(kFirstRtpTimestamp)
851 .ReceivedTime(kStartTime)
852 .AsLast()
853 .Build();
854 auto delta_frame = test::FakeFrameBuilder()
855 .Id(1)
Sergio Garcia Murillob5289d72024-10-24 12:10:53856 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:07857 .Time(RtpTimestampForFrame(1))
858 .ReceivedTime(ReceiveTimeForFrame(1))
859 .Refs({0})
860 .AsLast()
861 .Build();
862
863 // Expect frames are decoded in order.
864 InSequence seq;
Tony Herre55b593f2023-08-29 14:05:49865 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp), _));
Evan Shrubsolea4062722022-05-02 15:46:07866 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp +
867 k30FpsRtpTimestampDelta),
Tony Herre55b593f2023-08-29 14:05:49868 _))
Evan Shrubsolea4062722022-05-02 15:46:07869 .Times(1);
870 video_receive_stream_->OnCompleteFrame(std::move(key_frame));
871 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
872
873 time_controller_.AdvanceTime(k30FpsDelay);
874 video_receive_stream_->OnCompleteFrame(std::move(delta_frame));
875 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
876
877 video_receive_stream_->Stop();
878}
879
880TEST_P(VideoReceiveStream2Test, FramesScheduledInOrder) {
881 video_receive_stream_->Start();
882
883 auto key_frame = test::FakeFrameBuilder()
884 .Id(0)
Sergio Garcia Murillob5289d72024-10-24 12:10:53885 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:07886 .Time(kFirstRtpTimestamp)
887 .AsLast()
888 .Build();
889 auto delta_frame1 = test::FakeFrameBuilder()
890 .Id(1)
Sergio Garcia Murillob5289d72024-10-24 12:10:53891 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:07892 .Time(RtpTimestampForFrame(1))
893 .Refs({0})
894 .AsLast()
895 .Build();
896 auto delta_frame2 = test::FakeFrameBuilder()
897 .Id(2)
Sergio Garcia Murillob5289d72024-10-24 12:10:53898 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:07899 .Time(RtpTimestampForFrame(2))
900 .Refs({1})
901 .AsLast()
902 .Build();
903
904 // Expect frames are decoded in order despite delta_frame1 arriving first.
905 InSequence seq;
Tony Herre55b593f2023-08-29 14:05:49906 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp), _))
Evan Shrubsolea4062722022-05-02 15:46:07907 .Times(1);
908 EXPECT_CALL(mock_decoder_,
Tony Herre55b593f2023-08-29 14:05:49909 Decode(test::RtpTimestamp(RtpTimestampForFrame(1)), _))
Evan Shrubsolea4062722022-05-02 15:46:07910 .Times(1);
911 EXPECT_CALL(mock_decoder_,
Tony Herre55b593f2023-08-29 14:05:49912 Decode(test::RtpTimestamp(RtpTimestampForFrame(2)), _))
Evan Shrubsolea4062722022-05-02 15:46:07913 .Times(1);
Rasmus Brandtf6e100b2025-10-13 07:42:36914
915 // `key_frame` arrives at the start time...
Danil Chapovalov22333492023-12-07 11:10:46916 key_frame->SetReceivedTime(env_.clock().CurrentTime().ms());
Evan Shrubsolea4062722022-05-02 15:46:07917 video_receive_stream_->OnCompleteFrame(std::move(key_frame));
Rasmus Brandtf6e100b2025-10-13 07:42:36918 // ...and it is decoded and rendered right away.
Evan Shrubsolea4062722022-05-02 15:46:07919 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
920
Rasmus Brandtf6e100b2025-10-13 07:42:36921 // `delta_frame2` arrives on time, which is two inter-frame durations later...
922 time_controller_.AdvanceTime(2 * k30FpsDelay);
923 int64_t delta_frame2_received_time_ms = env_.clock().CurrentTime().ms();
924 delta_frame2->SetReceivedTime(delta_frame2_received_time_ms);
Evan Shrubsolea4062722022-05-02 15:46:07925 video_receive_stream_->OnCompleteFrame(std::move(delta_frame2));
Rasmus Brandtf6e100b2025-10-13 07:42:36926 // ...but it doesn't render, since it is not yet decodable.
927 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
928 DidNotReceiveFrame());
929
930 // `delta_frame1` arrives back-to-back with `delta_frame2`...
Danil Chapovalov22333492023-12-07 11:10:46931 delta_frame1->SetReceivedTime(env_.clock().CurrentTime().ms());
Rasmus Brandtf6e100b2025-10-13 07:42:36932 EXPECT_EQ(delta_frame1->ReceivedTime(), delta_frame2_received_time_ms);
Evan Shrubsolea4062722022-05-02 15:46:07933 video_receive_stream_->OnCompleteFrame(std::move(delta_frame1));
Rasmus Brandtf6e100b2025-10-13 07:42:36934 // ...so it is decoded and rendered right away (since it is late).
935 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
936 // And then a bit later, `delta_frame2` is decoded and rendered.
Evan Shrubsolea4062722022-05-02 15:46:07937 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
Rasmus Brandtf6e100b2025-10-13 07:42:36938
Evan Shrubsolea4062722022-05-02 15:46:07939 video_receive_stream_->Stop();
940}
941
942TEST_P(VideoReceiveStream2Test, WaitsforAllSpatialLayers) {
943 video_receive_stream_->Start();
944 auto sl0 = test::FakeFrameBuilder()
945 .Id(0)
Sergio Garcia Murillob5289d72024-10-24 12:10:53946 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:07947 .Time(kFirstRtpTimestamp)
948 .ReceivedTime(kStartTime)
949 .Build();
950 auto sl1 = test::FakeFrameBuilder()
951 .Id(1)
Sergio Garcia Murillob5289d72024-10-24 12:10:53952 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:07953 .ReceivedTime(kStartTime)
954 .Time(kFirstRtpTimestamp)
955 .Refs({0})
956 .Build();
957 auto sl2 = test::FakeFrameBuilder()
958 .Id(2)
Sergio Garcia Murillob5289d72024-10-24 12:10:53959 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:07960 .ReceivedTime(kStartTime)
961 .Time(kFirstRtpTimestamp)
962 .Refs({0, 1})
963 .AsLast()
964 .Build();
965
966 // No decodes should be called until `sl2` is received.
Tony Herre55b593f2023-08-29 14:05:49967 EXPECT_CALL(mock_decoder_, Decode(_, _)).Times(0);
Danil Chapovalov22333492023-12-07 11:10:46968 sl0->SetReceivedTime(env_.clock().CurrentTime().ms());
Evan Shrubsolea4062722022-05-02 15:46:07969 video_receive_stream_->OnCompleteFrame(std::move(sl0));
970 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
971 DidNotReceiveFrame());
972 video_receive_stream_->OnCompleteFrame(std::move(sl1));
973 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
974 DidNotReceiveFrame());
975 // When `sl2` arrives decode should happen.
Tony Herre55b593f2023-08-29 14:05:49976 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp), _))
Evan Shrubsolea4062722022-05-02 15:46:07977 .Times(1);
978 video_receive_stream_->OnCompleteFrame(std::move(sl2));
979 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
980 video_receive_stream_->Stop();
981}
982
983TEST_P(VideoReceiveStream2Test, FramesFastForwardOnSystemHalt) {
984 video_receive_stream_->Start();
985
986 // The frame structure looks like this,
987 // F1
988 // /
989 // F0 --> F2
990 //
991 // In this case we will have a system halt simulated. By the time the system
992 // resumes, F1 will be old and so F2 should be decoded.
993 auto key_frame = test::FakeFrameBuilder()
994 .Id(0)
Sergio Garcia Murillob5289d72024-10-24 12:10:53995 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:07996 .Time(kFirstRtpTimestamp)
997 .AsLast()
998 .Build();
999 auto ffwd_frame = test::FakeFrameBuilder()
1000 .Id(1)
Sergio Garcia Murillob5289d72024-10-24 12:10:531001 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:071002 .Time(RtpTimestampForFrame(1))
1003 .Refs({0})
1004 .AsLast()
1005 .Build();
1006 auto rendered_frame = test::FakeFrameBuilder()
1007 .Id(2)
Sergio Garcia Murillob5289d72024-10-24 12:10:531008 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:071009 .Time(RtpTimestampForFrame(2))
1010 .Refs({0})
1011 .AsLast()
1012 .Build();
1013 InSequence seq;
Tony Herre55b593f2023-08-29 14:05:491014 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp), _))
Ilya Nikolaevskiy47ad4b62025-10-17 10:13:551015 .WillOnce(testing::DoAll(
1016 [&] {
1017 // System halt will be simulated in the decode.
1018 time_controller_.AdvanceTime(k30FpsDelay * 2);
1019 },
1020 DefaultDecodeAction()));
Evan Shrubsolea4062722022-05-02 15:46:071021 EXPECT_CALL(mock_decoder_,
Tony Herre55b593f2023-08-29 14:05:491022 Decode(test::RtpTimestamp(RtpTimestampForFrame(2)), _));
Evan Shrubsolea4062722022-05-02 15:46:071023 video_receive_stream_->OnCompleteFrame(std::move(key_frame));
1024 video_receive_stream_->OnCompleteFrame(std::move(ffwd_frame));
1025 video_receive_stream_->OnCompleteFrame(std::move(rendered_frame));
Evan Shrubsole7cbd8de2022-08-16 08:08:531026 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
1027 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(0))));
1028 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
1029 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(2))));
Evan Shrubsolea4062722022-05-02 15:46:071030
1031 // Check stats show correct dropped frames.
1032 auto stats = video_receive_stream_->GetStats();
1033 EXPECT_EQ(stats.frames_dropped, 1u);
1034
1035 video_receive_stream_->Stop();
1036}
1037
1038TEST_P(VideoReceiveStream2Test, BetterFrameInsertedWhileWaitingToDecodeFrame) {
1039 video_receive_stream_->Start();
1040
1041 auto key_frame = test::FakeFrameBuilder()
1042 .Id(0)
Sergio Garcia Murillob5289d72024-10-24 12:10:531043 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:071044 .Time(kFirstRtpTimestamp)
1045 .ReceivedTime(ReceiveTimeForFrame(0))
1046 .AsLast()
1047 .Build();
1048 auto f1 = test::FakeFrameBuilder()
1049 .Id(1)
Sergio Garcia Murillob5289d72024-10-24 12:10:531050 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:071051 .Time(RtpTimestampForFrame(1))
1052 .ReceivedTime(ReceiveTimeForFrame(1))
1053 .Refs({0})
1054 .AsLast()
1055 .Build();
1056 auto f2 = test::FakeFrameBuilder()
1057 .Id(2)
Sergio Garcia Murillob5289d72024-10-24 12:10:531058 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:071059 .Time(RtpTimestampForFrame(2))
1060 .ReceivedTime(ReceiveTimeForFrame(2))
1061 .Refs({0})
1062 .AsLast()
1063 .Build();
1064
1065 video_receive_stream_->OnCompleteFrame(std::move(key_frame));
1066 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
1067
1068 InSequence seq;
1069 EXPECT_CALL(mock_decoder_,
Tony Herre55b593f2023-08-29 14:05:491070 Decode(test::RtpTimestamp(RtpTimestampForFrame(1)), _))
Evan Shrubsolea4062722022-05-02 15:46:071071 .Times(1);
1072 EXPECT_CALL(mock_decoder_,
Tony Herre55b593f2023-08-29 14:05:491073 Decode(test::RtpTimestamp(RtpTimestampForFrame(2)), _))
Evan Shrubsolea4062722022-05-02 15:46:071074 .Times(1);
1075 // Simulate f1 arriving after f2 but before f2 is decoded.
1076 video_receive_stream_->OnCompleteFrame(std::move(f2));
1077 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), DidNotReceiveFrame());
1078 video_receive_stream_->OnCompleteFrame(std::move(f1));
1079 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
1080 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
1081
1082 video_receive_stream_->Stop();
1083}
1084
1085// Note: This test takes a long time (~10s) to run if the fake metronome is
1086// active. Since the test needs to wait for the timestamp to rollover, it has a
1087// fake delay of around 6.5 hours. Even though time is simulated, this will be
1088// around 1,500,000 metronome tick invocations.
1089TEST_P(VideoReceiveStream2Test, RtpTimestampWrapAround) {
Evan Shrubsole7cbd8de2022-08-16 08:08:531090 EXPECT_CALL(mock_transport_, SendRtcp).Times(AnyNumber());
Evan Shrubsolea4062722022-05-02 15:46:071091 video_receive_stream_->Start();
1092
1093 constexpr uint32_t kBaseRtp = std::numeric_limits<uint32_t>::max() / 2;
1094 video_receive_stream_->OnCompleteFrame(
1095 test::FakeFrameBuilder()
1096 .Id(0)
Sergio Garcia Murillob5289d72024-10-24 12:10:531097 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:071098 .Time(kBaseRtp)
Danil Chapovalov22333492023-12-07 11:10:461099 .ReceivedTime(env_.clock().CurrentTime())
Evan Shrubsolea4062722022-05-02 15:46:071100 .AsLast()
1101 .Build());
1102 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
1103 time_controller_.AdvanceTime(k30FpsDelay);
1104 video_receive_stream_->OnCompleteFrame(
1105 test::FakeFrameBuilder()
1106 .Id(1)
Sergio Garcia Murillob5289d72024-10-24 12:10:531107 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:071108 .Time(kBaseRtp + k30FpsRtpTimestampDelta)
Danil Chapovalov22333492023-12-07 11:10:461109 .ReceivedTime(env_.clock().CurrentTime())
Evan Shrubsolea4062722022-05-02 15:46:071110 .AsLast()
1111 .Build());
1112 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
1113
1114 // Pause stream so that RTP timestamp wraps around.
1115 constexpr uint32_t kLastRtp = kBaseRtp + k30FpsRtpTimestampDelta;
1116 constexpr uint32_t kWrapAroundRtp =
1117 kLastRtp + std::numeric_limits<uint32_t>::max() / 2 + 1;
1118 // Pause for corresponding delay such that RTP timestamp would increase this
1119 // much at 30fps.
1120 constexpr TimeDelta kWrapAroundDelay =
1121 (std::numeric_limits<uint32_t>::max() / 2 + 1) / kRtpTimestampHz;
1122
1123 time_controller_.AdvanceTime(kWrapAroundDelay);
1124 video_receive_stream_->OnCompleteFrame(
1125 test::FakeFrameBuilder()
1126 .Id(2)
Sergio Garcia Murillob5289d72024-10-24 12:10:531127 .PayloadType(kH264PayloadType)
Evan Shrubsolea4062722022-05-02 15:46:071128 .Time(kWrapAroundRtp)
Danil Chapovalov22333492023-12-07 11:10:461129 .ReceivedTime(env_.clock().CurrentTime())
Evan Shrubsolea4062722022-05-02 15:46:071130 .AsLast()
1131 .Build());
Tony Herre55b593f2023-08-29 14:05:491132 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kWrapAroundRtp), _))
Evan Shrubsolea4062722022-05-02 15:46:071133 .Times(1);
philipel2f3b75d2024-03-13 12:02:151134 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Seconds(1)),
1135 RenderedFrame());
Evan Shrubsolea4062722022-05-02 15:46:071136
1137 video_receive_stream_->Stop();
1138}
1139
Evan Shrubsoledcb9c5d2022-06-13 15:39:531140// If a frame was lost causing the stream to become temporarily non-decodable
1141// and the sender reduces their framerate during this time, the video stream
1142// should start decoding at the new framerate. However, if the connection is
1143// poor, a keyframe will take a long time to send. If the timing of the incoming
1144// frames was not kept up to date with the new framerate while the stream was
1145// decodable, this late frame will have a large delay as the rtp timestamp of
1146// this keyframe will look like the frame arrived early if the frame-rate was
1147// not updated.
1148TEST_P(VideoReceiveStream2Test, PoorConnectionWithFpsChangeDuringLostFrame) {
1149 video_receive_stream_->Start();
1150
1151 constexpr Frequency k15Fps = Frequency::Hertz(15);
1152 constexpr TimeDelta k15FpsDelay = 1 / k15Fps;
1153 constexpr uint32_t k15FpsRtpTimestampDelta = kRtpTimestampHz / k15Fps;
1154
1155 // Initial keyframe and frames at 30fps.
1156 video_receive_stream_->OnCompleteFrame(
1157 test::FakeFrameBuilder()
1158 .Id(0)
Sergio Garcia Murillob5289d72024-10-24 12:10:531159 .PayloadType(kH264PayloadType)
Evan Shrubsoledcb9c5d2022-06-13 15:39:531160 .Time(RtpTimestampForFrame(0))
1161 .ReceivedTime(ReceiveTimeForFrame(0))
1162 .AsLast()
1163 .Build());
1164 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1165 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(0))));
1166
1167 video_receive_stream_->OnCompleteFrame(
1168 test::FakeFrameBuilder()
1169 .Id(1)
Sergio Garcia Murillob5289d72024-10-24 12:10:531170 .PayloadType(kH264PayloadType)
Evan Shrubsoledcb9c5d2022-06-13 15:39:531171 .Time(RtpTimestampForFrame(1))
1172 .ReceivedTime(ReceiveTimeForFrame(1))
1173 .Refs({0})
1174 .AsLast()
1175 .Build());
1176 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1177 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(1))));
1178
1179 // Simulate lost frame 2, followed by 2 second of frames at 30fps, followed by
1180 // 2 second of frames at 15 fps, and then a keyframe.
1181 time_controller_.AdvanceTime(k30FpsDelay);
1182
Danil Chapovalov22333492023-12-07 11:10:461183 Timestamp send_30fps_end_time =
1184 env_.clock().CurrentTime() + TimeDelta::Seconds(2);
Evan Shrubsoledcb9c5d2022-06-13 15:39:531185 int id = 3;
1186 EXPECT_CALL(mock_transport_, SendRtcp).Times(AnyNumber());
Danil Chapovalov22333492023-12-07 11:10:461187 while (env_.clock().CurrentTime() < send_30fps_end_time) {
Evan Shrubsoledcb9c5d2022-06-13 15:39:531188 ++id;
1189 video_receive_stream_->OnCompleteFrame(
1190 test::FakeFrameBuilder()
1191 .Id(id)
Sergio Garcia Murillob5289d72024-10-24 12:10:531192 .PayloadType(kH264PayloadType)
Evan Shrubsoledcb9c5d2022-06-13 15:39:531193 .Time(RtpTimestampForFrame(id))
1194 .ReceivedTime(ReceiveTimeForFrame(id))
1195 .Refs({id - 1})
1196 .AsLast()
1197 .Build());
1198 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
Florent Castelli8037fc62024-08-29 13:00:401199 Eq(std::nullopt));
Evan Shrubsoledcb9c5d2022-06-13 15:39:531200 }
1201 uint32_t current_rtp = RtpTimestampForFrame(id);
Danil Chapovalov22333492023-12-07 11:10:461202 Timestamp send_15fps_end_time =
1203 env_.clock().CurrentTime() + TimeDelta::Seconds(2);
1204 while (env_.clock().CurrentTime() < send_15fps_end_time) {
Evan Shrubsoledcb9c5d2022-06-13 15:39:531205 ++id;
1206 current_rtp += k15FpsRtpTimestampDelta;
1207 video_receive_stream_->OnCompleteFrame(
1208 test::FakeFrameBuilder()
1209 .Id(id)
Sergio Garcia Murillob5289d72024-10-24 12:10:531210 .PayloadType(kH264PayloadType)
Evan Shrubsoledcb9c5d2022-06-13 15:39:531211 .Time(current_rtp)
Danil Chapovalov22333492023-12-07 11:10:461212 .ReceivedTime(env_.clock().CurrentTime())
Evan Shrubsoledcb9c5d2022-06-13 15:39:531213 .Refs({id - 1})
1214 .AsLast()
1215 .Build());
1216 EXPECT_THAT(fake_renderer_.WaitForFrame(k15FpsDelay, /*advance_time=*/true),
Florent Castelli8037fc62024-08-29 13:00:401217 Eq(std::nullopt));
Evan Shrubsoledcb9c5d2022-06-13 15:39:531218 }
1219
1220 ++id;
1221 current_rtp += k15FpsRtpTimestampDelta;
1222 // Insert keyframe which will recover the stream. However, on a poor
1223 // connection the keyframe will take significant time to send.
1224 constexpr TimeDelta kKeyframeDelay = TimeDelta::Millis(200);
1225 video_receive_stream_->OnCompleteFrame(
1226 test::FakeFrameBuilder()
1227 .Id(id)
Sergio Garcia Murillob5289d72024-10-24 12:10:531228 .PayloadType(kH264PayloadType)
Evan Shrubsoledcb9c5d2022-06-13 15:39:531229 .Time(current_rtp)
Danil Chapovalov22333492023-12-07 11:10:461230 .ReceivedTime(env_.clock().CurrentTime() + kKeyframeDelay)
Evan Shrubsoledcb9c5d2022-06-13 15:39:531231 .AsLast()
1232 .Build());
1233 // If the framerate was not updated to be 15fps from the frames that arrived
1234 // previously, this will fail, as the delay will be longer.
1235 EXPECT_THAT(fake_renderer_.WaitForFrame(k15FpsDelay, /*advance_time=*/true),
1236 RenderedFrameWith(RtpTimestamp(current_rtp)));
1237
1238 video_receive_stream_->Stop();
1239}
1240
Evan Shrubsole4d3ba772022-06-22 14:32:361241TEST_P(VideoReceiveStream2Test, StreamShouldNotTimeoutWhileWaitingForFrame) {
1242 // Disable smoothing since this makes it hard to test frame timing.
1243 config_.enable_prerenderer_smoothing = false;
1244 RecreateReceiveStream();
1245
1246 video_receive_stream_->Start();
1247 EXPECT_CALL(mock_transport_, SendRtcp).Times(AnyNumber());
1248
1249 video_receive_stream_->OnCompleteFrame(
1250 test::FakeFrameBuilder()
1251 .Id(0)
Sergio Garcia Murillob5289d72024-10-24 12:10:531252 .PayloadType(kH264PayloadType)
Evan Shrubsole4d3ba772022-06-22 14:32:361253 .Time(RtpTimestampForFrame(0))
1254 .ReceivedTime(ReceiveTimeForFrame(0))
1255 .AsLast()
1256 .Build());
1257 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1258 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(0))));
1259
1260 for (int id = 1; id < 30; ++id) {
1261 video_receive_stream_->OnCompleteFrame(
1262 test::FakeFrameBuilder()
1263 .Id(id)
Sergio Garcia Murillob5289d72024-10-24 12:10:531264 .PayloadType(kH264PayloadType)
Evan Shrubsole4d3ba772022-06-22 14:32:361265 .Time(RtpTimestampForFrame(id))
1266 .ReceivedTime(ReceiveTimeForFrame(id))
1267 .Refs({0})
1268 .AsLast()
1269 .Build());
1270 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1271 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(id))));
1272 }
1273
1274 // Simulate a pause in the stream, followed by a decodable frame that is ready
1275 // long in the future. The stream should not timeout in this case, but rather
1276 // decode the frame just before the timeout.
1277 time_controller_.AdvanceTime(TimeDelta::Millis(2900));
1278 uint32_t late_decode_rtp = kFirstRtpTimestamp + 200 * k30FpsRtpTimestampDelta;
1279 video_receive_stream_->OnCompleteFrame(
1280 test::FakeFrameBuilder()
1281 .Id(121)
Sergio Garcia Murillob5289d72024-10-24 12:10:531282 .PayloadType(kH264PayloadType)
Evan Shrubsole4d3ba772022-06-22 14:32:361283 .Time(late_decode_rtp)
Danil Chapovalov22333492023-12-07 11:10:461284 .ReceivedTime(env_.clock().CurrentTime())
Evan Shrubsole4d3ba772022-06-22 14:32:361285 .AsLast()
1286 .Build());
1287 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Millis(100),
1288 /*advance_time=*/true),
1289 RenderedFrameWith(RtpTimestamp(late_decode_rtp)));
1290
1291 video_receive_stream_->Stop();
1292}
1293
Evan Shrubsolea4062722022-05-02 15:46:071294INSTANTIATE_TEST_SUITE_P(VideoReceiveStream2Test,
1295 VideoReceiveStream2Test,
1296 testing::Bool(),
1297 [](const auto& test_param_info) {
Evan Shrubsoledcb9c5d2022-06-13 15:39:531298 return (test_param_info.param
1299 ? "ScheduleDecodesWithMetronome"
1300 : "ScheduleDecodesWithPostTask");
Evan Shrubsolea4062722022-05-02 15:46:071301 });
1302
Tommiae4d0972020-05-18 06:45:381303} // namespace webrtc