Change how TimingFrameInfo is passed to ReceiveStatisticsProxy. Instead of being stored in VCMTiming and pulled by VideoStreamBufferController, it is now passed directly after a frame is decoded through OnFrameToRender and OnDecodedFrame callbacks. Bug: b/493549134 Change-Id: I3e2ce002805b3d3f30411848bfb0144b10338605 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/459780 Commit-Queue: Åsa Persson <asapersson@webrtc.org> Reviewed-by: Rasmus Brandt <brandtr@webrtc.org> Reviewed-by: Danil Chapovalov <danilchap@webrtc.org> Cr-Commit-Position: refs/heads/main@{#47265}
diff --git a/modules/video_coding/generic_decoder.cc b/modules/video_coding/generic_decoder.cc index 2320dfd..dac95bf 100644 --- a/modules/video_coding/generic_decoder.cc +++ b/modules/video_coding/generic_decoder.cc
@@ -236,7 +236,6 @@ RTC_HISTOGRAM_COUNTS_1000( "WebRTC.Video.GenericDecoder.DecodeDelay", timing_frame_info.decode_finish_ms - timing_frame_info.decode_start_ms); - timing_->SetTimingFrameInfo(timing_frame_info); decodedImage.set_timestamp_us( frame_info->render_time ? frame_info->render_time->us() : -1); decodedImage.set_content_type(frame_info->content_type); @@ -244,7 +243,8 @@ .qp = qp, .decode_time = decode_time, .content_type = frame_info->content_type, - .frame_type = frame_info->frame_type}); + .frame_type = frame_info->frame_type, + .timing_frame_info = timing_frame_info}); if (corruption_score_calculator_ && frame_info->frame_instrumentation_data.has_value()) {
diff --git a/modules/video_coding/include/video_coding_defines.h b/modules/video_coding/include/video_coding_defines.h index 88c16ca..f2e852f 100644 --- a/modules/video_coding/include/video_coding_defines.h +++ b/modules/video_coding/include/video_coding_defines.h
@@ -20,6 +20,7 @@ #include "api/video/video_content_type.h" #include "api/video/video_frame.h" #include "api/video/video_frame_type.h" +#include "api/video/video_timing.h" #include "api/video_codecs/video_decoder.h" namespace webrtc { @@ -58,6 +59,7 @@ TimeDelta decode_time; VideoContentType content_type; VideoFrameType frame_type; + TimingFrameInfo timing_frame_info; }; virtual int32_t OnFrameToRender(const FrameToRender& arguments) = 0;
diff --git a/modules/video_coding/timing/timing.cc b/modules/video_coding/timing/timing.cc index 63a7bef..79f71f3 100644 --- a/modules/video_coding/timing/timing.cc +++ b/modules/video_coding/timing/timing.cc
@@ -259,16 +259,6 @@ .current_delay = current_delay_}; } -void VCMTiming::SetTimingFrameInfo(const TimingFrameInfo& info) { - MutexLock lock(&mutex_); - timing_frame_info_.emplace(info); -} - -std::optional<TimingFrameInfo> VCMTiming::GetTimingFrameInfo() { - MutexLock lock(&mutex_); - return timing_frame_info_; -} - void VCMTiming::SetMaxCompositionDelayInFrames( std::optional<int> max_composition_delay_in_frames) { MutexLock lock(&mutex_);
diff --git a/modules/video_coding/timing/timing.h b/modules/video_coding/timing/timing.h index d0b8638..9ee0d8a 100644 --- a/modules/video_coding/timing/timing.h +++ b/modules/video_coding/timing/timing.h
@@ -114,9 +114,6 @@ // Returns current timing information. VideoDelayTimings GetTimings() const; - void SetTimingFrameInfo(const TimingFrameInfo& info); - std::optional<TimingFrameInfo> GetTimingFrameInfo(); - void SetMaxCompositionDelayInFrames( std::optional<int> max_composition_delay_in_frames); @@ -150,7 +147,6 @@ TimeDelta max_playout_delay_ RTC_GUARDED_BY(mutex_); TimeDelta jitter_delay_ RTC_GUARDED_BY(mutex_); TimeDelta current_delay_ RTC_GUARDED_BY(mutex_); - std::optional<TimingFrameInfo> timing_frame_info_ RTC_GUARDED_BY(mutex_); size_t num_decoded_frames_ RTC_GUARDED_BY(mutex_); std::optional<int> max_composition_delay_in_frames_ RTC_GUARDED_BY(mutex_); // Set by the field trial WebRTC-ZeroPlayoutDelay. The parameter min_pacing
diff --git a/video/receive_statistics_proxy.cc b/video/receive_statistics_proxy.cc index 2d1a283..ca4b7df 100644 --- a/video/receive_statistics_proxy.cc +++ b/video/receive_statistics_proxy.cc
@@ -609,11 +609,13 @@ stats_.c_name = std::string(cname); } -void ReceiveStatisticsProxy::OnDecodedFrame(const VideoFrame& frame, - std::optional<uint8_t> qp, - TimeDelta decode_time, - VideoContentType content_type, - VideoFrameType frame_type) { +void ReceiveStatisticsProxy::OnDecodedFrame( + const VideoFrame& frame, + std::optional<uint8_t> qp, + TimeDelta decode_time, + VideoContentType content_type, + VideoFrameType frame_type, + const TimingFrameInfo& timing_frame_info) { TimeDelta processing_delay = TimeDelta::Zero(); Timestamp current_time = clock_->CurrentTime(); // TODO(bugs.webrtc.org/13984): some tests do not fill packet_infos(). @@ -637,10 +639,12 @@ // "com.apple.coremedia.decompressionsession.clientcallback" VideoFrameMetaData meta(frame, current_time); worker_thread_->PostTask(SafeTask( - task_safety_.flag(), [meta, qp, decode_time, processing_delay, - assembly_time, content_type, frame_type, this]() { + task_safety_.flag(), + [meta, qp, decode_time, processing_delay, assembly_time, content_type, + frame_type, timing_frame_info, this]() { OnDecodedFrame(meta, qp, decode_time, processing_delay, assembly_time, content_type, frame_type); + OnTimingFrameInfoUpdated(timing_frame_info); })); }
diff --git a/video/receive_statistics_proxy.h b/video/receive_statistics_proxy.h index b72ed92..16f42bd 100644 --- a/video/receive_statistics_proxy.h +++ b/video/receive_statistics_proxy.h
@@ -70,7 +70,8 @@ std::optional<uint8_t> qp, TimeDelta decode_time, VideoContentType content_type, - VideoFrameType frame_type); + VideoFrameType frame_type, + const TimingFrameInfo& timing_frame_info); // Called asyncronously on the worker thread as a result of a call to the // above OnDecodedFrame method, which is called back on the thread where @@ -82,6 +83,7 @@ TimeDelta assembly_time, VideoContentType content_type, VideoFrameType frame_type); + void OnTimingFrameInfoUpdated(const TimingFrameInfo& info); void OnSyncOffsetUpdated(int64_t video_playout_ntp_ms, int64_t sync_offset_ms, @@ -111,7 +113,6 @@ int jitter_delay_ms, int min_playout_delay_ms, int render_delay_ms) override; - void OnTimingFrameInfoUpdated(const TimingFrameInfo& info) override; // Implements RtcpCnameCallback. void OnCname(uint32_t ssrc, absl::string_view cname) override;
diff --git a/video/receive_statistics_proxy_unittest.cc b/video/receive_statistics_proxy_unittest.cc index 40ba033..b38b310 100644 --- a/video/receive_statistics_proxy_unittest.cc +++ b/video/receive_statistics_proxy_unittest.cc
@@ -48,6 +48,8 @@ namespace internal { namespace { using ::testing::DoubleEq; +using ::testing::Field; +using ::testing::Optional; constexpr TimeDelta kFreqOffsetProcessInterval = TimeDelta::Seconds(40); constexpr uint32_t kRemoteSsrc = 456; @@ -69,7 +71,7 @@ ~ReceiveStatisticsProxyTest() override { statistics_proxy_.reset(); } protected: - // Convenience method to avoid too many explict flushes. + // Convenience method to avoid too many explicit flushes. VideoReceiveStreamInterface::Stats FlushAndGetStats() { time_controller_.AdvanceTime(TimeDelta::Zero()); return statistics_proxy_->GetStats(); @@ -120,6 +122,13 @@ return VideoFrameMetaData(frame, Now()); } + // Creates a TimingFrameInfo with invalid flag. + TimingFrameInfo Info() { + TimingFrameInfo info; + info.flags = VideoSendTiming::kInvalid; + return info; + } + GlobalSimulatedTimeController time_controller_; std::unique_ptr<ReceiveStatisticsProxy> statistics_proxy_; }; @@ -131,7 +140,7 @@ for (uint32_t i = 1; i <= 3; ++i) { statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); EXPECT_EQ(i, FlushAndGetStats().frames_decoded); EXPECT_EQ(0, FlushAndGetStats().frame_counts.key_frames); } @@ -144,7 +153,7 @@ for (int i = 0; i < kRequiredSamples; ++i) { statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); time_controller_.AdvanceTime(1 / kFps); } FlushAndUpdateHistograms(std::nullopt, StreamDataCounters(), nullptr); @@ -161,7 +170,7 @@ for (int i = 0; i < kRequiredSamples - 1; ++i) { statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); time_controller_.AdvanceTime(1 / kFps); } FlushAndUpdateHistograms(std::nullopt, StreamDataCounters(), nullptr); @@ -178,7 +187,7 @@ for (uint32_t i = 1; i <= 3; ++i) { statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Millis(1), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); expected_total_decode_time += TimeDelta::Millis(1); ++expected_frames_decoded; time_controller_.AdvanceTime(TimeDelta::Zero()); @@ -189,7 +198,7 @@ } statistics_proxy_->OnDecodedFrame(frame, 1u, TimeDelta::Millis(3), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); ++expected_frames_decoded; expected_total_decode_time += TimeDelta::Millis(3); time_controller_.AdvanceTime(TimeDelta::Zero()); @@ -215,7 +224,7 @@ time_controller_.AdvanceTime(kProcessingDelay); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Millis(1), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); expected_total_processing_delay += i * kProcessingDelay; ++expected_frames_decoded; time_controller_.AdvanceTime(TimeDelta::Zero()); @@ -227,7 +236,7 @@ time_controller_.AdvanceTime(kProcessingDelay); statistics_proxy_->OnDecodedFrame(frame, 1u, TimeDelta::Millis(3), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); ++expected_frames_decoded; expected_total_processing_delay += 4 * kProcessingDelay; time_controller_.AdvanceTime(TimeDelta::Zero()); @@ -252,7 +261,7 @@ frame.set_packet_infos(RtpPacketInfos(single_packet_frame)); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Millis(1), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); ++expected_frames_decoded; time_controller_.AdvanceTime(TimeDelta::Zero()); EXPECT_EQ(expected_total_assembly_time, @@ -273,7 +282,7 @@ frame.set_packet_infos(RtpPacketInfos(ordered_frame)); statistics_proxy_->OnDecodedFrame(frame, 1u, TimeDelta::Millis(3), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); ++expected_frames_decoded; ++expected_frames_assembled_from_multiple_packets; expected_total_assembly_time += 2 * kAssemblyTime; @@ -298,7 +307,7 @@ frame.set_packet_infos(RtpPacketInfos(unordered_frame)); statistics_proxy_->OnDecodedFrame(frame, 1u, TimeDelta::Millis(3), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); ++expected_frames_decoded; ++expected_frames_assembled_from_multiple_packets; expected_total_assembly_time += 2 * kAssemblyTime; @@ -317,11 +326,11 @@ VideoFrame frame = CreateFrame(kWidth, kHeight); statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); EXPECT_EQ(3u, FlushAndGetStats().qp_sum); statistics_proxy_->OnDecodedFrame(frame, 127u, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameDelta); + VideoFrameType::kVideoFrameDelta, Info()); EXPECT_EQ(130u, FlushAndGetStats().qp_sum); } @@ -330,11 +339,11 @@ VideoFrame frame = CreateFrame(kWidth, kHeight); statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Millis(4), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); EXPECT_EQ(4u, FlushAndGetStats().total_decode_time.ms()); statistics_proxy_->OnDecodedFrame(frame, 127u, TimeDelta::Millis(7), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameDelta); + VideoFrameType::kVideoFrameDelta, Info()); EXPECT_EQ(11u, FlushAndGetStats().total_decode_time.ms()); } @@ -346,12 +355,12 @@ statistics_proxy_->GetStats().content_type)); statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Zero(), VideoContentType::SCREENSHARE, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); EXPECT_EQ(kScreenshareString, videocontenttypehelpers::ToString(FlushAndGetStats().content_type)); statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameDelta); + VideoFrameType::kVideoFrameDelta, Info()); EXPECT_EQ(kRealtimeString, videocontenttypehelpers::ToString(FlushAndGetStats().content_type)); } @@ -364,25 +373,25 @@ EXPECT_EQ(-1, statistics_proxy_->GetStats().interframe_delay_max_ms); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); EXPECT_EQ(-1, FlushAndGetStats().interframe_delay_max_ms); time_controller_.AdvanceTime(kInterframeDelay1); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameDelta); + VideoFrameType::kVideoFrameDelta, Info()); EXPECT_EQ(kInterframeDelay1.ms(), FlushAndGetStats().interframe_delay_max_ms); time_controller_.AdvanceTime(kInterframeDelay2); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameDelta); + VideoFrameType::kVideoFrameDelta, Info()); EXPECT_EQ(kInterframeDelay2.ms(), FlushAndGetStats().interframe_delay_max_ms); time_controller_.AdvanceTime(kInterframeDelay3); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameDelta); + VideoFrameType::kVideoFrameDelta, Info()); // kInterframeDelay3 is smaller than kInterframeDelay2. EXPECT_EQ(kInterframeDelay2.ms(), FlushAndGetStats().interframe_delay_max_ms); } @@ -395,26 +404,26 @@ EXPECT_EQ(-1, statistics_proxy_->GetStats().interframe_delay_max_ms); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); EXPECT_EQ(-1, FlushAndGetStats().interframe_delay_max_ms); time_controller_.AdvanceTime(kInterframeDelay1); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameDelta); + VideoFrameType::kVideoFrameDelta, Info()); EXPECT_EQ(kInterframeDelay1.ms(), FlushAndGetStats().interframe_delay_max_ms); time_controller_.AdvanceTime(kInterframeDelay2); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameDelta); + VideoFrameType::kVideoFrameDelta, Info()); // Still first delay is the maximum EXPECT_EQ(kInterframeDelay1.ms(), FlushAndGetStats().interframe_delay_max_ms); time_controller_.AdvanceTime(kInterframeDelay3); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameDelta); + VideoFrameType::kVideoFrameDelta, Info()); // Now the first sample is out of the window, so the second is the maximum. EXPECT_EQ(kInterframeDelay2.ms(), FlushAndGetStats().interframe_delay_max_ms); } @@ -524,7 +533,7 @@ EXPECT_EQ(std::nullopt, statistics_proxy_->GetStats().qp_sum); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); EXPECT_EQ(std::nullopt, FlushAndGetStats().qp_sum); } @@ -533,11 +542,11 @@ EXPECT_EQ(std::nullopt, statistics_proxy_->GetStats().qp_sum); statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); EXPECT_EQ(3u, FlushAndGetStats().qp_sum); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameDelta); + VideoFrameType::kVideoFrameDelta, Info()); EXPECT_EQ(std::nullopt, FlushAndGetStats().qp_sum); } @@ -618,8 +627,7 @@ statistics_proxy_->OnDroppedFrames(i); dropped_frames += i; } - VideoReceiveStreamInterface::Stats stats = FlushAndGetStats(); - EXPECT_EQ(dropped_frames, stats.frames_dropped); + EXPECT_EQ(dropped_frames, FlushAndGetStats().frames_dropped); } TEST_F(ReceiveStatisticsProxyTest, GetStatsReportsDecodeTimingStats) { @@ -697,12 +705,12 @@ for (int i = 0; i < kKeyFrames; i++) { statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); } for (int i = 0; i < kDeltaFrames; i++) { statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameDelta); + VideoFrameType::kVideoFrameDelta, Info()); } VideoReceiveStreamInterface::Stats stats = statistics_proxy_->GetStats(); @@ -722,13 +730,43 @@ EXPECT_STREQ("", statistics_proxy_->GetStats().c_name.c_str()); } +TEST_F(ReceiveStatisticsProxyTest, ReportsTimingFrameInfo) { + TimingFrameInfo info; + info.rtp_timestamp = 123u; + statistics_proxy_->OnDecodedFrame( + CreateFrame(kWidth, kHeight), std::nullopt, TimeDelta::Zero(), + VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameDelta, info); + EXPECT_THAT(FlushAndGetStats().timing_frame_info, + Optional(Field(&TimingFrameInfo::rtp_timestamp, 123u))); +} + +TEST_F(ReceiveStatisticsProxyTest, ReportsNoTimingFrameInfoWhenInvalid) { + TimingFrameInfo info; + info.flags = VideoSendTiming::kInvalid; + statistics_proxy_->OnDecodedFrame( + CreateFrame(kWidth, kHeight), std::nullopt, TimeDelta::Zero(), + VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameDelta, info); + EXPECT_EQ(FlushAndGetStats().timing_frame_info, std::nullopt); +} + +TEST_F(ReceiveStatisticsProxyTest, UpdatesFirstFrameReceivedToDecodedTime) { + const TimeDelta kDecodeTime = TimeDelta::Millis(14); + TimingFrameInfo info; + info.receive_finish_ms = Now().ms(); + time_controller_.AdvanceTime(kDecodeTime); + statistics_proxy_->OnDecodedFrame( + CreateFrame(kWidth, kHeight), std::nullopt, TimeDelta::Zero(), + VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameDelta, info); + EXPECT_EQ(FlushAndGetStats().first_frame_received_to_decoded_ms, + kDecodeTime.ms()); +} + TEST_F(ReceiveStatisticsProxyTest, ReportsLongestTimingFrameInfo) { const int64_t kShortEndToEndDelay = 10; const int64_t kMedEndToEndDelay = 20; const int64_t kLongEndToEndDelay = 100; const uint32_t kExpectedRtpTimestamp = 2; TimingFrameInfo info; - std::optional<TimingFrameInfo> result; info.rtp_timestamp = kExpectedRtpTimestamp - 1; info.capture_time_ms = 0; info.decode_finish_ms = kShortEndToEndDelay; @@ -742,29 +780,27 @@ info.capture_time_ms = 0; info.decode_finish_ms = kMedEndToEndDelay; statistics_proxy_->OnTimingFrameInfoUpdated(info); - result = FlushAndGetStats().timing_frame_info; - EXPECT_TRUE(result); - EXPECT_EQ(kExpectedRtpTimestamp, result->rtp_timestamp); + EXPECT_THAT( + FlushAndGetStats().timing_frame_info, + Optional(Field(&TimingFrameInfo::rtp_timestamp, kExpectedRtpTimestamp))); } TEST_F(ReceiveStatisticsProxyTest, RespectsReportingIntervalForTimingFrames) { - TimingFrameInfo info; const int64_t kShortEndToEndDelay = 10; const uint32_t kExpectedRtpTimestamp = 2; const TimeDelta kShortDelay = TimeDelta::Seconds(1); const TimeDelta kLongDelay = TimeDelta::Seconds(10); - std::optional<TimingFrameInfo> result; + TimingFrameInfo info; info.rtp_timestamp = kExpectedRtpTimestamp; info.capture_time_ms = 0; info.decode_finish_ms = kShortEndToEndDelay; statistics_proxy_->OnTimingFrameInfoUpdated(info); time_controller_.AdvanceTime(kShortDelay); - result = FlushAndGetStats().timing_frame_info; - EXPECT_TRUE(result); - EXPECT_EQ(kExpectedRtpTimestamp, result->rtp_timestamp); + EXPECT_THAT( + FlushAndGetStats().timing_frame_info, + Optional(Field(&TimingFrameInfo::rtp_timestamp, kExpectedRtpTimestamp))); time_controller_.AdvanceTime(kLongDelay); - result = statistics_proxy_->GetStats().timing_frame_info; - EXPECT_FALSE(result); + EXPECT_EQ(statistics_proxy_->GetStats().timing_frame_info, std::nullopt); } TEST_F(ReceiveStatisticsProxyTest, LifetimeHistogramIsUpdated) { @@ -776,7 +812,7 @@ statistics_proxy_->OnCompleteFrame(true, 1000, VideoContentType::UNSPECIFIED); statistics_proxy_->OnDecodedFrame( frame, std::nullopt, TimeDelta::Millis(1000), - VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameKey); + VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameKey, Info()); FlushAndGetStats(); statistics_proxy_->UpdateHistograms(std::nullopt, StreamDataCounters(), @@ -922,9 +958,10 @@ for (int i = 0; i < kMinRequiredSamples - 1; ++i) { statistics_proxy_->OnCompleteFrame(kIsKeyFrame, kFrameSizeBytes, VideoContentType::UNSPECIFIED); - statistics_proxy_->OnDecodedFrame( - frame, std::nullopt, TimeDelta::Millis(1000), - VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameDelta); + statistics_proxy_->OnDecodedFrame(frame, std::nullopt, + TimeDelta::Millis(1000), + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameDelta, Info()); } FlushAndGetStats(); @@ -947,9 +984,10 @@ for (int i = 0; i < kMinRequiredSamples; ++i) { statistics_proxy_->OnCompleteFrame(kIsKeyFrame, kFrameSizeBytes, VideoContentType::UNSPECIFIED); - statistics_proxy_->OnDecodedFrame( - frame, std::nullopt, TimeDelta::Millis(1000), - VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameDelta); + statistics_proxy_->OnDecodedFrame(frame, std::nullopt, + TimeDelta::Millis(1000), + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameDelta, Info()); } FlushAndGetStats(); @@ -974,14 +1012,15 @@ VideoContentType::UNSPECIFIED); statistics_proxy_->OnDecodedFrame( frame, std::nullopt, TimeDelta::Millis(1000), - VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameKey); + VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameKey, Info()); } for (int i = 0; i < kMinRequiredSamples; ++i) { statistics_proxy_->OnCompleteFrame(false, kFrameSizeBytes, VideoContentType::UNSPECIFIED); - statistics_proxy_->OnDecodedFrame( - frame, std::nullopt, TimeDelta::Millis(1000), - VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameDelta); + statistics_proxy_->OnDecodedFrame(frame, std::nullopt, + TimeDelta::Millis(1000), + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameDelta, Info()); } FlushAndGetStats(); @@ -1064,7 +1103,7 @@ time_controller_.GetClock()->CurrentNtpInMilliseconds()); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); statistics_proxy_->OnRenderedFrame(MetaData(frame)); time_controller_.AdvanceTime(1 / kDefaultFps); } @@ -1138,7 +1177,7 @@ VideoFrame frame = CreateFrame(kWidth, kHeight); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); // Frame not delayed, delayed frames to render: 0%. statistics_proxy_->OnRenderedFrame( @@ -1160,7 +1199,7 @@ VideoFrame frame = CreateFrame(kWidth, kHeight); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); // Frame not delayed, delayed frames to render: 0%. statistics_proxy_->OnRenderedFrame( @@ -1181,7 +1220,7 @@ VideoFrame frame = CreateFrame(kWidth, kHeight); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); // Min run time has passed. No rendered frames. time_controller_.AdvanceTime(metrics::kMinRunTime); @@ -1197,7 +1236,7 @@ VideoFrame frame = CreateFrame(kWidth, kHeight); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); // Frame delayed 1 ms, delayed frames to render: 100%. statistics_proxy_->OnRenderedFrame( @@ -1221,7 +1260,7 @@ VideoFrame frame = CreateFrame(kWidth, kHeight); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), VideoContentType::UNSPECIFIED, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); // Two frames delayed (6 ms, 10 ms), delayed frames to render: 50%. const int64_t kNowMs = Now().ms(); @@ -1372,14 +1411,14 @@ for (int i = 0; i < kMinRequiredSamples; ++i) { statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), content_type_, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); time_controller_.AdvanceTime(kInterFrameDelay); } // One extra with double the interval. time_controller_.AdvanceTime(kInterFrameDelay); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), content_type_, - VideoFrameType::kVideoFrameDelta); + VideoFrameType::kVideoFrameDelta, Info()); FlushAndUpdateHistograms(std::nullopt, StreamDataCounters(), nullptr); const TimeDelta kExpectedInterFrame = @@ -1410,20 +1449,20 @@ time_controller_.AdvanceTime(kInterFrameDelay); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), content_type_, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); } // Last 5% of intervals are double in size. for (int i = 0; i < kLastFivePercentsSamples; ++i) { time_controller_.AdvanceTime(2 * kInterFrameDelay); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), content_type_, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); } // Final sample is outlier and 10 times as big. time_controller_.AdvanceTime(10 * kInterFrameDelay); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), content_type_, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); FlushAndUpdateHistograms(std::nullopt, StreamDataCounters(), nullptr); const TimeDelta kExpectedInterFrame = kInterFrameDelay * 2; @@ -1447,7 +1486,7 @@ for (int i = 0; i < kMinRequiredSamples; ++i) { statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), content_type_, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); time_controller_.AdvanceTime(kInterFrameDelay); } @@ -1471,7 +1510,7 @@ for (int i = 0; i <= kMinRequiredSamples; ++i) { statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), content_type_, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); time_controller_.AdvanceTime(kInterFrameDelay); } @@ -1483,11 +1522,11 @@ // disregarded in the stats. statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), content_type_, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); time_controller_.AdvanceTime(kInterFrameDelay); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), content_type_, - VideoFrameType::kVideoFrameDelta); + VideoFrameType::kVideoFrameDelta, Info()); FlushAndUpdateHistograms(std::nullopt, StreamDataCounters(), nullptr); if (videocontenttypehelpers::IsScreenshare(content_type_)) { @@ -1609,7 +1648,7 @@ time_controller_.AdvanceTime(kFrameDuration); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), content_type_, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); statistics_proxy_->OnRenderedFrame(MetaData(frame)); } @@ -1618,7 +1657,7 @@ time_controller_.AdvanceTime(kFreezeDuration); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), content_type_, - VideoFrameType::kVideoFrameDelta); + VideoFrameType::kVideoFrameDelta, Info()); statistics_proxy_->OnRenderedFrame(MetaData(frame)); // Add pause. @@ -1626,7 +1665,7 @@ statistics_proxy_->OnStreamInactive(); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), content_type_, - VideoFrameType::kVideoFrameDelta); + VideoFrameType::kVideoFrameDelta, Info()); statistics_proxy_->OnRenderedFrame(MetaData(frame)); FlushAndUpdateHistograms(std::nullopt, StreamDataCounters(), nullptr); @@ -1702,13 +1741,13 @@ for (int i = 0; i <= kMinRequiredSamples; ++i) { statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), content_type_, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); time_controller_.AdvanceTime(kInterFrameDelay); statistics_proxy_->OnStreamInactive(); time_controller_.AdvanceTime(kPauseDuration); statistics_proxy_->OnDecodedFrame(frame, std::nullopt, TimeDelta::Zero(), content_type_, - VideoFrameType::kVideoFrameDelta); + VideoFrameType::kVideoFrameDelta, Info()); time_controller_.AdvanceTime(kInterFrameDelay); } @@ -1791,7 +1830,7 @@ // Extra last frame. statistics_proxy_->OnDecodedFrame(frame, kHighQp, TimeDelta::Zero(), content_type_, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); statistics_proxy_->OnRenderedFrame(MetaData(frame)); FlushAndUpdateHistograms(std::nullopt, StreamDataCounters(), nullptr); @@ -1819,7 +1858,7 @@ // Call once to pass content type. statistics_proxy_->OnDecodedFrame(frame_hd, std::nullopt, TimeDelta::Zero(), content_type_, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); time_controller_.AdvanceTime(TimeDelta::Zero()); statistics_proxy_->OnRenderedFrame(MetaData(frame_hd)); @@ -1849,7 +1888,7 @@ for (int i = 0; i < kMinRequiredSamples; ++i) { statistics_proxy_->OnDecodedFrame(frame, kLowQp, kDecodeTime, content_type_, - VideoFrameType::kVideoFrameKey); + VideoFrameType::kVideoFrameKey, Info()); time_controller_.AdvanceTime(kInterFrameDelay); } FlushAndUpdateHistograms(std::nullopt, StreamDataCounters(), nullptr);
diff --git a/video/timing/simulator/rendering_tracker.h b/video/timing/simulator/rendering_tracker.h index de33fae..0e0dc75 100644 --- a/video/timing/simulator/rendering_tracker.h +++ b/video/timing/simulator/rendering_tracker.h
@@ -26,7 +26,6 @@ #include "api/video/video_content_type.h" #include "api/video/video_frame.h" #include "api/video/video_sink_interface.h" -#include "api/video/video_timing.h" #include "modules/video_coding/timing/timing.h" #include "rtc_base/thread_annotations.h" #include "video/timing/simulator/assembler.h" @@ -99,7 +98,6 @@ TimeDelta jitter_buffer_target_delay, TimeDelta jitter_buffer_minimum_delay) override; void OnFrameBufferTimingsUpdated(int, int, int, int, int, int) override {} - void OnTimingFrameInfoUpdated(const TimingFrameInfo&) override {} // Implements `VideoSinkInterface<VideoFrame>`. void OnFrame(const VideoFrame& decoded_frame) override;
diff --git a/video/video_stream_buffer_controller.cc b/video/video_stream_buffer_controller.cc index ff77863..21e9424 100644 --- a/video/video_stream_buffer_controller.cc +++ b/video/video_stream_buffer_controller.cc
@@ -28,7 +28,6 @@ #include "api/video/encoded_frame.h" #include "api/video/frame_buffer.h" #include "api/video/video_content_type.h" -#include "api/video/video_timing.h" #include "modules/video_coding/frame_helpers.h" #include "modules/video_coding/include/video_coding_defines.h" #include "modules/video_coding/timing/inter_frame_delay_variation_calculator.h" @@ -265,7 +264,6 @@ // Update stats. UpdateDroppedFrames(); UpdateFrameBufferTimings(min_receive_time, now); - UpdateTimingFrameInfo(); std::unique_ptr<EncodedFrame> frame = CombineAndDeleteFrames(std::move(frames)); @@ -356,12 +354,6 @@ timings.minimum_delay); } -void VideoStreamBufferController::UpdateTimingFrameInfo() { - std::optional<TimingFrameInfo> info = timing_->GetTimingFrameInfo(); - if (info) - stats_proxy_->OnTimingFrameInfoUpdated(*info); -} - bool VideoStreamBufferController::IsTooManyFramesQueued() const RTC_RUN_ON(&worker_sequence_checker_) { return buffer_->CurrentSize() > zero_playout_delay_max_decode_queue_size_;
diff --git a/video/video_stream_buffer_controller.h b/video/video_stream_buffer_controller.h index 1722787..93a98fc 100644 --- a/video/video_stream_buffer_controller.h +++ b/video/video_stream_buffer_controller.h
@@ -26,7 +26,6 @@ #include "api/video/encoded_frame.h" #include "api/video/frame_buffer.h" #include "api/video/video_content_type.h" -#include "api/video/video_timing.h" #include "modules/video_coding/include/video_coding_defines.h" #include "modules/video_coding/timing/inter_frame_delay_variation_calculator.h" #include "modules/video_coding/timing/jitter_estimator.h" @@ -73,8 +72,6 @@ int jitter_delay_ms, int min_playout_delay_ms, int render_delay_ms) = 0; - - virtual void OnTimingFrameInfoUpdated(const TimingFrameInfo& info) = 0; }; class VideoStreamBufferController { @@ -109,7 +106,6 @@ void FrameReadyForDecode(uint32_t rtp_timestamp, Timestamp render_time); void UpdateDroppedFrames() RTC_RUN_ON(&worker_sequence_checker_); void UpdateFrameBufferTimings(Timestamp min_receive_time, Timestamp now); - void UpdateTimingFrameInfo(); bool IsTooManyFramesQueued() const RTC_RUN_ON(&worker_sequence_checker_); void ForceKeyFrameReleaseImmediately() RTC_RUN_ON(&worker_sequence_checker_); void MaybeScheduleFrameForRelease() RTC_RUN_ON(&worker_sequence_checker_);
diff --git a/video/video_stream_buffer_controller_unittest.cc b/video/video_stream_buffer_controller_unittest.cc index e7e8dc1..26aacb7 100644 --- a/video/video_stream_buffer_controller_unittest.cc +++ b/video/video_stream_buffer_controller_unittest.cc
@@ -123,10 +123,6 @@ int min_playout_delay_ms, int render_delay_ms), (override)); - MOCK_METHOD(void, - OnTimingFrameInfoUpdated, - (const TimingFrameInfo& info), - (override)); }; } // namespace
diff --git a/video/video_stream_decoder2.cc b/video/video_stream_decoder2.cc index 45379a0..f8b8a82 100644 --- a/video/video_stream_decoder2.cc +++ b/video/video_stream_decoder2.cc
@@ -52,7 +52,8 @@ int32_t VideoStreamDecoder::OnFrameToRender(const FrameToRender& arguments) { receive_stats_callback_->OnDecodedFrame( arguments.video_frame, arguments.qp, arguments.decode_time, - arguments.content_type, arguments.frame_type); + arguments.content_type, arguments.frame_type, + arguments.timing_frame_info); incoming_video_stream_->OnFrame(arguments.video_frame); return 0; }