blob: d12242f913ed0a3454401cd05a2fda79e6e9c3cc [file] [log] [blame]
Sebastian Janssonc5017132018-02-02 15:24:161/*
2 * Copyright 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Danil Chapovalovb9b146c2018-06-15 10:28:0711#include "absl/types/optional.h"
Danil Chapovalov99b71df2018-10-26 13:57:4812#include "api/test/video/function_video_encoder_factory.h"
Sebastian Janssonc5017132018-02-02 15:24:1613#include "modules/video_coding/codecs/vp8/include/vp8.h"
Markus Handell9bbff072020-07-07 12:23:1814#include "rtc_base/synchronization/mutex.h"
Sebastian Janssonc5017132018-02-02 15:24:1615#include "system_wrappers/include/metrics.h"
Sebastian Janssonc5017132018-02-02 15:24:1616#include "test/call_test.h"
17#include "test/gtest.h"
Artem Titov8a9f3a82023-04-25 07:56:4918#include "test/video_test_constants.h"
Sebastian Janssonc5017132018-02-02 15:24:1619
20namespace webrtc {
Elad Alond8d32482019-02-18 22:45:5721namespace {
22enum : int { // The first valid value is 1.
23 kTransportSequenceNumberExtensionId = 1,
24 kVideoContentTypeExtensionId,
25};
26} // namespace
Sebastian Janssonc5017132018-02-02 15:24:1627
28class HistogramTest : public test::CallTest {
Elad Alond8d32482019-02-18 22:45:5729 public:
30 HistogramTest() {
31 RegisterRtpExtension(RtpExtension(RtpExtension::kTransportSequenceNumberUri,
32 kTransportSequenceNumberExtensionId));
33 RegisterRtpExtension(RtpExtension(RtpExtension::kVideoContentTypeUri,
34 kVideoContentTypeExtensionId));
35 }
36
Sebastian Janssonc5017132018-02-02 15:24:1637 protected:
38 void VerifyHistogramStats(bool use_rtx, bool use_fec, bool screenshare);
39};
40
41void HistogramTest::VerifyHistogramStats(bool use_rtx,
42 bool use_fec,
43 bool screenshare) {
Åsa Persson5b2b6922018-05-07 07:55:5144 class FrameObserver : public test::EndToEndTest,
Sebastian Janssonc5017132018-02-02 15:24:1645 public rtc::VideoSinkInterface<VideoFrame> {
46 public:
Åsa Persson5b2b6922018-05-07 07:55:5147 FrameObserver(bool use_rtx, bool use_fec, bool screenshare)
Artem Titov8a9f3a82023-04-25 07:56:4948 : EndToEndTest(test::VideoTestConstants::kLongTimeout),
Sebastian Janssonc5017132018-02-02 15:24:1649 use_rtx_(use_rtx),
50 use_fec_(use_fec),
51 screenshare_(screenshare),
52 // This test uses NACK, so to send FEC we can't use a fake encoder.
Danil Chapovalovbe9d13a2024-03-22 12:33:5953 encoder_factory_(
54 [](const Environment& env, const SdpVideoFormat& format) {
55 return CreateVp8Encoder(env);
56 }),
Sebastian Janssonc5017132018-02-02 15:24:1657 num_frames_received_(0) {}
58
59 private:
60 void OnFrame(const VideoFrame& video_frame) override {
Artem Titovab30d722021-07-27 14:22:1161 // The RTT is needed to estimate `ntp_time_ms` which is used by
Sebastian Janssonc5017132018-02-02 15:24:1662 // end-to-end delay stats. Therefore, start counting received frames once
Artem Titovab30d722021-07-27 14:22:1163 // `ntp_time_ms` is valid.
Sebastian Janssonc5017132018-02-02 15:24:1664 if (video_frame.ntp_time_ms() > 0 &&
65 Clock::GetRealTimeClock()->CurrentNtpInMilliseconds() >=
66 video_frame.ntp_time_ms()) {
Markus Handell9bbff072020-07-07 12:23:1867 MutexLock lock(&mutex_);
Sebastian Janssonc5017132018-02-02 15:24:1668 ++num_frames_received_;
69 }
70 }
71
Harald Alvestrandd43af912023-08-15 11:41:4572 Action OnSendRtp(rtc::ArrayView<const uint8_t> packet) override {
Sebastian Janssonc5017132018-02-02 15:24:1673 if (MinMetricRunTimePassed() && MinNumberOfFramesReceived())
74 observation_complete_.Set();
75
76 return SEND_PACKET;
77 }
78
79 bool MinMetricRunTimePassed() {
Åsa Persson5b2b6922018-05-07 07:55:5180 int64_t now_ms = Clock::GetRealTimeClock()->TimeInMilliseconds();
Åsa Persson81327d52018-06-05 11:34:3381 if (!start_runtime_ms_)
Åsa Persson5b2b6922018-05-07 07:55:5182 start_runtime_ms_ = now_ms;
Åsa Persson81327d52018-06-05 11:34:3383
Åsa Persson5b2b6922018-05-07 07:55:5184 int64_t elapsed_sec = (now_ms - *start_runtime_ms_) / 1000;
Sebastian Janssonc5017132018-02-02 15:24:1685 return elapsed_sec > metrics::kMinRunTimeInSeconds * 2;
86 }
87
88 bool MinNumberOfFramesReceived() const {
89 const int kMinRequiredHistogramSamples = 200;
Markus Handell9bbff072020-07-07 12:23:1890 MutexLock lock(&mutex_);
Sebastian Janssonc5017132018-02-02 15:24:1691 return num_frames_received_ > kMinRequiredHistogramSamples;
92 }
93
94 void ModifyVideoConfigs(
95 VideoSendStream::Config* send_config,
Tommif6f45432022-05-20 13:21:2096 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
Sebastian Janssonc5017132018-02-02 15:24:1697 VideoEncoderConfig* encoder_config) override {
98 // NACK
Artem Titov8a9f3a82023-04-25 07:56:4999 send_config->rtp.nack.rtp_history_ms =
100 test::VideoTestConstants::kNackRtpHistoryMs;
101 (*receive_configs)[0].rtp.nack.rtp_history_ms =
102 test::VideoTestConstants::kNackRtpHistoryMs;
Sebastian Janssonc5017132018-02-02 15:24:16103 (*receive_configs)[0].renderer = this;
104 // FEC
105 if (use_fec_) {
Artem Titov8a9f3a82023-04-25 07:56:49106 send_config->rtp.ulpfec.ulpfec_payload_type =
107 test::VideoTestConstants::kUlpfecPayloadType;
108 send_config->rtp.ulpfec.red_payload_type =
109 test::VideoTestConstants::kRedPayloadType;
Niels Möller4db138e2018-04-19 07:04:13110 send_config->encoder_settings.encoder_factory = &encoder_factory_;
Niels Möller259a4972018-04-05 13:36:51111 send_config->rtp.payload_name = "VP8";
112 encoder_config->codec_type = kVideoCodecVP8;
Philipp Hanckebbff58d2024-02-27 11:18:33113 (*receive_configs)[0].decoders[0].video_format = SdpVideoFormat::VP8();
Artem Titov8a9f3a82023-04-25 07:56:49114 (*receive_configs)[0].rtp.red_payload_type =
115 test::VideoTestConstants::kRedPayloadType;
116 (*receive_configs)[0].rtp.ulpfec_payload_type =
117 test::VideoTestConstants::kUlpfecPayloadType;
Sebastian Janssonc5017132018-02-02 15:24:16118 }
119 // RTX
120 if (use_rtx_) {
Artem Titov8a9f3a82023-04-25 07:56:49121 send_config->rtp.rtx.ssrcs.push_back(
122 test::VideoTestConstants::kSendRtxSsrcs[0]);
123 send_config->rtp.rtx.payload_type =
124 test::VideoTestConstants::kSendRtxPayloadType;
125 (*receive_configs)[0].rtp.rtx_ssrc =
126 test::VideoTestConstants::kSendRtxSsrcs[0];
127 (*receive_configs)[0].rtp.rtx_associated_payload_types
128 [test::VideoTestConstants::kSendRtxPayloadType] =
129 test::VideoTestConstants::kFakeVideoSendPayloadType;
Sebastian Janssonc5017132018-02-02 15:24:16130 if (use_fec_) {
Artem Titov8a9f3a82023-04-25 07:56:49131 send_config->rtp.ulpfec.red_rtx_payload_type =
132 test::VideoTestConstants::kRtxRedPayloadType;
133 (*receive_configs)[0].rtp.rtx_associated_payload_types
134 [test::VideoTestConstants::kRtxRedPayloadType] =
135 test::VideoTestConstants::kSendRtxPayloadType;
Sebastian Janssonc5017132018-02-02 15:24:16136 }
137 }
138 // RTT needed for RemoteNtpTimeEstimator for the receive stream.
139 (*receive_configs)[0].rtp.rtcp_xr.receiver_reference_time_report = true;
140 encoder_config->content_type =
141 screenshare_ ? VideoEncoderConfig::ContentType::kScreen
142 : VideoEncoderConfig::ContentType::kRealtimeVideo;
143 }
144
Sebastian Janssonc5017132018-02-02 15:24:16145 void PerformTest() override {
Åsa Persson5b2b6922018-05-07 07:55:51146 EXPECT_TRUE(Wait()) << "Timed out waiting for min frames to be received.";
Sebastian Janssonc5017132018-02-02 15:24:16147 }
148
Markus Handell9bbff072020-07-07 12:23:18149 mutable Mutex mutex_;
Sebastian Janssonc5017132018-02-02 15:24:16150 const bool use_rtx_;
151 const bool use_fec_;
152 const bool screenshare_;
Niels Möller4db138e2018-04-19 07:04:13153 test::FunctionVideoEncoderFactory encoder_factory_;
Danil Chapovalovb9b146c2018-06-15 10:28:07154 absl::optional<int64_t> start_runtime_ms_;
Markus Handell9bbff072020-07-07 12:23:18155 int num_frames_received_ RTC_GUARDED_BY(&mutex_);
Sebastian Janssonc5017132018-02-02 15:24:16156 } test(use_rtx, use_fec, screenshare);
157
158 metrics::Reset();
159 RunBaseTest(&test);
160
Åsa Persson5b2b6922018-05-07 07:55:51161 const std::string video_prefix =
Sebastian Janssonc5017132018-02-02 15:24:16162 screenshare ? "WebRTC.Video.Screenshare." : "WebRTC.Video.";
Åsa Persson5b2b6922018-05-07 07:55:51163
Sebastian Janssonc5017132018-02-02 15:24:16164 // Verify that stats have been updated once.
Ying Wangef3998f2019-12-09 12:06:53165 EXPECT_METRIC_EQ(2, metrics::NumSamples("WebRTC.Call.LifetimeInSeconds"));
166 EXPECT_METRIC_EQ(1, metrics::NumSamples(
167 "WebRTC.Call.TimeReceivingVideoRtpPacketsInSeconds"));
168 EXPECT_METRIC_EQ(
169 1, metrics::NumSamples("WebRTC.Call.VideoBitrateReceivedInKbps"));
170 EXPECT_METRIC_EQ(1,
171 metrics::NumSamples("WebRTC.Call.RtcpBitrateReceivedInBps"));
172 EXPECT_METRIC_EQ(1, metrics::NumSamples("WebRTC.Call.BitrateReceivedInKbps"));
173 EXPECT_METRIC_EQ(
174 1, metrics::NumSamples("WebRTC.Call.EstimatedSendBitrateInKbps"));
175 EXPECT_METRIC_EQ(1, metrics::NumSamples("WebRTC.Call.PacerBitrateInKbps"));
Sebastian Janssonc5017132018-02-02 15:24:16176
Ying Wangef3998f2019-12-09 12:06:53177 EXPECT_METRIC_EQ(
178 1, metrics::NumSamples("WebRTC.Video.SendStreamLifetimeInSeconds"));
179 EXPECT_METRIC_EQ(
180 1, metrics::NumSamples("WebRTC.Video.ReceiveStreamLifetimeInSeconds"));
Sebastian Janssonc5017132018-02-02 15:24:16181
Ying Wangef3998f2019-12-09 12:06:53182 EXPECT_METRIC_EQ(
183 1, metrics::NumSamples("WebRTC.Video.NackPacketsSentPerMinute"));
184 EXPECT_METRIC_EQ(
185 1, metrics::NumSamples(video_prefix + "NackPacketsReceivedPerMinute"));
186 EXPECT_METRIC_EQ(1,
187 metrics::NumSamples("WebRTC.Video.FirPacketsSentPerMinute"));
188 EXPECT_METRIC_EQ(
189 1, metrics::NumSamples(video_prefix + "FirPacketsReceivedPerMinute"));
190 EXPECT_METRIC_EQ(1,
191 metrics::NumSamples("WebRTC.Video.PliPacketsSentPerMinute"));
192 EXPECT_METRIC_EQ(
193 1, metrics::NumSamples(video_prefix + "PliPacketsReceivedPerMinute"));
Sebastian Janssonc5017132018-02-02 15:24:16194
Ying Wangef3998f2019-12-09 12:06:53195 EXPECT_METRIC_EQ(
196 1, metrics::NumSamples(video_prefix + "KeyFramesSentInPermille"));
197 EXPECT_METRIC_EQ(
198 1, metrics::NumSamples("WebRTC.Video.KeyFramesReceivedInPermille"));
Sebastian Janssonc5017132018-02-02 15:24:16199
Ying Wangef3998f2019-12-09 12:06:53200 EXPECT_METRIC_EQ(
201 1, metrics::NumSamples(video_prefix + "SentPacketsLostInPercent"));
202 EXPECT_METRIC_EQ(
203 1, metrics::NumSamples("WebRTC.Video.ReceivedPacketsLostInPercent"));
Sebastian Janssonc5017132018-02-02 15:24:16204
Ying Wangef3998f2019-12-09 12:06:53205 EXPECT_METRIC_EQ(1, metrics::NumSamples(video_prefix + "InputWidthInPixels"));
206 EXPECT_METRIC_EQ(1,
207 metrics::NumSamples(video_prefix + "InputHeightInPixels"));
208 EXPECT_METRIC_EQ(1, metrics::NumSamples(video_prefix + "SentWidthInPixels"));
209 EXPECT_METRIC_EQ(1, metrics::NumSamples(video_prefix + "SentHeightInPixels"));
210 EXPECT_METRIC_EQ(1,
211 metrics::NumSamples(video_prefix + "ReceivedWidthInPixels"));
212 EXPECT_METRIC_EQ(
213 1, metrics::NumSamples(video_prefix + "ReceivedHeightInPixels"));
Sebastian Janssonc5017132018-02-02 15:24:16214
Artem Titov01716662023-04-21 15:35:49215 EXPECT_METRIC_EQ(1,
Artem Titov8a9f3a82023-04-25 07:56:49216 metrics::NumEvents(video_prefix + "InputWidthInPixels",
217 test::VideoTestConstants::kDefaultWidth));
218 EXPECT_METRIC_EQ(
219 1, metrics::NumEvents(video_prefix + "InputHeightInPixels",
220 test::VideoTestConstants::kDefaultHeight));
221 EXPECT_METRIC_EQ(1,
222 metrics::NumEvents(video_prefix + "SentWidthInPixels",
223 test::VideoTestConstants::kDefaultWidth));
224 EXPECT_METRIC_EQ(
225 1, metrics::NumEvents(video_prefix + "SentHeightInPixels",
226 test::VideoTestConstants::kDefaultHeight));
227 EXPECT_METRIC_EQ(1,
228 metrics::NumEvents(video_prefix + "ReceivedWidthInPixels",
229 test::VideoTestConstants::kDefaultWidth));
230 EXPECT_METRIC_EQ(
231 1, metrics::NumEvents(video_prefix + "ReceivedHeightInPixels",
232 test::VideoTestConstants::kDefaultHeight));
Sebastian Janssonc5017132018-02-02 15:24:16233
Ying Wangef3998f2019-12-09 12:06:53234 EXPECT_METRIC_EQ(1,
235 metrics::NumSamples(video_prefix + "InputFramesPerSecond"));
236 EXPECT_METRIC_EQ(1,
237 metrics::NumSamples(video_prefix + "SentFramesPerSecond"));
238 EXPECT_METRIC_EQ(1,
239 metrics::NumSamples("WebRTC.Video.DecodedFramesPerSecond"));
240 EXPECT_METRIC_EQ(1,
241 metrics::NumSamples("WebRTC.Video.RenderFramesPerSecond"));
242 EXPECT_METRIC_EQ(1,
243 metrics::NumSamples("WebRTC.Video.DelayedFramesToRenderer"));
Sebastian Janssonc5017132018-02-02 15:24:16244
Ying Wangef3998f2019-12-09 12:06:53245 EXPECT_METRIC_EQ(1,
246 metrics::NumSamples("WebRTC.Video.JitterBufferDelayInMs"));
247 EXPECT_METRIC_EQ(1, metrics::NumSamples("WebRTC.Video.TargetDelayInMs"));
248 EXPECT_METRIC_EQ(1, metrics::NumSamples("WebRTC.Video.CurrentDelayInMs"));
249 EXPECT_METRIC_EQ(1, metrics::NumSamples("WebRTC.Video.OnewayDelayInMs"));
Sebastian Janssonc5017132018-02-02 15:24:16250
Harald Alvestrand00f11222023-07-21 07:23:01251 EXPECT_METRIC_EQ(1, metrics::NumSamples(video_prefix + "EndToEndDelayInMs"));
Ying Wangef3998f2019-12-09 12:06:53252 EXPECT_METRIC_EQ(1,
Harald Alvestrand00f11222023-07-21 07:23:01253 metrics::NumSamples(video_prefix + "EndToEndDelayMaxInMs"));
Ying Wangef3998f2019-12-09 12:06:53254 EXPECT_METRIC_EQ(1,
Harald Alvestrand00f11222023-07-21 07:23:01255 metrics::NumSamples(video_prefix + "InterframeDelayInMs"));
256 EXPECT_METRIC_EQ(
257 1, metrics::NumSamples(video_prefix + "InterframeDelayMaxInMs"));
Ying Wangef3998f2019-12-09 12:06:53258 EXPECT_METRIC_EQ(
259 1, metrics::NumSamples("WebRTC.Video.RenderSqrtPixelsPerSecond"));
Sebastian Janssonc5017132018-02-02 15:24:16260
Ying Wangef3998f2019-12-09 12:06:53261 EXPECT_METRIC_EQ(1, metrics::NumSamples(video_prefix + "EncodeTimeInMs"));
262 EXPECT_METRIC_EQ(1, metrics::NumSamples("WebRTC.Video.DecodeTimeInMs"));
Sebastian Janssonc5017132018-02-02 15:24:16263
Ying Wangef3998f2019-12-09 12:06:53264 EXPECT_METRIC_EQ(1,
265 metrics::NumSamples(video_prefix + "NumberOfPauseEvents"));
266 EXPECT_METRIC_EQ(1,
267 metrics::NumSamples(video_prefix + "PausedTimeInPercent"));
Sebastian Janssonc5017132018-02-02 15:24:16268
Ying Wangef3998f2019-12-09 12:06:53269 EXPECT_METRIC_EQ(1, metrics::NumSamples(video_prefix + "BitrateSentInKbps"));
270 EXPECT_METRIC_EQ(1,
271 metrics::NumSamples("WebRTC.Video.BitrateReceivedInKbps"));
272 EXPECT_METRIC_EQ(
273 1, metrics::NumSamples(video_prefix + "MediaBitrateSentInKbps"));
274 EXPECT_METRIC_EQ(
275 1, metrics::NumSamples("WebRTC.Video.MediaBitrateReceivedInKbps"));
276 EXPECT_METRIC_EQ(
277 1, metrics::NumSamples(video_prefix + "PaddingBitrateSentInKbps"));
278 EXPECT_METRIC_EQ(
279 1, metrics::NumSamples("WebRTC.Video.PaddingBitrateReceivedInKbps"));
280 EXPECT_METRIC_EQ(
Sebastian Janssonc5017132018-02-02 15:24:16281 1, metrics::NumSamples(video_prefix + "RetransmittedBitrateSentInKbps"));
Ying Wangef3998f2019-12-09 12:06:53282 EXPECT_METRIC_EQ(1, metrics::NumSamples(
283 "WebRTC.Video.RetransmittedBitrateReceivedInKbps"));
Sebastian Janssonc5017132018-02-02 15:24:16284
Ying Wangef3998f2019-12-09 12:06:53285 EXPECT_METRIC_EQ(1, metrics::NumSamples("WebRTC.Video.SendDelayInMs"));
286 EXPECT_METRIC_EQ(1, metrics::NumSamples(video_prefix + "SendSideDelayInMs"));
287 EXPECT_METRIC_EQ(1,
288 metrics::NumSamples(video_prefix + "SendSideDelayMaxInMs"));
Sebastian Janssonc5017132018-02-02 15:24:16289
290 int num_rtx_samples = use_rtx ? 1 : 0;
Ying Wangef3998f2019-12-09 12:06:53291 EXPECT_METRIC_EQ(num_rtx_samples,
292 metrics::NumSamples("WebRTC.Video.RtxBitrateSentInKbps"));
293 EXPECT_METRIC_EQ(
294 num_rtx_samples,
295 metrics::NumSamples("WebRTC.Video.RtxBitrateReceivedInKbps"));
Sebastian Janssonc5017132018-02-02 15:24:16296
297 int num_red_samples = use_fec ? 1 : 0;
Ying Wangef3998f2019-12-09 12:06:53298 EXPECT_METRIC_EQ(num_red_samples,
299 metrics::NumSamples("WebRTC.Video.FecBitrateSentInKbps"));
300 EXPECT_METRIC_EQ(
301 num_red_samples,
302 metrics::NumSamples("WebRTC.Video.FecBitrateReceivedInKbps"));
303 EXPECT_METRIC_EQ(
304 num_red_samples,
305 metrics::NumSamples("WebRTC.Video.ReceivedFecPacketsInPercent"));
Sebastian Janssonc5017132018-02-02 15:24:16306}
307
Åsa Persson5b2b6922018-05-07 07:55:51308TEST_F(HistogramTest, VerifyStatsWithRtx) {
Sebastian Janssonc5017132018-02-02 15:24:16309 const bool kEnabledRtx = true;
310 const bool kEnabledRed = false;
311 const bool kScreenshare = false;
312 VerifyHistogramStats(kEnabledRtx, kEnabledRed, kScreenshare);
313}
314
Åsa Persson5b2b6922018-05-07 07:55:51315TEST_F(HistogramTest, VerifyStatsWithRed) {
Sebastian Janssonc5017132018-02-02 15:24:16316 const bool kEnabledRtx = false;
317 const bool kEnabledRed = true;
318 const bool kScreenshare = false;
319 VerifyHistogramStats(kEnabledRtx, kEnabledRed, kScreenshare);
320}
321
Åsa Persson5b2b6922018-05-07 07:55:51322TEST_F(HistogramTest, VerifyStatsWithScreenshare) {
Sebastian Janssonc5017132018-02-02 15:24:16323 const bool kEnabledRtx = false;
324 const bool kEnabledRed = false;
325 const bool kScreenshare = true;
326 VerifyHistogramStats(kEnabledRtx, kEnabledRed, kScreenshare);
327}
328
329} // namespace webrtc