Reland "Adds more performance stats collection to scenario tests."
This is a reland of 63b0b2cf307b47bae3c10b295ece9a5f6d9bd7a4
Original change's description:
> Adds more performance stats collection to scenario tests.
>
> Bug: webrtc:10365
> Change-Id: I66dce6ff21242c30af674f89fc9fd19172d4a3af
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/131948
> Commit-Queue: Sebastian Jansson <srte@webrtc.org>
> Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
> Cr-Commit-Position: refs/heads/master@{#27585}
Bug: webrtc:10365
Change-Id: Id7ddb64ac17ecbb4de223dec497bc562040ba7c4
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/132711
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Commit-Queue: Sebastian Jansson <srte@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27618}diff --git a/test/scenario/BUILD.gn b/test/scenario/BUILD.gn
index eba344b..1918a2b 100644
--- a/test/scenario/BUILD.gn
+++ b/test/scenario/BUILD.gn
@@ -59,6 +59,7 @@
"hardware_codecs.h",
"network_node.cc",
"network_node.h",
+ "performance_stats.cc",
"performance_stats.h",
"scenario.cc",
"scenario.h",
diff --git a/test/scenario/performance_stats.cc b/test/scenario/performance_stats.cc
new file mode 100644
index 0000000..3ff66cd
--- /dev/null
+++ b/test/scenario/performance_stats.cc
@@ -0,0 +1,193 @@
+/*
+ * Copyright 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/scenario/performance_stats.h"
+
+#include <algorithm>
+
+namespace webrtc {
+namespace test {
+void EventRateCounter::AddEvent(Timestamp event_time) {
+ if (first_time_.IsInfinite()) {
+ first_time_ = event_time;
+ } else {
+ RTC_DCHECK(event_time >= last_time_);
+ interval_.AddSample(event_time - last_time_);
+ }
+ last_time_ = event_time;
+ event_count_++;
+}
+
+void EventRateCounter::AddEvents(EventRateCounter other) {
+ first_time_ = std::min(first_time_, other.first_time_);
+ last_time_ = std::max(last_time_, other.last_time_);
+ event_count_ += other.event_count_;
+ interval_.AddSamples(other.interval_);
+}
+
+bool EventRateCounter::IsEmpty() const {
+ return first_time_ == last_time_;
+}
+
+double EventRateCounter::Rate() const {
+ if (event_count_ == 0)
+ return 0;
+ if (event_count_ == 1)
+ return NAN;
+ return (event_count_ - 1) / (last_time_ - first_time_).seconds<double>();
+}
+
+double SampleStats<double>::Max() {
+ if (IsEmpty())
+ return INFINITY;
+ return GetMax();
+}
+
+double SampleStats<double>::Mean() {
+ if (IsEmpty())
+ return 0;
+ return GetAverage();
+}
+
+double SampleStats<double>::Median() {
+ return Quantile(0.5);
+}
+
+double SampleStats<double>::Quantile(double quantile) {
+ if (IsEmpty())
+ return 0;
+ return GetPercentile(quantile);
+}
+
+double SampleStats<double>::Min() {
+ if (IsEmpty())
+ return -INFINITY;
+ return GetMin();
+}
+
+double SampleStats<double>::Variance() {
+ if (IsEmpty())
+ return 0;
+ return GetVariance();
+}
+
+double SampleStats<double>::StandardDeviation() {
+ return sqrt(Variance());
+}
+
+void SampleStats<TimeDelta>::AddSample(TimeDelta delta) {
+ RTC_DCHECK(delta.IsFinite());
+ stats_.AddSample(delta.seconds<double>());
+}
+
+void SampleStats<TimeDelta>::AddSampleMs(double delta_ms) {
+ AddSample(TimeDelta::ms(delta_ms));
+}
+void SampleStats<TimeDelta>::AddSamples(const SampleStats<TimeDelta>& other) {
+ stats_.AddSamples(other.stats_);
+}
+
+TimeDelta SampleStats<TimeDelta>::Max() {
+ return TimeDelta::seconds(stats_.Max());
+}
+
+TimeDelta SampleStats<TimeDelta>::Mean() {
+ return TimeDelta::seconds(stats_.Mean());
+}
+
+TimeDelta SampleStats<TimeDelta>::Median() {
+ return Quantile(0.5);
+}
+
+TimeDelta SampleStats<TimeDelta>::Quantile(double quantile) {
+ return TimeDelta::seconds(stats_.Quantile(quantile));
+}
+
+TimeDelta SampleStats<TimeDelta>::Min() {
+ return TimeDelta::seconds(stats_.Min());
+}
+
+TimeDelta SampleStats<TimeDelta>::Variance() {
+ return TimeDelta::seconds(stats_.Variance());
+}
+
+TimeDelta SampleStats<TimeDelta>::StandardDeviation() {
+ return TimeDelta::seconds(stats_.StandardDeviation());
+}
+
+void SampleStats<DataRate>::AddSample(DataRate sample) {
+ stats_.AddSample(sample.bps<double>());
+}
+
+void SampleStats<DataRate>::AddSampleBps(double rate_bps) {
+ stats_.AddSample(rate_bps);
+}
+
+void SampleStats<DataRate>::AddSamples(const SampleStats<DataRate>& other) {
+ stats_.AddSamples(other.stats_);
+}
+
+DataRate SampleStats<DataRate>::Max() {
+ return DataRate::bps(stats_.Max());
+}
+
+DataRate SampleStats<DataRate>::Mean() {
+ return DataRate::bps(stats_.Mean());
+}
+
+DataRate SampleStats<DataRate>::Median() {
+ return Quantile(0.5);
+}
+
+DataRate SampleStats<DataRate>::Quantile(double quantile) {
+ return DataRate::bps(stats_.Quantile(quantile));
+}
+
+DataRate SampleStats<DataRate>::Min() {
+ return DataRate::bps(stats_.Min());
+}
+
+DataRate SampleStats<DataRate>::Variance() {
+ return DataRate::bps(stats_.Variance());
+}
+
+DataRate SampleStats<DataRate>::StandardDeviation() {
+ return DataRate::bps(stats_.StandardDeviation());
+}
+
+void VideoFramesStats::AddFrameInfo(const VideoFrameBuffer& frame,
+ Timestamp at_time) {
+ ++count;
+ RTC_DCHECK(at_time.IsFinite());
+ pixels.AddSample(frame.width() * frame.height());
+ resolution.AddSample(std::max(frame.width(), frame.height()));
+ frames.AddEvent(at_time);
+}
+
+void VideoFramesStats::AddStats(const VideoFramesStats& other) {
+ count += other.count;
+ pixels.AddSamples(other.pixels);
+ resolution.AddSamples(other.resolution);
+ frames.AddEvents(other.frames);
+}
+
+void VideoQualityStats::AddStats(const VideoQualityStats& other) {
+ capture.AddStats(other.capture);
+ render.AddStats(other.render);
+ lost_count += other.lost_count;
+ freeze_count += other.freeze_count;
+ end_to_end_delay.AddSamples(other.end_to_end_delay);
+ psnr.AddSamples(other.psnr);
+ skipped_between_rendered.AddSamples(other.skipped_between_rendered);
+ freeze_duration.AddSamples(other.freeze_duration);
+ time_between_freezes.AddSamples(other.time_between_freezes);
+}
+
+} // namespace test
+} // namespace webrtc
diff --git a/test/scenario/performance_stats.h b/test/scenario/performance_stats.h
index b00c62c..1979876 100644
--- a/test/scenario/performance_stats.h
+++ b/test/scenario/performance_stats.h
@@ -14,7 +14,7 @@
#include "api/units/time_delta.h"
#include "api/units/timestamp.h"
#include "api/video/video_frame_buffer.h"
-#include "rtc_base/numerics/running_statistics.h"
+#include "rtc_base/numerics/samples_stats_counter.h"
namespace webrtc {
namespace test {
@@ -35,13 +35,125 @@
int repeated = 0;
};
+template <typename T>
+class SampleStats;
+
+template <>
+class SampleStats<double> : public SamplesStatsCounter {
+ public:
+ double Max();
+ double Mean();
+ double Median();
+ double Quantile(double quantile);
+ double Min();
+ double Variance();
+ double StandardDeviation();
+};
+
+template <>
+class SampleStats<TimeDelta> {
+ public:
+ void AddSample(TimeDelta delta);
+ void AddSampleMs(double delta_ms);
+ void AddSamples(const SampleStats<TimeDelta>& other);
+ TimeDelta Max();
+ TimeDelta Mean();
+ TimeDelta Median();
+ TimeDelta Quantile(double quantile);
+ TimeDelta Min();
+ TimeDelta Variance();
+ TimeDelta StandardDeviation();
+
+ private:
+ SampleStats<double> stats_;
+};
+
+template <>
+class SampleStats<DataRate> {
+ public:
+ void AddSample(DataRate rate);
+ void AddSampleBps(double rate_bps);
+ void AddSamples(const SampleStats<DataRate>& other);
+ DataRate Max();
+ DataRate Mean();
+ DataRate Median();
+ DataRate Quantile(double quantile);
+ DataRate Min();
+ DataRate Variance();
+ DataRate StandardDeviation();
+
+ private:
+ SampleStats<double> stats_;
+};
+
+class EventRateCounter {
+ public:
+ void AddEvent(Timestamp event_time);
+ void AddEvents(EventRateCounter other);
+ bool IsEmpty() const;
+ double Rate() const;
+ SampleStats<TimeDelta>& interval() { return interval_; }
+
+ private:
+ Timestamp first_time_ = Timestamp::PlusInfinity();
+ Timestamp last_time_ = Timestamp::MinusInfinity();
+ int64_t event_count_ = 0;
+ SampleStats<TimeDelta> interval_;
+};
+
+struct VideoFramesStats {
+ int count = 0;
+ SampleStats<double> pixels;
+ SampleStats<double> resolution;
+ EventRateCounter frames;
+ void AddFrameInfo(const VideoFrameBuffer& frame, Timestamp at_time);
+ void AddStats(const VideoFramesStats& other);
+};
+
struct VideoQualityStats {
- int captures_count = 0;
- int valid_count = 0;
int lost_count = 0;
- RunningStatistics<double> end_to_end_seconds;
- RunningStatistics<int> frame_size;
- RunningStatistics<double> psnr;
+ int freeze_count = 0;
+ VideoFramesStats capture;
+ VideoFramesStats render;
+ // Time from frame was captured on device to time frame was displayed on
+ // device.
+ SampleStats<TimeDelta> end_to_end_delay;
+ SampleStats<double> psnr;
+ // Frames skipped between two nearest.
+ SampleStats<double> skipped_between_rendered;
+ // In the next 2 metrics freeze is a pause that is longer, than maximum:
+ // 1. 150ms
+ // 2. 3 * average time between two sequential frames.
+ // Item 1 will cover high fps video and is a duration, that is noticeable by
+ // human eye. Item 2 will cover low fps video like screen sharing.
+ SampleStats<TimeDelta> freeze_duration;
+ // Mean time between one freeze end and next freeze start.
+ SampleStats<TimeDelta> time_between_freezes;
+ void AddStats(const VideoQualityStats& other);
+};
+
+struct CollectedCallStats {
+ SampleStats<DataRate> target_rate;
+ SampleStats<double> memory_usage;
+};
+
+struct CollectedAudioReceiveStats {
+ SampleStats<double> expand_rate;
+ SampleStats<double> accelerate_rate;
+ SampleStats<TimeDelta> jitter_buffer;
+};
+struct CollectedVideoSendStats {
+ SampleStats<double> encode_frame_rate;
+ SampleStats<TimeDelta> encode_time;
+ SampleStats<double> encode_usage;
+ SampleStats<DataRate> media_bitrate;
+ SampleStats<DataRate> fec_bitrate;
+};
+struct CollectedVideoReceiveStats {
+ SampleStats<TimeDelta> decode_time;
+ SampleStats<TimeDelta> decode_time_max;
+ SampleStats<double> decode_pixels;
+ SampleStats<double> resolution;
};
} // namespace test
diff --git a/test/scenario/scenario_unittest.cc b/test/scenario/scenario_unittest.cc
index aa65a12..c0f8508 100644
--- a/test/scenario/scenario_unittest.cc
+++ b/test/scenario/scenario_unittest.cc
@@ -102,8 +102,7 @@
}
// Regression tests based on previous runs.
EXPECT_EQ(analyzer.stats().lost_count, 0);
- ASSERT_TRUE(analyzer.stats().psnr.GetMean());
- EXPECT_NEAR(*analyzer.stats().psnr.GetMean(), 38, 2);
+ EXPECT_NEAR(analyzer.stats().psnr.Mean(), 38, 2);
}
// TODO(bugs.webrtc.org/10515): Remove this when performance has been improved.
@@ -123,8 +122,7 @@
}
// Regression tests based on previous runs.
EXPECT_LT(analyzer.stats().lost_count, 2);
- ASSERT_TRUE(analyzer.stats().psnr.GetMean());
- EXPECT_NEAR(*analyzer.stats().psnr.GetMean(), 38, 10);
+ EXPECT_NEAR(analyzer.stats().psnr.Mean(), 38, 10);
}
TEST(ScenarioTest, SimTimeFakeing) {
diff --git a/test/scenario/stats_collection.cc b/test/scenario/stats_collection.cc
index 77b2ae0..104c824 100644
--- a/test/scenario/stats_collection.cc
+++ b/test/scenario/stats_collection.cc
@@ -9,10 +9,8 @@
*/
#include "test/scenario/stats_collection.h"
-
-#include <utility>
-
#include "common_video/libyuv/include/webrtc_libyuv.h"
+#include "rtc_base/memory_usage.h"
namespace webrtc {
namespace test {
@@ -39,20 +37,47 @@
}
void VideoQualityAnalyzer::HandleFramePair(VideoFramePair sample) {
+ layer_analyzers_[sample.layer_id].HandleFramePair(sample, writer_.get());
+ cached_.reset();
+}
+
+std::vector<VideoQualityStats> VideoQualityAnalyzer::layer_stats() const {
+ std::vector<VideoQualityStats> res;
+ for (auto& layer : layer_analyzers_)
+ res.push_back(layer.second.stats_);
+ return res;
+}
+
+VideoQualityStats& VideoQualityAnalyzer::stats() {
+ if (!cached_) {
+ cached_ = VideoQualityStats();
+ for (auto& layer : layer_analyzers_)
+ cached_->AddStats(layer.second.stats_);
+ }
+ return *cached_;
+}
+
+void VideoLayerAnalyzer::HandleFramePair(VideoFramePair sample,
+ RtcEventLogOutput* writer) {
double psnr = NAN;
RTC_CHECK(sample.captured);
- ++stats_.captures_count;
+ HandleCapturedFrame(sample);
if (!sample.decoded) {
++stats_.lost_count;
+ ++skip_count_;
} else {
psnr = I420PSNR(*sample.captured->ToI420(), *sample.decoded->ToI420());
- ++stats_.valid_count;
- stats_.end_to_end_seconds.AddSample(
- (sample.render_time - sample.capture_time).seconds<double>());
+ stats_.end_to_end_delay.AddSample(sample.render_time - sample.capture_time);
stats_.psnr.AddSample(psnr);
+ if (sample.repeated) {
+ ++stats_.freeze_count;
+ ++skip_count_;
+ } else {
+ HandleRenderedFrame(sample);
+ }
}
- if (writer_) {
- LogWriteFormat(writer_.get(), "%.3f %.3f %.3f %i %i %i %i %.3f\n",
+ if (writer) {
+ LogWriteFormat(writer, "%.3f %.3f %.3f %i %i %i %i %.3f\n",
sample.capture_time.seconds<double>(),
sample.render_time.seconds<double>(),
sample.captured->width(), sample.captured->height(),
@@ -60,9 +85,78 @@
}
}
-VideoQualityStats VideoQualityAnalyzer::stats() const {
- return stats_;
+void VideoLayerAnalyzer::HandleCapturedFrame(const VideoFramePair& sample) {
+ stats_.capture.AddFrameInfo(*sample.captured, sample.capture_time);
+ if (last_freeze_time_.IsInfinite())
+ last_freeze_time_ = sample.capture_time;
}
+void VideoLayerAnalyzer::HandleRenderedFrame(const VideoFramePair& sample) {
+ stats_.render.AddFrameInfo(*sample.decoded, sample.render_time);
+ stats_.skipped_between_rendered.AddSample(skip_count_);
+ skip_count_ = 0;
+
+ if (last_render_time_.IsFinite()) {
+ RTC_DCHECK(sample.render_time.IsFinite());
+ TimeDelta render_interval = sample.render_time - last_render_time_;
+ TimeDelta mean_interval = stats_.render.frames.interval().Mean();
+ if (render_interval > TimeDelta::ms(150) + mean_interval ||
+ render_interval > 3 * mean_interval) {
+ stats_.freeze_duration.AddSample(render_interval);
+ stats_.time_between_freezes.AddSample(last_render_time_ -
+ last_freeze_time_);
+ last_freeze_time_ = sample.render_time;
+ }
+ }
+ last_render_time_ = sample.render_time;
+}
+
+void CallStatsCollector::AddStats(Call::Stats sample) {
+ stats_.target_rate.AddSampleBps(sample.send_bandwidth_bps);
+ stats_.memory_usage.AddSample(rtc::GetProcessResidentSizeBytes());
+}
+
+void AudioReceiveStatsCollector::AddStats(AudioReceiveStream::Stats sample) {
+ stats_.expand_rate.AddSample(sample.expand_rate);
+ stats_.accelerate_rate.AddSample(sample.accelerate_rate);
+ stats_.jitter_buffer.AddSampleMs(sample.jitter_buffer_ms);
+}
+
+void VideoSendStatsCollector::AddStats(VideoSendStream::Stats sample,
+ Timestamp at_time) {
+ // It's not certain that we yet have estimates for any of these stats.
+ // Check that they are positive before mixing them in.
+ if (sample.encode_frame_rate <= 0)
+ return;
+
+ stats_.encode_frame_rate.AddSample(sample.encode_frame_rate);
+ stats_.encode_time.AddSampleMs(sample.avg_encode_time_ms);
+ stats_.encode_usage.AddSample(sample.encode_usage_percent / 100.0);
+ stats_.media_bitrate.AddSampleBps(sample.media_bitrate_bps);
+
+ size_t fec_bytes = 0;
+ for (const auto& kv : sample.substreams) {
+ fec_bytes += kv.second.rtp_stats.fec.payload_bytes +
+ kv.second.rtp_stats.fec.padding_bytes;
+ }
+ if (last_update_.IsFinite()) {
+ auto fec_delta = DataSize::bytes(fec_bytes - last_fec_bytes_);
+ auto time_delta = at_time - last_update_;
+ stats_.fec_bitrate.AddSample(fec_delta / time_delta);
+ }
+ last_fec_bytes_ = fec_bytes;
+ last_update_ = at_time;
+}
+
+void VideoReceiveStatsCollector::AddStats(VideoReceiveStream::Stats sample) {
+ if (sample.decode_ms > 0)
+ stats_.decode_time.AddSampleMs(sample.decode_ms);
+ if (sample.max_decode_ms > 0)
+ stats_.decode_time_max.AddSampleMs(sample.max_decode_ms);
+ if (sample.width > 0 && sample.height > 0) {
+ stats_.decode_pixels.AddSample(sample.width * sample.height);
+ stats_.resolution.AddSample(sample.height);
+ }
+}
} // namespace test
} // namespace webrtc
diff --git a/test/scenario/stats_collection.h b/test/scenario/stats_collection.h
index d1b46e4..0b8b4a3 100644
--- a/test/scenario/stats_collection.h
+++ b/test/scenario/stats_collection.h
@@ -10,8 +10,11 @@
#ifndef TEST_SCENARIO_STATS_COLLECTION_H_
#define TEST_SCENARIO_STATS_COLLECTION_H_
+#include <map>
#include <memory>
+#include "absl/types/optional.h"
+#include "call/call.h"
#include "test/logging/log_writer.h"
#include "test/scenario/performance_stats.h"
@@ -22,6 +25,18 @@
double psnr_coverage = 1;
};
+class VideoLayerAnalyzer {
+ public:
+ void HandleCapturedFrame(const VideoFramePair& sample);
+ void HandleRenderedFrame(const VideoFramePair& sample);
+ void HandleFramePair(VideoFramePair sample, RtcEventLogOutput* writer);
+ VideoQualityStats stats_;
+ Timestamp last_capture_time_ = Timestamp::MinusInfinity();
+ Timestamp last_render_time_ = Timestamp::MinusInfinity();
+ Timestamp last_freeze_time_ = Timestamp::MinusInfinity();
+ int skip_count_ = 0;
+};
+
class VideoQualityAnalyzer {
public:
explicit VideoQualityAnalyzer(
@@ -29,15 +44,59 @@
std::unique_ptr<RtcEventLogOutput> writer = nullptr);
~VideoQualityAnalyzer();
void HandleFramePair(VideoFramePair sample);
- VideoQualityStats stats() const;
+ std::vector<VideoQualityStats> layer_stats() const;
+ VideoQualityStats& stats();
void PrintHeaders();
void PrintFrameInfo(const VideoFramePair& sample);
std::function<void(const VideoFramePair&)> Handler();
private:
const VideoQualityAnalyzerConfig config_;
- VideoQualityStats stats_;
+ std::map<int, VideoLayerAnalyzer> layer_analyzers_;
const std::unique_ptr<RtcEventLogOutput> writer_;
+ absl::optional<VideoQualityStats> cached_;
+};
+
+class CallStatsCollector {
+ public:
+ void AddStats(Call::Stats sample);
+ CollectedCallStats& stats() { return stats_; }
+
+ private:
+ CollectedCallStats stats_;
+};
+class AudioReceiveStatsCollector {
+ public:
+ void AddStats(AudioReceiveStream::Stats sample);
+ CollectedAudioReceiveStats& stats() { return stats_; }
+
+ private:
+ CollectedAudioReceiveStats stats_;
+};
+class VideoSendStatsCollector {
+ public:
+ void AddStats(VideoSendStream::Stats sample, Timestamp at_time);
+ CollectedVideoSendStats& stats() { return stats_; }
+
+ private:
+ CollectedVideoSendStats stats_;
+ Timestamp last_update_ = Timestamp::MinusInfinity();
+ size_t last_fec_bytes_ = 0;
+};
+class VideoReceiveStatsCollector {
+ public:
+ void AddStats(VideoReceiveStream::Stats sample);
+ CollectedVideoReceiveStats& stats() { return stats_; }
+
+ private:
+ CollectedVideoReceiveStats stats_;
+};
+
+struct CallStatsCollectors {
+ CallStatsCollector call;
+ AudioReceiveStatsCollector audio_receive;
+ VideoSendStatsCollector video_send;
+ VideoReceiveStatsCollector video_receive;
};
} // namespace test
diff --git a/test/scenario/stats_collection_unittest.cc b/test/scenario/stats_collection_unittest.cc
index 7e65ec1..48a4bc3 100644
--- a/test/scenario/stats_collection_unittest.cc
+++ b/test/scenario/stats_collection_unittest.cc
@@ -16,49 +16,71 @@
namespace {
void CreateAnalyzedStream(Scenario* s,
NetworkNodeConfig network_config,
- VideoQualityAnalyzer* analyzer) {
+ VideoQualityAnalyzer* analyzer,
+ CallStatsCollectors* collectors) {
VideoStreamConfig config;
config.encoder.codec = VideoStreamConfig::Encoder::Codec::kVideoCodecVP8;
config.encoder.implementation =
VideoStreamConfig::Encoder::Implementation::kSoftware;
config.hooks.frame_pair_handlers = {analyzer->Handler()};
- auto route = s->CreateRoutes(s->CreateClient("caller", CallClientConfig()),
- {s->CreateSimulationNode(network_config)},
- s->CreateClient("callee", CallClientConfig()),
- {s->CreateSimulationNode(NetworkNodeConfig())});
- s->CreateVideoStream(route->forward(), config);
+ auto* caller = s->CreateClient("caller", CallClientConfig());
+ auto route =
+ s->CreateRoutes(caller, {s->CreateSimulationNode(network_config)},
+ s->CreateClient("callee", CallClientConfig()),
+ {s->CreateSimulationNode(NetworkNodeConfig())});
+ auto* video = s->CreateVideoStream(route->forward(), config);
+ auto* audio = s->CreateAudioStream(route->forward(), AudioStreamConfig());
+ if (collectors) {
+ s->Every(TimeDelta::seconds(1), [=] {
+ collectors->call.AddStats(caller->GetStats());
+ collectors->audio_receive.AddStats(audio->receive()->GetStats());
+ collectors->video_send.AddStats(video->send()->GetStats(), s->Now());
+ collectors->video_receive.AddStats(video->receive()->GetStats());
+ });
+ }
}
} // namespace
TEST(ScenarioAnalyzerTest, PsnrIsHighWhenNetworkIsGood) {
VideoQualityAnalyzer analyzer;
+ CallStatsCollectors stats;
{
- Scenario s("", /*real_time*/ false);
+ Scenario s;
NetworkNodeConfig good_network;
good_network.simulation.bandwidth = DataRate::kbps(1000);
- CreateAnalyzedStream(&s, good_network, &analyzer);
- s.RunFor(TimeDelta::seconds(1));
+ CreateAnalyzedStream(&s, good_network, &analyzer, &stats);
+ s.RunFor(TimeDelta::seconds(3));
}
- // This is mainty a regression test, the target is based on previous runs and
- // might change due to changes in configuration and encoder etc.
- ASSERT_TRUE(analyzer.stats().psnr.GetMean());
- EXPECT_GT(*analyzer.stats().psnr.GetMean(), 40);
+ // This is a change detecting test, the targets are based on previous runs and
+ // might change due to changes in configuration and encoder etc. The main
+ // purpose is to show how the stats can be used. To avoid being overly
+ // sensistive to change, the ranges are chosen to be quite large.
+ EXPECT_NEAR(analyzer.stats().psnr.Mean(), 43, 10);
+ EXPECT_NEAR(stats.call.stats().target_rate.Mean().kbps(), 700, 300);
+ EXPECT_NEAR(stats.video_send.stats().media_bitrate.Mean().kbps(), 500, 200);
+ EXPECT_NEAR(stats.video_receive.stats().resolution.Mean(), 180, 10);
+ EXPECT_NEAR(stats.audio_receive.stats().jitter_buffer.Mean().ms(), 40, 20);
}
TEST(ScenarioAnalyzerTest, PsnrIsLowWhenNetworkIsBad) {
VideoQualityAnalyzer analyzer;
+ CallStatsCollectors stats;
{
- Scenario s("", /*real_time*/ false);
+ Scenario s;
NetworkNodeConfig bad_network;
bad_network.simulation.bandwidth = DataRate::kbps(100);
bad_network.simulation.loss_rate = 0.02;
- CreateAnalyzedStream(&s, bad_network, &analyzer);
- s.RunFor(TimeDelta::seconds(1));
+ CreateAnalyzedStream(&s, bad_network, &analyzer, &stats);
+ s.RunFor(TimeDelta::seconds(3));
}
- // This is mainty a regression test, the target is based on previous runs and
+ // This is a change detecting test, the targets are based on previous runs and
// might change due to changes in configuration and encoder etc.
- ASSERT_TRUE(analyzer.stats().psnr.GetMean());
- EXPECT_LT(*analyzer.stats().psnr.GetMean(), 30);
+ EXPECT_NEAR(analyzer.stats().psnr.Mean(), 16, 10);
+ EXPECT_NEAR(stats.call.stats().target_rate.Mean().kbps(), 75, 50);
+ EXPECT_NEAR(stats.video_send.stats().media_bitrate.Mean().kbps(), 100, 50);
+ EXPECT_NEAR(stats.video_receive.stats().resolution.Mean(), 180, 10);
+ EXPECT_NEAR(stats.audio_receive.stats().jitter_buffer.Mean().ms(), 45, 20);
}
+
} // namespace test
} // namespace webrtc
