Logging basic bad call detection BUG=webrtc:6814 Review-Url: https://codereview.webrtc.org/2474913002 Cr-Commit-Position: refs/heads/master@{#15568}
diff --git a/webrtc/video/BUILD.gn b/webrtc/video/BUILD.gn index 9709e45..0161479 100644 --- a/webrtc/video/BUILD.gn +++ b/webrtc/video/BUILD.gn
@@ -18,6 +18,8 @@ "overuse_frame_detector.h", "payload_router.cc", "payload_router.h", + "quality_threshold.cc", + "quality_threshold.h", "receive_statistics_proxy.cc", "receive_statistics_proxy.h", "report_block_stats.cc", @@ -151,6 +153,7 @@ "end_to_end_tests.cc", "overuse_frame_detector_unittest.cc", "payload_router_unittest.cc", + "quality_threshold_unittest.cc", "receive_statistics_proxy_unittest.cc", "report_block_stats_unittest.cc", "send_delay_stats_unittest.cc",
diff --git a/webrtc/video/quality_threshold.cc b/webrtc/video/quality_threshold.cc new file mode 100644 index 0000000..3f2d835 --- /dev/null +++ b/webrtc/video/quality_threshold.cc
@@ -0,0 +1,86 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/video/quality_threshold.h" + +#include "webrtc/base/checks.h" +#include "webrtc/base/logging.h" + +namespace webrtc { + +QualityThreshold::QualityThreshold(int low_threshold, + int high_threshold, + float fraction, + int max_measurements) + : buffer_(new int[max_measurements]), + max_measurements_(max_measurements), + fraction_(fraction), + low_threshold_(low_threshold), + high_threshold_(high_threshold), + until_full_(max_measurements), + next_index_(0), + sum_(0), + count_low_(0), + count_high_(0) { + RTC_CHECK_GT(fraction, 0.5f); + RTC_CHECK_GT(max_measurements, 1); + RTC_CHECK_LT(low_threshold, high_threshold); +} + +void QualityThreshold::AddMeasurement(int measurement) { + int prev_val = until_full_ > 0 ? 0 : buffer_[next_index_]; + buffer_[next_index_] = measurement; + next_index_ = (next_index_ + 1) % max_measurements_; + + sum_ += measurement - prev_val; + + if (until_full_ == 0) { + if (prev_val <= low_threshold_) { + --count_low_; + } else if (prev_val >= high_threshold_) { + --count_high_; + } + } + + if (measurement <= low_threshold_) { + ++count_low_; + } else if (measurement >= high_threshold_) { + ++count_high_; + } + + float sufficient_majority = fraction_ * max_measurements_; + if (count_high_ >= sufficient_majority) { + is_high_ = rtc::Optional<bool>(true); + } else if (count_low_ >= sufficient_majority) { + is_high_ = rtc::Optional<bool>(false); + } + + if (until_full_ > 0) + --until_full_; +} + +rtc::Optional<bool> QualityThreshold::IsHigh() const { + return is_high_; +} + +rtc::Optional<double> QualityThreshold::CalculateVariance() const { + if (until_full_ > 0) { + return rtc::Optional<double>(); + } + + double variance = 0; + double mean = static_cast<double>(sum_) / max_measurements_; + for (int i = 0; i < max_measurements_; ++i) { + variance += (buffer_[i] - mean) * (buffer_[i] - mean); + } + return rtc::Optional<double>(variance / (max_measurements_ - 1)); +} + +} // namespace webrtc
diff --git a/webrtc/video/quality_threshold.h b/webrtc/video/quality_threshold.h new file mode 100644 index 0000000..9bd0aaee --- /dev/null +++ b/webrtc/video/quality_threshold.h
@@ -0,0 +1,49 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_VIDEO_QUALITY_THRESHOLD_H_ +#define WEBRTC_VIDEO_QUALITY_THRESHOLD_H_ + +#include <memory> + +#include "webrtc/base/optional.h" + +namespace webrtc { + +class QualityThreshold { + public: + // Both thresholds are inclusive, i.e. measurement >= high signifies a high + // state, while measurement <= low signifies a low state. + QualityThreshold(int low_threshold, + int high_threshold, + float fraction, + int max_measurements); + + void AddMeasurement(int measurement); + rtc::Optional<bool> IsHigh() const; + rtc::Optional<double> CalculateVariance() const; + + private: + const std::unique_ptr<int[]> buffer_; + const int max_measurements_; + const float fraction_; + const int low_threshold_; + const int high_threshold_; + int until_full_; + int next_index_; + rtc::Optional<bool> is_high_; + int sum_; + int count_low_; + int count_high_; +}; + +} // namespace webrtc + +#endif // WEBRTC_VIDEO_QUALITY_THRESHOLD_H_
diff --git a/webrtc/video/quality_threshold_unittest.cc b/webrtc/video/quality_threshold_unittest.cc new file mode 100644 index 0000000..dcd094c --- /dev/null +++ b/webrtc/video/quality_threshold_unittest.cc
@@ -0,0 +1,97 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/video/quality_threshold.h" + +#include "webrtc/test/gtest.h" + +namespace webrtc { + +TEST(QualityThresholdTest, BackAndForth) { + const int kLowThreshold = 0; + const int kHighThreshold = 1; + const float kFraction = 0.75f; + const int kMaxMeasurements = 10; + + QualityThreshold thresh(kLowThreshold, kHighThreshold, kFraction, + kMaxMeasurements); + + const int kNeededMeasurements = + static_cast<int>(kFraction * kMaxMeasurements + 1); + for (int i = 0; i < kNeededMeasurements; ++i) { + EXPECT_FALSE(thresh.IsHigh()); + thresh.AddMeasurement(kLowThreshold); + } + ASSERT_TRUE(thresh.IsHigh()); + for (int i = 0; i < kNeededMeasurements; ++i) { + EXPECT_FALSE(*thresh.IsHigh()); + thresh.AddMeasurement(kHighThreshold); + } + EXPECT_TRUE(*thresh.IsHigh()); + + for (int i = 0; i < kNeededMeasurements; ++i) { + EXPECT_TRUE(*thresh.IsHigh()); + thresh.AddMeasurement(kLowThreshold); + } + EXPECT_FALSE(*thresh.IsHigh()); +} + +TEST(QualityThresholdTest, Variance) { + const int kLowThreshold = 0; + const int kHighThreshold = 1; + const float kFraction = 0.8f; + const int kMaxMeasurements = 10; + const double kMaxError = 0.01; + + // Previously randomly generated values... + int values[] = {51, 79, 80, 56, 19, 20, 48, 57, 48, 25, 2, 25, 38, 37, 25}; + // ...with precomputed variances. + double variances[] = {476.9, 687.6, 552, 336.4, 278.767, 265.167}; + + QualityThreshold thresh(kLowThreshold, kHighThreshold, kFraction, + kMaxMeasurements); + + for (int i = 0; i < kMaxMeasurements; ++i) { + EXPECT_FALSE(thresh.CalculateVariance()); + thresh.AddMeasurement(values[i]); + } + + ASSERT_TRUE(thresh.CalculateVariance()); + EXPECT_NEAR(variances[0], *thresh.CalculateVariance(), kMaxError); + for (unsigned int i = 1; i < sizeof(variances) / sizeof(double); ++i) { + thresh.AddMeasurement(values[i + kMaxMeasurements - 1]); + EXPECT_NEAR(variances[i], *thresh.CalculateVariance(), kMaxError); + } + + for (int i = 0; i < kMaxMeasurements; ++i) { + thresh.AddMeasurement(42); + } + EXPECT_NEAR(0, *thresh.CalculateVariance(), kMaxError); +} + +TEST(QualityThresholdTest, BetweenThresholds) { + const int kLowThreshold = 0; + const int kHighThreshold = 2; + const float kFraction = 0.6f; + const int kMaxMeasurements = 10; + + const int kBetweenThresholds = (kLowThreshold + kHighThreshold) / 2; + + QualityThreshold thresh(kLowThreshold, kHighThreshold, kFraction, + kMaxMeasurements); + + for (int i = 0; i < 2 * kMaxMeasurements; ++i) { + EXPECT_FALSE(thresh.IsHigh()); + thresh.AddMeasurement(kBetweenThresholds); + } + EXPECT_FALSE(thresh.IsHigh()); +} + +} // namespace webrtc
diff --git a/webrtc/video/receive_statistics_proxy.cc b/webrtc/video/receive_statistics_proxy.cc index 15a2206..6bf9a74 100644 --- a/webrtc/video/receive_statistics_proxy.cc +++ b/webrtc/video/receive_statistics_proxy.cc
@@ -13,6 +13,7 @@ #include <cmath> #include "webrtc/base/checks.h" +#include "webrtc/base/logging.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/system_wrappers/include/clock.h" #include "webrtc/system_wrappers/include/field_trial.h" @@ -22,6 +23,22 @@ namespace { // Periodic time interval for processing samples for |freq_offset_counter_|. const int64_t kFreqOffsetProcessIntervalMs = 40000; + +// Configuration for bad call detection. +const int kMinSampleLengthMs = 990; +const int kNumMeasurements = 10; +const int kNumMeasurementsVariance = kNumMeasurements * 1.5; +const float kBadFraction = 0.8f; +// For fps: +// Low means low enough to be bad, high means high enough to be good +const int kLowFpsThreshold = 12; +const int kHighFpsThreshold = 14; +// For qp and fps variance: +// Low means low enough to be good, high means high enough to be bad +const int kLowQpThresholdVp8 = 60; +const int kHighQpThresholdVp8 = 70; +const int kLowVarianceThreshold = 1; +const int kHighVarianceThreshold = 2; } // namespace ReceiveStatisticsProxy::ReceiveStatisticsProxy( @@ -30,6 +47,19 @@ : clock_(clock), config_(*config), start_ms_(clock->TimeInMilliseconds()), + last_sample_time_(clock->TimeInMilliseconds()), + fps_threshold_(kLowFpsThreshold, + kHighFpsThreshold, + kBadFraction, + kNumMeasurements), + qp_threshold_(kLowQpThresholdVp8, + kHighQpThresholdVp8, + kBadFraction, + kNumMeasurements), + variance_threshold_(kLowVarianceThreshold, + kHighVarianceThreshold, + kBadFraction, + kNumMeasurementsVariance), // 1000ms window, scale 1000 for ms to s. decode_fps_estimator_(1000, 1000), renders_fps_estimator_(1000, 1000), @@ -173,6 +203,67 @@ } } +void ReceiveStatisticsProxy::QualitySample() { + int64_t now = clock_->TimeInMilliseconds(); + if (last_sample_time_ + kMinSampleLengthMs > now) + return; + + double fps = + render_fps_tracker_.ComputeRateForInterval(now - last_sample_time_); + int qp = qp_sample_.Avg(1); + + bool prev_fps_bad = !fps_threshold_.IsHigh().value_or(true); + bool prev_qp_bad = qp_threshold_.IsHigh().value_or(false); + bool prev_variance_bad = variance_threshold_.IsHigh().value_or(false); + bool prev_any_bad = prev_fps_bad || prev_qp_bad || prev_variance_bad; + + fps_threshold_.AddMeasurement(static_cast<int>(fps)); + if (qp != -1) + qp_threshold_.AddMeasurement(qp); + rtc::Optional<double> fps_variance_opt = fps_threshold_.CalculateVariance(); + double fps_variance = fps_variance_opt.value_or(0); + if (fps_variance_opt) { + variance_threshold_.AddMeasurement(static_cast<int>(fps_variance)); + } + + bool fps_bad = !fps_threshold_.IsHigh().value_or(true); + bool qp_bad = qp_threshold_.IsHigh().value_or(false); + bool variance_bad = variance_threshold_.IsHigh().value_or(false); + bool any_bad = fps_bad || qp_bad || variance_bad; + + if (!prev_any_bad && any_bad) { + LOG(LS_WARNING) << "Bad call (any) start: " << now; + } else if (prev_any_bad && !any_bad) { + LOG(LS_WARNING) << "Bad call (any) end: " << now; + } + + if (!prev_fps_bad && fps_bad) { + LOG(LS_WARNING) << "Bad call (fps) start: " << now; + } else if (prev_fps_bad && !fps_bad) { + LOG(LS_WARNING) << "Bad call (fps) end: " << now; + } + + if (!prev_qp_bad && qp_bad) { + LOG(LS_WARNING) << "Bad call (qp) start: " << now; + } else if (prev_qp_bad && !qp_bad) { + LOG(LS_WARNING) << "Bad call (qp) end: " << now; + } + + if (!prev_variance_bad && variance_bad) { + LOG(LS_WARNING) << "Bad call (variance) start: " << now; + } else if (prev_variance_bad && !variance_bad) { + LOG(LS_WARNING) << "Bad call (variance) end: " << now; + } + + LOG(LS_INFO) << "SAMPLE: sample_length: " << (now - last_sample_time_) + << " fps: " << fps << " fps_bad: " << fps_bad << " qp: " << qp + << " qp_bad: " << qp_bad << " variance_bad: " << variance_bad + << " fps_variance: " << fps_variance; + + last_sample_time_ = now; + qp_sample_.Reset(); +} + VideoReceiveStream::Stats ReceiveStatisticsProxy::GetStats() const { rtc::CritScope lock(&crit_); return stats_; @@ -191,6 +282,7 @@ void ReceiveStatisticsProxy::OnIncomingRate(unsigned int framerate, unsigned int bitrate_bps) { rtc::CritScope lock(&crit_); + QualitySample(); stats_.network_frame_rate = framerate; stats_.total_bitrate_bps = bitrate_bps; } @@ -340,6 +432,8 @@ } if (codec_specific_info->codecType == kVideoCodecVP8) { qp_counters_.vp8.Add(encoded_image.qp_); + rtc::CritScope lock(&crit_); + qp_sample_.Add(encoded_image.qp_); } } @@ -354,4 +448,9 @@ return sum / num_samples; } +void ReceiveStatisticsProxy::SampleCounter::Reset() { + num_samples = 0; + sum = 0; +} + } // namespace webrtc
diff --git a/webrtc/video/receive_statistics_proxy.h b/webrtc/video/receive_statistics_proxy.h index b75ad8c..33b13f7 100644 --- a/webrtc/video/receive_statistics_proxy.h +++ b/webrtc/video/receive_statistics_proxy.h
@@ -21,6 +21,7 @@ #include "webrtc/common_types.h" #include "webrtc/common_video/include/frame_callback.h" #include "webrtc/modules/video_coding/include/video_coding_defines.h" +#include "webrtc/video/quality_threshold.h" #include "webrtc/video/report_block_stats.h" #include "webrtc/video/stats_counter.h" #include "webrtc/video/video_stream_decoder.h" @@ -85,6 +86,7 @@ SampleCounter() : sum(0), num_samples(0) {} void Add(int sample); int Avg(int min_required_samples) const; + void Reset(); private: int sum; @@ -96,6 +98,8 @@ void UpdateHistograms() EXCLUSIVE_LOCKS_REQUIRED(crit_); + void QualitySample() EXCLUSIVE_LOCKS_REQUIRED(crit_); + Clock* const clock_; // Ownership of this object lies with the owner of the ReceiveStatisticsProxy // instance. Lifetime is guaranteed to outlive |this|. @@ -108,6 +112,11 @@ const int64_t start_ms_; rtc::CriticalSection crit_; + int64_t last_sample_time_ GUARDED_BY(crit_); + QualityThreshold fps_threshold_ GUARDED_BY(crit_); + QualityThreshold qp_threshold_ GUARDED_BY(crit_); + QualityThreshold variance_threshold_ GUARDED_BY(crit_); + SampleCounter qp_sample_ GUARDED_BY(crit_); VideoReceiveStream::Stats stats_ GUARDED_BY(crit_); RateStatistics decode_fps_estimator_ GUARDED_BY(crit_); RateStatistics renders_fps_estimator_ GUARDED_BY(crit_);