Move calculation of target_encode_bitrate to DefaultVideoQualityAnalyzer
To migrate on new GetStats API and properly support target encode bitrate
for regular, simulcast and svc cases we need to calculate it inside video
quality analyzer getting values from SetRates in VideoEncoder.
Bug: webrtc:11381
Change-Id: Ia37acac764ed3c30f64cdbfda8906d543fa03ae2
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/171501
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Patrik Höglund <phoglund@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30881}
diff --git a/api/test/video_quality_analyzer_interface.h b/api/test/video_quality_analyzer_interface.h
index 990548a..0d3f441 100644
--- a/api/test/video_quality_analyzer_interface.h
+++ b/api/test/video_quality_analyzer_interface.h
@@ -53,6 +53,20 @@
// The analyzer will be injected in all points from A to F.
class VideoQualityAnalyzerInterface : public StatsObserverInterface {
public:
+ // Contains extra statistic provided by video encoder.
+ struct EncoderStats {
+ // TODO(hbos) https://crbug.com/webrtc/9547,
+ // https://crbug.com/webrtc/11443: improve stats API to make available
+ // there.
+ uint32_t target_encode_bitrate;
+ };
+ // Contains extra statistic provided by video decoder.
+ struct DecoderStats {
+ // Decode time provided by decoder itself. If decoder doesn’t produce such
+ // information can be omitted.
+ absl::optional<int32_t> decode_time_ms;
+ };
+
~VideoQualityAnalyzerInterface() override = default;
// Will be called by framework before test.
@@ -74,18 +88,16 @@
// VideoFrame can produce multiple EncodedImages. Each encoded image will
// have id from VideoFrame.
virtual void OnFrameEncoded(uint16_t frame_id,
- const EncodedImage& encoded_image) {}
+ const EncodedImage& encoded_image,
+ const EncoderStats& stats) {}
// Will be called for each frame dropped by encoder.
virtual void OnFrameDropped(EncodedImageCallback::DropReason reason) {}
// Will be called before calling the decoder.
virtual void OnFramePreDecode(uint16_t frame_id,
const EncodedImage& encoded_image) {}
- // Will be called after decoding the frame. |decode_time_ms| is a decode
- // time provided by decoder itself. If decoder doesn’t produce such
- // information can be omitted.
+ // Will be called after decoding the frame.
virtual void OnFrameDecoded(const VideoFrame& frame,
- absl::optional<int32_t> decode_time_ms,
- absl::optional<uint8_t> qp) {}
+ const DecoderStats& stats) {}
// Will be called when frame will be obtained from PeerConnection stack.
virtual void OnFrameRendered(const VideoFrame& frame) {}
// Will be called if encoder return not WEBRTC_VIDEO_CODEC_OK.
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
index 81a0448..40a885f 100644
--- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
@@ -26,6 +26,7 @@
constexpr int kMaxActiveComparisons = 10;
constexpr int kFreezeThresholdMs = 150;
constexpr int kMicrosPerSecond = 1000000;
+constexpr int kBitsInByte = 8;
void LogFrameCounters(const std::string& name, const FrameCounters& counters) {
RTC_LOG(INFO) << "[" << name << "] Captured : " << counters.captured;
@@ -180,7 +181,8 @@
void DefaultVideoQualityAnalyzer::OnFrameEncoded(
uint16_t frame_id,
- const webrtc::EncodedImage& encoded_image) {
+ const webrtc::EncodedImage& encoded_image,
+ const EncoderStats& stats) {
rtc::CritScope crit(&lock_);
auto it = frame_stats_.find(frame_id);
RTC_DCHECK(it != frame_stats_.end());
@@ -193,6 +195,7 @@
}
it->second.encoded_time = Now();
it->second.encoded_image_size = encoded_image.size();
+ it->second.target_encode_bitrate = stats.target_encode_bitrate;
}
void DefaultVideoQualityAnalyzer::OnFrameDropped(
@@ -226,8 +229,7 @@
void DefaultVideoQualityAnalyzer::OnFrameDecoded(
const webrtc::VideoFrame& frame,
- absl::optional<int32_t> decode_time_ms,
- absl::optional<uint8_t> qp) {
+ const DecoderStats& stats) {
rtc::CritScope crit(&lock_);
auto it = frame_stats_.find(frame.id());
RTC_DCHECK(it != frame_stats_.end());
@@ -517,6 +519,7 @@
(frame_stats.encoded_time - frame_stats.pre_encode_time).ms());
stats->encode_frame_rate.AddEvent(frame_stats.encoded_time);
stats->total_encoded_images_payload += frame_stats.encoded_image_size;
+ stats->target_encode_bitrate.AddSample(frame_stats.target_encode_bitrate);
} else {
if (frame_stats.pre_encode_time.IsFinite()) {
stats->dropped_by_encoder++;
@@ -670,6 +673,9 @@
/*important=*/false, ImproveDirection::kSmallerIsBetter);
ReportResult("max_skipped", test_case_name, stats.skipped_between_rendered,
"count", ImproveDirection::kSmallerIsBetter);
+ ReportResult("target_encode_bitrate", test_case_name,
+ stats.target_encode_bitrate / kBitsInByte, "bytesPerSecond",
+ ImproveDirection::kNone);
test::PrintResult(
"actual_encode_bitrate", "", test_case_name,
static_cast<double>(stats.total_encoded_images_payload) /
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
index 31e4267..778ccb3 100644
--- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
@@ -101,6 +101,7 @@
// Mean time between one freeze end and next freeze start.
SamplesStatsCounter time_between_freezes_ms;
SamplesStatsCounter resolution_of_rendered_frame;
+ SamplesStatsCounter target_encode_bitrate;
int64_t total_encoded_images_payload = 0;
int64_t dropped_by_encoder = 0;
@@ -138,13 +139,13 @@
const VideoFrame& frame) override;
void OnFramePreEncode(const VideoFrame& frame) override;
void OnFrameEncoded(uint16_t frame_id,
- const EncodedImage& encoded_image) override;
+ const EncodedImage& encoded_image,
+ const EncoderStats& stats) override;
void OnFrameDropped(EncodedImageCallback::DropReason reason) override;
void OnFramePreDecode(uint16_t frame_id,
const EncodedImage& input_image) override;
void OnFrameDecoded(const VideoFrame& frame,
- absl::optional<int32_t> decode_time_ms,
- absl::optional<uint8_t> qp) override;
+ const DecoderStats& stats) override;
void OnFrameRendered(const VideoFrame& frame) override;
void OnEncoderError(const VideoFrame& frame, int32_t error_code) override;
void OnDecoderError(uint16_t frame_id, int32_t error_code) override;
@@ -181,6 +182,8 @@
Timestamp rendered_time = Timestamp::MinusInfinity();
Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity();
+ uint32_t target_encode_bitrate = 0;
+
absl::optional<int> rendered_frame_width = absl::nullopt;
absl::optional<int> rendered_frame_height = absl::nullopt;
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc
index 1a59015..1bc29c5 100644
--- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc
@@ -83,14 +83,15 @@
frames_order.push_back(frame.id());
captured_frames.insert({frame.id(), frame});
analyzer.OnFramePreEncode(frame);
- analyzer.OnFrameEncoded(frame.id(), FakeEncode(frame));
+ analyzer.OnFrameEncoded(frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats());
}
for (const uint16_t& frame_id : frames_order) {
VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
analyzer.OnFramePreDecode(received_frame.id(), FakeEncode(received_frame));
- analyzer.OnFrameDecoded(received_frame, /*decode_time_ms=*/absl::nullopt,
- /*qp=*/absl::nullopt);
+ analyzer.OnFrameDecoded(received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
analyzer.OnFrameRendered(received_frame);
}
@@ -129,15 +130,16 @@
frames_order.push_back(frame.id());
captured_frames.insert({frame.id(), frame});
analyzer.OnFramePreEncode(frame);
- analyzer.OnFrameEncoded(frame.id(), FakeEncode(frame));
+ analyzer.OnFrameEncoded(frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats());
}
for (size_t i = kMaxFramesInFlightPerStream; i < frames_order.size(); ++i) {
uint16_t frame_id = frames_order.at(i);
VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
analyzer.OnFramePreDecode(received_frame.id(), FakeEncode(received_frame));
- analyzer.OnFrameDecoded(received_frame, /*decode_time_ms=*/absl::nullopt,
- /*qp=*/absl::nullopt);
+ analyzer.OnFrameDecoded(received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
analyzer.OnFrameRendered(received_frame);
}
@@ -174,15 +176,16 @@
frames_order.push_back(frame.id());
captured_frames.insert({frame.id(), frame});
analyzer.OnFramePreEncode(frame);
- analyzer.OnFrameEncoded(frame.id(), FakeEncode(frame));
+ analyzer.OnFrameEncoded(frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats());
}
for (size_t i = 1; i < frames_order.size(); i += 2) {
uint16_t frame_id = frames_order.at(i);
VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
analyzer.OnFramePreDecode(received_frame.id(), FakeEncode(received_frame));
- analyzer.OnFrameDecoded(received_frame, /*decode_time_ms=*/absl::nullopt,
- /*qp=*/absl::nullopt);
+ analyzer.OnFrameDecoded(received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
analyzer.OnFrameRendered(received_frame);
}
diff --git a/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc
index caa639a..d1d1bfa 100644
--- a/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc
+++ b/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc
@@ -52,7 +52,8 @@
void ExampleVideoQualityAnalyzer::OnFrameEncoded(
uint16_t frame_id,
- const webrtc::EncodedImage& encoded_image) {
+ const webrtc::EncodedImage& encoded_image,
+ const EncoderStats& stats) {
rtc::CritScope crit(&lock_);
++frames_encoded_;
}
@@ -73,8 +74,7 @@
void ExampleVideoQualityAnalyzer::OnFrameDecoded(
const webrtc::VideoFrame& frame,
- absl::optional<int32_t> decode_time_ms,
- absl::optional<uint8_t> qp) {
+ const DecoderStats& stats) {
rtc::CritScope crit(&lock_);
++frames_decoded_;
}
diff --git a/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h
index 8b29e12..0d6169f 100644
--- a/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h
+++ b/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h
@@ -38,13 +38,13 @@
const VideoFrame& frame) override;
void OnFramePreEncode(const VideoFrame& frame) override;
void OnFrameEncoded(uint16_t frame_id,
- const EncodedImage& encoded_image) override;
+ const EncodedImage& encoded_image,
+ const EncoderStats& stats) override;
void OnFrameDropped(EncodedImageCallback::DropReason reason) override;
void OnFramePreDecode(uint16_t frame_id,
const EncodedImage& encoded_image) override;
void OnFrameDecoded(const VideoFrame& frame,
- absl::optional<int32_t> decode_time_ms,
- absl::optional<uint8_t> qp) override;
+ const DecoderStats& stats) override;
void OnFrameRendered(const VideoFrame& frame) override;
void OnEncoderError(const VideoFrame& frame, int32_t error_code) override;
void OnDecoderError(uint16_t frame_id, int32_t error_code) override;
diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc
index d4d9611..228ab8a 100644
--- a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc
+++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc
@@ -222,7 +222,9 @@
// Set frame id to the value, that was extracted from corresponding encoded
// image.
frame->set_id(frame_id);
- analyzer_->OnFrameDecoded(*frame, decode_time_ms, qp);
+ VideoQualityAnalyzerInterface::DecoderStats stats;
+ stats.decode_time_ms = decode_time_ms;
+ analyzer_->OnFrameDecoded(*frame, stats);
}
QualityAnalyzingVideoDecoderFactory::QualityAnalyzingVideoDecoderFactory(
diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc
index 96b23b4..6ab2938 100644
--- a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc
+++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc
@@ -161,6 +161,10 @@
const VideoEncoder::RateControlParameters& parameters) {
RTC_DCHECK_GT(bitrate_multiplier_, 0.0);
if (fabs(bitrate_multiplier_ - kNoMultiplier) < kEps) {
+ {
+ rtc::CritScope crit(&lock_);
+ bitrate_allocation_ = parameters.bitrate;
+ }
return delegate_->SetRates(parameters);
}
@@ -200,6 +204,10 @@
RateControlParameters adjusted_params = parameters;
adjusted_params.bitrate = multiplied_allocation;
+ {
+ rtc::CritScope crit(&lock_);
+ bitrate_allocation_ = adjusted_params.bitrate;
+ }
return delegate_->SetRates(adjusted_params);
}
@@ -226,6 +234,7 @@
const RTPFragmentationHeader* fragmentation) {
uint16_t frame_id;
bool discard = false;
+ uint32_t target_encode_bitrate = 0;
{
rtc::CritScope crit(&lock_);
std::pair<uint32_t, uint16_t> timestamp_frame_id;
@@ -257,11 +266,20 @@
frame_id = timestamp_frame_id.second;
discard = ShouldDiscard(frame_id, encoded_image);
+ if (!discard) {
+ std::string stream_label = analyzer_->GetStreamLabel(frame_id);
+ absl::optional<int> required_spatial_index =
+ stream_required_spatial_index_[stream_label];
+ target_encode_bitrate = bitrate_allocation_.GetSpatialLayerSum(
+ required_spatial_index.value_or(0));
+ }
}
if (!discard) {
// Analyzer should see only encoded images, that weren't discarded.
- analyzer_->OnFrameEncoded(frame_id, encoded_image);
+ VideoQualityAnalyzerInterface::EncoderStats stats;
+ stats.target_encode_bitrate = target_encode_bitrate;
+ analyzer_->OnFrameEncoded(frame_id, encoded_image, stats);
}
// Image data injector injects frame id and discard flag into provided
diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h
index 247be73..03231be 100644
--- a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h
+++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h
@@ -150,6 +150,7 @@
EncodedImageCallback* delegate_callback_ RTC_GUARDED_BY(lock_);
std::list<std::pair<uint32_t, uint16_t>> timestamp_to_frame_id_list_
RTC_GUARDED_BY(lock_);
+ VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(lock_);
};
// Produces QualityAnalyzingVideoEncoder, which hold decoders, produced by
diff --git a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc
index b461c6a..754a0a4 100644
--- a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc
+++ b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc
@@ -42,13 +42,9 @@
const webrtc::StatsReport::Value* transmission_bitrate =
stats_report->FindValue(
StatsReport::StatsValueName::kStatsValueNameTransmitBitrate);
- const webrtc::StatsReport::Value* target_encode_bitrate =
- stats_report->FindValue(
- StatsReport::StatsValueName::kStatsValueNameTargetEncBitrate);
RTC_CHECK(available_send_bandwidth);
RTC_CHECK(retransmission_bitrate);
RTC_CHECK(transmission_bitrate);
- RTC_CHECK(target_encode_bitrate);
rtc::CritScope crit(&video_bwe_stats_lock_);
VideoBweStats& video_bwe_stats = video_bwe_stats_[pc_label];
@@ -58,8 +54,6 @@
transmission_bitrate->int_val());
video_bwe_stats.retransmission_bitrate.AddSample(
retransmission_bitrate->int_val());
- video_bwe_stats.target_encode_bitrate.AddSample(
- target_encode_bitrate->int_val());
}
}
@@ -87,9 +81,6 @@
ReportResult("retransmission_bitrate", test_case_name,
video_bwe_stats.retransmission_bitrate / kBitsInByte,
"bytesPerSecond");
- ReportResult("target_encode_bitrate", test_case_name,
- video_bwe_stats.target_encode_bitrate / kBitsInByte,
- "bytesPerSecond");
}
void VideoQualityMetricsReporter::ReportResult(
diff --git a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h
index fe2f169..1688a7b 100644
--- a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h
+++ b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h
@@ -26,7 +26,6 @@
SamplesStatsCounter available_send_bandwidth;
SamplesStatsCounter transmission_bitrate;
SamplesStatsCounter retransmission_bitrate;
- SamplesStatsCounter target_encode_bitrate;
};
class VideoQualityMetricsReporter
