Analyze quality of dropped frames in VideoProcessor.
Calculate quality metrics for dropped frames by comparing original
frame against last decoded one.
This feature makes comparison of encoders which do/don't drop frames
more fair.
The feature is controlled by analyze_quality_of_dropped_frames flag
and is disabled by default.
Bug: none
Change-Id: Ifab8df92d0b76e743ff3193c05d7c8dbd14921c4
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/190660
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Sebastian Jansson <srte@webrtc.org>
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32518}
diff --git a/api/test/videocodec_test_fixture.h b/api/test/videocodec_test_fixture.h
index 2ed4091..395c5cb 100644
--- a/api/test/videocodec_test_fixture.h
+++ b/api/test/videocodec_test_fixture.h
@@ -138,6 +138,9 @@
bool save_encoded_ivf = false;
bool save_decoded_y4m = false;
} visualization_params;
+
+ // Enables quality analysis for dropped frames.
+ bool analyze_quality_of_dropped_frames = false;
};
virtual ~VideoCodecTestFixture() = default;
diff --git a/api/test/videocodec_test_stats.h b/api/test/videocodec_test_stats.h
index 63e1576..df1aed7 100644
--- a/api/test/videocodec_test_stats.h
+++ b/api/test/videocodec_test_stats.h
@@ -67,6 +67,7 @@
int qp = -1;
// Quality.
+ bool quality_analysis_successful = false;
float psnr_y = 0.0f;
float psnr_u = 0.0f;
float psnr_v = 0.0f;
diff --git a/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc
index aa3ffed..ebe90b8 100644
--- a/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc
+++ b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc
@@ -450,6 +450,8 @@
}
}
+ task_queue->PostTask([this] { processor_->Finalize(); });
+
// Wait until we know that the last frame has been sent for encode.
task_queue->SendTask([] {}, RTC_FROM_HERE);
diff --git a/modules/video_coding/codecs/test/videocodec_test_stats_impl.cc b/modules/video_coding/codecs/test/videocodec_test_stats_impl.cc
index 7290d29..aa0ff0b 100644
--- a/modules/video_coding/codecs/test/videocodec_test_stats_impl.cc
+++ b/modules/video_coding/codecs/test/videocodec_test_stats_impl.cc
@@ -252,12 +252,6 @@
video_stat.height =
std::max(video_stat.height, frame_stat.decoded_height);
- psnr_y.AddSample(frame_stat.psnr_y);
- psnr_u.AddSample(frame_stat.psnr_u);
- psnr_v.AddSample(frame_stat.psnr_v);
- psnr.AddSample(frame_stat.psnr);
- ssim.AddSample(frame_stat.ssim);
-
if (video_stat.num_decoded_frames > 1) {
if (last_successfully_decoded_frame.decoded_width !=
frame_stat.decoded_width ||
@@ -271,6 +265,14 @@
last_successfully_decoded_frame = frame_stat;
}
+ if (frame_stat.quality_analysis_successful) {
+ psnr_y.AddSample(frame_stat.psnr_y);
+ psnr_u.AddSample(frame_stat.psnr_u);
+ psnr_v.AddSample(frame_stat.psnr_v);
+ psnr.AddSample(frame_stat.psnr);
+ ssim.AddSample(frame_stat.ssim);
+ }
+
if (video_stat.num_input_frames > 0) {
if (video_stat.time_to_reach_target_bitrate_sec == 0.0f) {
RTC_CHECK_GT(time_since_first_frame_sec, 0);
diff --git a/modules/video_coding/codecs/test/videoprocessor.cc b/modules/video_coding/codecs/test/videoprocessor.cc
index f495bdd..f433268 100644
--- a/modules/video_coding/codecs/test/videoprocessor.cc
+++ b/modules/video_coding/codecs/test/videoprocessor.cc
@@ -41,8 +41,6 @@
namespace webrtc {
namespace test {
-using FrameStatistics = VideoCodecTestStats::FrameStatistics;
-
namespace {
const int kMsToRtpTimestamp = kVideoPayloadTypeFrequency / 1000;
const int kMaxBufferedInputFrames = 20;
@@ -86,34 +84,9 @@
return static_cast<int>(diff_us);
}
-void ExtractI420BufferWithSize(const VideoFrame& image,
- int width,
- int height,
- rtc::Buffer* buffer) {
- if (image.width() != width || image.height() != height) {
- EXPECT_DOUBLE_EQ(static_cast<double>(width) / height,
- static_cast<double>(image.width()) / image.height());
- // Same aspect ratio, no cropping needed.
- rtc::scoped_refptr<I420Buffer> scaled(I420Buffer::Create(width, height));
- scaled->ScaleFrom(*image.video_frame_buffer()->ToI420());
-
- size_t length =
- CalcBufferSize(VideoType::kI420, scaled->width(), scaled->height());
- buffer->SetSize(length);
- RTC_CHECK_NE(ExtractBuffer(scaled, length, buffer->data()), -1);
- return;
- }
-
- // No resize.
- size_t length =
- CalcBufferSize(VideoType::kI420, image.width(), image.height());
- buffer->SetSize(length);
- RTC_CHECK_NE(ExtractBuffer(image, length, buffer->data()), -1);
-}
-
void CalculateFrameQuality(const I420BufferInterface& ref_buffer,
const I420BufferInterface& dec_buffer,
- FrameStatistics* frame_stat,
+ VideoCodecTestStats::FrameStatistics* frame_stat,
bool calc_ssim) {
if (ref_buffer.width() != dec_buffer.width() ||
ref_buffer.height() != dec_buffer.height()) {
@@ -174,6 +147,7 @@
num_simulcast_or_spatial_layers_(
std::max(config_.NumberOfSimulcastStreams(),
config_.NumberOfSpatialLayers())),
+ analyze_frame_quality_(!config_.measure_cpu),
stats_(stats),
encoder_(encoder),
decoders_(decoders),
@@ -192,8 +166,9 @@
last_encoded_frame_num_(num_simulcast_or_spatial_layers_),
first_decoded_frame_(num_simulcast_or_spatial_layers_, true),
last_decoded_frame_num_(num_simulcast_or_spatial_layers_),
- decoded_frame_buffer_(num_simulcast_or_spatial_layers_),
- post_encode_time_ns_(0) {
+ last_decoded_frame_buffer_(num_simulcast_or_spatial_layers_),
+ post_encode_time_ns_(0),
+ is_finalized_(false) {
// Sanity checks.
RTC_CHECK(TaskQueueBase::Current())
<< "VideoProcessor must be run on a task queue.";
@@ -234,6 +209,10 @@
VideoProcessor::~VideoProcessor() {
RTC_DCHECK_RUN_ON(&sequence_checker_);
+ if (!is_finalized_) {
+ Finalize();
+ }
+
// Explicitly reset codecs, in case they don't do that themselves when they
// go out of scope.
RTC_CHECK_EQ(encoder_->Release(), WEBRTC_VIDEO_CODEC_OK);
@@ -249,6 +228,8 @@
void VideoProcessor::ProcessFrame() {
RTC_DCHECK_RUN_ON(&sequence_checker_);
+ RTC_DCHECK(!is_finalized_);
+
const size_t frame_number = last_inputed_frame_num_++;
// Get input frame and store for future quality calculation.
@@ -304,6 +285,8 @@
void VideoProcessor::SetRates(size_t bitrate_kbps, double framerate_fps) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
+ RTC_DCHECK(!is_finalized_);
+
framerate_fps_ = framerate_fps;
bitrate_allocation_ =
bitrate_allocator_->Allocate(VideoBitrateAllocationParameters(
@@ -460,6 +443,56 @@
}
}
+void VideoProcessor::CalcFrameQuality(const I420BufferInterface& decoded_frame,
+ FrameStatistics* frame_stat) {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
+
+ const auto reference_frame = input_frames_.find(frame_stat->frame_number);
+ RTC_CHECK(reference_frame != input_frames_.cend())
+ << "The codecs are either buffering too much, dropping too much, or "
+ "being too slow relative to the input frame rate.";
+
+ // SSIM calculation is not optimized. Skip it in real-time mode.
+ const bool calc_ssim = !config_.encode_in_real_time;
+ CalculateFrameQuality(*reference_frame->second.video_frame_buffer()->ToI420(),
+ decoded_frame, frame_stat, calc_ssim);
+
+ frame_stat->quality_analysis_successful = true;
+}
+
+void VideoProcessor::WriteDecodedFrame(const I420BufferInterface& decoded_frame,
+ FrameWriter& frame_writer) {
+ int input_video_width = config_.codec_settings.width;
+ int input_video_height = config_.codec_settings.height;
+
+ rtc::scoped_refptr<I420Buffer> scaled_buffer;
+ const I420BufferInterface* scaled_frame;
+
+ if (decoded_frame.width() == input_video_width &&
+ decoded_frame.height() == input_video_height) {
+ scaled_frame = &decoded_frame;
+ } else {
+ EXPECT_DOUBLE_EQ(
+ static_cast<double>(input_video_width) / input_video_height,
+ static_cast<double>(decoded_frame.width()) / decoded_frame.height());
+
+ scaled_buffer = I420Buffer::Create(input_video_width, input_video_height);
+ scaled_buffer->ScaleFrom(decoded_frame);
+
+ scaled_frame = scaled_buffer;
+ }
+
+ // Ensure there is no padding.
+ RTC_CHECK_EQ(scaled_frame->StrideY(), input_video_width);
+ RTC_CHECK_EQ(scaled_frame->StrideU(), input_video_width / 2);
+ RTC_CHECK_EQ(scaled_frame->StrideV(), input_video_width / 2);
+
+ RTC_CHECK_EQ(3 * input_video_width * input_video_height / 2,
+ frame_writer.FrameLength());
+
+ RTC_CHECK(frame_writer.WriteFrame(scaled_frame->DataY()));
+}
+
void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame,
size_t spatial_idx) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
@@ -472,13 +505,24 @@
stats_->GetFrameWithTimestamp(decoded_frame.timestamp(), spatial_idx);
const size_t frame_number = frame_stat->frame_number;
- if (decoded_frame_writers_ && !first_decoded_frame_[spatial_idx]) {
- // Fill drops with last decoded frame to make them look like freeze at
- // playback and to keep decoded layers in sync.
- for (size_t i = last_decoded_frame_num_[spatial_idx] + 1; i < frame_number;
- ++i) {
- RTC_CHECK(decoded_frame_writers_->at(spatial_idx)
- ->WriteFrame(decoded_frame_buffer_[spatial_idx].data()));
+ if (!first_decoded_frame_[spatial_idx]) {
+ for (size_t dropped_frame_number = last_decoded_frame_num_[spatial_idx] + 1;
+ dropped_frame_number < frame_number; ++dropped_frame_number) {
+ FrameStatistics* dropped_frame_stat =
+ stats_->GetFrame(dropped_frame_number, spatial_idx);
+
+ if (analyze_frame_quality_ && config_.analyze_quality_of_dropped_frames) {
+ // Calculate frame quality comparing input frame with last decoded one.
+ CalcFrameQuality(*last_decoded_frame_buffer_[spatial_idx],
+ dropped_frame_stat);
+ }
+
+ if (decoded_frame_writers_ != nullptr) {
+ // Fill drops with last decoded frame to make them look like freeze at
+ // playback and to keep decoded layers in sync.
+ WriteDecodedFrame(*last_decoded_frame_buffer_[spatial_idx],
+ *decoded_frame_writers_->at(spatial_idx));
+ }
}
}
@@ -497,41 +541,40 @@
frame_stat->decoded_height = decoded_frame.height();
// Skip quality metrics calculation to not affect CPU usage.
- if (!config_.measure_cpu) {
- const auto reference_frame = input_frames_.find(frame_number);
- RTC_CHECK(reference_frame != input_frames_.cend())
- << "The codecs are either buffering too much, dropping too much, or "
- "being too slow relative the input frame rate.";
+ if (analyze_frame_quality_ || decoded_frame_writers_) {
+ // Save last decoded frame to handle possible future drops.
+ rtc::scoped_refptr<I420BufferInterface> i420buffer =
+ decoded_frame.video_frame_buffer()->ToI420();
- // SSIM calculation is not optimized. Skip it in real-time mode.
- const bool calc_ssim = !config_.encode_in_real_time;
- CalculateFrameQuality(
- *reference_frame->second.video_frame_buffer()->ToI420(),
- *decoded_frame.video_frame_buffer()->ToI420(), frame_stat, calc_ssim);
-
- // Erase all buffered input frames that we have moved past for all
- // simulcast/spatial layers. Never buffer more than
- // |kMaxBufferedInputFrames| frames, to protect against long runs of
- // consecutive frame drops for a particular layer.
- const auto min_last_decoded_frame_num = std::min_element(
- last_decoded_frame_num_.cbegin(), last_decoded_frame_num_.cend());
- const size_t min_buffered_frame_num = std::max(
- 0, static_cast<int>(frame_number) - kMaxBufferedInputFrames + 1);
- RTC_CHECK(min_last_decoded_frame_num != last_decoded_frame_num_.cend());
- const auto input_frames_erase_before = input_frames_.lower_bound(
- std::max(*min_last_decoded_frame_num, min_buffered_frame_num));
- input_frames_.erase(input_frames_.cbegin(), input_frames_erase_before);
+ // Copy decoded frame to a buffer without padding/stride such that we can
+ // dump Y, U and V planes into a file in one shot.
+ last_decoded_frame_buffer_[spatial_idx] = I420Buffer::Copy(
+ i420buffer->width(), i420buffer->height(), i420buffer->DataY(),
+ i420buffer->StrideY(), i420buffer->DataU(), i420buffer->StrideU(),
+ i420buffer->DataV(), i420buffer->StrideV());
}
- if (decoded_frame_writers_) {
- ExtractI420BufferWithSize(decoded_frame, config_.codec_settings.width,
- config_.codec_settings.height,
- &decoded_frame_buffer_[spatial_idx]);
- RTC_CHECK_EQ(decoded_frame_buffer_[spatial_idx].size(),
- decoded_frame_writers_->at(spatial_idx)->FrameLength());
- RTC_CHECK(decoded_frame_writers_->at(spatial_idx)
- ->WriteFrame(decoded_frame_buffer_[spatial_idx].data()));
+ if (analyze_frame_quality_) {
+ CalcFrameQuality(*decoded_frame.video_frame_buffer()->ToI420(), frame_stat);
}
+
+ if (decoded_frame_writers_ != nullptr) {
+ WriteDecodedFrame(*last_decoded_frame_buffer_[spatial_idx],
+ *decoded_frame_writers_->at(spatial_idx));
+ }
+
+ // Erase all buffered input frames that we have moved past for all
+ // simulcast/spatial layers. Never buffer more than
+ // |kMaxBufferedInputFrames| frames, to protect against long runs of
+ // consecutive frame drops for a particular layer.
+ const auto min_last_decoded_frame_num = std::min_element(
+ last_decoded_frame_num_.cbegin(), last_decoded_frame_num_.cend());
+ const size_t min_buffered_frame_num =
+ std::max(0, static_cast<int>(frame_number) - kMaxBufferedInputFrames + 1);
+ RTC_CHECK(min_last_decoded_frame_num != last_decoded_frame_num_.cend());
+ const auto input_frames_erase_before = input_frames_.lower_bound(
+ std::max(*min_last_decoded_frame_num, min_buffered_frame_num));
+ input_frames_.erase(input_frames_.cbegin(), input_frames_erase_before);
}
void VideoProcessor::DecodeFrame(const EncodedImage& encoded_image,
@@ -588,5 +631,41 @@
return &merged_encoded_frames_.at(spatial_idx);
}
+void VideoProcessor::Finalize() {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
+ RTC_DCHECK(!is_finalized_);
+ is_finalized_ = true;
+
+ if (!(analyze_frame_quality_ && config_.analyze_quality_of_dropped_frames) &&
+ decoded_frame_writers_ == nullptr) {
+ return;
+ }
+
+ for (size_t spatial_idx = 0; spatial_idx < num_simulcast_or_spatial_layers_;
+ ++spatial_idx) {
+ if (first_decoded_frame_[spatial_idx]) {
+ continue; // No decoded frames on this spatial layer.
+ }
+
+ for (size_t dropped_frame_number = last_decoded_frame_num_[spatial_idx] + 1;
+ dropped_frame_number < last_inputed_frame_num_;
+ ++dropped_frame_number) {
+ FrameStatistics* frame_stat =
+ stats_->GetFrame(dropped_frame_number, spatial_idx);
+
+ RTC_DCHECK(!frame_stat->decoding_successful);
+
+ if (analyze_frame_quality_ && config_.analyze_quality_of_dropped_frames) {
+ CalcFrameQuality(*last_decoded_frame_buffer_[spatial_idx], frame_stat);
+ }
+
+ if (decoded_frame_writers_ != nullptr) {
+ WriteDecodedFrame(*last_decoded_frame_buffer_[spatial_idx],
+ *decoded_frame_writers_->at(spatial_idx));
+ }
+ }
+ }
+}
+
} // namespace test
} // namespace webrtc
diff --git a/modules/video_coding/codecs/test/videoprocessor.h b/modules/video_coding/codecs/test/videoprocessor.h
index a1c96c6..cd755ea 100644
--- a/modules/video_coding/codecs/test/videoprocessor.h
+++ b/modules/video_coding/codecs/test/videoprocessor.h
@@ -24,6 +24,7 @@
#include "api/task_queue/task_queue_base.h"
#include "api/test/videocodec_test_fixture.h"
#include "api/video/encoded_image.h"
+#include "api/video/i420_buffer.h"
#include "api/video/video_bitrate_allocation.h"
#include "api/video/video_bitrate_allocator.h"
#include "api/video/video_frame.h"
@@ -58,6 +59,7 @@
// TODO(brandtr): Consider changing FrameWriterList to be a FrameWriterMap,
// to be able to save different TLs separately.
using FrameWriterList = std::vector<std::unique_ptr<FrameWriter>>;
+ using FrameStatistics = VideoCodecTestStats::FrameStatistics;
VideoProcessor(webrtc::VideoEncoder* encoder,
VideoDecoderList* decoders,
@@ -77,6 +79,11 @@
// Updates the encoder with target rates. Must be called at least once.
void SetRates(size_t bitrate_kbps, double framerate_fps);
+ // Signals processor to finalize frame processing and handle possible tail
+ // drops. If not called expelicitly, this will be called in dtor. It is
+ // unexpected to get ProcessFrame() or SetRates() calls after Finalize().
+ void Finalize();
+
private:
class VideoProcessorEncodeCompleteCallback
: public webrtc::EncodedImageCallback {
@@ -182,9 +189,20 @@
size_t simulcast_svc_idx,
bool inter_layer_predicted) RTC_RUN_ON(sequence_checker_);
- // Test input/output.
- VideoCodecTestFixture::Config config_ RTC_GUARDED_BY(sequence_checker_);
+ void CalcFrameQuality(const I420BufferInterface& decoded_frame,
+ FrameStatistics* frame_stat);
+
+ void WriteDecodedFrame(const I420BufferInterface& decoded_frame,
+ FrameWriter& frame_writer);
+
+ void HandleTailDrops();
+
+ // Test config.
+ const VideoCodecTestFixture::Config config_;
const size_t num_simulcast_or_spatial_layers_;
+ const bool analyze_frame_quality_;
+
+ // Frame statistics.
VideoCodecTestStatsImpl* const stats_;
// Codecs.
@@ -240,7 +258,7 @@
// simulcast_svc_idx -> frame_number.
std::vector<size_t> last_decoded_frame_num_ RTC_GUARDED_BY(sequence_checker_);
// simulcast_svc_idx -> buffer.
- std::vector<rtc::Buffer> decoded_frame_buffer_
+ std::vector<rtc::scoped_refptr<I420Buffer>> last_decoded_frame_buffer_
RTC_GUARDED_BY(sequence_checker_);
// Time spent in frame encode callback. It is accumulated for layers and
@@ -248,6 +266,9 @@
// is substracted from measured encode time. Thus we get pure encode time.
int64_t post_encode_time_ns_ RTC_GUARDED_BY(sequence_checker_);
+ // Indicates whether Finalize() was called or not.
+ bool is_finalized_ RTC_GUARDED_BY(sequence_checker_);
+
// This class must be operated on a TaskQueue.
SequenceChecker sequence_checker_;
diff --git a/test/testsupport/frame_writer.h b/test/testsupport/frame_writer.h
index b91e57c..5f85d8b 100644
--- a/test/testsupport/frame_writer.h
+++ b/test/testsupport/frame_writer.h
@@ -32,7 +32,7 @@
// Writes a frame of the configured frame length to the output file.
// Returns true if the write was successful, false otherwise.
- virtual bool WriteFrame(uint8_t* frame_buffer) = 0;
+ virtual bool WriteFrame(const uint8_t* frame_buffer) = 0;
// Closes the output file if open. Essentially makes this class impossible
// to use anymore. Will also be invoked by the destructor.
@@ -54,7 +54,7 @@
YuvFrameWriterImpl(std::string output_filename, int width, int height);
~YuvFrameWriterImpl() override;
bool Init() override;
- bool WriteFrame(uint8_t* frame_buffer) override;
+ bool WriteFrame(const uint8_t* frame_buffer) override;
void Close() override;
size_t FrameLength() override;
@@ -76,7 +76,7 @@
int frame_rate);
~Y4mFrameWriterImpl() override;
bool Init() override;
- bool WriteFrame(uint8_t* frame_buffer) override;
+ bool WriteFrame(const uint8_t* frame_buffer) override;
private:
const int frame_rate_;
diff --git a/test/testsupport/y4m_frame_writer.cc b/test/testsupport/y4m_frame_writer.cc
index 25106d8..896524f 100644
--- a/test/testsupport/y4m_frame_writer.cc
+++ b/test/testsupport/y4m_frame_writer.cc
@@ -41,7 +41,7 @@
return true;
}
-bool Y4mFrameWriterImpl::WriteFrame(uint8_t* frame_buffer) {
+bool Y4mFrameWriterImpl::WriteFrame(const uint8_t* frame_buffer) {
if (output_file_ == nullptr) {
fprintf(stderr,
"Y4mFrameWriterImpl is not initialized (output file is NULL)\n");
diff --git a/test/testsupport/yuv_frame_writer.cc b/test/testsupport/yuv_frame_writer.cc
index 24f4f53..59cb74e 100644
--- a/test/testsupport/yuv_frame_writer.cc
+++ b/test/testsupport/yuv_frame_writer.cc
@@ -50,7 +50,7 @@
return true;
}
-bool YuvFrameWriterImpl::WriteFrame(uint8_t* frame_buffer) {
+bool YuvFrameWriterImpl::WriteFrame(const uint8_t* frame_buffer) {
RTC_DCHECK(frame_buffer);
if (output_file_ == nullptr) {
fprintf(stderr,
