| /* |
| * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. |
| * |
| * Use of this source code is governed by a BSD-style license |
| * that can be found in the LICENSE file in the root of the source |
| * tree. An additional intellectual property rights grant can be found |
| * in the file PATENTS. All contributing project authors may |
| * be found in the AUTHORS file in the root of the source tree. |
| */ |
| |
| #include "test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h" |
| |
| #include <cstdint> |
| #include <cstring> |
| #include <utility> |
| |
| #include "absl/memory/memory.h" |
| #include "absl/types/optional.h" |
| #include "api/video/i420_buffer.h" |
| #include "modules/video_coding/include/video_error_codes.h" |
| #include "rtc_base/logging.h" |
| #include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h" |
| |
| namespace webrtc { |
| namespace webrtc_pc_e2e { |
| |
| QualityAnalyzingVideoDecoder::QualityAnalyzingVideoDecoder( |
| int id, |
| std::unique_ptr<VideoDecoder> delegate, |
| EncodedImageDataExtractor* extractor, |
| VideoQualityAnalyzerInterface* analyzer) |
| : id_(id), |
| implementation_name_("AnalyzingDecoder-" + |
| std::string(delegate->ImplementationName())), |
| delegate_(std::move(delegate)), |
| extractor_(extractor), |
| analyzer_(analyzer) { |
| analyzing_callback_ = absl::make_unique<DecoderCallback>(this); |
| } |
| QualityAnalyzingVideoDecoder::~QualityAnalyzingVideoDecoder() = default; |
| |
| int32_t QualityAnalyzingVideoDecoder::InitDecode( |
| const VideoCodec* codec_settings, |
| int32_t number_of_cores) { |
| return delegate_->InitDecode(codec_settings, number_of_cores); |
| } |
| |
| int32_t QualityAnalyzingVideoDecoder::Decode( |
| const EncodedImage& input_image, |
| bool missing_frames, |
| int64_t render_time_ms) { |
| // Image extractor extracts id from provided EncodedImage and also returns |
| // the image with the original buffer. Buffer can be modified in place, so |
| // owner of original buffer will be responsible for deleting it, or extractor |
| // can create a new buffer. In such case extractor will be responsible for |
| // deleting it. |
| EncodedImageExtractionResult out = extractor_->ExtractData(input_image, id_); |
| |
| if (out.discard) { |
| // To partly emulate behavior of Selective Forwarding Unit (SFU) in the |
| // test, on receiver side we will "discard" frames from irrelevant streams. |
| // When all encoded images were marked to discarded, black frame have to be |
| // returned. Because simulcast streams will be received by receiver as 3 |
| // different independent streams we don't want that irrelevant streams |
| // affect video quality metrics and also we don't want to use CPU time to |
| // decode them to prevent regressions on relevant streams. Also we can't |
| // just drop frame, because in such case, receiving part will be confused |
| // with all frames missing and will request a key frame, which will result |
| // into extra load on network and sender side. Because of it, discarded |
| // image will be always decoded as black frame and will be passed to |
| // callback directly without reaching decoder and video quality analyzer. |
| // |
| // For more details see QualityAnalyzingVideoEncoder. |
| return analyzing_callback_->IrrelevantSimulcastStreamDecoded( |
| out.id, input_image.Timestamp()); |
| } |
| |
| EncodedImage* origin_image; |
| { |
| rtc::CritScope crit(&lock_); |
| // Store id to be able to retrieve it in analyzing callback. |
| timestamp_to_frame_id_.insert({input_image.Timestamp(), out.id}); |
| // Store encoded image to prevent its destruction while it is used in |
| // decoder. |
| origin_image = &( |
| decoding_images_.insert({out.id, std::move(out.image)}).first->second); |
| } |
| // We can safely dereference |origin_image|, because it can be removed from |
| // the map only after |delegate_| Decode method will be invoked. Image will be |
| // removed inside DecodedImageCallback, which can be done on separate thread. |
| analyzer_->OnFrameReceived(out.id, *origin_image); |
| int32_t result = |
| delegate_->Decode(*origin_image, missing_frames, render_time_ms); |
| if (result != WEBRTC_VIDEO_CODEC_OK) { |
| // If delegate decoder failed, then cleanup data for this image. |
| { |
| rtc::CritScope crit(&lock_); |
| timestamp_to_frame_id_.erase(input_image.Timestamp()); |
| decoding_images_.erase(out.id); |
| } |
| analyzer_->OnDecoderError(out.id, result); |
| } |
| return result; |
| } |
| |
| int32_t QualityAnalyzingVideoDecoder::RegisterDecodeCompleteCallback( |
| DecodedImageCallback* callback) { |
| analyzing_callback_->SetDelegateCallback(callback); |
| return delegate_->RegisterDecodeCompleteCallback(analyzing_callback_.get()); |
| } |
| |
| int32_t QualityAnalyzingVideoDecoder::Release() { |
| rtc::CritScope crit(&lock_); |
| analyzing_callback_->SetDelegateCallback(nullptr); |
| timestamp_to_frame_id_.clear(); |
| decoding_images_.clear(); |
| return delegate_->Release(); |
| } |
| |
| bool QualityAnalyzingVideoDecoder::PrefersLateDecoding() const { |
| return delegate_->PrefersLateDecoding(); |
| } |
| |
| const char* QualityAnalyzingVideoDecoder::ImplementationName() const { |
| return implementation_name_.c_str(); |
| } |
| |
| QualityAnalyzingVideoDecoder::DecoderCallback::DecoderCallback( |
| QualityAnalyzingVideoDecoder* decoder) |
| : decoder_(decoder), delegate_callback_(nullptr) {} |
| QualityAnalyzingVideoDecoder::DecoderCallback::~DecoderCallback() = default; |
| |
| void QualityAnalyzingVideoDecoder::DecoderCallback::SetDelegateCallback( |
| DecodedImageCallback* delegate) { |
| rtc::CritScope crit(&callback_lock_); |
| delegate_callback_ = delegate; |
| } |
| |
| // We have to implement all next 3 methods because we don't know which one |
| // exactly is implemented in |delegate_callback_|, so we need to call the same |
| // method on |delegate_callback_|, as was called on |this| callback. |
| int32_t QualityAnalyzingVideoDecoder::DecoderCallback::Decoded( |
| VideoFrame& decodedImage) { |
| decoder_->OnFrameDecoded(&decodedImage, /*decode_time_ms=*/absl::nullopt, |
| /*qp=*/absl::nullopt); |
| |
| rtc::CritScope crit(&callback_lock_); |
| RTC_DCHECK(delegate_callback_); |
| return delegate_callback_->Decoded(decodedImage); |
| } |
| |
| int32_t QualityAnalyzingVideoDecoder::DecoderCallback::Decoded( |
| VideoFrame& decodedImage, |
| int64_t decode_time_ms) { |
| decoder_->OnFrameDecoded(&decodedImage, decode_time_ms, /*qp=*/absl::nullopt); |
| |
| rtc::CritScope crit(&callback_lock_); |
| RTC_DCHECK(delegate_callback_); |
| return delegate_callback_->Decoded(decodedImage, decode_time_ms); |
| } |
| |
| void QualityAnalyzingVideoDecoder::DecoderCallback::Decoded( |
| VideoFrame& decodedImage, |
| absl::optional<int32_t> decode_time_ms, |
| absl::optional<uint8_t> qp) { |
| decoder_->OnFrameDecoded(&decodedImage, decode_time_ms, qp); |
| |
| rtc::CritScope crit(&callback_lock_); |
| RTC_DCHECK(delegate_callback_); |
| delegate_callback_->Decoded(decodedImage, decode_time_ms, qp); |
| } |
| |
| int32_t |
| QualityAnalyzingVideoDecoder::DecoderCallback::IrrelevantSimulcastStreamDecoded( |
| uint16_t frame_id, |
| uint32_t timestamp_ms) { |
| webrtc::VideoFrame dummy_frame = |
| webrtc::VideoFrame::Builder() |
| .set_video_frame_buffer(GetDummyFrameBuffer()) |
| .set_timestamp_rtp(timestamp_ms) |
| .set_id(frame_id) |
| .build(); |
| rtc::CritScope crit(&callback_lock_); |
| RTC_DCHECK(delegate_callback_); |
| delegate_callback_->Decoded(dummy_frame, absl::nullopt, absl::nullopt); |
| return WEBRTC_VIDEO_CODEC_OK; |
| } |
| |
| rtc::scoped_refptr<webrtc::VideoFrameBuffer> |
| QualityAnalyzingVideoDecoder::DecoderCallback::GetDummyFrameBuffer() { |
| if (!dummy_frame_buffer_) { |
| dummy_frame_buffer_ = CreateDummyFrameBuffer(); |
| } |
| |
| return dummy_frame_buffer_; |
| } |
| |
| void QualityAnalyzingVideoDecoder::OnFrameDecoded( |
| VideoFrame* frame, |
| absl::optional<int32_t> decode_time_ms, |
| absl::optional<uint8_t> qp) { |
| uint16_t frame_id; |
| { |
| rtc::CritScope crit(&lock_); |
| auto it = timestamp_to_frame_id_.find(frame->timestamp()); |
| if (it == timestamp_to_frame_id_.end()) { |
| // Ensure, that we have info about this frame. It can happen that for some |
| // reasons decoder response, that he failed to decode, when we were |
| // posting frame to it, but then call the callback for this frame. |
| RTC_LOG(LS_ERROR) << "QualityAnalyzingVideoDecoder::OnFrameDecoded: No " |
| "frame id for frame for frame->timestamp()=" |
| << frame->timestamp(); |
| return; |
| } |
| frame_id = it->second; |
| timestamp_to_frame_id_.erase(it); |
| decoding_images_.erase(frame_id); |
| } |
| // Set frame id to the value, that was extracted from corresponding encoded |
| // image. |
| frame->set_id(frame_id); |
| analyzer_->OnFrameDecoded(*frame, decode_time_ms, qp); |
| } |
| |
| QualityAnalyzingVideoDecoderFactory::QualityAnalyzingVideoDecoderFactory( |
| std::unique_ptr<VideoDecoderFactory> delegate, |
| IdGenerator<int>* id_generator, |
| EncodedImageDataExtractor* extractor, |
| VideoQualityAnalyzerInterface* analyzer) |
| : delegate_(std::move(delegate)), |
| id_generator_(id_generator), |
| extractor_(extractor), |
| analyzer_(analyzer) {} |
| QualityAnalyzingVideoDecoderFactory::~QualityAnalyzingVideoDecoderFactory() = |
| default; |
| |
| std::vector<SdpVideoFormat> |
| QualityAnalyzingVideoDecoderFactory::GetSupportedFormats() const { |
| return delegate_->GetSupportedFormats(); |
| } |
| |
| std::unique_ptr<VideoDecoder> |
| QualityAnalyzingVideoDecoderFactory::CreateVideoDecoder( |
| const SdpVideoFormat& format) { |
| std::unique_ptr<VideoDecoder> decoder = delegate_->CreateVideoDecoder(format); |
| return absl::make_unique<QualityAnalyzingVideoDecoder>( |
| id_generator_->GetNextId(), std::move(decoder), extractor_, analyzer_); |
| } |
| |
| std::unique_ptr<VideoDecoder> |
| QualityAnalyzingVideoDecoderFactory::LegacyCreateVideoDecoder( |
| const SdpVideoFormat& format, |
| const std::string& receive_stream_id) { |
| std::unique_ptr<VideoDecoder> decoder = |
| delegate_->LegacyCreateVideoDecoder(format, receive_stream_id); |
| return absl::make_unique<QualityAnalyzingVideoDecoder>( |
| id_generator_->GetNextId(), std::move(decoder), extractor_, analyzer_); |
| } |
| |
| } // namespace webrtc_pc_e2e |
| } // namespace webrtc |