| /* |
| * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
| * |
| * Use of this source code is governed by a BSD-style license |
| * that can be found in the LICENSE file in the root of the source |
| * tree. An additional intellectual property rights grant can be found |
| * in the file PATENTS. All contributing project authors may |
| * be found in the AUTHORS file in the root of the source tree. |
| */ |
| |
| #include "webrtc/video/vie_encoder.h" |
| |
| #include <algorithm> |
| #include <limits> |
| |
| #include "webrtc/base/checks.h" |
| #include "webrtc/base/logging.h" |
| #include "webrtc/base/trace_event.h" |
| #include "webrtc/base/timeutils.h" |
| #include "webrtc/modules/pacing/paced_sender.h" |
| #include "webrtc/modules/video_coding/include/video_coding.h" |
| #include "webrtc/modules/video_coding/include/video_coding_defines.h" |
| #include "webrtc/system_wrappers/include/metrics.h" |
| #include "webrtc/video/overuse_frame_detector.h" |
| #include "webrtc/video/send_statistics_proxy.h" |
| #include "webrtc/video_frame.h" |
| |
| namespace webrtc { |
| |
| ViEEncoder::ViEEncoder(uint32_t number_of_cores, |
| ProcessThread* module_process_thread, |
| SendStatisticsProxy* stats_proxy, |
| OveruseFrameDetector* overuse_detector, |
| EncodedImageCallback* sink) |
| : number_of_cores_(number_of_cores), |
| sink_(sink), |
| vp_(VideoProcessing::Create()), |
| video_sender_(Clock::GetRealTimeClock(), this, this), |
| stats_proxy_(stats_proxy), |
| overuse_detector_(overuse_detector), |
| time_of_last_frame_activity_ms_(std::numeric_limits<int64_t>::max()), |
| encoder_config_(), |
| last_observed_bitrate_bps_(0), |
| encoder_paused_and_dropped_frame_(false), |
| module_process_thread_(module_process_thread), |
| has_received_sli_(false), |
| picture_id_sli_(0), |
| has_received_rpsi_(false), |
| picture_id_rpsi_(0), |
| video_suspended_(false) { |
| module_process_thread_->RegisterModule(&video_sender_); |
| vp_->EnableTemporalDecimation(true); |
| } |
| |
| vcm::VideoSender* ViEEncoder::video_sender() { |
| return &video_sender_; |
| } |
| |
| ViEEncoder::~ViEEncoder() { |
| module_process_thread_->DeRegisterModule(&video_sender_); |
| } |
| |
| int32_t ViEEncoder::RegisterExternalEncoder(webrtc::VideoEncoder* encoder, |
| uint8_t pl_type, |
| bool internal_source) { |
| video_sender_.RegisterExternalEncoder(encoder, pl_type, internal_source); |
| return 0; |
| } |
| |
| int32_t ViEEncoder::DeRegisterExternalEncoder(uint8_t pl_type) { |
| video_sender_.RegisterExternalEncoder(nullptr, pl_type, false); |
| return 0; |
| } |
| |
| void ViEEncoder::SetEncoder(const webrtc::VideoCodec& video_codec, |
| size_t max_data_payload_length) { |
| // Setting target width and height for VPM. |
| RTC_CHECK_EQ(VPM_OK, |
| vp_->SetTargetResolution(video_codec.width, video_codec.height, |
| video_codec.maxFramerate)); |
| { |
| rtc::CritScope lock(&data_cs_); |
| encoder_config_ = video_codec; |
| } |
| |
| bool success = video_sender_.RegisterSendCodec( |
| &video_codec, number_of_cores_, |
| static_cast<uint32_t>(max_data_payload_length)) == VCM_OK; |
| |
| if (!success) { |
| LOG(LS_ERROR) << "Failed to configure encoder."; |
| RTC_DCHECK(success); |
| } |
| |
| if (stats_proxy_) { |
| VideoEncoderConfig::ContentType content_type = |
| VideoEncoderConfig::ContentType::kRealtimeVideo; |
| switch (video_codec.mode) { |
| case kRealtimeVideo: |
| content_type = VideoEncoderConfig::ContentType::kRealtimeVideo; |
| break; |
| case kScreensharing: |
| content_type = VideoEncoderConfig::ContentType::kScreen; |
| break; |
| default: |
| RTC_NOTREACHED(); |
| break; |
| } |
| stats_proxy_->SetContentType(content_type); |
| } |
| } |
| |
| bool ViEEncoder::EncoderPaused() const { |
| // Pause video if paused by caller or as long as the network is down or the |
| // pacer queue has grown too large in buffered mode. |
| // If the pacer queue has grown too large or the network is down, |
| // last_observed_bitrate_bps_ will be 0. |
| return video_suspended_ || last_observed_bitrate_bps_ == 0; |
| } |
| |
| void ViEEncoder::TraceFrameDropStart() { |
| // Start trace event only on the first frame after encoder is paused. |
| if (!encoder_paused_and_dropped_frame_) { |
| TRACE_EVENT_ASYNC_BEGIN0("webrtc", "EncoderPaused", this); |
| } |
| encoder_paused_and_dropped_frame_ = true; |
| return; |
| } |
| |
| void ViEEncoder::TraceFrameDropEnd() { |
| // End trace event on first frame after encoder resumes, if frame was dropped. |
| if (encoder_paused_and_dropped_frame_) { |
| TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this); |
| } |
| encoder_paused_and_dropped_frame_ = false; |
| } |
| |
| void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame) { |
| VideoCodecType codec_type; |
| { |
| rtc::CritScope lock(&data_cs_); |
| time_of_last_frame_activity_ms_ = rtc::TimeMillis(); |
| if (EncoderPaused()) { |
| TraceFrameDropStart(); |
| return; |
| } |
| TraceFrameDropEnd(); |
| codec_type = encoder_config_.codecType; |
| } |
| |
| TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), |
| "Encode"); |
| const VideoFrame* frame_to_send = &video_frame; |
| // TODO(wuchengli): support texture frames. |
| if (!video_frame.video_frame_buffer()->native_handle()) { |
| // Pass frame via preprocessor. |
| frame_to_send = vp_->PreprocessFrame(video_frame); |
| if (!frame_to_send) { |
| // Drop this frame, or there was an error processing it. |
| return; |
| } |
| } |
| |
| if (codec_type == webrtc::kVideoCodecVP8) { |
| webrtc::CodecSpecificInfo codec_specific_info; |
| codec_specific_info.codecType = webrtc::kVideoCodecVP8; |
| { |
| rtc::CritScope lock(&data_cs_); |
| codec_specific_info.codecSpecific.VP8.hasReceivedRPSI = |
| has_received_rpsi_; |
| codec_specific_info.codecSpecific.VP8.hasReceivedSLI = |
| has_received_sli_; |
| codec_specific_info.codecSpecific.VP8.pictureIdRPSI = |
| picture_id_rpsi_; |
| codec_specific_info.codecSpecific.VP8.pictureIdSLI = |
| picture_id_sli_; |
| has_received_sli_ = false; |
| has_received_rpsi_ = false; |
| } |
| |
| video_sender_.AddVideoFrame(*frame_to_send, &codec_specific_info); |
| return; |
| } |
| video_sender_.AddVideoFrame(*frame_to_send, nullptr); |
| } |
| |
| void ViEEncoder::SendKeyFrame() { |
| video_sender_.IntraFrameRequest(0); |
| } |
| |
| int64_t ViEEncoder::time_of_last_frame_activity_ms() { |
| rtc::CritScope lock(&data_cs_); |
| return time_of_last_frame_activity_ms_; |
| } |
| |
| int32_t ViEEncoder::Encoded(const EncodedImage& encoded_image, |
| const CodecSpecificInfo* codec_specific_info, |
| const RTPFragmentationHeader* fragmentation) { |
| { |
| rtc::CritScope lock(&data_cs_); |
| time_of_last_frame_activity_ms_ = rtc::TimeMillis(); |
| } |
| if (stats_proxy_) { |
| stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); |
| } |
| |
| int success = |
| sink_->Encoded(encoded_image, codec_specific_info, fragmentation); |
| |
| overuse_detector_->FrameSent(encoded_image._timeStamp); |
| return success; |
| } |
| |
| void ViEEncoder::SendStatistics(uint32_t bit_rate, |
| uint32_t frame_rate, |
| const std::string& encoder_name) { |
| if (stats_proxy_) |
| stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate, encoder_name); |
| } |
| |
| void ViEEncoder::OnReceivedSLI(uint8_t picture_id) { |
| rtc::CritScope lock(&data_cs_); |
| picture_id_sli_ = picture_id; |
| has_received_sli_ = true; |
| } |
| |
| void ViEEncoder::OnReceivedRPSI(uint64_t picture_id) { |
| rtc::CritScope lock(&data_cs_); |
| picture_id_rpsi_ = picture_id; |
| has_received_rpsi_ = true; |
| } |
| |
| void ViEEncoder::OnReceivedIntraFrameRequest(size_t stream_index) { |
| // Key frame request from remote side, signal to VCM. |
| TRACE_EVENT0("webrtc", "OnKeyFrameRequest"); |
| video_sender_.IntraFrameRequest(stream_index); |
| } |
| |
| void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps, |
| uint8_t fraction_lost, |
| int64_t round_trip_time_ms) { |
| LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate " << bitrate_bps |
| << " packet loss " << static_cast<int>(fraction_lost) |
| << " rtt " << round_trip_time_ms; |
| video_sender_.SetChannelParameters(bitrate_bps, fraction_lost, |
| round_trip_time_ms); |
| bool video_suspension_changed; |
| bool video_is_suspended = bitrate_bps == 0; |
| { |
| rtc::CritScope lock(&data_cs_); |
| last_observed_bitrate_bps_ = bitrate_bps; |
| video_suspension_changed = video_suspended_ != video_is_suspended; |
| video_suspended_ = video_is_suspended; |
| // Set |time_of_last_frame_activity_ms_| to now if this is the first time |
| // the encoder is supposed to produce encoded frames. |
| // TODO(perkj): Remove this hack. It is here to avoid a race that the |
| // encoder report that it has timed out before it has processed the first |
| // frame. |
| if (last_observed_bitrate_bps_ != 0 && |
| time_of_last_frame_activity_ms_ == |
| std::numeric_limits<int64_t>::max()) { |
| time_of_last_frame_activity_ms_ = rtc::TimeMillis(); |
| } |
| } |
| |
| if (stats_proxy_ && video_suspension_changed) { |
| LOG(LS_INFO) << "Video suspend state changed to: " |
| << (video_is_suspended ? "suspended" : "not suspended"); |
| stats_proxy_->OnSuspendChange(video_is_suspended); |
| } |
| } |
| |
| } // namespace webrtc |
