niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 1 | /* |
stefan@webrtc.org | 07b45a5 | 2012-02-02 08:37:48 | [diff] [blame] | 2 | * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
pbos@webrtc.org | f5d4cb1 | 2013-05-17 13:44:48 | [diff] [blame] | 11 | #include "webrtc/video_engine/vie_encoder.h" |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 12 | |
pbos@webrtc.org | 12dc1a3 | 2013-08-05 16:22:53 | [diff] [blame] | 13 | #include <assert.h> |
| 14 | |
stefan@webrtc.org | c3cc375 | 2013-06-04 09:36:56 | [diff] [blame] | 15 | #include <algorithm> |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 16 | |
mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 | [diff] [blame] | 17 | #include "webrtc/base/checks.h" |
sprang@webrtc.org | 4070935 | 2013-11-26 11:41:59 | [diff] [blame] | 18 | #include "webrtc/common_video/interface/video_image.h" |
pbos@webrtc.org | f5d4cb1 | 2013-05-17 13:44:48 | [diff] [blame] | 19 | #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" |
pbos@webrtc.org | 273a414 | 2014-12-01 15:23:21 | [diff] [blame] | 20 | #include "webrtc/frame_callback.h" |
Peter Boström | 8e4e8b0 | 2015-09-15 13:08:03 | [diff] [blame] | 21 | #include "webrtc/modules/bitrate_controller/include/bitrate_controller.h" |
pbos@webrtc.org | f5d4cb1 | 2013-05-17 13:44:48 | [diff] [blame] | 22 | #include "webrtc/modules/pacing/include/paced_sender.h" |
pbos@webrtc.org | f5d4cb1 | 2013-05-17 13:44:48 | [diff] [blame] | 23 | #include "webrtc/modules/utility/interface/process_thread.h" |
| 24 | #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h" |
| 25 | #include "webrtc/modules/video_coding/main/interface/video_coding.h" |
| 26 | #include "webrtc/modules/video_coding/main/interface/video_coding_defines.h" |
sprang@webrtc.org | 4070935 | 2013-11-26 11:41:59 | [diff] [blame] | 27 | #include "webrtc/modules/video_coding/main/source/encoded_frame.h" |
stefan@webrtc.org | 88e0dda | 2014-07-04 09:20:42 | [diff] [blame] | 28 | #include "webrtc/system_wrappers/interface/clock.h" |
pbos@webrtc.org | f5d4cb1 | 2013-05-17 13:44:48 | [diff] [blame] | 29 | #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" |
| 30 | #include "webrtc/system_wrappers/interface/logging.h" |
asapersson@webrtc.org | 96dc685 | 2014-11-03 14:40:38 | [diff] [blame] | 31 | #include "webrtc/system_wrappers/interface/metrics.h" |
pbos@webrtc.org | f5d4cb1 | 2013-05-17 13:44:48 | [diff] [blame] | 32 | #include "webrtc/system_wrappers/interface/tick_util.h" |
pbos@webrtc.org | f5d4cb1 | 2013-05-17 13:44:48 | [diff] [blame] | 33 | #include "webrtc/system_wrappers/interface/trace_event.h" |
pbos@webrtc.org | 273a414 | 2014-12-01 15:23:21 | [diff] [blame] | 34 | #include "webrtc/video/send_statistics_proxy.h" |
mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 | [diff] [blame] | 35 | #include "webrtc/video_engine/payload_router.h" |
pbos@webrtc.org | f5d4cb1 | 2013-05-17 13:44:48 | [diff] [blame] | 36 | #include "webrtc/video_engine/vie_defines.h" |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 37 | |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 38 | namespace webrtc { |
| 39 | |
pwestin@webrtc.org | 52b4e88 | 2013-05-02 19:02:17 | [diff] [blame] | 40 | // Margin on when we pause the encoder when the pacing buffer overflows relative |
| 41 | // to the configured buffer delay. |
| 42 | static const float kEncoderPausePacerMargin = 2.0f; |
| 43 | |
pwestin@webrtc.org | 91563e4 | 2013-04-25 22:20:08 | [diff] [blame] | 44 | // Don't stop the encoder unless the delay is above this configured value. |
| 45 | static const int kMinPacingDelayMs = 200; |
| 46 | |
stefan@webrtc.org | 3e00505 | 2013-10-18 15:05:29 | [diff] [blame] | 47 | static const float kStopPaddingThresholdMs = 2000; |
| 48 | |
stefan@webrtc.org | b2c8a95 | 2013-09-06 13:58:01 | [diff] [blame] | 49 | std::vector<uint32_t> AllocateStreamBitrates( |
| 50 | uint32_t total_bitrate, |
| 51 | const SimulcastStream* stream_configs, |
| 52 | size_t number_of_streams) { |
| 53 | if (number_of_streams == 0) { |
| 54 | std::vector<uint32_t> stream_bitrates(1, 0); |
| 55 | stream_bitrates[0] = total_bitrate; |
| 56 | return stream_bitrates; |
| 57 | } |
| 58 | std::vector<uint32_t> stream_bitrates(number_of_streams, 0); |
| 59 | uint32_t bitrate_remainder = total_bitrate; |
| 60 | for (size_t i = 0; i < stream_bitrates.size() && bitrate_remainder > 0; ++i) { |
| 61 | if (stream_configs[i].maxBitrate * 1000 > bitrate_remainder) { |
| 62 | stream_bitrates[i] = bitrate_remainder; |
| 63 | } else { |
| 64 | stream_bitrates[i] = stream_configs[i].maxBitrate * 1000; |
| 65 | } |
| 66 | bitrate_remainder -= stream_bitrates[i]; |
| 67 | } |
| 68 | return stream_bitrates; |
| 69 | } |
| 70 | |
stefan@webrtc.org | 439be29 | 2012-02-16 14:45:37 | [diff] [blame] | 71 | class QMVideoSettingsCallback : public VCMQMSettingsCallback { |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 72 | public: |
marpan@webrtc.org | efd01fd | 2012-04-18 15:56:34 | [diff] [blame] | 73 | explicit QMVideoSettingsCallback(VideoProcessingModule* vpm); |
mflodman@webrtc.org | 6879c8a | 2013-07-23 11:35:00 | [diff] [blame] | 74 | |
stefan@webrtc.org | 439be29 | 2012-02-16 14:45:37 | [diff] [blame] | 75 | ~QMVideoSettingsCallback(); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 76 | |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 77 | // Update VPM with QM (quality modes: frame size & frame rate) settings. |
pbos@webrtc.org | b238d12 | 2013-04-09 13:41:51 | [diff] [blame] | 78 | int32_t SetVideoQMSettings(const uint32_t frame_rate, |
| 79 | const uint32_t width, |
| 80 | const uint32_t height); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 81 | |
jackychen | 6e2ce6e | 2015-07-13 23:26:33 | [diff] [blame] | 82 | // Update target frame rate. |
| 83 | void SetTargetFramerate(int frame_rate); |
| 84 | |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 85 | private: |
| 86 | VideoProcessingModule* vpm_; |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 87 | }; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 88 | |
pwestin@webrtc.org | 49888ce | 2012-04-27 05:25:53 | [diff] [blame] | 89 | class ViEBitrateObserver : public BitrateObserver { |
| 90 | public: |
mflodman@webrtc.org | f5e99db | 2012-06-27 09:49:37 | [diff] [blame] | 91 | explicit ViEBitrateObserver(ViEEncoder* owner) |
pwestin@webrtc.org | 49888ce | 2012-04-27 05:25:53 | [diff] [blame] | 92 | : owner_(owner) { |
| 93 | } |
mflodman@webrtc.org | 6879c8a | 2013-07-23 11:35:00 | [diff] [blame] | 94 | virtual ~ViEBitrateObserver() {} |
pwestin@webrtc.org | 49888ce | 2012-04-27 05:25:53 | [diff] [blame] | 95 | // Implements BitrateObserver. |
stefan@webrtc.org | edeea91 | 2014-12-08 19:46:23 | [diff] [blame] | 96 | virtual void OnNetworkChanged(uint32_t bitrate_bps, |
| 97 | uint8_t fraction_lost, |
pkasting@chromium.org | 16825b1 | 2015-01-12 21:51:21 | [diff] [blame] | 98 | int64_t rtt) { |
pwestin@webrtc.org | 49888ce | 2012-04-27 05:25:53 | [diff] [blame] | 99 | owner_->OnNetworkChanged(bitrate_bps, fraction_lost, rtt); |
| 100 | } |
| 101 | private: |
| 102 | ViEEncoder* owner_; |
| 103 | }; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 104 | |
mflodman | 0dbf009 | 2015-10-19 15:12:12 | [diff] [blame] | 105 | ViEEncoder::ViEEncoder(uint32_t number_of_cores, |
Peter Boström | 7083e11 | 2015-09-22 14:28:51 | [diff] [blame] | 106 | ProcessThread* module_process_thread, |
| 107 | SendStatisticsProxy* stats_proxy, |
| 108 | I420FrameCallback* pre_encode_callback, |
Stefan Holmer | e590416 | 2015-03-26 10:11:06 | [diff] [blame] | 109 | PacedSender* pacer, |
Peter Boström | 8e4e8b0 | 2015-09-15 13:08:03 | [diff] [blame] | 110 | BitrateAllocator* bitrate_allocator) |
mflodman | 0dbf009 | 2015-10-19 15:12:12 | [diff] [blame] | 111 | : number_of_cores_(number_of_cores), |
Peter Boström | f4aa4c2 | 2015-09-18 10:24:25 | [diff] [blame] | 112 | vpm_(VideoProcessingModule::Create()), |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 113 | qm_callback_(new QMVideoSettingsCallback(vpm_.get())), |
| 114 | vcm_(VideoCodingModule::Create(Clock::GetRealTimeClock(), |
| 115 | this, |
| 116 | qm_callback_.get())), |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 | [diff] [blame] | 117 | send_payload_router_(NULL), |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 | [diff] [blame] | 118 | data_cs_(CriticalSectionWrapper::CreateCriticalSection()), |
Peter Boström | 7083e11 | 2015-09-22 14:28:51 | [diff] [blame] | 119 | stats_proxy_(stats_proxy), |
| 120 | pre_encode_callback_(pre_encode_callback), |
Stefan Holmer | e590416 | 2015-03-26 10:11:06 | [diff] [blame] | 121 | pacer_(pacer), |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 | [diff] [blame] | 122 | bitrate_allocator_(bitrate_allocator), |
Noah Richards | 099323e | 2015-04-15 16:14:12 | [diff] [blame] | 123 | time_of_last_frame_activity_ms_(0), |
Peter Boström | a753177 | 2015-09-15 11:12:24 | [diff] [blame] | 124 | simulcast_enabled_(false), |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 | [diff] [blame] | 125 | min_transmit_bitrate_kbps_(0), |
pbos@webrtc.org | 143451d | 2015-03-18 14:40:03 | [diff] [blame] | 126 | last_observed_bitrate_bps_(0), |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 | [diff] [blame] | 127 | target_delay_ms_(0), |
| 128 | network_is_transmitting_(true), |
| 129 | encoder_paused_(false), |
| 130 | encoder_paused_and_dropped_frame_(false), |
| 131 | fec_enabled_(false), |
| 132 | nack_enabled_(false), |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 | [diff] [blame] | 133 | module_process_thread_(module_process_thread), |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 | [diff] [blame] | 134 | has_received_sli_(false), |
| 135 | picture_id_sli_(0), |
| 136 | has_received_rpsi_(false), |
| 137 | picture_id_rpsi_(0), |
asapersson | dec5ebf | 2015-10-05 09:36:17 | [diff] [blame] | 138 | video_suspended_(false) { |
pwestin@webrtc.org | 49888ce | 2012-04-27 05:25:53 | [diff] [blame] | 139 | bitrate_observer_.reset(new ViEBitrateObserver(this)); |
wu@webrtc.org | 5d8c102 | 2012-04-10 16:54:05 | [diff] [blame] | 140 | } |
| 141 | |
| 142 | bool ViEEncoder::Init() { |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 143 | vpm_->EnableTemporalDecimation(true); |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 144 | |
| 145 | // Enable/disable content analysis: off by default for now. |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 146 | vpm_->EnableContentAnalysis(false); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 147 | |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 148 | if (vcm_->RegisterTransportCallback(this) != 0) { |
wu@webrtc.org | 5d8c102 | 2012-04-10 16:54:05 | [diff] [blame] | 149 | return false; |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 150 | } |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 151 | if (vcm_->RegisterSendStatisticsCallback(this) != 0) { |
wu@webrtc.org | 5d8c102 | 2012-04-10 16:54:05 | [diff] [blame] | 152 | return false; |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 153 | } |
wu@webrtc.org | 5d8c102 | 2012-04-10 16:54:05 | [diff] [blame] | 154 | return true; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 155 | } |
| 156 | |
mflodman@webrtc.org | 96abda0 | 2015-02-25 13:50:10 | [diff] [blame] | 157 | void ViEEncoder::StartThreadsAndSetSharedMembers( |
Peter Boström | 26b0860 | 2015-06-04 13:18:17 | [diff] [blame] | 158 | rtc::scoped_refptr<PayloadRouter> send_payload_router, |
mflodman@webrtc.org | 96abda0 | 2015-02-25 13:50:10 | [diff] [blame] | 159 | VCMProtectionCallback* vcm_protection_callback) { |
henrikg | 91d6ede | 2015-09-17 07:24:34 | [diff] [blame] | 160 | RTC_DCHECK(send_payload_router_ == NULL); |
mflodman@webrtc.org | 96abda0 | 2015-02-25 13:50:10 | [diff] [blame] | 161 | |
mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 | [diff] [blame] | 162 | send_payload_router_ = send_payload_router; |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 163 | vcm_->RegisterProtectionCallback(vcm_protection_callback); |
Peter Boström | 7083e11 | 2015-09-22 14:28:51 | [diff] [blame] | 164 | module_process_thread_->RegisterModule(vcm_.get()); |
mflodman@webrtc.org | 290cb56 | 2015-02-17 10:15:06 | [diff] [blame] | 165 | } |
| 166 | |
mflodman@webrtc.org | 96abda0 | 2015-02-25 13:50:10 | [diff] [blame] | 167 | void ViEEncoder::StopThreadsAndRemoveSharedMembers() { |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 168 | if (bitrate_allocator_) |
| 169 | bitrate_allocator_->RemoveBitrateObserver(bitrate_observer_.get()); |
Peter Boström | 7083e11 | 2015-09-22 14:28:51 | [diff] [blame] | 170 | module_process_thread_->DeRegisterModule(vcm_.get()); |
| 171 | module_process_thread_->DeRegisterModule(vpm_.get()); |
mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 | [diff] [blame] | 172 | } |
| 173 | |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 174 | ViEEncoder::~ViEEncoder() { |
asapersson@webrtc.org | 96dc685 | 2014-11-03 14:40:38 | [diff] [blame] | 175 | } |
| 176 | |
stefan@webrtc.org | bfacda6 | 2013-03-27 16:36:01 | [diff] [blame] | 177 | void ViEEncoder::SetNetworkTransmissionState(bool is_transmitting) { |
stefan@webrtc.org | bfacda6 | 2013-03-27 16:36:01 | [diff] [blame] | 178 | { |
| 179 | CriticalSectionScoped cs(data_cs_.get()); |
| 180 | network_is_transmitting_ = is_transmitting; |
| 181 | } |
stefan@webrtc.org | bfacda6 | 2013-03-27 16:36:01 | [diff] [blame] | 182 | } |
| 183 | |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 184 | void ViEEncoder::Pause() { |
mflodman@webrtc.org | d32c447 | 2011-12-22 14:17:53 | [diff] [blame] | 185 | CriticalSectionScoped cs(data_cs_.get()); |
stefan@webrtc.org | bfacda6 | 2013-03-27 16:36:01 | [diff] [blame] | 186 | encoder_paused_ = true; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 187 | } |
| 188 | |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 189 | void ViEEncoder::Restart() { |
mflodman@webrtc.org | d32c447 | 2011-12-22 14:17:53 | [diff] [blame] | 190 | CriticalSectionScoped cs(data_cs_.get()); |
stefan@webrtc.org | bfacda6 | 2013-03-27 16:36:01 | [diff] [blame] | 191 | encoder_paused_ = false; |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 192 | } |
| 193 | |
pbos@webrtc.org | b238d12 | 2013-04-09 13:41:51 | [diff] [blame] | 194 | uint8_t ViEEncoder::NumberOfCodecs() { |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 195 | return vcm_->NumberOfCodecs(); |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 196 | } |
| 197 | |
pbos@webrtc.org | b238d12 | 2013-04-09 13:41:51 | [diff] [blame] | 198 | int32_t ViEEncoder::GetCodec(uint8_t list_index, VideoCodec* video_codec) { |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 199 | if (vcm_->Codec(list_index, video_codec) != 0) { |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 200 | return -1; |
| 201 | } |
| 202 | return 0; |
| 203 | } |
| 204 | |
pbos@webrtc.org | b238d12 | 2013-04-09 13:41:51 | [diff] [blame] | 205 | int32_t ViEEncoder::RegisterExternalEncoder(webrtc::VideoEncoder* encoder, |
| 206 | uint8_t pl_type, |
| 207 | bool internal_source) { |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 208 | if (encoder == NULL) |
| 209 | return -1; |
| 210 | |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 211 | if (vcm_->RegisterExternalEncoder(encoder, pl_type, internal_source) != |
mflodman@webrtc.org | 5574dac | 2014-04-07 10:56:31 | [diff] [blame] | 212 | VCM_OK) { |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 213 | return -1; |
| 214 | } |
| 215 | return 0; |
| 216 | } |
| 217 | |
pbos@webrtc.org | b238d12 | 2013-04-09 13:41:51 | [diff] [blame] | 218 | int32_t ViEEncoder::DeRegisterExternalEncoder(uint8_t pl_type) { |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 219 | if (vcm_->RegisterExternalEncoder(NULL, pl_type) != VCM_OK) { |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 220 | return -1; |
| 221 | } |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 222 | return 0; |
| 223 | } |
| 224 | |
pbos@webrtc.org | b238d12 | 2013-04-09 13:41:51 | [diff] [blame] | 225 | int32_t ViEEncoder::SetEncoder(const webrtc::VideoCodec& video_codec) { |
henrikg | 91d6ede | 2015-09-17 07:24:34 | [diff] [blame] | 226 | RTC_DCHECK(send_payload_router_ != NULL); |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 227 | // Setting target width and height for VPM. |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 228 | if (vpm_->SetTargetResolution(video_codec.width, video_codec.height, |
| 229 | video_codec.maxFramerate) != VPM_OK) { |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 230 | return -1; |
| 231 | } |
| 232 | |
stefan@webrtc.org | 9075d51 | 2014-02-14 09:45:58 | [diff] [blame] | 233 | { |
| 234 | CriticalSectionScoped cs(data_cs_.get()); |
Peter Boström | a753177 | 2015-09-15 11:12:24 | [diff] [blame] | 235 | simulcast_enabled_ = video_codec.numberOfSimulcastStreams > 1; |
stefan@webrtc.org | 9075d51 | 2014-02-14 09:45:58 | [diff] [blame] | 236 | } |
Stefan Holmer | e590416 | 2015-03-26 10:11:06 | [diff] [blame] | 237 | |
| 238 | // Add a bitrate observer to the allocator and update the start, max and |
| 239 | // min bitrates of the bitrate controller as needed. |
Peter Boström | 8e4e8b0 | 2015-09-15 13:08:03 | [diff] [blame] | 240 | int allocated_bitrate_bps = bitrate_allocator_->AddBitrateObserver( |
| 241 | bitrate_observer_.get(), video_codec.minBitrate * 1000, |
| 242 | video_codec.maxBitrate * 1000); |
Stefan Holmer | e590416 | 2015-03-26 10:11:06 | [diff] [blame] | 243 | |
| 244 | webrtc::VideoCodec modified_video_codec = video_codec; |
| 245 | modified_video_codec.startBitrate = allocated_bitrate_bps / 1000; |
| 246 | |
mflodman@webrtc.org | a4ef2ce | 2015-02-12 09:54:18 | [diff] [blame] | 247 | size_t max_data_payload_length = send_payload_router_->MaxPayloadLength(); |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 248 | if (vcm_->RegisterSendCodec(&modified_video_codec, number_of_cores_, |
Peter Boström | ae37abb | 2015-06-18 17:00:34 | [diff] [blame] | 249 | static_cast<uint32_t>(max_data_payload_length)) != |
| 250 | VCM_OK) { |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 251 | return -1; |
| 252 | } |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 253 | return 0; |
| 254 | } |
| 255 | |
pbos@webrtc.org | b238d12 | 2013-04-09 13:41:51 | [diff] [blame] | 256 | int32_t ViEEncoder::GetEncoder(VideoCodec* video_codec) { |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 257 | *video_codec = vcm_->GetSendCodec(); |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 258 | return 0; |
| 259 | } |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 260 | |
pbos@webrtc.org | b238d12 | 2013-04-09 13:41:51 | [diff] [blame] | 261 | int32_t ViEEncoder::GetCodecConfigParameters( |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 262 | unsigned char config_parameters[kConfigParameterSize], |
| 263 | unsigned char& config_parameters_size) { |
pbos@webrtc.org | b238d12 | 2013-04-09 13:41:51 | [diff] [blame] | 264 | int32_t num_parameters = |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 265 | vcm_->CodecConfigParameters(config_parameters, kConfigParameterSize); |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 266 | if (num_parameters <= 0) { |
| 267 | config_parameters_size = 0; |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 268 | return -1; |
| 269 | } |
| 270 | config_parameters_size = static_cast<unsigned char>(num_parameters); |
| 271 | return 0; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 272 | } |
| 273 | |
pbos@webrtc.org | b238d12 | 2013-04-09 13:41:51 | [diff] [blame] | 274 | int32_t ViEEncoder::ScaleInputImage(bool enable) { |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 275 | VideoFrameResampling resampling_mode = kFastRescaling; |
mflodman@webrtc.org | 5574dac | 2014-04-07 10:56:31 | [diff] [blame] | 276 | // TODO(mflodman) What? |
| 277 | if (enable) { |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 278 | // kInterpolation is currently not supported. |
mflodman@webrtc.org | 5574dac | 2014-04-07 10:56:31 | [diff] [blame] | 279 | LOG_F(LS_ERROR) << "Not supported."; |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 280 | return -1; |
| 281 | } |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 282 | vpm_->SetInputFrameResampleMode(resampling_mode); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 283 | |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 284 | return 0; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 285 | } |
| 286 | |
stefan | a4a8d4a | 2015-07-15 11:39:22 | [diff] [blame] | 287 | int ViEEncoder::GetPaddingNeededBps() const { |
Noah Richards | 099323e | 2015-04-15 16:14:12 | [diff] [blame] | 288 | int64_t time_of_last_frame_activity_ms; |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 | [diff] [blame] | 289 | int min_transmit_bitrate_bps; |
stefan | a4a8d4a | 2015-07-15 11:39:22 | [diff] [blame] | 290 | int bitrate_bps; |
henrik.lundin@webrtc.org | 331d440 | 2013-11-21 14:05:40 | [diff] [blame] | 291 | { |
| 292 | CriticalSectionScoped cs(data_cs_.get()); |
Peter Boström | a753177 | 2015-09-15 11:12:24 | [diff] [blame] | 293 | bool send_padding = simulcast_enabled_ || video_suspended_ || |
| 294 | min_transmit_bitrate_kbps_ > 0; |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 | [diff] [blame] | 295 | if (!send_padding) |
| 296 | return 0; |
Noah Richards | 099323e | 2015-04-15 16:14:12 | [diff] [blame] | 297 | time_of_last_frame_activity_ms = time_of_last_frame_activity_ms_; |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 | [diff] [blame] | 298 | min_transmit_bitrate_bps = 1000 * min_transmit_bitrate_kbps_; |
stefan | a4a8d4a | 2015-07-15 11:39:22 | [diff] [blame] | 299 | bitrate_bps = last_observed_bitrate_bps_; |
henrik.lundin@webrtc.org | 331d440 | 2013-11-21 14:05:40 | [diff] [blame] | 300 | } |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 | [diff] [blame] | 301 | |
| 302 | VideoCodec send_codec; |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 303 | if (vcm_->SendCodec(&send_codec) != 0) |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 | [diff] [blame] | 304 | return 0; |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 | [diff] [blame] | 305 | |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 306 | bool video_is_suspended = vcm_->VideoSuspended(); |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 | [diff] [blame] | 307 | |
| 308 | // Find the max amount of padding we can allow ourselves to send at this |
| 309 | // point, based on which streams are currently active and what our current |
| 310 | // available bandwidth is. |
| 311 | int pad_up_to_bitrate_bps = 0; |
| 312 | if (send_codec.numberOfSimulcastStreams == 0) { |
| 313 | pad_up_to_bitrate_bps = send_codec.minBitrate * 1000; |
| 314 | } else { |
stefan | a4a8d4a | 2015-07-15 11:39:22 | [diff] [blame] | 315 | SimulcastStream* stream_configs = send_codec.simulcastStream; |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 | [diff] [blame] | 316 | pad_up_to_bitrate_bps = |
| 317 | stream_configs[send_codec.numberOfSimulcastStreams - 1].minBitrate * |
| 318 | 1000; |
| 319 | for (int i = 0; i < send_codec.numberOfSimulcastStreams - 1; ++i) { |
| 320 | pad_up_to_bitrate_bps += stream_configs[i].targetBitrate * 1000; |
| 321 | } |
stefan@webrtc.org | 508a84b | 2013-06-17 12:53:37 | [diff] [blame] | 322 | } |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 | [diff] [blame] | 323 | |
| 324 | // Disable padding if only sending one stream and video isn't suspended and |
| 325 | // min-transmit bitrate isn't used (applied later). |
| 326 | if (!video_is_suspended && send_codec.numberOfSimulcastStreams <= 1) |
| 327 | pad_up_to_bitrate_bps = 0; |
| 328 | |
| 329 | // The amount of padding should decay to zero if no frames are being |
Noah Richards | 099323e | 2015-04-15 16:14:12 | [diff] [blame] | 330 | // captured/encoded unless a min-transmit bitrate is used. |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 | [diff] [blame] | 331 | int64_t now_ms = TickTime::MillisecondTimestamp(); |
Noah Richards | 099323e | 2015-04-15 16:14:12 | [diff] [blame] | 332 | if (now_ms - time_of_last_frame_activity_ms > kStopPaddingThresholdMs) |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 | [diff] [blame] | 333 | pad_up_to_bitrate_bps = 0; |
| 334 | |
| 335 | // Pad up to min bitrate. |
| 336 | if (pad_up_to_bitrate_bps < min_transmit_bitrate_bps) |
| 337 | pad_up_to_bitrate_bps = min_transmit_bitrate_bps; |
| 338 | |
| 339 | // Padding may never exceed bitrate estimate. |
| 340 | if (pad_up_to_bitrate_bps > bitrate_bps) |
| 341 | pad_up_to_bitrate_bps = bitrate_bps; |
| 342 | |
| 343 | return pad_up_to_bitrate_bps; |
stefan@webrtc.org | 508a84b | 2013-06-17 12:53:37 | [diff] [blame] | 344 | } |
| 345 | |
stefan@webrtc.org | bfacda6 | 2013-03-27 16:36:01 | [diff] [blame] | 346 | bool ViEEncoder::EncoderPaused() const { |
pwestin@webrtc.org | 91563e4 | 2013-04-25 22:20:08 | [diff] [blame] | 347 | // Pause video if paused by caller or as long as the network is down or the |
| 348 | // pacer queue has grown too large in buffered mode. |
| 349 | if (encoder_paused_) { |
| 350 | return true; |
| 351 | } |
| 352 | if (target_delay_ms_ > 0) { |
| 353 | // Buffered mode. |
| 354 | // TODO(pwestin): Workaround until nack is configured as a time and not |
| 355 | // number of packets. |
Stefan Holmer | e590416 | 2015-03-26 10:11:06 | [diff] [blame] | 356 | return pacer_->QueueInMs() >= |
| 357 | std::max( |
| 358 | static_cast<int>(target_delay_ms_ * kEncoderPausePacerMargin), |
| 359 | kMinPacingDelayMs); |
pwestin@webrtc.org | 91563e4 | 2013-04-25 22:20:08 | [diff] [blame] | 360 | } |
Stefan Holmer | e590416 | 2015-03-26 10:11:06 | [diff] [blame] | 361 | if (pacer_->ExpectedQueueTimeMs() > PacedSender::kDefaultMaxQueueLengthMs) { |
sprang@webrtc.org | dcebf2d | 2014-11-04 16:27:16 | [diff] [blame] | 362 | // Too much data in pacer queue, drop frame. |
| 363 | return true; |
| 364 | } |
pwestin@webrtc.org | 91563e4 | 2013-04-25 22:20:08 | [diff] [blame] | 365 | return !network_is_transmitting_; |
stefan@webrtc.org | bfacda6 | 2013-03-27 16:36:01 | [diff] [blame] | 366 | } |
| 367 | |
sprang@webrtc.org | dcebf2d | 2014-11-04 16:27:16 | [diff] [blame] | 368 | void ViEEncoder::TraceFrameDropStart() { |
| 369 | // Start trace event only on the first frame after encoder is paused. |
| 370 | if (!encoder_paused_and_dropped_frame_) { |
| 371 | TRACE_EVENT_ASYNC_BEGIN0("webrtc", "EncoderPaused", this); |
| 372 | } |
| 373 | encoder_paused_and_dropped_frame_ = true; |
| 374 | return; |
| 375 | } |
| 376 | |
| 377 | void ViEEncoder::TraceFrameDropEnd() { |
| 378 | // End trace event on first frame after encoder resumes, if frame was dropped. |
| 379 | if (encoder_paused_and_dropped_frame_) { |
| 380 | TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this); |
| 381 | } |
| 382 | encoder_paused_and_dropped_frame_ = false; |
| 383 | } |
| 384 | |
Miguel Casas-Sanchez | 4765070 | 2015-05-30 00:21:40 | [diff] [blame] | 385 | void ViEEncoder::DeliverFrame(VideoFrame video_frame) { |
henrikg | 91d6ede | 2015-09-17 07:24:34 | [diff] [blame] | 386 | RTC_DCHECK(send_payload_router_ != NULL); |
mflodman@webrtc.org | 47d657b | 2015-02-19 10:29:32 | [diff] [blame] | 387 | if (!send_payload_router_->active()) { |
mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 | [diff] [blame] | 388 | // We've paused or we have no channels attached, don't waste resources on |
| 389 | // encoding. |
wuchengli@chromium.org | ac4b87c | 2014-03-19 03:44:20 | [diff] [blame] | 390 | return; |
| 391 | } |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 392 | { |
mflodman@webrtc.org | d32c447 | 2011-12-22 14:17:53 | [diff] [blame] | 393 | CriticalSectionScoped cs(data_cs_.get()); |
Noah Richards | 099323e | 2015-04-15 16:14:12 | [diff] [blame] | 394 | time_of_last_frame_activity_ms_ = TickTime::MillisecondTimestamp(); |
pwestin@webrtc.org | 52b4e88 | 2013-05-02 19:02:17 | [diff] [blame] | 395 | if (EncoderPaused()) { |
sprang@webrtc.org | dcebf2d | 2014-11-04 16:27:16 | [diff] [blame] | 396 | TraceFrameDropStart(); |
pwestin@webrtc.org | 52b4e88 | 2013-05-02 19:02:17 | [diff] [blame] | 397 | return; |
| 398 | } |
sprang@webrtc.org | dcebf2d | 2014-11-04 16:27:16 | [diff] [blame] | 399 | TraceFrameDropEnd(); |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 400 | } |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 401 | |
Magnus Jedvert | 26679d6 | 2015-04-07 12:07:41 | [diff] [blame] | 402 | TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), |
hclam@chromium.org | 1a7b9b9 | 2013-07-08 21:31:18 | [diff] [blame] | 403 | "Encode"); |
Miguel Casas-Sanchez | 4765070 | 2015-05-30 00:21:40 | [diff] [blame] | 404 | VideoFrame* decimated_frame = NULL; |
wuchengli@chromium.org | f425b55 | 2014-06-20 12:04:05 | [diff] [blame] | 405 | // TODO(wuchengli): support texture frames. |
Magnus Jedvert | 26679d6 | 2015-04-07 12:07:41 | [diff] [blame] | 406 | if (video_frame.native_handle() == NULL) { |
wuchengli@chromium.org | f425b55 | 2014-06-20 12:04:05 | [diff] [blame] | 407 | // Pass frame via preprocessor. |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 408 | const int ret = vpm_->PreprocessFrame(video_frame, &decimated_frame); |
wuchengli@chromium.org | f425b55 | 2014-06-20 12:04:05 | [diff] [blame] | 409 | if (ret == 1) { |
| 410 | // Drop this frame. |
| 411 | return; |
| 412 | } |
| 413 | if (ret != VPM_OK) { |
| 414 | return; |
| 415 | } |
pwestin@webrtc.org | 2f476ed | 2012-10-30 16:21:52 | [diff] [blame] | 416 | } |
pbos@webrtc.org | fe1ef93 | 2013-10-21 10:34:43 | [diff] [blame] | 417 | |
Magnus Jedvert | 26679d6 | 2015-04-07 12:07:41 | [diff] [blame] | 418 | // If we haven't resampled the frame and we have a FrameCallback, we need to |
| 419 | // make a deep copy of |video_frame|. |
Miguel Casas-Sanchez | 4765070 | 2015-05-30 00:21:40 | [diff] [blame] | 420 | VideoFrame copied_frame; |
Peter Boström | 7083e11 | 2015-09-22 14:28:51 | [diff] [blame] | 421 | if (pre_encode_callback_) { |
| 422 | // If the frame was not resampled or scaled => use copy of original. |
| 423 | if (decimated_frame == NULL) { |
| 424 | copied_frame.CopyFrame(video_frame); |
| 425 | decimated_frame = &copied_frame; |
Magnus Jedvert | 26679d6 | 2015-04-07 12:07:41 | [diff] [blame] | 426 | } |
Peter Boström | 7083e11 | 2015-09-22 14:28:51 | [diff] [blame] | 427 | pre_encode_callback_->FrameCallback(decimated_frame); |
pbos@webrtc.org | fe1ef93 | 2013-10-21 10:34:43 | [diff] [blame] | 428 | } |
| 429 | |
Magnus Jedvert | 26679d6 | 2015-04-07 12:07:41 | [diff] [blame] | 430 | // If the frame was not resampled, scaled, or touched by FrameCallback => use |
| 431 | // original. The frame is const from here. |
Miguel Casas-Sanchez | 4765070 | 2015-05-30 00:21:40 | [diff] [blame] | 432 | const VideoFrame* output_frame = |
Magnus Jedvert | 26679d6 | 2015-04-07 12:07:41 | [diff] [blame] | 433 | (decimated_frame != NULL) ? decimated_frame : &video_frame; |
| 434 | |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 435 | #ifdef VIDEOCODEC_VP8 |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 436 | if (vcm_->SendCodec() == webrtc::kVideoCodecVP8) { |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 437 | webrtc::CodecSpecificInfo codec_specific_info; |
| 438 | codec_specific_info.codecType = webrtc::kVideoCodecVP8; |
stefan@webrtc.org | 7af12be | 2014-07-09 14:46:31 | [diff] [blame] | 439 | { |
| 440 | CriticalSectionScoped cs(data_cs_.get()); |
| 441 | codec_specific_info.codecSpecific.VP8.hasReceivedRPSI = |
| 442 | has_received_rpsi_; |
| 443 | codec_specific_info.codecSpecific.VP8.hasReceivedSLI = |
| 444 | has_received_sli_; |
| 445 | codec_specific_info.codecSpecific.VP8.pictureIdRPSI = |
| 446 | picture_id_rpsi_; |
| 447 | codec_specific_info.codecSpecific.VP8.pictureIdSLI = |
| 448 | picture_id_sli_; |
| 449 | has_received_sli_ = false; |
| 450 | has_received_rpsi_ = false; |
| 451 | } |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 452 | |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 453 | vcm_->AddVideoFrame(*output_frame, vpm_->ContentMetrics(), |
| 454 | &codec_specific_info); |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 455 | return; |
| 456 | } |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 457 | #endif |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 458 | vcm_->AddVideoFrame(*output_frame); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 459 | } |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 460 | |
pwestin@webrtc.org | ce33035 | 2012-04-12 06:59:14 | [diff] [blame] | 461 | int ViEEncoder::SendKeyFrame() { |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 462 | return vcm_->IntraFrameRequest(0); |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 463 | } |
| 464 | |
pbos@webrtc.org | 143451d | 2015-03-18 14:40:03 | [diff] [blame] | 465 | uint32_t ViEEncoder::LastObservedBitrateBps() const { |
| 466 | CriticalSectionScoped cs(data_cs_.get()); |
| 467 | return last_observed_bitrate_bps_; |
| 468 | } |
| 469 | |
pbos@webrtc.org | b238d12 | 2013-04-09 13:41:51 | [diff] [blame] | 470 | int ViEEncoder::CodecTargetBitrate(uint32_t* bitrate) const { |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 471 | if (vcm_->Bitrate(bitrate) != 0) |
stefan@webrtc.org | 439be29 | 2012-02-16 14:45:37 | [diff] [blame] | 472 | return -1; |
| 473 | return 0; |
stefan@webrtc.org | 07b45a5 | 2012-02-02 08:37:48 | [diff] [blame] | 474 | } |
| 475 | |
mflodman@webrtc.org | 96abda0 | 2015-02-25 13:50:10 | [diff] [blame] | 476 | int32_t ViEEncoder::UpdateProtectionMethod(bool nack, bool fec) { |
henrikg | 91d6ede | 2015-09-17 07:24:34 | [diff] [blame] | 477 | RTC_DCHECK(send_payload_router_ != NULL); |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 478 | |
mflodman@webrtc.org | 96abda0 | 2015-02-25 13:50:10 | [diff] [blame] | 479 | if (fec_enabled_ == fec && nack_enabled_ == nack) { |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 480 | // No change needed, we're already in correct state. |
| 481 | return 0; |
| 482 | } |
mflodman@webrtc.org | 96abda0 | 2015-02-25 13:50:10 | [diff] [blame] | 483 | fec_enabled_ = fec; |
| 484 | nack_enabled_ = nack; |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 485 | |
| 486 | // Set Video Protection for VCM. |
pbos | ba8c15b | 2015-07-14 16:36:34 | [diff] [blame] | 487 | VCMVideoProtection protection_mode; |
| 488 | if (fec_enabled_) { |
| 489 | protection_mode = |
| 490 | nack_enabled_ ? webrtc::kProtectionNackFEC : kProtectionFEC; |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 491 | } else { |
pbos | ba8c15b | 2015-07-14 16:36:34 | [diff] [blame] | 492 | protection_mode = nack_enabled_ ? kProtectionNack : kProtectionNone; |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 493 | } |
pbos | ba8c15b | 2015-07-14 16:36:34 | [diff] [blame] | 494 | vcm_->SetVideoProtection(protection_mode, true); |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 495 | |
wu@webrtc.org | 822fbd8 | 2013-08-15 23:38:54 | [diff] [blame] | 496 | if (fec_enabled_ || nack_enabled_) { |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 497 | // The send codec must be registered to set correct MTU. |
| 498 | webrtc::VideoCodec codec; |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 499 | if (vcm_->SendCodec(&codec) == 0) { |
stefan@webrtc.org | 3d0b0d6 | 2013-03-19 10:04:57 | [diff] [blame] | 500 | uint32_t current_bitrate_bps = 0; |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 501 | if (vcm_->Bitrate(¤t_bitrate_bps) != 0) { |
mflodman@webrtc.org | 5574dac | 2014-04-07 10:56:31 | [diff] [blame] | 502 | LOG_F(LS_WARNING) << |
| 503 | "Failed to get the current encoder target bitrate."; |
stefan@webrtc.org | 439be29 | 2012-02-16 14:45:37 | [diff] [blame] | 504 | } |
stefan@webrtc.org | 3d0b0d6 | 2013-03-19 10:04:57 | [diff] [blame] | 505 | // Convert to start bitrate in kbps. |
| 506 | codec.startBitrate = (current_bitrate_bps + 500) / 1000; |
mflodman@webrtc.org | a4ef2ce | 2015-02-12 09:54:18 | [diff] [blame] | 507 | size_t max_payload_length = send_payload_router_->MaxPayloadLength(); |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 508 | if (vcm_->RegisterSendCodec(&codec, number_of_cores_, |
Peter Boström | ae37abb | 2015-06-18 17:00:34 | [diff] [blame] | 509 | static_cast<uint32_t>(max_payload_length)) != |
| 510 | 0) { |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 511 | return -1; |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 512 | } |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 513 | } |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 514 | } |
| 515 | return 0; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 516 | } |
| 517 | |
mikhal@webrtc.org | ef9f76a | 2013-02-15 23:22:18 | [diff] [blame] | 518 | void ViEEncoder::SetSenderBufferingMode(int target_delay_ms) { |
stefan@webrtc.org | bfacda6 | 2013-03-27 16:36:01 | [diff] [blame] | 519 | { |
| 520 | CriticalSectionScoped cs(data_cs_.get()); |
| 521 | target_delay_ms_ = target_delay_ms; |
| 522 | } |
mikhal@webrtc.org | 3d305c6 | 2013-02-10 18:42:55 | [diff] [blame] | 523 | if (target_delay_ms > 0) { |
stefan@webrtc.org | bfacda6 | 2013-03-27 16:36:01 | [diff] [blame] | 524 | // Disable external frame-droppers. |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 525 | vcm_->EnableFrameDropper(false); |
| 526 | vpm_->EnableTemporalDecimation(false); |
mikhal@webrtc.org | 3d305c6 | 2013-02-10 18:42:55 | [diff] [blame] | 527 | } else { |
mikhal@webrtc.org | ef9f76a | 2013-02-15 23:22:18 | [diff] [blame] | 528 | // Real-time mode - enable frame droppers. |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 529 | vpm_->EnableTemporalDecimation(true); |
| 530 | vcm_->EnableFrameDropper(true); |
mikhal@webrtc.org | 3d305c6 | 2013-02-10 18:42:55 | [diff] [blame] | 531 | } |
| 532 | } |
| 533 | |
pbos@webrtc.org | 891d483 | 2015-02-26 13:15:22 | [diff] [blame] | 534 | void ViEEncoder::OnSetRates(uint32_t bitrate_bps, int framerate) { |
Peter Boström | 7083e11 | 2015-09-22 14:28:51 | [diff] [blame] | 535 | if (stats_proxy_) |
| 536 | stats_proxy_->OnSetRates(bitrate_bps, framerate); |
pbos@webrtc.org | 891d483 | 2015-02-26 13:15:22 | [diff] [blame] | 537 | } |
| 538 | |
pbos@webrtc.org | b238d12 | 2013-04-09 13:41:51 | [diff] [blame] | 539 | int32_t ViEEncoder::SendData( |
pbos@webrtc.org | b238d12 | 2013-04-09 13:41:51 | [diff] [blame] | 540 | const uint8_t payload_type, |
pbos@webrtc.org | 273a414 | 2014-12-01 15:23:21 | [diff] [blame] | 541 | const EncodedImage& encoded_image, |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 542 | const webrtc::RTPFragmentationHeader& fragmentation_header, |
| 543 | const RTPVideoHeader* rtp_video_hdr) { |
henrikg | 91d6ede | 2015-09-17 07:24:34 | [diff] [blame] | 544 | RTC_DCHECK(send_payload_router_ != NULL); |
mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 | [diff] [blame] | 545 | |
pbos@webrtc.org | 891d483 | 2015-02-26 13:15:22 | [diff] [blame] | 546 | { |
Noah Richards | 099323e | 2015-04-15 16:14:12 | [diff] [blame] | 547 | CriticalSectionScoped cs(data_cs_.get()); |
| 548 | time_of_last_frame_activity_ms_ = TickTime::MillisecondTimestamp(); |
| 549 | } |
| 550 | |
Peter Boström | 7083e11 | 2015-09-22 14:28:51 | [diff] [blame] | 551 | if (stats_proxy_ != NULL) |
| 552 | stats_proxy_->OnSendEncodedImage(encoded_image, rtp_video_hdr); |
mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 | [diff] [blame] | 553 | |
| 554 | return send_payload_router_->RoutePayload( |
pbos | 22993e1 | 2015-10-19 09:39:06 | [diff] [blame] | 555 | encoded_image._frameType, payload_type, encoded_image._timeStamp, |
| 556 | encoded_image.capture_time_ms_, encoded_image._buffer, |
| 557 | encoded_image._length, &fragmentation_header, rtp_video_hdr) |
| 558 | ? 0 |
| 559 | : -1; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 560 | } |
| 561 | |
pbos@webrtc.org | b238d12 | 2013-04-09 13:41:51 | [diff] [blame] | 562 | int32_t ViEEncoder::SendStatistics(const uint32_t bit_rate, |
| 563 | const uint32_t frame_rate) { |
Peter Boström | 7083e11 | 2015-09-22 14:28:51 | [diff] [blame] | 564 | if (stats_proxy_) |
| 565 | stats_proxy_->OnOutgoingRate(frame_rate, bit_rate); |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 566 | return 0; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 567 | } |
| 568 | |
andrew@webrtc.org | 9663686 | 2012-09-20 23:33:17 | [diff] [blame] | 569 | void ViEEncoder::OnReceivedSLI(uint32_t /*ssrc*/, |
| 570 | uint8_t picture_id) { |
stefan@webrtc.org | 7af12be | 2014-07-09 14:46:31 | [diff] [blame] | 571 | CriticalSectionScoped cs(data_cs_.get()); |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 572 | picture_id_sli_ = picture_id; |
| 573 | has_received_sli_ = true; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 574 | } |
| 575 | |
andrew@webrtc.org | 9663686 | 2012-09-20 23:33:17 | [diff] [blame] | 576 | void ViEEncoder::OnReceivedRPSI(uint32_t /*ssrc*/, |
| 577 | uint64_t picture_id) { |
stefan@webrtc.org | 7af12be | 2014-07-09 14:46:31 | [diff] [blame] | 578 | CriticalSectionScoped cs(data_cs_.get()); |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 579 | picture_id_rpsi_ = picture_id; |
| 580 | has_received_rpsi_ = true; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 581 | } |
| 582 | |
mflodman@webrtc.org | d6ec386 | 2012-10-25 11:30:29 | [diff] [blame] | 583 | void ViEEncoder::OnReceivedIntraFrameRequest(uint32_t ssrc) { |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 584 | // Key frame request from remote side, signal to VCM. |
justinlin@chromium.org | 7bfb3a3 | 2013-05-13 22:59:00 | [diff] [blame] | 585 | TRACE_EVENT0("webrtc", "OnKeyFrameRequest"); |
pwestin@webrtc.org | 1da1ce0 | 2011-10-13 15:19:55 | [diff] [blame] | 586 | |
mflodman@webrtc.org | d6ec386 | 2012-10-25 11:30:29 | [diff] [blame] | 587 | int idx = 0; |
| 588 | { |
| 589 | CriticalSectionScoped cs(data_cs_.get()); |
Peter Boström | 5cb9ce4 | 2015-05-05 13:16:30 | [diff] [blame] | 590 | auto stream_it = ssrc_streams_.find(ssrc); |
mflodman@webrtc.org | d6ec386 | 2012-10-25 11:30:29 | [diff] [blame] | 591 | if (stream_it == ssrc_streams_.end()) { |
mflodman@webrtc.org | d73527c | 2012-12-20 09:26:17 | [diff] [blame] | 592 | LOG_F(LS_WARNING) << "ssrc not found: " << ssrc << ", map size " |
| 593 | << ssrc_streams_.size(); |
mflodman@webrtc.org | d6ec386 | 2012-10-25 11:30:29 | [diff] [blame] | 594 | return; |
| 595 | } |
pbos@webrtc.org | b238d12 | 2013-04-09 13:41:51 | [diff] [blame] | 596 | std::map<unsigned int, int64_t>::iterator time_it = |
mflodman@webrtc.org | d6ec386 | 2012-10-25 11:30:29 | [diff] [blame] | 597 | time_last_intra_request_ms_.find(ssrc); |
| 598 | if (time_it == time_last_intra_request_ms_.end()) { |
| 599 | time_last_intra_request_ms_[ssrc] = 0; |
| 600 | } |
| 601 | |
pbos@webrtc.org | b238d12 | 2013-04-09 13:41:51 | [diff] [blame] | 602 | int64_t now = TickTime::MillisecondTimestamp(); |
mflodman@webrtc.org | d6ec386 | 2012-10-25 11:30:29 | [diff] [blame] | 603 | if (time_last_intra_request_ms_[ssrc] + kViEMinKeyRequestIntervalMs > now) { |
mflodman@webrtc.org | d6ec386 | 2012-10-25 11:30:29 | [diff] [blame] | 604 | return; |
| 605 | } |
| 606 | time_last_intra_request_ms_[ssrc] = now; |
| 607 | idx = stream_it->second; |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 608 | } |
mflodman@webrtc.org | d6ec386 | 2012-10-25 11:30:29 | [diff] [blame] | 609 | // Release the critsect before triggering key frame. |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 610 | vcm_->IntraFrameRequest(idx); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 611 | } |
| 612 | |
mflodman@webrtc.org | aca2629 | 2012-10-05 16:17:41 | [diff] [blame] | 613 | void ViEEncoder::OnLocalSsrcChanged(uint32_t old_ssrc, uint32_t new_ssrc) { |
mflodman@webrtc.org | d6ec386 | 2012-10-25 11:30:29 | [diff] [blame] | 614 | CriticalSectionScoped cs(data_cs_.get()); |
| 615 | std::map<unsigned int, int>::iterator it = ssrc_streams_.find(old_ssrc); |
| 616 | if (it == ssrc_streams_.end()) { |
| 617 | return; |
| 618 | } |
| 619 | |
| 620 | ssrc_streams_[new_ssrc] = it->second; |
| 621 | ssrc_streams_.erase(it); |
| 622 | |
| 623 | std::map<unsigned int, int64_t>::iterator time_it = |
| 624 | time_last_intra_request_ms_.find(old_ssrc); |
| 625 | int64_t last_intra_request_ms = 0; |
| 626 | if (time_it != time_last_intra_request_ms_.end()) { |
| 627 | last_intra_request_ms = time_it->second; |
| 628 | time_last_intra_request_ms_.erase(time_it); |
| 629 | } |
| 630 | time_last_intra_request_ms_[new_ssrc] = last_intra_request_ms; |
| 631 | } |
| 632 | |
Peter Boström | 5cb9ce4 | 2015-05-05 13:16:30 | [diff] [blame] | 633 | bool ViEEncoder::SetSsrcs(const std::vector<uint32_t>& ssrcs) { |
mflodman@webrtc.org | d6ec386 | 2012-10-25 11:30:29 | [diff] [blame] | 634 | VideoCodec codec; |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 635 | if (vcm_->SendCodec(&codec) != 0) |
mflodman@webrtc.org | d6ec386 | 2012-10-25 11:30:29 | [diff] [blame] | 636 | return false; |
| 637 | |
| 638 | if (codec.numberOfSimulcastStreams > 0 && |
| 639 | ssrcs.size() != codec.numberOfSimulcastStreams) { |
| 640 | return false; |
| 641 | } |
| 642 | |
| 643 | CriticalSectionScoped cs(data_cs_.get()); |
| 644 | ssrc_streams_.clear(); |
| 645 | time_last_intra_request_ms_.clear(); |
| 646 | int idx = 0; |
Peter Boström | 5cb9ce4 | 2015-05-05 13:16:30 | [diff] [blame] | 647 | for (uint32_t ssrc : ssrcs) { |
| 648 | ssrc_streams_[ssrc] = idx++; |
mflodman@webrtc.org | d6ec386 | 2012-10-25 11:30:29 | [diff] [blame] | 649 | } |
| 650 | return true; |
mflodman@webrtc.org | aca2629 | 2012-10-05 16:17:41 | [diff] [blame] | 651 | } |
| 652 | |
pbos@webrtc.org | 3349ae0 | 2014-03-13 12:52:27 | [diff] [blame] | 653 | void ViEEncoder::SetMinTransmitBitrate(int min_transmit_bitrate_kbps) { |
| 654 | assert(min_transmit_bitrate_kbps >= 0); |
| 655 | CriticalSectionScoped crit(data_cs_.get()); |
| 656 | min_transmit_bitrate_kbps_ = min_transmit_bitrate_kbps; |
| 657 | } |
| 658 | |
pwestin@webrtc.org | 49888ce | 2012-04-27 05:25:53 | [diff] [blame] | 659 | // Called from ViEBitrateObserver. |
stefan@webrtc.org | edeea91 | 2014-12-08 19:46:23 | [diff] [blame] | 660 | void ViEEncoder::OnNetworkChanged(uint32_t bitrate_bps, |
| 661 | uint8_t fraction_lost, |
pkasting@chromium.org | 16825b1 | 2015-01-12 21:51:21 | [diff] [blame] | 662 | int64_t round_trip_time_ms) { |
mflodman@webrtc.org | 5574dac | 2014-04-07 10:56:31 | [diff] [blame] | 663 | LOG(LS_VERBOSE) << "OnNetworkChanged, bitrate" << bitrate_bps |
mflodman | 8602a3d | 2015-05-20 22:54:42 | [diff] [blame] | 664 | << " packet loss " << static_cast<int>(fraction_lost) |
mflodman@webrtc.org | 5574dac | 2014-04-07 10:56:31 | [diff] [blame] | 665 | << " rtt " << round_trip_time_ms; |
henrikg | 91d6ede | 2015-09-17 07:24:34 | [diff] [blame] | 666 | RTC_DCHECK(send_payload_router_ != NULL); |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 667 | vcm_->SetChannelParameters(bitrate_bps, fraction_lost, round_trip_time_ms); |
| 668 | bool video_is_suspended = vcm_->VideoSuspended(); |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 | [diff] [blame] | 669 | |
stefan@webrtc.org | 508a84b | 2013-06-17 12:53:37 | [diff] [blame] | 670 | VideoCodec send_codec; |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 671 | if (vcm_->SendCodec(&send_codec) != 0) { |
stefan@webrtc.org | 508a84b | 2013-06-17 12:53:37 | [diff] [blame] | 672 | return; |
| 673 | } |
stefan@webrtc.org | b2c8a95 | 2013-09-06 13:58:01 | [diff] [blame] | 674 | SimulcastStream* stream_configs = send_codec.simulcastStream; |
| 675 | // Allocate the bandwidth between the streams. |
| 676 | std::vector<uint32_t> stream_bitrates = AllocateStreamBitrates( |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 | [diff] [blame] | 677 | bitrate_bps, stream_configs, send_codec.numberOfSimulcastStreams); |
| 678 | send_payload_router_->SetTargetSendBitrates(stream_bitrates); |
pbos@webrtc.org | 709e297 | 2014-03-19 10:59:52 | [diff] [blame] | 679 | |
stefan@webrtc.org | 3e00505 | 2013-10-18 15:05:29 | [diff] [blame] | 680 | { |
stefan@webrtc.org | 3e00505 | 2013-10-18 15:05:29 | [diff] [blame] | 681 | CriticalSectionScoped cs(data_cs_.get()); |
pbos@webrtc.org | 143451d | 2015-03-18 14:40:03 | [diff] [blame] | 682 | last_observed_bitrate_bps_ = bitrate_bps; |
pbos@webrtc.org | 484ee96 | 2013-11-21 18:44:23 | [diff] [blame] | 683 | if (video_suspended_ == video_is_suspended) |
| 684 | return; |
| 685 | video_suspended_ = video_is_suspended; |
mflodman | 0dbf009 | 2015-10-19 15:12:12 | [diff] [blame] | 686 | |
| 687 | LOG(LS_INFO) << "Video suspend state changed " << video_is_suspended |
| 688 | << " for ssrc " << ssrc_streams_.begin()->first; |
henrik.lundin@webrtc.org | 7ea4f24 | 2013-10-02 13:34:26 | [diff] [blame] | 689 | } |
Peter Boström | 7083e11 | 2015-09-22 14:28:51 | [diff] [blame] | 690 | // Video suspend-state changed, inform codec observer. |
| 691 | if (stats_proxy_) |
| 692 | stats_proxy_->OnSuspendChange(video_is_suspended); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 693 | } |
| 694 | |
henrik.lundin@webrtc.org | ce8e093 | 2013-11-18 12:18:43 | [diff] [blame] | 695 | void ViEEncoder::SuspendBelowMinBitrate() { |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 696 | vcm_->SuspendBelowMinBitrate(); |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 | [diff] [blame] | 697 | bitrate_allocator_->EnforceMinBitrate(false); |
henrik.lundin@webrtc.org | 7ea4f24 | 2013-10-02 13:34:26 | [diff] [blame] | 698 | } |
| 699 | |
sprang@webrtc.org | 4070935 | 2013-11-26 11:41:59 | [diff] [blame] | 700 | void ViEEncoder::RegisterPostEncodeImageCallback( |
| 701 | EncodedImageCallback* post_encode_callback) { |
mflodman | fcf54bd | 2015-04-14 19:28:08 | [diff] [blame] | 702 | vcm_->RegisterPostEncodeImageCallback(post_encode_callback); |
sprang@webrtc.org | 4070935 | 2013-11-26 11:41:59 | [diff] [blame] | 703 | } |
| 704 | |
marpan@webrtc.org | efd01fd | 2012-04-18 15:56:34 | [diff] [blame] | 705 | QMVideoSettingsCallback::QMVideoSettingsCallback(VideoProcessingModule* vpm) |
| 706 | : vpm_(vpm) { |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 707 | } |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 708 | |
stefan@webrtc.org | 439be29 | 2012-02-16 14:45:37 | [diff] [blame] | 709 | QMVideoSettingsCallback::~QMVideoSettingsCallback() { |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 710 | } |
| 711 | |
pbos@webrtc.org | b238d12 | 2013-04-09 13:41:51 | [diff] [blame] | 712 | int32_t QMVideoSettingsCallback::SetVideoQMSettings( |
| 713 | const uint32_t frame_rate, |
| 714 | const uint32_t width, |
| 715 | const uint32_t height) { |
marpan@webrtc.org | cf706c2 | 2012-03-27 21:04:13 | [diff] [blame] | 716 | return vpm_->SetTargetResolution(width, height, frame_rate); |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 717 | } |
| 718 | |
jackychen | 6e2ce6e | 2015-07-13 23:26:33 | [diff] [blame] | 719 | void QMVideoSettingsCallback::SetTargetFramerate(int frame_rate) { |
| 720 | vpm_->SetTargetFramerate(frame_rate); |
| 721 | } |
| 722 | |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 | [diff] [blame] | 723 | } // namespace webrtc |