modules/video_coding/media_optimization.cc - src/webrtc - Git at Google

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */

 #include "webrtc/modules/video_coding/media_optimization.h"

 #include "webrtc/base/logging.h"
 #include "webrtc/modules/video_coding/content_metrics_processing.h"
 #include "webrtc/modules/video_coding/qm_select.h"
 #include "webrtc/modules/video_coding/utility/frame_dropper.h"
 #include "webrtc/system_wrappers/include/clock.h"

 namespace webrtc {
 namespace media_optimization {
 namespace {
 void UpdateProtectionCallback(
     VCMProtectionMethod* selected_method,
     uint32_t* video_rate_bps,
     uint32_t* nack_overhead_rate_bps,
     uint32_t* fec_overhead_rate_bps,
     VCMProtectionCallback* video_protection_callback) {
   FecProtectionParams delta_fec_params;
   FecProtectionParams key_fec_params;
   // Get the FEC code rate for Key frames (set to 0 when NA).
   key_fec_params.fec_rate = selected_method->RequiredProtectionFactorK();

   // Get the FEC code rate for Delta frames (set to 0 when NA).
   delta_fec_params.fec_rate = selected_method->RequiredProtectionFactorD();

   // Get the FEC-UEP protection status for Key frames: UEP on/off.
   key_fec_params.use_uep_protection = selected_method->RequiredUepProtectionK();

   // Get the FEC-UEP protection status for Delta frames: UEP on/off.
   delta_fec_params.use_uep_protection =
       selected_method->RequiredUepProtectionD();

   // The RTP module currently requires the same |max_fec_frames| for both
   // key and delta frames.
   delta_fec_params.max_fec_frames = selected_method->MaxFramesFec();
   key_fec_params.max_fec_frames = selected_method->MaxFramesFec();

   // Set the FEC packet mask type. |kFecMaskBursty| is more effective for
   // consecutive losses and little/no packet re-ordering. As we currently
   // do not have feedback data on the degree of correlated losses and packet
   // re-ordering, we keep default setting to |kFecMaskRandom| for now.
   delta_fec_params.fec_mask_type = kFecMaskRandom;
   key_fec_params.fec_mask_type = kFecMaskRandom;

   // TODO(Marco): Pass FEC protection values per layer.
   video_protection_callback->ProtectionRequest(
       &delta_fec_params, &key_fec_params, video_rate_bps,
       nack_overhead_rate_bps, fec_overhead_rate_bps);
 }
 }  // namespace

 struct MediaOptimization::EncodedFrameSample {
   EncodedFrameSample(size_t size_bytes,
                      uint32_t timestamp,
                      int64_t time_complete_ms)
       : size_bytes(size_bytes),
         timestamp(timestamp),
         time_complete_ms(time_complete_ms) {}

   size_t size_bytes;
   uint32_t timestamp;
   int64_t time_complete_ms;
 };

 MediaOptimization::MediaOptimization(Clock* clock)
     : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
       clock_(clock),
       max_bit_rate_(0),
       send_codec_type_(kVideoCodecUnknown),
       codec_width_(0),
       codec_height_(0),
       user_frame_rate_(0),
       frame_dropper_(new FrameDropper),
       loss_prot_logic_(
           new VCMLossProtectionLogic(clock_->TimeInMilliseconds())),
       fraction_lost_(0),
       send_statistics_zero_encode_(0),
       max_payload_size_(1460),
       video_target_bitrate_(0),
       incoming_frame_rate_(0),
       enable_qm_(false),
       encoded_frame_samples_(),
       avg_sent_bit_rate_bps_(0),
       avg_sent_framerate_(0),
       key_frame_cnt_(0),
       delta_frame_cnt_(0),
       content_(new VCMContentMetricsProcessing()),
       qm_resolution_(new VCMQmResolution()),
       last_qm_update_time_(0),
       last_change_time_(0),
       num_layers_(0),
       suspension_enabled_(false),
       video_suspended_(false),
       suspension_threshold_bps_(0),
       suspension_window_bps_(0) {
   memset(send_statistics_, 0, sizeof(send_statistics_));
   memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
 }

 MediaOptimization::~MediaOptimization(void) {
   loss_prot_logic_->Release();
 }

 void MediaOptimization::Reset() {
   CriticalSectionScoped lock(crit_sect_.get());
   SetEncodingDataInternal(kVideoCodecUnknown, 0, 0, 0, 0, 0, 0,
                           max_payload_size_);
   memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
   incoming_frame_rate_ = 0.0;
   frame_dropper_->Reset();
   loss_prot_logic_->Reset(clock_->TimeInMilliseconds());
   frame_dropper_->SetRates(0, 0);
   content_->Reset();
   qm_resolution_->Reset();
   loss_prot_logic_->UpdateFrameRate(incoming_frame_rate_);
   loss_prot_logic_->Reset(clock_->TimeInMilliseconds());
   send_statistics_zero_encode_ = 0;
   video_target_bitrate_ = 0;
   codec_width_ = 0;
   codec_height_ = 0;
   user_frame_rate_ = 0;
   key_frame_cnt_ = 0;
   delta_frame_cnt_ = 0;
   last_qm_update_time_ = 0;
   last_change_time_ = 0;
   encoded_frame_samples_.clear();
   avg_sent_bit_rate_bps_ = 0;
   num_layers_ = 1;
 }

 void MediaOptimization::SetEncodingData(VideoCodecType send_codec_type,
                                         int32_t max_bit_rate,
                                         uint32_t target_bitrate,
                                         uint16_t width,
                                         uint16_t height,
                                         uint32_t frame_rate,
                                         int num_layers,
                                         int32_t mtu) {
   CriticalSectionScoped lock(crit_sect_.get());
   SetEncodingDataInternal(send_codec_type, max_bit_rate, frame_rate,
                           target_bitrate, width, height, num_layers, mtu);
 }

 void MediaOptimization::SetEncodingDataInternal(VideoCodecType send_codec_type,
                                                 int32_t max_bit_rate,
                                                 uint32_t frame_rate,
                                                 uint32_t target_bitrate,
                                                 uint16_t width,
                                                 uint16_t height,
                                                 int num_layers,
                                                 int32_t mtu) {
   // Everything codec specific should be reset here since this means the codec
   // has changed. If native dimension values have changed, then either user
   // initiated change, or QM initiated change. Will be able to determine only
   // after the processing of the first frame.
   last_change_time_ = clock_->TimeInMilliseconds();
   content_->Reset();
   content_->UpdateFrameRate(frame_rate);

   max_bit_rate_ = max_bit_rate;
   send_codec_type_ = send_codec_type;
   video_target_bitrate_ = target_bitrate;
   float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f;
   loss_prot_logic_->UpdateBitRate(target_bitrate_kbps);
   loss_prot_logic_->UpdateFrameRate(static_cast<float>(frame_rate));
   loss_prot_logic_->UpdateFrameSize(width, height);
   loss_prot_logic_->UpdateNumLayers(num_layers);
   frame_dropper_->Reset();
   frame_dropper_->SetRates(target_bitrate_kbps, static_cast<float>(frame_rate));
   user_frame_rate_ = static_cast<float>(frame_rate);
   codec_width_ = width;
   codec_height_ = height;
   num_layers_ = (num_layers <= 1) ? 1 : num_layers;  // Can also be zero.
   max_payload_size_ = mtu;
   qm_resolution_->Initialize(target_bitrate_kbps, user_frame_rate_,
                              codec_width_, codec_height_, num_layers_);
 }

 uint32_t MediaOptimization::SetTargetRates(
     uint32_t target_bitrate,
     uint8_t fraction_lost,
     int64_t round_trip_time_ms,
     VCMProtectionCallback* protection_callback,
     VCMQMSettingsCallback* qmsettings_callback) {
   CriticalSectionScoped lock(crit_sect_.get());
   VCMProtectionMethod* selected_method = loss_prot_logic_->SelectedMethod();
   float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f;
   loss_prot_logic_->UpdateBitRate(target_bitrate_kbps);
   loss_prot_logic_->UpdateRtt(round_trip_time_ms);

   // Get frame rate for encoder: this is the actual/sent frame rate.
   float actual_frame_rate = SentFrameRateInternal();

   // Sanity check.
   if (actual_frame_rate < 1.0) {
     actual_frame_rate = 1.0;
   }

   // Update frame rate for the loss protection logic class: frame rate should
   // be the actual/sent rate.
   loss_prot_logic_->UpdateFrameRate(actual_frame_rate);

   fraction_lost_ = fraction_lost;

   // Returns the filtered packet loss, used for the protection setting.
   // The filtered loss may be the received loss (no filter), or some
   // filtered value (average or max window filter).
   // Use max window filter for now.
   FilterPacketLossMode filter_mode = kMaxFilter;
   uint8_t packet_loss_enc = loss_prot_logic_->FilteredLoss(
       clock_->TimeInMilliseconds(), filter_mode, fraction_lost);

   // For now use the filtered loss for computing the robustness settings.
   loss_prot_logic_->UpdateFilteredLossPr(packet_loss_enc);

   // Rate cost of the protection methods.
   float protection_overhead_rate = 0.0f;

   // Update protection settings, when applicable.
   float sent_video_rate_kbps = 0.0f;
   if (loss_prot_logic_->SelectedType() != kNone) {
     // Update protection method with content metrics.
     selected_method->UpdateContentMetrics(content_->ShortTermAvgData());

     // Update method will compute the robustness settings for the given
     // protection method and the overhead cost
     // the protection method is set by the user via SetVideoProtection.
     loss_prot_logic_->UpdateMethod();

     // Update protection callback with protection settings.
     uint32_t sent_video_rate_bps = 0;
     uint32_t sent_nack_rate_bps = 0;
     uint32_t sent_fec_rate_bps = 0;
     // Get the bit cost of protection method, based on the amount of
     // overhead data actually transmitted (including headers) the last
     // second.
     if (protection_callback) {
       UpdateProtectionCallback(selected_method, &sent_video_rate_bps,
                                &sent_nack_rate_bps, &sent_fec_rate_bps,
                                protection_callback);
     }
     uint32_t sent_total_rate_bps =
         sent_video_rate_bps + sent_nack_rate_bps + sent_fec_rate_bps;
     // Estimate the overhead costs of the next second as staying the same
     // wrt the source bitrate.
     if (sent_total_rate_bps > 0) {
       protection_overhead_rate =
           static_cast<float>(sent_nack_rate_bps + sent_fec_rate_bps) /
           sent_total_rate_bps;
     }
     // Cap the overhead estimate to 50%.
     if (protection_overhead_rate > 0.5)
       protection_overhead_rate = 0.5;

     // Get the effective packet loss for encoder ER when applicable. Should be
     // passed to encoder via fraction_lost.
     packet_loss_enc = selected_method->RequiredPacketLossER();
     sent_video_rate_kbps = static_cast<float>(sent_video_rate_bps) / 1000.0f;
   }

   // Source coding rate: total rate - protection overhead.
   video_target_bitrate_ = target_bitrate * (1.0 - protection_overhead_rate);

   // Cap target video bitrate to codec maximum.
   if (max_bit_rate_ > 0 && video_target_bitrate_ > max_bit_rate_) {
     video_target_bitrate_ = max_bit_rate_;
   }

   // Update encoding rates following protection settings.
   float target_video_bitrate_kbps =
       static_cast<float>(video_target_bitrate_) / 1000.0f;
   frame_dropper_->SetRates(target_video_bitrate_kbps, incoming_frame_rate_);

   if (enable_qm_ && qmsettings_callback) {
     // Update QM with rates.
     qm_resolution_->UpdateRates(target_video_bitrate_kbps, sent_video_rate_kbps,
                                 incoming_frame_rate_, fraction_lost_);
     // Check for QM selection.
     bool select_qm = CheckStatusForQMchange();
     if (select_qm) {
       SelectQuality(qmsettings_callback);
     }
     // Reset the short-term averaged content data.
     content_->ResetShortTermAvgData();
   }

   CheckSuspendConditions();

   return video_target_bitrate_;
 }

 void MediaOptimization::SetProtectionMethod(VCMProtectionMethodEnum method) {
   CriticalSectionScoped lock(crit_sect_.get());
   loss_prot_logic_->SetMethod(method);
 }

 uint32_t MediaOptimization::InputFrameRate() {
   CriticalSectionScoped lock(crit_sect_.get());
   return InputFrameRateInternal();
 }

 uint32_t MediaOptimization::InputFrameRateInternal() {
   ProcessIncomingFrameRate(clock_->TimeInMilliseconds());
   return uint32_t(incoming_frame_rate_ + 0.5f);
 }

 uint32_t MediaOptimization::SentFrameRate() {
   CriticalSectionScoped lock(crit_sect_.get());
   return SentFrameRateInternal();
 }

 uint32_t MediaOptimization::SentFrameRateInternal() {
   PurgeOldFrameSamples(clock_->TimeInMilliseconds());
   UpdateSentFramerate();
   return avg_sent_framerate_;
 }

 uint32_t MediaOptimization::SentBitRate() {
   CriticalSectionScoped lock(crit_sect_.get());
   const int64_t now_ms = clock_->TimeInMilliseconds();
   PurgeOldFrameSamples(now_ms);
   UpdateSentBitrate(now_ms);
   return avg_sent_bit_rate_bps_;
 }

 int32_t MediaOptimization::UpdateWithEncodedData(
     const EncodedImage& encoded_image) {
   size_t encoded_length = encoded_image._length;
   uint32_t timestamp = encoded_image._timeStamp;
   CriticalSectionScoped lock(crit_sect_.get());
   const int64_t now_ms = clock_->TimeInMilliseconds();
   PurgeOldFrameSamples(now_ms);
   if (encoded_frame_samples_.size() > 0 &&
       encoded_frame_samples_.back().timestamp == timestamp) {
     // Frames having the same timestamp are generated from the same input
     // frame. We don't want to double count them, but only increment the
     // size_bytes.
     encoded_frame_samples_.back().size_bytes += encoded_length;
     encoded_frame_samples_.back().time_complete_ms = now_ms;
   } else {
     encoded_frame_samples_.push_back(
         EncodedFrameSample(encoded_length, timestamp, now_ms));
   }
   UpdateSentBitrate(now_ms);
   UpdateSentFramerate();
   if (encoded_length > 0) {
     const bool delta_frame = encoded_image._frameType != kVideoFrameKey;

     frame_dropper_->Fill(encoded_length, delta_frame);
     if (max_payload_size_ > 0 && encoded_length > 0) {
       const float min_packets_per_frame =
           encoded_length / static_cast<float>(max_payload_size_);
       if (delta_frame) {
         loss_prot_logic_->UpdatePacketsPerFrame(min_packets_per_frame,
                                                 clock_->TimeInMilliseconds());
       } else {
         loss_prot_logic_->UpdatePacketsPerFrameKey(
             min_packets_per_frame, clock_->TimeInMilliseconds());
       }

       if (enable_qm_) {
         // Update quality select with encoded length.
         qm_resolution_->UpdateEncodedSize(encoded_length);
       }
     }
     if (!delta_frame && encoded_length > 0) {
       loss_prot_logic_->UpdateKeyFrameSize(static_cast<float>(encoded_length));
     }

     // Updating counters.
     if (delta_frame) {
       delta_frame_cnt_++;
     } else {
       key_frame_cnt_++;
     }
   }

   return VCM_OK;
 }

 void MediaOptimization::EnableQM(bool enable) {
   CriticalSectionScoped lock(crit_sect_.get());
   enable_qm_ = enable;
 }

 void MediaOptimization::EnableFrameDropper(bool enable) {
   CriticalSectionScoped lock(crit_sect_.get());
   frame_dropper_->Enable(enable);
 }

 void MediaOptimization::SuspendBelowMinBitrate(int threshold_bps,
                                                int window_bps) {
   CriticalSectionScoped lock(crit_sect_.get());
   assert(threshold_bps > 0 && window_bps >= 0);
   suspension_threshold_bps_ = threshold_bps;
   suspension_window_bps_ = window_bps;
   suspension_enabled_ = true;
   video_suspended_ = false;
 }

 bool MediaOptimization::IsVideoSuspended() const {
   CriticalSectionScoped lock(crit_sect_.get());
   return video_suspended_;
 }

 bool MediaOptimization::DropFrame() {
   CriticalSectionScoped lock(crit_sect_.get());
   UpdateIncomingFrameRate();
   // Leak appropriate number of bytes.
   frame_dropper_->Leak((uint32_t)(InputFrameRateInternal() + 0.5f));
   if (video_suspended_) {
     return true;  // Drop all frames when muted.
   }
   return frame_dropper_->DropFrame();
 }

 void MediaOptimization::UpdateContentData(
     const VideoContentMetrics* content_metrics) {
   CriticalSectionScoped lock(crit_sect_.get());
   // Updating content metrics.
   if (content_metrics == NULL) {
     // Disable QM if metrics are NULL.
     enable_qm_ = false;
     qm_resolution_->Reset();
   } else {
     content_->UpdateContentData(content_metrics);
   }
 }

 void MediaOptimization::UpdateIncomingFrameRate() {
   int64_t now = clock_->TimeInMilliseconds();
   if (incoming_frame_times_[0] == 0) {
     // No shifting if this is the first time.
   } else {
     // Shift all times one step.
     for (int32_t i = (kFrameCountHistorySize - 2); i >= 0; i--) {
       incoming_frame_times_[i + 1] = incoming_frame_times_[i];
     }
   }
   incoming_frame_times_[0] = now;
   ProcessIncomingFrameRate(now);
 }

 int32_t MediaOptimization::SelectQuality(
     VCMQMSettingsCallback* video_qmsettings_callback) {
   // Reset quantities for QM select.
   qm_resolution_->ResetQM();

   // Update QM will long-term averaged content metrics.
   qm_resolution_->UpdateContent(content_->LongTermAvgData());

   // Select quality mode.
   VCMResolutionScale* qm = NULL;
   int32_t ret = qm_resolution_->SelectResolution(&qm);
   if (ret < 0) {
     return ret;
   }

   // Check for updates to spatial/temporal modes.
   QMUpdate(qm, video_qmsettings_callback);

   // Reset all the rate and related frame counters quantities.
   qm_resolution_->ResetRates();

   // Reset counters.
   last_qm_update_time_ = clock_->TimeInMilliseconds();

   // Reset content metrics.
   content_->Reset();

   return VCM_OK;
 }

 void MediaOptimization::PurgeOldFrameSamples(int64_t now_ms) {
   while (!encoded_frame_samples_.empty()) {
     if (now_ms - encoded_frame_samples_.front().time_complete_ms >
         kBitrateAverageWinMs) {
       encoded_frame_samples_.pop_front();
     } else {
       break;
     }
   }
 }

 void MediaOptimization::UpdateSentBitrate(int64_t now_ms) {
   if (encoded_frame_samples_.empty()) {
     avg_sent_bit_rate_bps_ = 0;
     return;
   }
   size_t framesize_sum = 0;
   for (FrameSampleList::iterator it = encoded_frame_samples_.begin();
        it != encoded_frame_samples_.end(); ++it) {
     framesize_sum += it->size_bytes;
   }
   float denom = static_cast<float>(
       now_ms - encoded_frame_samples_.front().time_complete_ms);
   if (denom >= 1.0f) {
     avg_sent_bit_rate_bps_ =
         static_cast<uint32_t>(framesize_sum * 8.0f * 1000.0f / denom + 0.5f);
   } else {
     avg_sent_bit_rate_bps_ = framesize_sum * 8;
   }
 }

 void MediaOptimization::UpdateSentFramerate() {
   if (encoded_frame_samples_.size() <= 1) {
     avg_sent_framerate_ = encoded_frame_samples_.size();
     return;
   }
   int denom = encoded_frame_samples_.back().timestamp -
               encoded_frame_samples_.front().timestamp;
   if (denom > 0) {
     avg_sent_framerate_ =
         (90000 * (encoded_frame_samples_.size() - 1) + denom / 2) / denom;
   } else {
     avg_sent_framerate_ = encoded_frame_samples_.size();
   }
 }

 bool MediaOptimization::QMUpdate(
     VCMResolutionScale* qm,
     VCMQMSettingsCallback* video_qmsettings_callback) {
   // Check for no change.
   if (!qm->change_resolution_spatial && !qm->change_resolution_temporal) {
     return false;
   }

   // Check for change in frame rate.
   if (qm->change_resolution_temporal) {
     incoming_frame_rate_ = qm->frame_rate;
     // Reset frame rate estimate.
     memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
   }

   // Check for change in frame size.
   if (qm->change_resolution_spatial) {
     codec_width_ = qm->codec_width;
     codec_height_ = qm->codec_height;
   }

   LOG(LS_INFO) << "Media optimizer requests the video resolution to be changed "
                   "to "
                << qm->codec_width << "x" << qm->codec_height << "@"
                << qm->frame_rate;

   // Update VPM with new target frame rate and frame size.
   // Note: use |qm->frame_rate| instead of |_incoming_frame_rate| for updating
   // target frame rate in VPM frame dropper. The quantity |_incoming_frame_rate|
   // will vary/fluctuate, and since we don't want to change the state of the
   // VPM frame dropper, unless a temporal action was selected, we use the
   // quantity |qm->frame_rate| for updating.
   video_qmsettings_callback->SetVideoQMSettings(qm->frame_rate, codec_width_,
                                                 codec_height_);
   content_->UpdateFrameRate(qm->frame_rate);
   qm_resolution_->UpdateCodecParameters(qm->frame_rate, codec_width_,
                                         codec_height_);
   return true;
 }

 // Check timing constraints and look for significant change in:
 // (1) scene content,
 // (2) target bit rate.
 bool MediaOptimization::CheckStatusForQMchange() {
   bool status = true;

   // Check that we do not call QMSelect too often, and that we waited some time
   // (to sample the metrics) from the event last_change_time
   // last_change_time is the time where user changed the size/rate/frame rate
   // (via SetEncodingData).
   int64_t now = clock_->TimeInMilliseconds();
   if ((now - last_qm_update_time_) < kQmMinIntervalMs ||
       (now - last_change_time_) < kQmMinIntervalMs) {
     status = false;
   }

   return status;
 }

 // Allowing VCM to keep track of incoming frame rate.
 void MediaOptimization::ProcessIncomingFrameRate(int64_t now) {
   int32_t num = 0;
   int32_t nr_of_frames = 0;
   for (num = 1; num < (kFrameCountHistorySize - 1); ++num) {
     if (incoming_frame_times_[num] <= 0 ||
         // don't use data older than 2 s
         now - incoming_frame_times_[num] > kFrameHistoryWinMs) {
       break;
     } else {
       nr_of_frames++;
     }
   }
   if (num > 1) {
     const int64_t diff =
         incoming_frame_times_[0] - incoming_frame_times_[num - 1];
     incoming_frame_rate_ = 0.0;  // No frame rate estimate available.
     if (diff > 0) {
       incoming_frame_rate_ = nr_of_frames * 1000.0f / static_cast<float>(diff);
     }
   }
 }

 void MediaOptimization::CheckSuspendConditions() {
   // Check conditions for SuspendBelowMinBitrate. |video_target_bitrate_| is in
   // bps.
   if (suspension_enabled_) {
     if (!video_suspended_) {
       // Check if we just went below the threshold.
       if (video_target_bitrate_ < suspension_threshold_bps_) {
         video_suspended_ = true;
       }
     } else {
       // Video is already suspended. Check if we just went over the threshold
       // with a margin.
       if (video_target_bitrate_ >
           suspension_threshold_bps_ + suspension_window_bps_) {
         video_suspended_ = false;
       }
     }
   }
 }

 }  // namespace media_optimization
 }  // namespace webrtc
	/*
	* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
	*
	* Use of this source code is governed by a BSD-style license
	* that can be found in the LICENSE file in the root of the source
	* tree. An additional intellectual property rights grant can be found
	* in the file PATENTS. All contributing project authors may
	* be found in the AUTHORS file in the root of the source tree.
	*/

	#include "webrtc/modules/video_coding/media_optimization.h"

	#include "webrtc/base/logging.h"
	#include "webrtc/modules/video_coding/content_metrics_processing.h"
	#include "webrtc/modules/video_coding/qm_select.h"
	#include "webrtc/modules/video_coding/utility/frame_dropper.h"
	#include "webrtc/system_wrappers/include/clock.h"

	namespace webrtc {
	namespace media_optimization {
	namespace {
	void UpdateProtectionCallback(
	VCMProtectionMethod* selected_method,
	uint32_t* video_rate_bps,
	uint32_t* nack_overhead_rate_bps,
	uint32_t* fec_overhead_rate_bps,
	VCMProtectionCallback* video_protection_callback) {
	FecProtectionParams delta_fec_params;
	FecProtectionParams key_fec_params;
	// Get the FEC code rate for Key frames (set to 0 when NA).
	key_fec_params.fec_rate = selected_method->RequiredProtectionFactorK();

	// Get the FEC code rate for Delta frames (set to 0 when NA).
	delta_fec_params.fec_rate = selected_method->RequiredProtectionFactorD();

	// Get the FEC-UEP protection status for Key frames: UEP on/off.
	key_fec_params.use_uep_protection = selected_method->RequiredUepProtectionK();

	// Get the FEC-UEP protection status for Delta frames: UEP on/off.
	delta_fec_params.use_uep_protection =
	selected_method->RequiredUepProtectionD();

	// The RTP module currently requires the same \|max_fec_frames\| for both
	// key and delta frames.
	delta_fec_params.max_fec_frames = selected_method->MaxFramesFec();
	key_fec_params.max_fec_frames = selected_method->MaxFramesFec();

	// Set the FEC packet mask type. \|kFecMaskBursty\| is more effective for
	// consecutive losses and little/no packet re-ordering. As we currently
	// do not have feedback data on the degree of correlated losses and packet
	// re-ordering, we keep default setting to \|kFecMaskRandom\| for now.
	delta_fec_params.fec_mask_type = kFecMaskRandom;
	key_fec_params.fec_mask_type = kFecMaskRandom;

	// TODO(Marco): Pass FEC protection values per layer.
	video_protection_callback->ProtectionRequest(
	&delta_fec_params, &key_fec_params, video_rate_bps,
	nack_overhead_rate_bps, fec_overhead_rate_bps);
	}
	} // namespace

	struct MediaOptimization::EncodedFrameSample {
	EncodedFrameSample(size_t size_bytes,
	uint32_t timestamp,
	int64_t time_complete_ms)
	: size_bytes(size_bytes),
	timestamp(timestamp),
	time_complete_ms(time_complete_ms) {}

	size_t size_bytes;
	uint32_t timestamp;
	int64_t time_complete_ms;
	};

	MediaOptimization::MediaOptimization(Clock* clock)
	: crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
	clock_(clock),
	max_bit_rate_(0),
	send_codec_type_(kVideoCodecUnknown),
	codec_width_(0),
	codec_height_(0),
	user_frame_rate_(0),
	frame_dropper_(new FrameDropper),
	loss_prot_logic_(
	new VCMLossProtectionLogic(clock_->TimeInMilliseconds())),
	fraction_lost_(0),
	send_statistics_zero_encode_(0),
	max_payload_size_(1460),
	video_target_bitrate_(0),
	incoming_frame_rate_(0),
	enable_qm_(false),
	encoded_frame_samples_(),
	avg_sent_bit_rate_bps_(0),
	avg_sent_framerate_(0),
	key_frame_cnt_(0),
	delta_frame_cnt_(0),
	content_(new VCMContentMetricsProcessing()),
	qm_resolution_(new VCMQmResolution()),
	last_qm_update_time_(0),
	last_change_time_(0),
	num_layers_(0),
	suspension_enabled_(false),
	video_suspended_(false),
	suspension_threshold_bps_(0),
	suspension_window_bps_(0) {
	memset(send_statistics_, 0, sizeof(send_statistics_));
	memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
	}

	MediaOptimization::~MediaOptimization(void) {
	loss_prot_logic_->Release();
	}

	void MediaOptimization::Reset() {
	CriticalSectionScoped lock(crit_sect_.get());
	SetEncodingDataInternal(kVideoCodecUnknown, 0, 0, 0, 0, 0, 0,
	max_payload_size_);
	memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
	incoming_frame_rate_ = 0.0;
	frame_dropper_->Reset();
	loss_prot_logic_->Reset(clock_->TimeInMilliseconds());
	frame_dropper_->SetRates(0, 0);
	content_->Reset();
	qm_resolution_->Reset();
	loss_prot_logic_->UpdateFrameRate(incoming_frame_rate_);
	loss_prot_logic_->Reset(clock_->TimeInMilliseconds());
	send_statistics_zero_encode_ = 0;
	video_target_bitrate_ = 0;
	codec_width_ = 0;
	codec_height_ = 0;
	user_frame_rate_ = 0;
	key_frame_cnt_ = 0;
	delta_frame_cnt_ = 0;
	last_qm_update_time_ = 0;
	last_change_time_ = 0;
	encoded_frame_samples_.clear();
	avg_sent_bit_rate_bps_ = 0;
	num_layers_ = 1;
	}

	void MediaOptimization::SetEncodingData(VideoCodecType send_codec_type,
	int32_t max_bit_rate,
	uint32_t target_bitrate,
	uint16_t width,
	uint16_t height,
	uint32_t frame_rate,
	int num_layers,
	int32_t mtu) {
	CriticalSectionScoped lock(crit_sect_.get());
	SetEncodingDataInternal(send_codec_type, max_bit_rate, frame_rate,
	target_bitrate, width, height, num_layers, mtu);
	}

	void MediaOptimization::SetEncodingDataInternal(VideoCodecType send_codec_type,
	int32_t max_bit_rate,
	uint32_t frame_rate,
	uint32_t target_bitrate,
	uint16_t width,
	uint16_t height,
	int num_layers,
	int32_t mtu) {
	// Everything codec specific should be reset here since this means the codec
	// has changed. If native dimension values have changed, then either user
	// initiated change, or QM initiated change. Will be able to determine only
	// after the processing of the first frame.
	last_change_time_ = clock_->TimeInMilliseconds();
	content_->Reset();
	content_->UpdateFrameRate(frame_rate);

	max_bit_rate_ = max_bit_rate;
	send_codec_type_ = send_codec_type;
	video_target_bitrate_ = target_bitrate;
	float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f;
	loss_prot_logic_->UpdateBitRate(target_bitrate_kbps);
	loss_prot_logic_->UpdateFrameRate(static_cast<float>(frame_rate));
	loss_prot_logic_->UpdateFrameSize(width, height);
	loss_prot_logic_->UpdateNumLayers(num_layers);
	frame_dropper_->Reset();
	frame_dropper_->SetRates(target_bitrate_kbps, static_cast<float>(frame_rate));
	user_frame_rate_ = static_cast<float>(frame_rate);
	codec_width_ = width;
	codec_height_ = height;
	num_layers_ = (num_layers <= 1) ? 1 : num_layers; // Can also be zero.
	max_payload_size_ = mtu;
	qm_resolution_->Initialize(target_bitrate_kbps, user_frame_rate_,
	codec_width_, codec_height_, num_layers_);
	}

	uint32_t MediaOptimization::SetTargetRates(
	uint32_t target_bitrate,
	uint8_t fraction_lost,
	int64_t round_trip_time_ms,
	VCMProtectionCallback* protection_callback,
	VCMQMSettingsCallback* qmsettings_callback) {
	CriticalSectionScoped lock(crit_sect_.get());
	VCMProtectionMethod* selected_method = loss_prot_logic_->SelectedMethod();
	float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f;
	loss_prot_logic_->UpdateBitRate(target_bitrate_kbps);
	loss_prot_logic_->UpdateRtt(round_trip_time_ms);

	// Get frame rate for encoder: this is the actual/sent frame rate.
	float actual_frame_rate = SentFrameRateInternal();

	// Sanity check.
	if (actual_frame_rate < 1.0) {
	actual_frame_rate = 1.0;
	}

	// Update frame rate for the loss protection logic class: frame rate should
	// be the actual/sent rate.
	loss_prot_logic_->UpdateFrameRate(actual_frame_rate);

	fraction_lost_ = fraction_lost;

	// Returns the filtered packet loss, used for the protection setting.
	// The filtered loss may be the received loss (no filter), or some
	// filtered value (average or max window filter).
	// Use max window filter for now.
	FilterPacketLossMode filter_mode = kMaxFilter;
	uint8_t packet_loss_enc = loss_prot_logic_->FilteredLoss(
	clock_->TimeInMilliseconds(), filter_mode, fraction_lost);

	// For now use the filtered loss for computing the robustness settings.
	loss_prot_logic_->UpdateFilteredLossPr(packet_loss_enc);

	// Rate cost of the protection methods.
	float protection_overhead_rate = 0.0f;

	// Update protection settings, when applicable.
	float sent_video_rate_kbps = 0.0f;
	if (loss_prot_logic_->SelectedType() != kNone) {
	// Update protection method with content metrics.
	selected_method->UpdateContentMetrics(content_->ShortTermAvgData());

	// Update method will compute the robustness settings for the given
	// protection method and the overhead cost
	// the protection method is set by the user via SetVideoProtection.
	loss_prot_logic_->UpdateMethod();

	// Update protection callback with protection settings.
	uint32_t sent_video_rate_bps = 0;
	uint32_t sent_nack_rate_bps = 0;
	uint32_t sent_fec_rate_bps = 0;
	// Get the bit cost of protection method, based on the amount of
	// overhead data actually transmitted (including headers) the last
	// second.
	if (protection_callback) {
	UpdateProtectionCallback(selected_method, &sent_video_rate_bps,
	&sent_nack_rate_bps, &sent_fec_rate_bps,
	protection_callback);
	}
	uint32_t sent_total_rate_bps =
	sent_video_rate_bps + sent_nack_rate_bps + sent_fec_rate_bps;
	// Estimate the overhead costs of the next second as staying the same
	// wrt the source bitrate.
	if (sent_total_rate_bps > 0) {
	protection_overhead_rate =
	static_cast<float>(sent_nack_rate_bps + sent_fec_rate_bps) /
	sent_total_rate_bps;
	}
	// Cap the overhead estimate to 50%.
	if (protection_overhead_rate > 0.5)
	protection_overhead_rate = 0.5;

	// Get the effective packet loss for encoder ER when applicable. Should be
	// passed to encoder via fraction_lost.
	packet_loss_enc = selected_method->RequiredPacketLossER();
	sent_video_rate_kbps = static_cast<float>(sent_video_rate_bps) / 1000.0f;
	}

	// Source coding rate: total rate - protection overhead.
	video_target_bitrate_ = target_bitrate * (1.0 - protection_overhead_rate);

	// Cap target video bitrate to codec maximum.
	if (max_bit_rate_ > 0 && video_target_bitrate_ > max_bit_rate_) {
	video_target_bitrate_ = max_bit_rate_;
	}

	// Update encoding rates following protection settings.
	float target_video_bitrate_kbps =
	static_cast<float>(video_target_bitrate_) / 1000.0f;
	frame_dropper_->SetRates(target_video_bitrate_kbps, incoming_frame_rate_);

	if (enable_qm_ && qmsettings_callback) {
	// Update QM with rates.
	qm_resolution_->UpdateRates(target_video_bitrate_kbps, sent_video_rate_kbps,
	incoming_frame_rate_, fraction_lost_);
	// Check for QM selection.
	bool select_qm = CheckStatusForQMchange();
	if (select_qm) {
	SelectQuality(qmsettings_callback);
	}
	// Reset the short-term averaged content data.
	content_->ResetShortTermAvgData();
	}

	CheckSuspendConditions();

	return video_target_bitrate_;
	}

	void MediaOptimization::SetProtectionMethod(VCMProtectionMethodEnum method) {
	CriticalSectionScoped lock(crit_sect_.get());
	loss_prot_logic_->SetMethod(method);
	}

	uint32_t MediaOptimization::InputFrameRate() {
	CriticalSectionScoped lock(crit_sect_.get());
	return InputFrameRateInternal();
	}

	uint32_t MediaOptimization::InputFrameRateInternal() {
	ProcessIncomingFrameRate(clock_->TimeInMilliseconds());
	return uint32_t(incoming_frame_rate_ + 0.5f);
	}

	uint32_t MediaOptimization::SentFrameRate() {
	CriticalSectionScoped lock(crit_sect_.get());
	return SentFrameRateInternal();
	}

	uint32_t MediaOptimization::SentFrameRateInternal() {
	PurgeOldFrameSamples(clock_->TimeInMilliseconds());
	UpdateSentFramerate();
	return avg_sent_framerate_;
	}

	uint32_t MediaOptimization::SentBitRate() {
	CriticalSectionScoped lock(crit_sect_.get());
	const int64_t now_ms = clock_->TimeInMilliseconds();
	PurgeOldFrameSamples(now_ms);
	UpdateSentBitrate(now_ms);
	return avg_sent_bit_rate_bps_;
	}

	int32_t MediaOptimization::UpdateWithEncodedData(
	const EncodedImage& encoded_image) {
	size_t encoded_length = encoded_image._length;
	uint32_t timestamp = encoded_image._timeStamp;
	CriticalSectionScoped lock(crit_sect_.get());
	const int64_t now_ms = clock_->TimeInMilliseconds();
	PurgeOldFrameSamples(now_ms);
	if (encoded_frame_samples_.size() > 0 &&
	encoded_frame_samples_.back().timestamp == timestamp) {
	// Frames having the same timestamp are generated from the same input
	// frame. We don't want to double count them, but only increment the
	// size_bytes.
	encoded_frame_samples_.back().size_bytes += encoded_length;
	encoded_frame_samples_.back().time_complete_ms = now_ms;
	} else {
	encoded_frame_samples_.push_back(
	EncodedFrameSample(encoded_length, timestamp, now_ms));
	}
	UpdateSentBitrate(now_ms);
	UpdateSentFramerate();
	if (encoded_length > 0) {
	const bool delta_frame = encoded_image._frameType != kVideoFrameKey;

	frame_dropper_->Fill(encoded_length, delta_frame);
	if (max_payload_size_ > 0 && encoded_length > 0) {
	const float min_packets_per_frame =
	encoded_length / static_cast<float>(max_payload_size_);
	if (delta_frame) {
	loss_prot_logic_->UpdatePacketsPerFrame(min_packets_per_frame,
	clock_->TimeInMilliseconds());
	} else {
	loss_prot_logic_->UpdatePacketsPerFrameKey(
	min_packets_per_frame, clock_->TimeInMilliseconds());
	}

	if (enable_qm_) {
	// Update quality select with encoded length.
	qm_resolution_->UpdateEncodedSize(encoded_length);
	}
	}
	if (!delta_frame && encoded_length > 0) {
	loss_prot_logic_->UpdateKeyFrameSize(static_cast<float>(encoded_length));
	}

	// Updating counters.
	if (delta_frame) {
	delta_frame_cnt_++;
	} else {
	key_frame_cnt_++;
	}
	}

	return VCM_OK;
	}

	void MediaOptimization::EnableQM(bool enable) {
	CriticalSectionScoped lock(crit_sect_.get());
	enable_qm_ = enable;
	}

	void MediaOptimization::EnableFrameDropper(bool enable) {
	CriticalSectionScoped lock(crit_sect_.get());
	frame_dropper_->Enable(enable);
	}

	void MediaOptimization::SuspendBelowMinBitrate(int threshold_bps,
	int window_bps) {
	CriticalSectionScoped lock(crit_sect_.get());
	assert(threshold_bps > 0 && window_bps >= 0);
	suspension_threshold_bps_ = threshold_bps;
	suspension_window_bps_ = window_bps;
	suspension_enabled_ = true;
	video_suspended_ = false;
	}

	bool MediaOptimization::IsVideoSuspended() const {
	CriticalSectionScoped lock(crit_sect_.get());
	return video_suspended_;
	}

	bool MediaOptimization::DropFrame() {
	CriticalSectionScoped lock(crit_sect_.get());
	UpdateIncomingFrameRate();
	// Leak appropriate number of bytes.
	frame_dropper_->Leak((uint32_t)(InputFrameRateInternal() + 0.5f));
	if (video_suspended_) {
	return true; // Drop all frames when muted.
	}
	return frame_dropper_->DropFrame();
	}

	void MediaOptimization::UpdateContentData(
	const VideoContentMetrics* content_metrics) {
	CriticalSectionScoped lock(crit_sect_.get());
	// Updating content metrics.
	if (content_metrics == NULL) {
	// Disable QM if metrics are NULL.
	enable_qm_ = false;
	qm_resolution_->Reset();
	} else {
	content_->UpdateContentData(content_metrics);
	}
	}

	void MediaOptimization::UpdateIncomingFrameRate() {
	int64_t now = clock_->TimeInMilliseconds();
	if (incoming_frame_times_[0] == 0) {
	// No shifting if this is the first time.
	} else {
	// Shift all times one step.
	for (int32_t i = (kFrameCountHistorySize - 2); i >= 0; i--) {
	incoming_frame_times_[i + 1] = incoming_frame_times_[i];
	}
	}
	incoming_frame_times_[0] = now;
	ProcessIncomingFrameRate(now);
	}

	int32_t MediaOptimization::SelectQuality(
	VCMQMSettingsCallback* video_qmsettings_callback) {
	// Reset quantities for QM select.
	qm_resolution_->ResetQM();

	// Update QM will long-term averaged content metrics.
	qm_resolution_->UpdateContent(content_->LongTermAvgData());

	// Select quality mode.
	VCMResolutionScale* qm = NULL;
	int32_t ret = qm_resolution_->SelectResolution(&qm);
	if (ret < 0) {
	return ret;
	}

	// Check for updates to spatial/temporal modes.
	QMUpdate(qm, video_qmsettings_callback);

	// Reset all the rate and related frame counters quantities.
	qm_resolution_->ResetRates();

	// Reset counters.
	last_qm_update_time_ = clock_->TimeInMilliseconds();

	// Reset content metrics.
	content_->Reset();

	return VCM_OK;
	}

	void MediaOptimization::PurgeOldFrameSamples(int64_t now_ms) {
	while (!encoded_frame_samples_.empty()) {
	if (now_ms - encoded_frame_samples_.front().time_complete_ms >
	kBitrateAverageWinMs) {
	encoded_frame_samples_.pop_front();
	} else {
	break;
	}
	}
	}

	void MediaOptimization::UpdateSentBitrate(int64_t now_ms) {
	if (encoded_frame_samples_.empty()) {
	avg_sent_bit_rate_bps_ = 0;
	return;
	}
	size_t framesize_sum = 0;
	for (FrameSampleList::iterator it = encoded_frame_samples_.begin();
	it != encoded_frame_samples_.end(); ++it) {
	framesize_sum += it->size_bytes;
	}
	float denom = static_cast<float>(
	now_ms - encoded_frame_samples_.front().time_complete_ms);
	if (denom >= 1.0f) {
	avg_sent_bit_rate_bps_ =
	static_cast<uint32_t>(framesize_sum * 8.0f * 1000.0f / denom + 0.5f);
	} else {
	avg_sent_bit_rate_bps_ = framesize_sum * 8;
	}
	}

	void MediaOptimization::UpdateSentFramerate() {
	if (encoded_frame_samples_.size() <= 1) {
	avg_sent_framerate_ = encoded_frame_samples_.size();
	return;
	}
	int denom = encoded_frame_samples_.back().timestamp -
	encoded_frame_samples_.front().timestamp;
	if (denom > 0) {
	avg_sent_framerate_ =
	(90000 * (encoded_frame_samples_.size() - 1) + denom / 2) / denom;
	} else {
	avg_sent_framerate_ = encoded_frame_samples_.size();
	}
	}

	bool MediaOptimization::QMUpdate(
	VCMResolutionScale* qm,
	VCMQMSettingsCallback* video_qmsettings_callback) {
	// Check for no change.
	if (!qm->change_resolution_spatial && !qm->change_resolution_temporal) {
	return false;
	}

	// Check for change in frame rate.
	if (qm->change_resolution_temporal) {
	incoming_frame_rate_ = qm->frame_rate;
	// Reset frame rate estimate.
	memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
	}

	// Check for change in frame size.
	if (qm->change_resolution_spatial) {
	codec_width_ = qm->codec_width;
	codec_height_ = qm->codec_height;
	}

	LOG(LS_INFO) << "Media optimizer requests the video resolution to be changed "
	"to "
	<< qm->codec_width << "x" << qm->codec_height << "@"
	<< qm->frame_rate;

	// Update VPM with new target frame rate and frame size.
	// Note: use \|qm->frame_rate\| instead of \|_incoming_frame_rate\| for updating
	// target frame rate in VPM frame dropper. The quantity \|_incoming_frame_rate\|
	// will vary/fluctuate, and since we don't want to change the state of the
	// VPM frame dropper, unless a temporal action was selected, we use the
	// quantity \|qm->frame_rate\| for updating.
	video_qmsettings_callback->SetVideoQMSettings(qm->frame_rate, codec_width_,
	codec_height_);
	content_->UpdateFrameRate(qm->frame_rate);
	qm_resolution_->UpdateCodecParameters(qm->frame_rate, codec_width_,
	codec_height_);
	return true;
	}

	// Check timing constraints and look for significant change in:
	// (1) scene content,
	// (2) target bit rate.
	bool MediaOptimization::CheckStatusForQMchange() {
	bool status = true;

	// Check that we do not call QMSelect too often, and that we waited some time
	// (to sample the metrics) from the event last_change_time
	// last_change_time is the time where user changed the size/rate/frame rate
	// (via SetEncodingData).
	int64_t now = clock_->TimeInMilliseconds();
	if ((now - last_qm_update_time_) < kQmMinIntervalMs \|\|
	(now - last_change_time_) < kQmMinIntervalMs) {
	status = false;
	}

	return status;
	}

	// Allowing VCM to keep track of incoming frame rate.
	void MediaOptimization::ProcessIncomingFrameRate(int64_t now) {
	int32_t num = 0;
	int32_t nr_of_frames = 0;
	for (num = 1; num < (kFrameCountHistorySize - 1); ++num) {
	if (incoming_frame_times_[num] <= 0 \|\|
	// don't use data older than 2 s
	now - incoming_frame_times_[num] > kFrameHistoryWinMs) {
	break;
	} else {
	nr_of_frames++;
	}
	}
	if (num > 1) {
	const int64_t diff =
	incoming_frame_times_[0] - incoming_frame_times_[num - 1];
	incoming_frame_rate_ = 0.0; // No frame rate estimate available.
	if (diff > 0) {
	incoming_frame_rate_ = nr_of_frames * 1000.0f / static_cast<float>(diff);
	}
	}
	}

	void MediaOptimization::CheckSuspendConditions() {
	// Check conditions for SuspendBelowMinBitrate. \|video_target_bitrate_\| is in
	// bps.
	if (suspension_enabled_) {
	if (!video_suspended_) {
	// Check if we just went below the threshold.
	if (video_target_bitrate_ < suspension_threshold_bps_) {
	video_suspended_ = true;
	}
	} else {
	// Video is already suspended. Check if we just went over the threshold
	// with a margin.
	if (video_target_bitrate_ >
	suspension_threshold_bps_ + suspension_window_bps_) {
	video_suspended_ = false;
	}
	}
	}
	}

	} // namespace media_optimization
	} // namespace webrtc