webrtc/modules/video_coding/qm_select.h

/*
 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#ifndef WEBRTC_MODULES_VIDEO_CODING_QM_SELECT_H_
#define WEBRTC_MODULES_VIDEO_CODING_QM_SELECT_H_

#include "webrtc/common_types.h"
#include "webrtc/typedefs.h"

/******************************************************/
/* Quality Modes: Resolution and Robustness settings  */
/******************************************************/

namespace webrtc {
struct VideoContentMetrics;

struct VCMResolutionScale {
  VCMResolutionScale()
      : codec_width(640),
        codec_height(480),
        frame_rate(30.0f),
        spatial_width_fact(1.0f),
        spatial_height_fact(1.0f),
        temporal_fact(1.0f),
        change_resolution_spatial(false),
        change_resolution_temporal(false) {}
  uint16_t codec_width;
  uint16_t codec_height;
  float frame_rate;
  float spatial_width_fact;
  float spatial_height_fact;
  float temporal_fact;
  bool change_resolution_spatial;
  bool change_resolution_temporal;
};

enum ImageType {
  kQCIF = 0,  // 176x144
  kHCIF,      // 264x216 = half(~3/4x3/4) CIF.
  kQVGA,      // 320x240 = quarter VGA.
  kCIF,       // 352x288
  kHVGA,      // 480x360 = half(~3/4x3/4) VGA.
  kVGA,       // 640x480
  kQFULLHD,   // 960x540 = quarter FULLHD, and half(~3/4x3/4) WHD.
  kWHD,       // 1280x720
  kFULLHD,    // 1920x1080
  kNumImageTypes
};

const uint32_t kSizeOfImageType[kNumImageTypes] = {
    25344, 57024, 76800, 101376, 172800, 307200, 518400, 921600, 2073600};

enum FrameRateLevelClass {
  kFrameRateLow,
  kFrameRateMiddle1,
  kFrameRateMiddle2,
  kFrameRateHigh
};

enum ContentLevelClass { kLow, kHigh, kDefault };

struct VCMContFeature {
  VCMContFeature() : value(0.0f), level(kDefault) {}
  void Reset() {
    value = 0.0f;
    level = kDefault;
  }
  float value;
  ContentLevelClass level;
};

enum UpDownAction { kUpResolution, kDownResolution };

enum SpatialAction {
  kNoChangeSpatial,
  kOneHalfSpatialUniform,     // 3/4 x 3/4: 9/6 ~1/2 pixel reduction.
  kOneQuarterSpatialUniform,  // 1/2 x 1/2: 1/4 pixel reduction.
  kNumModesSpatial
};

enum TemporalAction {
  kNoChangeTemporal,
  kTwoThirdsTemporal,  // 2/3 frame rate reduction
  kOneHalfTemporal,    // 1/2 frame rate reduction
  kNumModesTemporal
};

struct ResolutionAction {
  ResolutionAction() : spatial(kNoChangeSpatial), temporal(kNoChangeTemporal) {}
  SpatialAction spatial;
  TemporalAction temporal;
};

// Down-sampling factors for spatial (width and height), and temporal.
const float kFactorWidthSpatial[kNumModesSpatial] = {1.0f, 4.0f / 3.0f, 2.0f};

const float kFactorHeightSpatial[kNumModesSpatial] = {1.0f, 4.0f / 3.0f, 2.0f};

const float kFactorTemporal[kNumModesTemporal] = {1.0f, 1.5f, 2.0f};

enum EncoderState {
  kStableEncoding,    // Low rate mis-match, stable buffer levels.
  kStressedEncoding,  // Significant over-shooting of target rate,
                      // Buffer under-flow, etc.
  kEasyEncoding       // Significant under-shooting of target rate.
};

// QmMethod class: main class for resolution and robustness settings

class VCMQmMethod {
 public:
  VCMQmMethod();
  virtual ~VCMQmMethod();

  // Reset values
  void ResetQM();
  virtual void Reset() = 0;

  // Compute content class.
  uint8_t ComputeContentClass();

  // Update with the content metrics.
  void UpdateContent(const VideoContentMetrics* content_metrics);

  // Compute spatial texture magnitude and level.
  // Spatial texture is a spatial prediction error measure.
  void ComputeSpatial();

  // Compute motion magnitude and level for NFD metric.
  // NFD is normalized frame difference (normalized by spatial variance).
  void ComputeMotionNFD();

  // Get the imageType (CIF, VGA, HD, etc) for the system width/height.
  ImageType GetImageType(uint16_t width, uint16_t height);

  // Return the closest image type.
  ImageType FindClosestImageType(uint16_t width, uint16_t height);

  // Get the frame rate level.
  FrameRateLevelClass FrameRateLevel(float frame_rate);

 protected:
  // Content Data.
  const VideoContentMetrics* content_metrics_;

  // Encoder frame sizes and native frame sizes.
  uint16_t width_;
  uint16_t height_;
  float user_frame_rate_;
  uint16_t native_width_;
  uint16_t native_height_;
  float native_frame_rate_;
  float aspect_ratio_;
  // Image type and frame rate leve, for the current encoder resolution.
  ImageType image_type_;
  FrameRateLevelClass framerate_level_;
  // Content class data.
  VCMContFeature motion_;
  VCMContFeature spatial_;
  uint8_t content_class_;
  bool init_;
};

// Resolution settings class

class VCMQmResolution : public VCMQmMethod {
 public:
  VCMQmResolution();
  virtual ~VCMQmResolution();

  // Reset all quantities.
  virtual void Reset();

  // Reset rate quantities and counters after every SelectResolution() call.
  void ResetRates();

  // Reset down-sampling state.
  void ResetDownSamplingState();

  // Get the encoder state.
  EncoderState GetEncoderState();

  // Initialize after SetEncodingData in media_opt.
  int Initialize(float bitrate,
                 float user_framerate,
                 uint16_t width,
                 uint16_t height,
                 int num_layers);

  // Update the encoder frame size.
  void UpdateCodecParameters(float frame_rate, uint16_t width, uint16_t height);

  // Update with actual bit rate (size of the latest encoded frame)
  // and frame type, after every encoded frame.
  void UpdateEncodedSize(size_t encoded_size);

  // Update with new target bitrate, actual encoder sent rate, frame_rate,
  // loss rate: every ~1 sec from SetTargetRates in media_opt.
  void UpdateRates(float target_bitrate,
                   float encoder_sent_rate,
                   float incoming_framerate,
                   uint8_t packet_loss);

  // Extract ST (spatio-temporal) resolution action.
  // Inputs: qm: Reference to the quality modes pointer.
  // Output: the spatial and/or temporal scale change.
  int SelectResolution(VCMResolutionScale** qm);

 private:
  // Set the default resolution action.
  void SetDefaultAction();

  // Compute rates for the selection of down-sampling action.
  void ComputeRatesForSelection();

  // Compute the encoder state.
  void ComputeEncoderState();

  // Return true if the action is to go back up in resolution.
  bool GoingUpResolution();

  // Return true if the action is to go down in resolution.
  bool GoingDownResolution();

  // Check the condition for going up in resolution by the scale factors:
  // |facWidth|, |facHeight|, |facTemp|.
  // |scaleFac| is a scale factor for the transition rate.
  bool ConditionForGoingUp(float fac_width,
                           float fac_height,
                           float fac_temp,
                           float scale_fac);

  // Get the bitrate threshold for the resolution action.
  // The case |facWidth|=|facHeight|=|facTemp|==1 is for down-sampling action.
  // |scaleFac| is a scale factor for the transition rate.
  float GetTransitionRate(float fac_width,
                          float fac_height,
                          float fac_temp,
                          float scale_fac);

  // Update the down-sampling state.
  void UpdateDownsamplingState(UpDownAction up_down);

  // Update the codec frame size and frame rate.
  void UpdateCodecResolution();

  // Return a state based on average target rate relative transition rate.
  uint8_t RateClass(float transition_rate);

  // Adjust the action selected from the table.
  void AdjustAction();

  // Covert 2 stages of 3/4 (=9/16) spatial decimation to 1/2.
  void ConvertSpatialFractionalToWhole();

  // Returns true if the new frame sizes, under the selected spatial action,
  // are of even size.
  bool EvenFrameSize();

  // Insert latest down-sampling action into the history list.
  void InsertLatestDownAction();

  // Remove the last (first element) down-sampling action from the list.
  void RemoveLastDownAction();

  // Check constraints on the amount of down-sampling allowed.
  void ConstrainAmountOfDownSampling();

  // For going up in resolution: pick spatial or temporal action,
  // if both actions were separately selected.
  void PickSpatialOrTemporal();

  // Select the directional (1x2 or 2x1) spatial down-sampling action.
  void SelectSpatialDirectionMode(float transition_rate);

  enum { kDownActionHistorySize = 10 };

  VCMResolutionScale* qm_;
  // Encoder rate control parameters.
  float target_bitrate_;
  float incoming_framerate_;
  float per_frame_bandwidth_;
  float buffer_level_;

  // Data accumulated every ~1sec from MediaOpt.
  float sum_target_rate_;
  float sum_incoming_framerate_;
  float sum_rate_MM_;
  float sum_rate_MM_sgn_;
  float sum_packet_loss_;
  // Counters.
  uint32_t frame_cnt_;
  uint32_t frame_cnt_delta_;
  uint32_t update_rate_cnt_;
  uint32_t low_buffer_cnt_;

  // Resolution state parameters.
  float state_dec_factor_spatial_;
  float state_dec_factor_temporal_;

  // Quantities used for selection.
  float avg_target_rate_;
  float avg_incoming_framerate_;
  float avg_ratio_buffer_low_;
  float avg_rate_mismatch_;
  float avg_rate_mismatch_sgn_;
  float avg_packet_loss_;
  EncoderState encoder_state_;
  ResolutionAction action_;
  // Short history of the down-sampling actions from the Initialize() state.
  // This is needed for going up in resolution. Since the total amount of
  // down-sampling actions are constrained, the length of the list need not be
  // large: i.e., (4/3) ^{kDownActionHistorySize} <= kMaxDownSample.
  ResolutionAction down_action_history_[kDownActionHistorySize];
  int num_layers_;
};

// Robustness settings class.

class VCMQmRobustness : public VCMQmMethod {
 public:
  VCMQmRobustness();
  ~VCMQmRobustness();

  virtual void Reset();

  // Adjust FEC rate based on content: every ~1 sec from SetTargetRates.
  // Returns an adjustment factor.
  float AdjustFecFactor(uint8_t code_rate_delta,
                        float total_rate,
                        float framerate,
                        int64_t rtt_time,
                        uint8_t packet_loss);

  // Set the UEP protection on/off.
  bool SetUepProtection(uint8_t code_rate_delta,
                        float total_rate,
                        uint8_t packet_loss,
                        bool frame_type);

 private:
  // Previous state of network parameters.
  float prev_total_rate_;
  int64_t prev_rtt_time_;
  uint8_t prev_packet_loss_;
  uint8_t prev_code_rate_delta_;
};
}  // namespace webrtc
#endif  // WEBRTC_MODULES_VIDEO_CODING_QM_SELECT_H_