Modified media_opt and qm_select to allow for robustness settings
based on the content metrics. Re-organized the class structure in qm_select
into a resolution class and robustness settings class, both derived from
a main (qm/content analysis) class.
Review URL: http://webrtc-codereview.appspot.com/55006
git-svn-id: http://webrtc.googlecode.com/svn/trunk@188 4adac7df-926f-26a2-2b94-8c16560cd09d
diff --git a/src/modules/video_coding/main/source/media_opt_util.cc b/src/modules/video_coding/main/source/media_opt_util.cc
index a0937b6..4f7f40c 100644
--- a/src/modules/video_coding/main/source/media_opt_util.cc
+++ b/src/modules/video_coding/main/source/media_opt_util.cc
@@ -21,6 +21,13 @@
namespace webrtc {
+void
+VCMProtectionMethod::UpdateContentMetrics(
+ const VideoContentMetrics* contentMetrics)
+{
+ _qmRobustness->UpdateContent(contentMetrics);
+}
+
bool
VCMProtectionMethod::BetterThan(VCMProtectionMethod *pm)
{
@@ -417,6 +424,10 @@
codeRateDelta = plossMax - 1;
}
+ codeRateDelta = _qmRobustness->AdjustFecFactor(codeRateDelta, bitRate,
+ parameters->frameRate,
+ parameters->rtt, packetLoss);
+
// For Key frame:
// Effectively at a higher rate, so we scale/boost the rate
// The boost factor may depend on several factors: ratio of packet
@@ -462,6 +473,14 @@
_protectionFactorK = codeRateKey;
_protectionFactorD = codeRateDelta;
+
+ // Set the UEP protection on/off for Key and Delta frames
+ _uepKey = _qmRobustness->SetUepProtection(codeRateKey, bitRate,
+ packetLoss, 0);
+
+ _uepDelta = _qmRobustness->SetUepProtection(codeRateKey, bitRate,
+ packetLoss, 1);
+
// DONE WITH FEC PROTECTION SETTINGS
return true;
}
diff --git a/src/modules/video_coding/main/source/media_opt_util.h b/src/modules/video_coding/main/source/media_opt_util.h
index 8298406..3f78dd7 100644
--- a/src/modules/video_coding/main/source/media_opt_util.h
+++ b/src/modules/video_coding/main/source/media_opt_util.h
@@ -17,6 +17,7 @@
#include "exp_filter.h"
#include "internal_defines.h"
#include "tick_time.h"
+#include "qm_select.h"
#include <cmath>
#include <cstdlib>
@@ -97,8 +98,10 @@
//friend VCMProtectionMethod;
VCMProtectionMethod(VCMProtectionMethodEnum type) : _protectionFactorK(0),
_protectionFactorD(0), _residualPacketLoss(0.0), _scaleProtKey(2.0),
- _maxPayloadSize(1460), _efficiency(0), _score(0), _type(type) {}
- virtual ~VCMProtectionMethod() {}
+ _maxPayloadSize(1460), _efficiency(0), _score(0), _type(type),
+ _uepKey(0), _uepDelta(1)
+ {_qmRobustness = new VCMQmRobustness();}
+ virtual ~VCMProtectionMethod() { delete _qmRobustness;}
// Updates the efficiency of the method using the parameters provided
//
@@ -142,6 +145,9 @@
// Return value : Required protectionFactor for delta frame
virtual WebRtc_UWord8 RequiredProtectionFactorD() { return _protectionFactorD; }
+ // Updates content metrics
+ void UpdateContentMetrics(const VideoContentMetrics* contentMetrics);
+
WebRtc_UWord8 _effectivePacketLoss;
WebRtc_UWord8 _protectionFactorK;
WebRtc_UWord8 _protectionFactorD;
@@ -149,6 +155,10 @@
float _scaleProtKey;
WebRtc_Word32 _maxPayloadSize;
+ VCMQmRobustness* _qmRobustness;
+ bool _uepKey;
+ bool _uepDelta;
+
protected:
float _efficiency;
float _score;
diff --git a/src/modules/video_coding/main/source/media_optimization.cc b/src/modules/video_coding/main/source/media_optimization.cc
index e215986..3b03515 100644
--- a/src/modules/video_coding/main/source/media_optimization.cc
+++ b/src/modules/video_coding/main/source/media_optimization.cc
@@ -46,7 +46,7 @@
_frameDropper = new VCMFrameDropper(_id);
_lossProtLogic = new VCMLossProtectionLogic();
_content = new VCMContentMetricsProcessing();
- _qms = new VCMQmSelect();
+ _qmResolution = new VCMQmResolution();
}
VCMMediaOptimization::~VCMMediaOptimization(void)
@@ -55,7 +55,7 @@
delete _lossProtLogic;
delete _frameDropper;
delete _content;
- delete _qms;
+ delete _qmResolution;
}
WebRtc_Word32
@@ -67,7 +67,7 @@
_lossProtLogic->Reset();
_frameDropper->SetRates(0, 0);
_content->Reset();
- _qms->Reset();
+ _qmResolution->Reset();
_lossProtLogic->UpdateFrameRate(_incomingFrameRate);
_lossProtLogic->Reset();
_sendStatisticsZeroEncode = 0;
@@ -135,6 +135,10 @@
selectedMethod->Type() == kNackFec ))
{
+ // Update protection method with content metrics
+ selectedMethod->UpdateContentMetrics(_content->ShortTermAvgData());
+
+
// Update method will compute the robustness settings for the given
// protection method and the overhead cost
// the protection method is set by the user via SetVideoProtection.
@@ -197,8 +201,8 @@
if (_enableQm)
{
//Update QM with rates
- _qms->UpdateRates((float)_targetBitRate, _avgSentBitRateBps,
- _incomingFrameRate, _fractionLost);
+ _qmResolution->UpdateRates((float)_targetBitRate, _avgSentBitRateBps,
+ _incomingFrameRate, _fractionLost);
//Check for QM selection
bool selectQM = checkStatusForQMchange();
if (selectQM)
@@ -249,11 +253,12 @@
_lossProtLogic->UpdateFrameSize(width, height);
_frameDropper->Reset();
_frameDropper->SetRates(static_cast<float>(bitRate), static_cast<float>(frameRate));
- _userFrameRate = (float)frameRate;
+ _userFrameRate = static_cast<float>(frameRate);
_codecWidth = width;
_codecHeight = height;
WebRtc_Word32 ret = VCM_OK;
- ret = _qms->Initialize((float)_targetBitRate, _userFrameRate, _codecWidth, _codecHeight);
+ ret = _qmResolution->Initialize((float)_targetBitRate, _userFrameRate,
+ _codecWidth, _codecHeight);
return ret;
}
@@ -420,7 +425,8 @@
if (_enableQm)
{
// update quality select with encoded length
- _qms->UpdateEncodedSize(encodedLength, encodedFrameType);
+ _qmResolution->UpdateEncodedSize(encodedLength,
+ encodedFrameType);
}
}
if (!deltaFrame && encodedLength > 0)
@@ -525,7 +531,7 @@
{
//No QM if metrics are NULL
_enableQm = false;
- _qms->Reset();
+ _qmResolution->Reset();
}
else
{
@@ -537,14 +543,14 @@
VCMMediaOptimization::SelectQuality()
{
// Reset quantities for QM select
- _qms->ResetQM();
+ _qmResolution->ResetQM();
// Update QM will long-term averaged content metrics.
- _qms->UpdateContent(_content->LongTermAvgData());
+ _qmResolution->UpdateContent(_content->LongTermAvgData());
// Select quality mode
- VCMQualityMode* qm = NULL;
- WebRtc_Word32 ret = _qms->SelectQuality(&qm);
+ VCMResolutionScale* qm = NULL;
+ WebRtc_Word32 ret = _qmResolution->SelectResolution(&qm);
if (ret < 0)
{
return ret;
@@ -554,7 +560,7 @@
QMUpdate(qm);
// Reset all the rate and related frame counters quantities
- _qms->ResetRates();
+ _qmResolution->ResetRates();
// Reset counters
_lastQMUpdateTime = VCMTickTime::MillisecondTimestamp();
@@ -592,7 +598,7 @@
}
bool
-VCMMediaOptimization::QMUpdate(VCMQualityMode* qm)
+VCMMediaOptimization::QMUpdate(VCMResolutionScale* qm)
{
// Check for no change
if (qm->spatialHeightFact == 1 &&
@@ -606,7 +612,9 @@
VideoContentMetrics* cm = _content->LongTermAvgData();
// Temporal
- WebRtc_UWord32 frameRate = static_cast<WebRtc_UWord32>(_incomingFrameRate + 0.5f);
+ WebRtc_UWord32 frameRate = static_cast<WebRtc_UWord32>
+ (_incomingFrameRate + 0.5f);
+
// Check if go back up in temporal resolution
if (qm->temporalFact == 0)
{
diff --git a/src/modules/video_coding/main/source/media_optimization.h b/src/modules/video_coding/main/source/media_optimization.h
index 8a9993e..eecf270 100644
--- a/src/modules/video_coding/main/source/media_optimization.h
+++ b/src/modules/video_coding/main/source/media_optimization.h
@@ -159,7 +159,7 @@
* verify if QM settings differ from default, i.e. if an update is required
* Compute actual values, as will be sent to the encoder
*/
- bool QMUpdate(VCMQualityMode* qm);
+ bool QMUpdate(VCMResolutionScale* qm);
/**
* check if we should make a QM change
* will return 1 if yes, 0 otherwise
@@ -207,7 +207,7 @@
WebRtc_UWord32 _deltaFrameCnt;
VCMContentMetricsProcessing* _content;
- VCMQmSelect* _qms;
+ VCMQmResolution* _qmResolution;
WebRtc_Word64 _lastQMUpdateTime;
WebRtc_Word64 _lastChangeTime; // content or user triggered
diff --git a/src/modules/video_coding/main/source/qm_select.cc b/src/modules/video_coding/main/source/qm_select.cc
index 3220db0..82c31da 100644
--- a/src/modules/video_coding/main/source/qm_select.cc
+++ b/src/modules/video_coding/main/source/qm_select.cc
@@ -20,36 +20,204 @@
namespace webrtc {
-VCMQmSelect::VCMQmSelect()
+// QM-METHOD class
+
+VCMQmMethod::VCMQmMethod()
{
- _qm = new VCMQualityMode();
_contentMetrics = new VideoContentMetrics();
- Reset();
+ ResetQM();
}
-VCMQmSelect::~VCMQmSelect()
+VCMQmMethod::~VCMQmMethod()
{
- delete _qm;
delete _contentMetrics;
}
void
-VCMQmSelect::ResetQM()
+VCMQmMethod::ResetQM()
{
_motion.Reset();
_spatial.Reset();
_coherence.Reset();
_stationaryMotion = 0;
_aspectRatio = 1;
- _maxRateQM = 0;
- _imageType = 1;
- _userResolutionPref = 50; // Neutral
- _qm->Reset();
+ _imageType = 2;
return;
}
void
-VCMQmSelect::ResetRates()
+VCMQmMethod::UpdateContent(const VideoContentMetrics* contentMetrics)
+{
+ _contentMetrics = contentMetrics;
+}
+
+void
+VCMQmMethod::MotionNFD()
+{
+ _motion.value = _contentMetrics->motionMagnitudeNZ;
+
+ // Determine motion level
+ if (_motion.value < LOW_MOTION_NFD)
+ {
+ _motion.level = kLow;
+ }
+ else if (_motion.value > HIGH_MOTION_NFD)
+ {
+ _motion.level = kHigh;
+ }
+ else
+ {
+ _motion.level = kDefault;
+ }
+
+}
+
+void
+VCMQmMethod::Motion()
+{
+
+ float sizeZeroMotion = _contentMetrics->sizeZeroMotion;
+ float motionMagNZ = _contentMetrics->motionMagnitudeNZ;
+
+ // Take product of size and magnitude with equal weight
+ _motion.value = (1.0f - sizeZeroMotion) * motionMagNZ;
+
+ // Stabilize: motionMagNZ could be large when only a
+ // few motion blocks are non-zero
+ _stationaryMotion = false;
+ if (sizeZeroMotion > HIGH_ZERO_MOTION_SIZE)
+ {
+ _motion.value = 0.0f;
+ _stationaryMotion = true;
+ }
+ // Determine motion level
+ if (_motion.value < LOW_MOTION)
+ {
+ _motion.level = kLow;
+ }
+ else if (_motion.value > HIGH_MOTION)
+ {
+ _motion.level = kHigh;
+ }
+ else
+ {
+ _motion.level = kDefault;
+ }
+}
+
+
+void
+VCMQmMethod::Spatial()
+{
+ float spatialErr = _contentMetrics->spatialPredErr;
+ float spatialErrH = _contentMetrics->spatialPredErrH;
+ float spatialErrV = _contentMetrics->spatialPredErrV;
+ // Spatial measure: take average of 3 prediction errors
+ _spatial.value = (spatialErr + spatialErrH + spatialErrV) / 3.0f;
+
+ float scale = 1.0f;
+ // Reduce thresholds for HD scenes
+ if (_imageType > 3)
+ {
+ scale = (float)SCALE_TEXTURE_HD;
+ }
+
+ if (_spatial.value > scale * HIGH_TEXTURE)
+ {
+ _spatial.level = kHigh;
+ }
+ else if (_spatial.value < scale * LOW_TEXTURE)
+ {
+ _spatial.level = kLow;
+ }
+ else
+ {
+ _spatial.level = kDefault;
+ }
+}
+
+void
+VCMQmMethod::Coherence()
+{
+ float horizNZ = _contentMetrics->motionHorizontalness;
+ float distortionNZ = _contentMetrics->motionClusterDistortion;
+
+ // Coherence measure: combine horizontalness with cluster distortion
+ _coherence.value = COH_MAX;
+ if (distortionNZ > 0.)
+ {
+ _coherence.value = horizNZ / distortionNZ;
+ }
+ _coherence.value = VCM_MIN(COH_MAX, _coherence.value);
+
+ if (_coherence.value < COHERENCE_THR)
+ {
+ _coherence.level = kLow;
+ }
+ else
+ {
+ _coherence.level = kHigh;
+ }
+
+}
+
+WebRtc_Word8
+VCMQmMethod::GetImageType(WebRtc_UWord32 width, WebRtc_UWord32 height)
+{
+ // Match image type
+ WebRtc_UWord32 imageSize = width * height;
+ WebRtc_Word8 imageType;
+
+ if (imageSize < kFrameSizeTh[0])
+ {
+ imageType = 0;
+ }
+ else if (imageSize < kFrameSizeTh[1])
+ {
+ imageType = 1;
+ }
+ else if (imageSize < kFrameSizeTh[2])
+ {
+ imageType = 2;
+ }
+ else if (imageSize < kFrameSizeTh[3])
+ {
+ imageType = 3;
+ }
+ else if (imageSize < kFrameSizeTh[4])
+ {
+ imageType = 4;
+ }
+ else if (imageSize < kFrameSizeTh[5])
+ {
+ imageType = 5;
+ }
+ else
+ {
+ imageType = 6;
+ }
+
+ return imageType;
+}
+
+// DONE WITH QM CLASS
+
+
+//RESOLUTION CLASS
+
+VCMQmResolution::VCMQmResolution()
+{
+ _qm = new VCMResolutionScale();
+ Reset();
+}
+
+VCMQmResolution::~VCMQmResolution()
+{
+ delete _qm;
+}
+
+void
+VCMQmResolution::ResetRates()
{
_sumEncodedBytes = 0;
_sumTargetRate = 0.0f;
@@ -65,7 +233,7 @@
}
void
-VCMQmSelect::Reset()
+VCMQmResolution::Reset()
{
_stateDecFactorSpatial = 1;
_stateDecFactorTemp = 1;
@@ -74,17 +242,14 @@
_incomingFrameRate = 0.0f;
_userFrameRate = 0.0f;
_perFrameBandwidth =0.0f;
- _prevTotalRate = 0.0f;
- _prevRttTime = 0;
- _prevPacketLoss = 0;
- ResetQM();
- ResetRates();
- return;
+ ResetRates();
+ ResetQM();
+ return;
}
-//Initialize after reset of encoder
+// Initialize rate control quantities after reset of encoder
WebRtc_Word32
-VCMQmSelect::Initialize(float bitRate, float userFrameRate,
+VCMQmResolution::Initialize(float bitRate, float userFrameRate,
WebRtc_UWord32 width, WebRtc_UWord32 height)
{
if (userFrameRate == 0.0f || width == 0 || height == 0)
@@ -98,6 +263,12 @@
_width = width;
_height = height;
+ // Aspect ratio: used for selection of 1x2,2x1,2x2
+ _aspectRatio = static_cast<float>(_width) / static_cast<float>(_height);
+
+ // Set the imageType for the encoder width/height.
+ _imageType = GetImageType(_width, _height);
+
// Initial buffer level
_bufferLevel = INIT_BUFFER_LEVEL * _targetBitRate;
@@ -121,19 +292,9 @@
return VCM_OK;
}
-WebRtc_Word32
-VCMQmSelect::SetPreferences(WebRtc_Word8 resolPref)
-{
- // Preference setting for temporal over spatial resolution
- // 100 means temporal, 0 means spatial, 50 is neutral
- _userResolutionPref = resolPref;
-
- return VCM_OK;
-}
-
-//Update after every encoded frame
+// Update after every encoded frame
void
-VCMQmSelect::UpdateEncodedSize(WebRtc_Word64 encodedSize,
+VCMQmResolution::UpdateEncodedSize(WebRtc_Word64 encodedSize,
FrameType encodedFrameType)
{
// Update encoded size;
@@ -167,16 +328,16 @@
*/
// Counter for occurrences of low buffer level
- if (_bufferLevel <= PERC_BUFFER_THR * INIT_BUFFER_LEVEL * _targetBitRate)
+ if (_bufferLevel <= PERC_BUFFER_THR * OPT_BUFFER_LEVEL * _targetBitRate)
{
_lowBufferCnt++;
}
}
-//Update various quantities after SetTargetRates in MediaOpt
+// Update various quantities after SetTargetRates in MediaOpt
void
-VCMQmSelect::UpdateRates(float targetBitRate, float avgSentBitRate,
+VCMQmResolution::UpdateRates(float targetBitRate, float avgSentBitRate,
float incomingFrameRate, WebRtc_UWord8 packetLoss)
{
@@ -219,42 +380,10 @@
}
-// Adjust the FEC rate based on the content and the network state
-// (packet loss rate, total rate/bandwidth, round trip time).
-// Note that packetLoss here is the filtered loss value.
-WebRtc_UWord8
-VCMQmSelect::AdjustFecFactor(WebRtc_UWord8 codeRateDelta, float totalRate,
- float frameRate,WebRtc_UWord16 rttTime,
- WebRtc_UWord8 packetLoss)
-{
- // Default: no adjustment
- WebRtc_UWord8 codeRateDeltaAdjust = codeRateDelta;
- float adjustFec = 1.0f;
-
- // TODO (marpan):
- // Set FEC adjustment factor
-
- codeRateDeltaAdjust = static_cast<WebRtc_UWord8>(codeRateDelta * adjustFec);
-
- // Keep track of previous values of network state:
- // adjustment may be also based on pattern of changes in network state
- _prevTotalRate = totalRate;
- _prevRttTime = rttTime;
- _prevPacketLoss = packetLoss;
-
- return codeRateDeltaAdjust;
-}
-
-void
-VCMQmSelect::UpdateContent(const VideoContentMetrics* contentMetrics)
-{
- _contentMetrics = contentMetrics;
-}
-
// Select the resolution factors: frame size and frame rate change: (QM modes)
// Selection is for going back up in resolution, or going down in.
WebRtc_Word32
-VCMQmSelect::SelectQuality(VCMQualityMode** qm)
+VCMQmResolution::SelectResolution(VCMResolutionScale** qm)
{
if (!_init)
{
@@ -272,15 +401,11 @@
_qm->spatialHeightFact = 1;
_qm->temporalFact = 1;
-
// Update native values
_nativeWidth = _contentMetrics->nativeWidth;
_nativeHeight = _contentMetrics->nativeHeight;
_nativeFrameRate = _contentMetrics->nativeFrameRate;
- // Aspect ratio: used for selection of 1x2,2x1,2x2
- _aspectRatio = (float)_width / (float)_height;
-
float avgTargetRate = 0.0f;
float avgIncomingFrameRate = 0.0f;
float ratioBufferLow = 0.0f;
@@ -317,13 +442,12 @@
// for up-sampled spatial dimensions.
// This is needed to get the transRate for going back up in
// spatial resolution (only 2x2 allowed in this version).
- SetMaxRateForQM(2 * _width, 2 * _height);
- WebRtc_UWord8 imageType2 = _imageType;
- WebRtc_UWord32 maxRateQM2 = _maxRateQM;
+ WebRtc_UWord8 imageType2 = GetImageType(2 * _width, 2 * _height);
+ WebRtc_UWord32 maxRateQM2 = kMaxRateQm[imageType2];
// Set the maximum transitional rate and image type:
// for the encoder spatial dimensions.
- SetMaxRateForQM(_width, _height);
+ WebRtc_UWord32 maxRateQM = kMaxRateQm[_imageType];
// Compute class state of the content.
MotionNFD();
@@ -342,18 +466,21 @@
// Get image class and content class: for going up spatially
WebRtc_UWord8 imageClass2 = 1;
- if (imageType2 <= 3) imageClass2 = 0;
+ if (imageType2 <= 3)
+ {
+ imageClass2 = 0;
+ }
WebRtc_UWord8 tableIndex2 = imageClass2 * 9 + contentClass;
float scaleTransRate2 = kScaleTransRateQm[tableIndex2];
// Transitonal rate for going down
WebRtc_UWord32 estimatedTransRateDown = static_cast<WebRtc_UWord32>
- (_incomingFrameRate * scaleTransRate * _maxRateQM / 30);
+ (_incomingFrameRate * scaleTransRate * maxRateQM / 30);
// Transitional rate for going up temporally
WebRtc_UWord32 estimatedTransRateUpT = static_cast<WebRtc_UWord32>
(TRANS_RATE_SCALE_UP_TEMP * 2 * _incomingFrameRate *
- scaleTransRate * _maxRateQM / 30);
+ scaleTransRate * maxRateQM / 30);
// Transitional rate for going up spatially
WebRtc_UWord32 estimatedTransRateUpS = static_cast<WebRtc_UWord32>
@@ -530,7 +657,7 @@
}
WebRtc_Word32
-VCMQmSelect::SelectSpatialDirectionMode(float transRate)
+VCMQmResolution::SelectSpatialDirectionMode(float transRate)
{
// Default is 1x2 (H)
@@ -579,156 +706,82 @@
return VCM_OK;
}
-void
-VCMQmSelect::Coherence()
+// DONE WITH RESOLUTION CLASS
+
+
+// ROBUSTNESS CLASS
+
+VCMQmRobustness::VCMQmRobustness()
{
- float horizNZ = _contentMetrics->motionHorizontalness;
- float distortionNZ = _contentMetrics->motionClusterDistortion;
+ Reset();
+}
- // Coherence measure: combine horizontalness with cluster distortion
- _coherence.value = COH_MAX;
- if (distortionNZ > 0.)
- {
- _coherence.value = horizNZ / distortionNZ;
- }
- _coherence.value = VCM_MIN(COH_MAX, _coherence.value);
-
- if (_coherence.value < COHERENCE_THR)
- {
- _coherence.level = kLow;
- }
- else
- {
- _coherence.level = kHigh;
- }
+VCMQmRobustness::~VCMQmRobustness()
+{
}
void
-VCMQmSelect::MotionNFD()
+VCMQmRobustness::Reset()
{
- _motion.value = _contentMetrics->motionMagnitudeNZ;
-
- // Determine motion level
- if (_motion.value < LOW_MOTION_NFD)
- {
- _motion.level = kLow;
- }
- else if (_motion.value > HIGH_MOTION_NFD)
- {
- _motion.level = kHigh;
- }
- else
- {
- _motion.level = kDefault;
- }
-
+ _prevTotalRate = 0.0f;
+ _prevRttTime = 0;
+ _prevPacketLoss = 0;
+ ResetQM();
+ return;
}
-void
-VCMQmSelect::Motion()
+// Adjust the FEC rate based on the content and the network state
+// (packet loss rate, total rate/bandwidth, round trip time).
+// Note that packetLoss here is the filtered loss value.
+WebRtc_UWord8
+VCMQmRobustness::AdjustFecFactor(WebRtc_UWord8 codeRateDelta, float totalRate,
+ float frameRate,WebRtc_UWord32 rttTime,
+ WebRtc_UWord8 packetLoss)
{
+ if (_contentMetrics == NULL)
+ {
+ return VCM_OK;
+ }
- float sizeZeroMotion = _contentMetrics->sizeZeroMotion;
- float motionMagNZ = _contentMetrics->motionMagnitudeNZ;
+ // Default: no adjustment
+ WebRtc_UWord8 codeRateDeltaAdjust = codeRateDelta;
+ float adjustFec = 1.0f;
- // Take product of size and magnitude with equal weight
- _motion.value = (1.0f - sizeZeroMotion) * motionMagNZ;
+ // Compute class state of the content.
+ MotionNFD();
+ Spatial();
- // Stabilize: motionMagNZ could be large when only a
- // few motion blocks are non-zero
- _stationaryMotion = false;
- if (sizeZeroMotion > HIGH_ZERO_MOTION_SIZE)
- {
- _motion.value = 0.0f;
- _stationaryMotion = true;
- }
- // Determine motion level
- if (_motion.value < LOW_MOTION)
- {
- _motion.level = kLow;
- }
- else if (_motion.value > HIGH_MOTION)
- {
- _motion.level = kHigh;
- }
- else
- {
- _motion.level = kDefault;
- }
+ // TODO (marpan):
+ // Set FEC adjustment factor
+
+ codeRateDeltaAdjust = static_cast<WebRtc_UWord8>(codeRateDelta * adjustFec);
+
+ // Keep track of previous values of network state:
+ // adjustment may be also based on pattern of changes in network state
+ _prevTotalRate = totalRate;
+ _prevRttTime = rttTime;
+ _prevPacketLoss = packetLoss;
+
+ _prevCodeRateDelta = codeRateDelta;
+
+ return codeRateDeltaAdjust;
}
-
-void
-VCMQmSelect::Spatial()
+// Set the UEP (unequal-protection) on/off for the FEC
+bool
+VCMQmRobustness::SetUepProtection(WebRtc_UWord8 codeRateDelta, float totalRate,
+ WebRtc_UWord8 packetLoss, bool frameType)
{
- float spatialErr = _contentMetrics->spatialPredErr;
- float spatialErrH = _contentMetrics->spatialPredErrH;
- float spatialErrV = _contentMetrics->spatialPredErrV;
- // Spatial measure: take average of 3 prediction errors
- _spatial.value = (spatialErr + spatialErrH + spatialErrV) / 3.0f;
-
- float scale = 1.0f;
- // Reduce thresholds for HD scenes
- if (_imageType > 3)
+ if (_contentMetrics == NULL)
{
- scale = (float)SCALE_TEXTURE_HD;
+ return VCM_OK;
}
- if (_spatial.value > scale * HIGH_TEXTURE)
- {
- _spatial.level = kHigh;
- }
- else if (_spatial.value < scale * LOW_TEXTURE)
- {
- _spatial.level = kLow;
- }
- else
- {
- _spatial.level = kDefault;
- }
-}
+ // Default: UEP on
+ bool uepProtection = true;
-
-WebRtc_Word32
-VCMQmSelect::SetMaxRateForQM(WebRtc_UWord32 width, WebRtc_UWord32 height)
-{
- // Match image type
- WebRtc_UWord32 imageSize = width * height;
-
- if (imageSize < kFrameSizeTh[0])
- {
- _imageType = 0;
- }
- else if (imageSize < kFrameSizeTh[1])
- {
- _imageType = 1;
- }
- else if (imageSize < kFrameSizeTh[2])
- {
- _imageType = 2;
- }
- else if (imageSize < kFrameSizeTh[3])
- {
- _imageType = 3;
- }
- else if (imageSize < kFrameSizeTh[4])
- {
- _imageType = 4;
- }
- else if (imageSize < kFrameSizeTh[5])
- {
- _imageType = 5;
- }
- else
- {
- _imageType = 6;
- }
-
- // Set max rate based on image size
- _maxRateQM = kMaxRateQm[_imageType];
-
- return VCM_OK;
+ return uepProtection;
}
} // end of namespace
diff --git a/src/modules/video_coding/main/source/qm_select.h b/src/modules/video_coding/main/source/qm_select.h
index 3bca4bc..589a16d 100644
--- a/src/modules/video_coding/main/source/qm_select.h
+++ b/src/modules/video_coding/main/source/qm_select.h
@@ -13,25 +13,19 @@
#include "typedefs.h"
#include "common_types.h"
-/************************/
-/* Quality Modes */
-/**********************/
+/******************************************************/
+/* Quality Modes: Resolution and Robustness settings */
+/******************************************************/
namespace webrtc
{
struct VideoContentMetrics;
-struct VCMQualityMode
+struct VCMResolutionScale
{
- VCMQualityMode():spatialWidthFact(1), spatialHeightFact(1),
+ VCMResolutionScale(): spatialWidthFact(1), spatialHeightFact(1),
temporalFact(1){}
- void Reset()
- {
- spatialWidthFact = 1;
- spatialHeightFact = 1;
- temporalFact = 1;
- }
WebRtc_UWord16 spatialWidthFact;
WebRtc_UWord16 spatialHeightFact;
@@ -59,53 +53,20 @@
VCMMagValues level;
};
-class VCMQmSelect
+// QmMethod class: main class for resolution and robustness settings
+
+class VCMQmMethod
{
public:
- VCMQmSelect();
- ~VCMQmSelect();
+ VCMQmMethod();
+ ~VCMQmMethod();
- // Initialize:
- WebRtc_Word32 Initialize(float bitRate, float userFrameRate,
- WebRtc_UWord32 width, WebRtc_UWord32 height);
-
- // Allow the user to set preferences: favor frame rate/resolution
- WebRtc_Word32 SetPreferences(WebRtc_Word8 resolPref);
-
- // Extract ST (spatio-temporal) QM behavior and make decision
- // Inputs: qm: Reference to the quality modes pointer
- WebRtc_Word32 SelectQuality(VCMQualityMode** qm);
-
- // Update QM with actual bit rate
- // (size of the latest encoded frame) and frame type.
- void UpdateEncodedSize(WebRtc_Word64 encodedSize,
- FrameType encodedFrameType);
-
- // Update QM with new bit/frame/loss rates from SetTargetRates
- void UpdateRates(float targetBitRate, float avgSentRate,
- float incomingFrameRate, WebRtc_UWord8 packetLoss);
-
- // Update QM with the content metrics
- void UpdateContent(const VideoContentMetrics* contentMetrics);
-
- // Adjust FEC rate based on content
- WebRtc_UWord8 AdjustFecFactor(WebRtc_UWord8 codeRateDelta, float totalRate,
- float frameRate, WebRtc_UWord16 rttTime,
- WebRtc_UWord8 packetLoss);
-
-
- // Select 1x2,2x2,2x2 spatial sampling mode
- WebRtc_Word32 SelectSpatialDirectionMode(float transRate);
-
- // Reset values prior to QMSelect
+ // Reset values
void ResetQM();
+ virtual void Reset() = 0;
- // Reset rate quantities and counter values after every QMSelect call
- void ResetRates();
-
- // Reset all
- void Reset();
-private:
+ // Update with the content metrics
+ void UpdateContent(const VideoContentMetrics* contentMetrics);
// Compute spatial texture magnitude and level
void Spatial();
@@ -119,36 +80,84 @@
// Compute coherence magnitude and level
void Coherence();
- // Set the max rate for QM selection
- WebRtc_Word32 SetMaxRateForQM(WebRtc_UWord32 width, WebRtc_UWord32 height);
+ // Get the imageType (CIF, VGA, HD, etc) for the system width/height
+ WebRtc_Word8 GetImageType(WebRtc_UWord32 width, WebRtc_UWord32 height);
// Content Data
- const VideoContentMetrics* _contentMetrics;
+ const VideoContentMetrics* _contentMetrics;
- // Encoder rate control parameters, network parameters
+ // Encoder and native frame sizes, frame rate, aspect ratio, imageType
+ WebRtc_UWord32 _width;
+ WebRtc_UWord32 _height;
+ WebRtc_UWord32 _nativeWidth;
+ WebRtc_UWord32 _nativeHeight;
+ WebRtc_UWord32 _nativeFrameRate;
+ float _aspectRatio;
+ // Image type for the current encoder system size.
+ WebRtc_UWord8 _imageType;
+
+ // Content L/M/H values. stationary flag
+ VCMContFeature _motion;
+ VCMContFeature _spatial;
+ VCMContFeature _coherence;
+ bool _stationaryMotion;
+ bool _init;
+
+};
+
+// Resolution settings class
+
+class VCMQmResolution : public VCMQmMethod
+{
+public:
+ VCMQmResolution();
+ ~VCMQmResolution();
+
+ // Reset all quantities
+ virtual void Reset();
+
+ // Reset rate quantities and counter values after every Select Quality call
+ void ResetRates();
+
+ // Initialize rate control quantities after re-init of encoder.
+ WebRtc_Word32 Initialize(float bitRate, float userFrameRate,
+ WebRtc_UWord32 width, WebRtc_UWord32 height);
+
+ // Update QM with actual bit rate (size of the latest encoded frame)
+ // and frame type, after every encoded frame.
+ void UpdateEncodedSize(WebRtc_Word64 encodedSize,
+ FrameType encodedFrameType);
+
+ // Update QM with new bit/frame/loss rates every ~1 sec from SetTargetRates
+ void UpdateRates(float targetBitRate, float avgSentRate,
+ float incomingFrameRate, WebRtc_UWord8 packetLoss);
+
+ // Extract ST (spatio-temporal) QM behavior and make decision
+ // Inputs: qm: Reference to the quality modes pointer
+ // Output: the spatial and/or temporal scale change
+ WebRtc_Word32 SelectResolution(VCMResolutionScale** qm);
+
+ // Select 1x2,2x2,2x2 spatial sampling mode
+ WebRtc_Word32 SelectSpatialDirectionMode(float transRate);
+
+private:
+ // Encoder rate control parameter
float _targetBitRate;
float _userFrameRate;
float _incomingFrameRate;
float _perFrameBandwidth;
float _bufferLevel;
+
+ // Data accumulated every ~1sec from MediaOpt
float _sumTargetRate;
float _sumIncomingFrameRate;
float _sumSeqRateMM;
float _sumFrameRateMM;
float _sumPacketLoss;
- float _prevTotalRate;
- WebRtc_UWord16 _prevRttTime;
- WebRtc_UWord8 _prevPacketLoss;
WebRtc_Word64 _sumEncodedBytes;
- // Encoder and native frame sizes
- WebRtc_UWord32 _width;
- WebRtc_UWord32 _height;
- WebRtc_UWord32 _nativeWidth;
- WebRtc_UWord32 _nativeHeight;
+ // Resolution state parameters
WebRtc_UWord8 _stateDecFactorSpatial;
-
- WebRtc_UWord32 _nativeFrameRate;
WebRtc_UWord8 _stateDecFactorTemp;
// Counters
@@ -157,24 +166,36 @@
WebRtc_UWord32 _updateRateCnt;
WebRtc_UWord32 _lowBufferCnt;
- // Content L/M/H values
- VCMContFeature _motion;
- VCMContFeature _spatial;
- VCMContFeature _coherence;
- bool _stationaryMotion;
+ VCMResolutionScale* _qm;
+};
- // Aspect ratio
- float _aspectRatio;
+// Robustness settings class
- // Max rate to saturate the transitionalRate
- WebRtc_UWord32 _maxRateQM;
- WebRtc_UWord8 _imageType;
+class VCMQmRobustness : public VCMQmMethod
+{
+public:
+ VCMQmRobustness();
+ ~VCMQmRobustness();
- // User preference for resolution or qmax change
- WebRtc_UWord8 _userResolutionPref;
- bool _init;
- VCMQualityMode* _qm;
+ virtual void Reset();
+ // Adjust FEC rate based on content: every ~1 sec from SetTargetRates
+ WebRtc_UWord8 AdjustFecFactor(WebRtc_UWord8 codeRateDelta, float totalRate,
+ float frameRate, WebRtc_UWord32 rttTime,
+ WebRtc_UWord8 packetLoss);
+
+ // Set the UEP protection on/off
+ bool SetUepProtection(WebRtc_UWord8 codeRateDelta, float totalRate,
+ WebRtc_UWord8 packetLoss, bool frameType);
+
+private:
+ // Previous state of network parameters
+ float _prevTotalRate;
+ WebRtc_UWord32 _prevRttTime;
+ WebRtc_UWord8 _prevPacketLoss;
+
+ // Previous FEC rate
+ WebRtc_UWord8 _prevCodeRateDelta;
};
} // namespace webrtc
diff --git a/src/modules/video_coding/main/source/qm_select_data.h b/src/modules/video_coding/main/source/qm_select_data.h
index 813c110..64870ea 100644
--- a/src/modules/video_coding/main/source/qm_select_data.h
+++ b/src/modules/video_coding/main/source/qm_select_data.h
@@ -28,6 +28,9 @@
// Initial level of buffer in secs: should corresponds to wrapper settings
#define INIT_BUFFER_LEVEL 0.5
+// Optimal level of buffer in secs: should corresponds to wrapper settings
+#define OPT_BUFFER_LEVEL 0.6
+
// Threshold of (max) buffer size below which we consider too low (underflow)
#define PERC_BUFFER_THR 0.10
