Refactoring temporal layers implementation and adding VideoCodecMode for easier control of codec settings.
Review URL: https://webrtc-codereview.appspot.com/1105007
git-svn-id: http://webrtc.googlecode.com/svn/trunk@3528 4adac7df-926f-26a2-2b94-8c16560cd09d
diff --git a/webrtc/modules/video_coding/utility/Android.mk b/webrtc/modules/video_coding/utility/Android.mk
new file mode 100644
index 0000000..4af007c
--- /dev/null
+++ b/webrtc/modules/video_coding/utility/Android.mk
@@ -0,0 +1,39 @@
+# Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+LOCAL_PATH := $(call my-dir)
+
+include $(CLEAR_VARS)
+
+include $(LOCAL_PATH)/../../../../android-webrtc.mk
+
+LOCAL_ARM_MODE := arm
+LOCAL_MODULE_CLASS := STATIC_LIBRARIES
+LOCAL_MODULE := libvideo_coding_utility
+LOCAL_MODULE_TAGS := optional
+LOCAL_CPP_EXTENSION := .cc
+LOCAL_SRC_FILES := \
+ exp_filter.cc \
+ frame_dropper.cc \
+
+# Flags passed to both C and C++ files.
+LOCAL_CFLAGS := \
+ $(MY_WEBRTC_COMMON_DEFS)
+
+LOCAL_C_INCLUDES := \
+ $(LOCAL_PATH)/../../../../system_wrappers/interface
+
+LOCAL_SHARED_LIBRARIES := \
+ libcutils \
+ libdl \
+ libstlport
+
+ifndef NDK_ROOT
+include external/stlport/libstlport.mk
+endif
+include $(BUILD_STATIC_LIBRARY)
diff --git a/webrtc/modules/video_coding/utility/exp_filter.cc b/webrtc/modules/video_coding/utility/exp_filter.cc
new file mode 100644
index 0000000..44f280bc
--- /dev/null
+++ b/webrtc/modules/video_coding/utility/exp_filter.cc
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/video_coding/utility/include/exp_filter.h"
+
+#include <math.h>
+
+namespace webrtc {
+
+void
+VCMExpFilter::Reset(float alpha)
+{
+ _alpha = alpha;
+ _filtered = -1.0;
+}
+
+float
+VCMExpFilter::Apply(float exp, float sample)
+{
+ if (_filtered == -1.0)
+ {
+ // Initialize filtered bit rates
+ _filtered = sample;
+ }
+ else if (exp == 1.0)
+ {
+ _filtered = _alpha * _filtered + (1 - _alpha) * sample;
+ }
+ else
+ {
+ float alpha = pow(_alpha, exp);
+ _filtered = alpha * _filtered + (1 - alpha) * sample;
+ }
+ if (_max != -1 && _filtered > _max)
+ {
+ _filtered = _max;
+ }
+ return _filtered;
+}
+
+void
+VCMExpFilter::UpdateBase(float alpha)
+{
+ _alpha = alpha;
+}
+
+float
+VCMExpFilter::Value() const
+{
+ return _filtered;
+}
+
+}
diff --git a/webrtc/modules/video_coding/utility/frame_dropper.cc b/webrtc/modules/video_coding/utility/frame_dropper.cc
new file mode 100644
index 0000000..b871022
--- /dev/null
+++ b/webrtc/modules/video_coding/utility/frame_dropper.cc
@@ -0,0 +1,348 @@
+/*
+ * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/video_coding/utility/include/frame_dropper.h"
+
+#include "webrtc/system_wrappers/interface/trace.h"
+
+namespace webrtc
+{
+
+FrameDropper::FrameDropper()
+:
+_keyFrameSizeAvgKbits(0.9f),
+_keyFrameRatio(0.99f),
+_dropRatio(0.9f, 0.96f),
+_enabled(true)
+{
+ Reset();
+}
+
+void
+FrameDropper::Reset()
+{
+ _keyFrameRatio.Reset(0.99f);
+ _keyFrameRatio.Apply(1.0f, 1.0f/300.0f); // 1 key frame every 10th second in 30 fps
+ _keyFrameSizeAvgKbits.Reset(0.9f);
+ _keyFrameCount = 0;
+ _accumulator = 0.0f;
+ _accumulatorMax = 150.0f; // assume 300 kb/s and 0.5 s window
+ _targetBitRate = 300.0f;
+ _incoming_frame_rate = 30;
+ _keyFrameSpreadFrames = 0.5f * _incoming_frame_rate;
+ _dropNext = false;
+ _dropRatio.Reset(0.9f);
+ _dropRatio.Apply(0.0f, 0.0f); // Initialize to 0
+ _dropCount = 0;
+ _windowSize = 0.5f;
+ _wasBelowMax = true;
+ _fastMode = false; // start with normal (non-aggressive) mode
+ // Cap for the encoder buffer level/accumulator, in secs.
+ _cap_buffer_size = 3.0f;
+ // Cap on maximum amount of dropped frames between kept frames, in secs.
+ _max_time_drops = 4.0f;
+}
+
+void
+FrameDropper::Enable(bool enable)
+{
+ _enabled = enable;
+}
+
+void
+FrameDropper::Fill(WebRtc_UWord32 frameSizeBytes, bool deltaFrame)
+{
+ if (!_enabled)
+ {
+ return;
+ }
+ float frameSizeKbits = 8.0f * static_cast<float>(frameSizeBytes) / 1000.0f;
+ if (!deltaFrame && !_fastMode) // fast mode does not treat key-frames any different
+ {
+ _keyFrameSizeAvgKbits.Apply(1, frameSizeKbits);
+ _keyFrameRatio.Apply(1.0, 1.0);
+ if (frameSizeKbits > _keyFrameSizeAvgKbits.Value())
+ {
+ // Remove the average key frame size since we
+ // compensate for key frames when adding delta
+ // frames.
+ frameSizeKbits -= _keyFrameSizeAvgKbits.Value();
+ }
+ else
+ {
+ // Shouldn't be negative, so zero is the lower bound.
+ frameSizeKbits = 0;
+ }
+ if (_keyFrameRatio.Value() > 1e-5 && 1 / _keyFrameRatio.Value() < _keyFrameSpreadFrames)
+ {
+ // We are sending key frames more often than our upper bound for
+ // how much we allow the key frame compensation to be spread
+ // out in time. Therefor we must use the key frame ratio rather
+ // than keyFrameSpreadFrames.
+ _keyFrameCount = static_cast<WebRtc_Word32>(1 / _keyFrameRatio.Value() + 0.5);
+ }
+ else
+ {
+ // Compensate for the key frame the following frames
+ _keyFrameCount = static_cast<WebRtc_Word32>(_keyFrameSpreadFrames + 0.5);
+ }
+ }
+ else
+ {
+ // Decrease the keyFrameRatio
+ _keyFrameRatio.Apply(1.0, 0.0);
+ }
+ // Change the level of the accumulator (bucket)
+ _accumulator += frameSizeKbits;
+ CapAccumulator();
+}
+
+void
+FrameDropper::Leak(WebRtc_UWord32 inputFrameRate)
+{
+ if (!_enabled)
+ {
+ return;
+ }
+ if (inputFrameRate < 1)
+ {
+ return;
+ }
+ if (_targetBitRate < 0.0f)
+ {
+ return;
+ }
+ _keyFrameSpreadFrames = 0.5f * inputFrameRate;
+ // T is the expected bits per frame (target). If all frames were the same size,
+ // we would get T bits per frame. Notice that T is also weighted to be able to
+ // force a lower frame rate if wanted.
+ float T = _targetBitRate / inputFrameRate;
+ if (_keyFrameCount > 0)
+ {
+ // Perform the key frame compensation
+ if (_keyFrameRatio.Value() > 0 && 1 / _keyFrameRatio.Value() < _keyFrameSpreadFrames)
+ {
+ T -= _keyFrameSizeAvgKbits.Value() * _keyFrameRatio.Value();
+ }
+ else
+ {
+ T -= _keyFrameSizeAvgKbits.Value() / _keyFrameSpreadFrames;
+ }
+ _keyFrameCount--;
+ }
+ _accumulator -= T;
+ UpdateRatio();
+}
+
+void
+FrameDropper::UpdateNack(WebRtc_UWord32 nackBytes)
+{
+ if (!_enabled)
+ {
+ return;
+ }
+ _accumulator += static_cast<float>(nackBytes) * 8.0f / 1000.0f;
+}
+
+void
+FrameDropper::FillBucket(float inKbits, float outKbits)
+{
+ _accumulator += (inKbits - outKbits);
+}
+
+void
+FrameDropper::UpdateRatio()
+{
+ if (_accumulator > 1.3f * _accumulatorMax)
+ {
+ // Too far above accumulator max, react faster
+ _dropRatio.UpdateBase(0.8f);
+ }
+ else
+ {
+ // Go back to normal reaction
+ _dropRatio.UpdateBase(0.9f);
+ }
+ if (_accumulator > _accumulatorMax)
+ {
+ // We are above accumulator max, and should ideally
+ // drop a frame. Increase the dropRatio and drop
+ // the frame later.
+ if (_wasBelowMax)
+ {
+ _dropNext = true;
+ }
+ if (_fastMode)
+ {
+ // always drop in aggressive mode
+ _dropNext = true;
+ }
+
+ _dropRatio.Apply(1.0f, 1.0f);
+ _dropRatio.UpdateBase(0.9f);
+ }
+ else
+ {
+ _dropRatio.Apply(1.0f, 0.0f);
+ }
+ if (_accumulator < 0.0f)
+ {
+ _accumulator = 0.0f;
+ }
+ _wasBelowMax = _accumulator < _accumulatorMax;
+}
+
+// This function signals when to drop frames to the caller. It makes use of the dropRatio
+// to smooth out the drops over time.
+bool
+FrameDropper::DropFrame()
+{
+ if (!_enabled)
+ {
+ return false;
+ }
+ if (_dropNext)
+ {
+ _dropNext = false;
+ _dropCount = 0;
+ }
+
+ if (_dropRatio.Value() >= 0.5f) // Drops per keep
+ {
+ // limit is the number of frames we should drop between each kept frame
+ // to keep our drop ratio. limit is positive in this case.
+ float denom = 1.0f - _dropRatio.Value();
+ if (denom < 1e-5)
+ {
+ denom = (float)1e-5;
+ }
+ WebRtc_Word32 limit = static_cast<WebRtc_Word32>(1.0f / denom - 1.0f + 0.5f);
+ // Put a bound on the max amount of dropped frames between each kept
+ // frame, in terms of frame rate and window size (secs).
+ int max_limit = static_cast<int>(_incoming_frame_rate *
+ _max_time_drops);
+ if (limit > max_limit) {
+ limit = max_limit;
+ }
+ if (_dropCount < 0)
+ {
+ // Reset the _dropCount since it was negative and should be positive.
+ if (_dropRatio.Value() > 0.4f)
+ {
+ _dropCount = -_dropCount;
+ }
+ else
+ {
+ _dropCount = 0;
+ }
+ }
+ if (_dropCount < limit)
+ {
+ // As long we are below the limit we should drop frames.
+ _dropCount++;
+ return true;
+ }
+ else
+ {
+ // Only when we reset _dropCount a frame should be kept.
+ _dropCount = 0;
+ return false;
+ }
+ }
+ else if (_dropRatio.Value() > 0.0f && _dropRatio.Value() < 0.5f) // Keeps per drop
+ {
+ // limit is the number of frames we should keep between each drop
+ // in order to keep the drop ratio. limit is negative in this case,
+ // and the _dropCount is also negative.
+ float denom = _dropRatio.Value();
+ if (denom < 1e-5)
+ {
+ denom = (float)1e-5;
+ }
+ WebRtc_Word32 limit = -static_cast<WebRtc_Word32>(1.0f / denom - 1.0f + 0.5f);
+ if (_dropCount > 0)
+ {
+ // Reset the _dropCount since we have a positive
+ // _dropCount, and it should be negative.
+ if (_dropRatio.Value() < 0.6f)
+ {
+ _dropCount = -_dropCount;
+ }
+ else
+ {
+ _dropCount = 0;
+ }
+ }
+ if (_dropCount > limit)
+ {
+ if (_dropCount == 0)
+ {
+ // Drop frames when we reset _dropCount.
+ _dropCount--;
+ return true;
+ }
+ else
+ {
+ // Keep frames as long as we haven't reached limit.
+ _dropCount--;
+ return false;
+ }
+ }
+ else
+ {
+ _dropCount = 0;
+ return false;
+ }
+ }
+ _dropCount = 0;
+ return false;
+
+ // A simpler version, unfiltered and quicker
+ //bool dropNext = _dropNext;
+ //_dropNext = false;
+ //return dropNext;
+}
+
+void
+FrameDropper::SetRates(float bitRate, float incoming_frame_rate)
+{
+ // Bit rate of -1 means infinite bandwidth.
+ _accumulatorMax = bitRate * _windowSize; // bitRate * windowSize (in seconds)
+ if (_targetBitRate > 0.0f && bitRate < _targetBitRate && _accumulator > _accumulatorMax)
+ {
+ // Rescale the accumulator level if the accumulator max decreases
+ _accumulator = bitRate / _targetBitRate * _accumulator;
+ }
+ _targetBitRate = bitRate;
+ CapAccumulator();
+ _incoming_frame_rate = incoming_frame_rate;
+}
+
+float
+FrameDropper::ActualFrameRate(WebRtc_UWord32 inputFrameRate) const
+{
+ if (!_enabled)
+ {
+ return static_cast<float>(inputFrameRate);
+ }
+ return inputFrameRate * (1.0f - _dropRatio.Value());
+}
+
+// Put a cap on the accumulator, i.e., don't let it grow beyond some level.
+// This is a temporary fix for screencasting where very large frames from
+// encoder will cause very slow response (too many frame drops).
+void FrameDropper::CapAccumulator() {
+ float max_accumulator = _targetBitRate * _cap_buffer_size;
+ if (_accumulator > max_accumulator) {
+ _accumulator = max_accumulator;
+ }
+}
+
+}
diff --git a/webrtc/modules/video_coding/utility/include/exp_filter.h b/webrtc/modules/video_coding/utility/include/exp_filter.h
new file mode 100644
index 0000000..42fbe3f
--- /dev/null
+++ b/webrtc/modules/video_coding/utility/include/exp_filter.h
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_MODULES_VIDEO_CODING_UTILITY_INCLUDE_EXP_FILTER_H_
+#define WEBRTC_MODULES_VIDEO_CODING_UTILITY_INCLUDE_EXP_FILTER_H_
+
+namespace webrtc
+{
+
+/**********************/
+/* ExpFilter class */
+/**********************/
+
+class VCMExpFilter
+{
+public:
+ VCMExpFilter(float alpha, float max = -1.0) : _alpha(alpha), _filtered(-1.0), _max(max) {}
+
+ // Resets the filter to its initial state, and resets alpha to the given value
+ //
+ // Input:
+ // - alpha : the new value of the filter factor base.
+ void Reset(float alpha);
+
+ // Applies the filter with the given exponent on the provided sample
+ //
+ // Input:
+ // - exp : Exponent T in y(k) = alpha^T * y(k-1) + (1 - alpha^T) * x(k)
+ // - sample : x(k) in the above filter equation
+ float Apply(float exp, float sample);
+
+ // Return current filtered value: y(k)
+ //
+ // Return value : The current filter output
+ float Value() const;
+
+ // Change the filter factor base
+ //
+ // Input:
+ // - alpha : The new filter factor base.
+ void UpdateBase(float alpha);
+
+private:
+ float _alpha; // Filter factor base
+ float _filtered; // Current filter output
+ const float _max;
+}; // end of ExpFilter class
+
+} // namespace webrtc
+
+#endif // WEBRTC_MODULES_VIDEO_CODING_UTILITY_INCLUDE_EXP_FILTER_H_
diff --git a/webrtc/modules/video_coding/utility/include/frame_dropper.h b/webrtc/modules/video_coding/utility/include/frame_dropper.h
new file mode 100644
index 0000000..fd15dd0
--- /dev/null
+++ b/webrtc/modules/video_coding/utility/include/frame_dropper.h
@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_MODULES_VIDEO_CODING_UTILITY_INCLUDE_FRAME_DROPPER_H_
+#define WEBRTC_MODULES_VIDEO_CODING_UTILITY_INCLUDE_FRAME_DROPPER_H_
+
+#include "webrtc/modules/video_coding/utility/include/exp_filter.h"
+#include "webrtc/typedefs.h"
+
+namespace webrtc
+{
+
+// The Frame Dropper implements a variant of the leaky bucket algorithm
+// for keeping track of when to drop frames to avoid bit rate
+// over use when the encoder can't keep its bit rate.
+class FrameDropper
+{
+public:
+ FrameDropper();
+ virtual ~FrameDropper() {}
+
+ // Resets the FrameDropper to its initial state.
+ // This means that the frameRateWeight is set to its
+ // default value as well.
+ virtual void Reset();
+
+ virtual void Enable(bool enable);
+ // Answers the question if it's time to drop a frame
+ // if we want to reach a given frame rate. Must be
+ // called for every frame.
+ //
+ // Return value : True if we should drop the current frame
+ virtual bool DropFrame();
+ // Updates the FrameDropper with the size of the latest encoded
+ // frame. The FrameDropper calculates a new drop ratio (can be
+ // seen as the probability to drop a frame) and updates its
+ // internal statistics.
+ //
+ // Input:
+ // - frameSizeBytes : The size of the latest frame
+ // returned from the encoder.
+ // - deltaFrame : True if the encoder returned
+ // a key frame.
+ virtual void Fill(WebRtc_UWord32 frameSizeBytes, bool deltaFrame);
+
+ virtual void Leak(WebRtc_UWord32 inputFrameRate);
+
+ void UpdateNack(WebRtc_UWord32 nackBytes);
+
+ // Sets the target bit rate and the frame rate produced by
+ // the camera.
+ //
+ // Input:
+ // - bitRate : The target bit rate
+ virtual void SetRates(float bitRate, float incoming_frame_rate);
+
+ // Return value : The current average frame rate produced
+ // if the DropFrame() function is used as
+ // instruction of when to drop frames.
+ virtual float ActualFrameRate(WebRtc_UWord32 inputFrameRate) const;
+
+
+private:
+ void FillBucket(float inKbits, float outKbits);
+ void UpdateRatio();
+ void CapAccumulator();
+
+ VCMExpFilter _keyFrameSizeAvgKbits;
+ VCMExpFilter _keyFrameRatio;
+ float _keyFrameSpreadFrames;
+ WebRtc_Word32 _keyFrameCount;
+ float _accumulator;
+ float _accumulatorMax;
+ float _targetBitRate;
+ bool _dropNext;
+ VCMExpFilter _dropRatio;
+ WebRtc_Word32 _dropCount;
+ float _windowSize;
+ float _incoming_frame_rate;
+ bool _wasBelowMax;
+ bool _enabled;
+ bool _fastMode;
+ float _cap_buffer_size;
+ float _max_time_drops;
+}; // end of VCMFrameDropper class
+
+} // namespace webrtc
+
+#endif // WEBRTC_MODULES_VIDEO_CODING_UTILITY_INCLUDE_FRAME_DROPPER_H_
diff --git a/webrtc/modules/video_coding/utility/include/mock/mock_frame_dropper.h b/webrtc/modules/video_coding/utility/include/mock/mock_frame_dropper.h
new file mode 100644
index 0000000..e8a6acc
--- /dev/null
+++ b/webrtc/modules/video_coding/utility/include/mock/mock_frame_dropper.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef WEBRTC_MODULES_VIDEO_CODING_UTILITY_INCLUDE_MOCK_MOCK_FRAME_DROPPER_H_
+#define WEBRTC_MODULES_VIDEO_CODING_UTILITY_INCLUDE_MOCK_MOCK_FRAME_DROPPER_H_
+
+#include <string>
+
+#include "gmock/gmock.h"
+#include "webrtc/modules/video_coding/utility/include/frame_dropper.h"
+#include "webrtc/typedefs.h"
+
+namespace webrtc {
+
+class MockFrameDropper : public FrameDropper {
+ public:
+ MOCK_METHOD0(Reset,
+ void());
+ MOCK_METHOD1(Enable,
+ void(bool enable));
+ MOCK_METHOD0(DropFrame,
+ bool());
+ MOCK_METHOD2(Fill,
+ void(WebRtc_UWord32 frameSizeBytes, bool deltaFrame));
+ MOCK_METHOD1(Leak,
+ void(WebRtc_UWord32 inputFrameRate));
+ MOCK_METHOD2(SetRates,
+ void(float bitRate, float incoming_frame_rate));
+ MOCK_CONST_METHOD1(ActualFrameRate,
+ float(WebRtc_UWord32 inputFrameRate));
+};
+
+} // namespace webrtc
+
+#endif // WEBRTC_MODULES_VIDEO_CODING_UTILITY_INCLUDE_MOCK_MOCK_FRAME_DROPPER_H_
diff --git a/webrtc/modules/video_coding/utility/video_coding_utility.gyp b/webrtc/modules/video_coding/utility/video_coding_utility.gyp
new file mode 100644
index 0000000..24f8880
--- /dev/null
+++ b/webrtc/modules/video_coding/utility/video_coding_utility.gyp
@@ -0,0 +1,28 @@
+# Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+{
+ 'includes': [
+ '../../../build/common.gypi',
+ ],
+ 'targets': [
+ {
+ 'target_name': 'video_coding_utility',
+ 'type': 'static_library',
+ 'dependencies': [
+ '<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers',
+ ],
+ 'sources': [
+ 'include/exp_filter.h',
+ 'include/frame_dropper.h',
+ 'exp_filter.cc',
+ 'frame_dropper.cc',
+ ],
+ },
+ ], # targets
+}
