blob: 6563c56e872c6af267acd9ec00048ea03f2f1291 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:251/*
stefan@webrtc.org29794612012-02-08 08:58:552 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:253 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 04:47:3111#ifndef MODULES_VIDEO_CODING_JITTER_BUFFER_H_
12#define MODULES_VIDEO_CODING_JITTER_BUFFER_H_
niklase@google.com470e71d2011-07-07 08:21:2513
stefan@webrtc.org4cf1a8a2013-06-27 15:20:1414#include <list>
stefan@webrtc.org50fb4af2013-06-17 07:33:5815#include <map>
kwiberg3f55dea2016-02-29 13:51:5916#include <memory>
stefan@webrtc.orga64300a2013-03-04 15:24:4017#include <set>
stefan@webrtc.orgbecf9c82013-02-01 15:09:5718#include <vector>
stefan@webrtc.org29794612012-02-08 08:58:5519
Jonas Orelande62c2f22022-03-29 09:04:4820#include "api/field_trials_view.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3121#include "modules/include/module_common_types.h"
Danil Chapovalov7c067772019-10-07 10:56:2422#include "modules/include/module_common_types_public.h"
Rasmus Brandtbc3a41e2023-03-09 12:12:2923#include "modules/video_coding/deprecated/decoding_state.h"
Rasmus Brandteec4fd12023-03-10 07:15:0824#include "modules/video_coding/deprecated/event_wrapper.h"
Rasmus Brandt78e13882023-03-06 10:10:2625#include "modules/video_coding/deprecated/jitter_buffer_common.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3126#include "modules/video_coding/include/video_coding.h"
27#include "modules/video_coding/include/video_coding_defines.h"
Rasmus Brandt65a6eca2023-03-03 08:22:1828#include "modules/video_coding/timing/inter_frame_delay_variation_calculator.h"
Rasmus Brandt10944e62022-05-25 08:12:4229#include "modules/video_coding/timing/jitter_estimator.h"
Markus Handell6deec382020-07-07 10:17:1230#include "rtc_base/synchronization/mutex.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3131#include "rtc_base/thread_annotations.h"
niklase@google.com470e71d2011-07-07 08:21:2532
stefan@webrtc.org912981f2012-10-12 07:04:5233namespace webrtc {
niklase@google.com470e71d2011-07-07 08:21:2534
niklase@google.com470e71d2011-07-07 08:21:2535// forward declarations
stefan@webrtc.orga678a3b2013-01-21 07:42:1136class Clock;
niklase@google.com470e71d2011-07-07 08:21:2537class VCMFrameBuffer;
38class VCMPacket;
39class VCMEncodedFrame;
40
stefan@webrtc.org4cf1a8a2013-06-27 15:20:1441typedef std::list<VCMFrameBuffer*> UnorderedFrameList;
42
stefan@webrtc.org912981f2012-10-12 07:04:5243struct VCMJitterSample {
44 VCMJitterSample() : timestamp(0), frame_size(0), latest_packet_time(-1) {}
45 uint32_t timestamp;
46 uint32_t frame_size;
47 int64_t latest_packet_time;
niklase@google.com470e71d2011-07-07 08:21:2548};
49
stefan@webrtc.org50fb4af2013-06-17 07:33:5850class TimestampLessThan {
51 public:
philipel9d3ab612015-12-21 12:12:3952 bool operator()(uint32_t timestamp1, uint32_t timestamp2) const {
stefan@webrtc.org50fb4af2013-06-17 07:33:5853 return IsNewerTimestamp(timestamp2, timestamp1);
54 }
55};
56
agalusza@google.comd818dcb2013-07-29 21:48:1157class FrameList
58 : public std::map<uint32_t, VCMFrameBuffer*, TimestampLessThan> {
stefan@webrtc.orgc8b29a22013-06-17 07:13:1659 public:
60 void InsertFrame(VCMFrameBuffer* frame);
stefan@webrtc.orgc8b29a22013-06-17 07:13:1661 VCMFrameBuffer* PopFrame(uint32_t timestamp);
stefan@webrtc.org50fb4af2013-06-17 07:33:5862 VCMFrameBuffer* Front() const;
63 VCMFrameBuffer* Back() const;
stefan@webrtc.org4cf1a8a2013-06-27 15:20:1464 int RecycleFramesUntilKeyFrame(FrameList::iterator* key_frame_it,
philipel9d3ab612015-12-21 12:12:3965 UnorderedFrameList* free_frames);
pbos@webrtc.org4f16c872014-11-24 09:06:4866 void CleanUpOldOrEmptyFrames(VCMDecodingState* decoding_state,
67 UnorderedFrameList* free_frames);
stefan@webrtc.org4cf1a8a2013-06-27 15:20:1468 void Reset(UnorderedFrameList* free_frames);
stefan@webrtc.orgc8b29a22013-06-17 07:13:1669};
70
stefan@webrtc.org912981f2012-10-12 07:04:5271class VCMJitterBuffer {
72 public:
Jonas Orelande02f9ee2022-03-25 11:43:1473 VCMJitterBuffer(Clock* clock,
74 std::unique_ptr<EventWrapper> event,
Jonas Orelande62c2f22022-03-29 09:04:4875 const FieldTrialsView& field_trials);
Qiang Chend4cec152015-06-19 16:17:0076
Wan-Teh Chang6a1ba8c2015-05-26 21:11:4177 ~VCMJitterBuffer();
niklase@google.com470e71d2011-07-07 08:21:2578
Byoungchan Lee604fd2f2022-01-21 00:49:3979 VCMJitterBuffer(const VCMJitterBuffer&) = delete;
80 VCMJitterBuffer& operator=(const VCMJitterBuffer&) = delete;
81
stefan@webrtc.org912981f2012-10-12 07:04:5282 // Initializes and starts jitter buffer.
83 void Start();
niklase@google.com470e71d2011-07-07 08:21:2584
stefan@webrtc.org912981f2012-10-12 07:04:5285 // Signals all internal events and stops the jitter buffer.
86 void Stop();
niklase@google.com470e71d2011-07-07 08:21:2587
stefan@webrtc.org912981f2012-10-12 07:04:5288 // Returns true if the jitter buffer is running.
89 bool Running() const;
niklase@google.com470e71d2011-07-07 08:21:2590
stefan@webrtc.org912981f2012-10-12 07:04:5291 // Empty the jitter buffer of all its data.
92 void Flush();
stefan@webrtc.org791eec72011-10-11 07:53:4393
asapersson@webrtc.org96dc6852014-11-03 14:40:3894 // Gets number of packets received.
95 int num_packets() const;
96
97 // Gets number of duplicated packets received.
98 int num_duplicated_packets() const;
99
Artem Titovdcd7fc72021-08-09 11:02:57100 // Wait `max_wait_time_ms` for a complete frame to arrive.
isheriff6b4b5f32016-06-08 07:24:21101 // If found, a pointer to the frame is returned. Returns nullptr otherwise.
102 VCMEncodedFrame* NextCompleteFrame(uint32_t max_wait_time_ms);
niklase@google.com470e71d2011-07-07 08:21:25103
mikhal@webrtc.org759b0412013-05-07 16:36:00104 // Extract frame corresponding to input timestamp.
105 // Frame will be set to a decoding state.
106 VCMEncodedFrame* ExtractAndSetDecode(uint32_t timestamp);
niklase@google.com470e71d2011-07-07 08:21:25107
stefan@webrtc.org912981f2012-10-12 07:04:52108 // Releases a frame returned from the jitter buffer, should be called when
109 // done with decoding.
110 void ReleaseFrame(VCMEncodedFrame* frame);
niklase@google.com470e71d2011-07-07 08:21:25111
stefan@webrtc.org912981f2012-10-12 07:04:52112 // Returns the time in ms when the latest packet was inserted into the frame.
113 // Retransmitted is set to true if any of the packets belonging to the frame
114 // has been retransmitted.
stefan@webrtc.org3417eb42013-05-21 15:25:53115 int64_t LastPacketTime(const VCMEncodedFrame* frame,
116 bool* retransmitted) const;
niklase@google.com470e71d2011-07-07 08:21:25117
stefan@webrtc.org912981f2012-10-12 07:04:52118 // Inserts a packet into a frame returned from GetFrame().
Artem Titovdcd7fc72021-08-09 11:02:57119 // If the return value is <= 0, `frame` is invalidated and the pointer must
stefan@webrtc.orgc8b29a22013-06-17 07:13:16120 // be dropped after this function returns.
philipel9d3ab612015-12-21 12:12:39121 VCMFrameBufferEnum InsertPacket(const VCMPacket& packet, bool* retransmitted);
niklase@google.com470e71d2011-07-07 08:21:25122
stefan@webrtc.org912981f2012-10-12 07:04:52123 // Returns the estimated jitter in milliseconds.
124 uint32_t EstimatedJitterMs();
niklase@google.com470e71d2011-07-07 08:21:25125
stefan@webrtc.orgbecf9c82013-02-01 15:09:57126 void SetNackSettings(size_t max_nack_list_size,
stefan@webrtc.orgef144882013-05-07 19:16:33127 int max_packet_age_to_nack,
128 int max_incomplete_time_ms);
stefan@webrtc.orgbecf9c82013-02-01 15:09:57129
stefan@webrtc.orga64300a2013-03-04 15:24:40130 // Returns a list of the sequence numbers currently missing.
Wan-Teh Changb1825a42015-06-03 22:03:35131 std::vector<uint16_t> GetNackList(bool* request_key_frame);
niklase@google.com470e71d2011-07-07 08:21:25132
stefan@webrtc.org912981f2012-10-12 07:04:52133 private:
stefan@webrtc.orga64300a2013-03-04 15:24:40134 class SequenceNumberLessThan {
135 public:
philipel9d3ab612015-12-21 12:12:39136 bool operator()(const uint16_t& sequence_number1,
137 const uint16_t& sequence_number2) const {
stefan@webrtc.org7bc465b2013-04-11 17:48:02138 return IsNewerSequenceNumber(sequence_number2, sequence_number1);
stefan@webrtc.orga64300a2013-03-04 15:24:40139 }
140 };
141 typedef std::set<uint16_t, SequenceNumberLessThan> SequenceNumberSet;
142
stefan@webrtc.org3417eb42013-05-21 15:25:53143 // Gets the frame assigned to the timestamp of the packet. May recycle
144 // existing frames if no free frames are available. Returns an error code if
Artem Titovdcd7fc72021-08-09 11:02:57145 // failing, or kNoError on success. `frame_list` contains which list the
pbos@webrtc.org4f16c872014-11-24 09:06:48146 // packet was in, or NULL if it was not in a FrameList (a new frame).
147 VCMFrameBufferEnum GetFrame(const VCMPacket& packet,
148 VCMFrameBuffer** frame,
149 FrameList** frame_list)
Markus Handell6deec382020-07-07 10:17:12150 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
asapersson@webrtc.org83b52002014-11-28 10:17:13151
Artem Titovdcd7fc72021-08-09 11:02:57152 // Returns true if `frame` is continuous in `decoding_state`, not taking
stefan@webrtc.orgc8b29a22013-06-17 07:13:16153 // decodable frames into account.
154 bool IsContinuousInState(const VCMFrameBuffer& frame,
pbos@webrtc.org4f16c872014-11-24 09:06:48155 const VCMDecodingState& decoding_state) const
Markus Handell6deec382020-07-07 10:17:12156 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
Artem Titovdcd7fc72021-08-09 11:02:57157 // Returns true if `frame` is continuous in the `last_decoded_state_`, taking
stefan@webrtc.orgc8b29a22013-06-17 07:13:16158 // all decodable frames into account.
pbos@webrtc.org4f16c872014-11-24 09:06:48159 bool IsContinuous(const VCMFrameBuffer& frame) const
Markus Handell6deec382020-07-07 10:17:12160 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
Artem Titovdcd7fc72021-08-09 11:02:57161 // Looks for frames in `incomplete_frames_` which are continuous in the
162 // provided `decoded_state`. Starts the search from the timestamp of
163 // `decoded_state`.
Noah Richardse4cb4e92015-05-22 21:03:00164 void FindAndInsertContinuousFramesWithState(
165 const VCMDecodingState& decoded_state)
Markus Handell6deec382020-07-07 10:17:12166 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
Artem Titovdcd7fc72021-08-09 11:02:57167 // Looks for frames in `incomplete_frames_` which are continuous in
168 // `last_decoded_state_` taking all decodable frames into account. Starts
169 // the search from `new_frame`.
pbos@webrtc.org4f16c872014-11-24 09:06:48170 void FindAndInsertContinuousFrames(const VCMFrameBuffer& new_frame)
Markus Handell6deec382020-07-07 10:17:12171 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
172 VCMFrameBuffer* NextFrame() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
stefan@webrtc.orga64300a2013-03-04 15:24:40173 // Returns true if the NACK list was updated to cover sequence numbers up to
Artem Titovdcd7fc72021-08-09 11:02:57174 // `sequence_number`. If false a key frame is needed to get into a state where
stefan@webrtc.orga64300a2013-03-04 15:24:40175 // we can continue decoding.
pbos@webrtc.org4f16c872014-11-24 09:06:48176 bool UpdateNackList(uint16_t sequence_number)
Markus Handell6deec382020-07-07 10:17:12177 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
stefan@webrtc.orga64300a2013-03-04 15:24:40178 bool TooLargeNackList() const;
179 // Returns true if the NACK list was reduced without problem. If false a key
180 // frame is needed to get into a state where we can continue decoding.
Markus Handell6deec382020-07-07 10:17:12181 bool HandleTooLargeNackList() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
pbos@webrtc.org4f16c872014-11-24 09:06:48182 bool MissingTooOldPacket(uint16_t latest_sequence_number) const
Markus Handell6deec382020-07-07 10:17:12183 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
stefan@webrtc.orga64300a2013-03-04 15:24:40184 // Returns true if the too old packets was successfully removed from the NACK
185 // list. If false, a key frame is needed to get into a state where we can
186 // continue decoding.
pbos@webrtc.org4f16c872014-11-24 09:06:48187 bool HandleTooOldPackets(uint16_t latest_sequence_number)
Markus Handell6deec382020-07-07 10:17:12188 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
Artem Titovdcd7fc72021-08-09 11:02:57189 // Drops all packets in the NACK list up until `last_decoded_sequence_number`.
stefan@webrtc.orga64300a2013-03-04 15:24:40190 void DropPacketsFromNackList(uint16_t last_decoded_sequence_number);
191
stefan@webrtc.org912981f2012-10-12 07:04:52192 // Gets an empty frame, creating a new frame if necessary (i.e. increases
193 // jitter buffer size).
Markus Handell6deec382020-07-07 10:17:12194 VCMFrameBuffer* GetEmptyFrame() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
niklase@google.com470e71d2011-07-07 08:21:25195
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14196 // Attempts to increase the size of the jitter buffer. Returns true on
197 // success, false otherwise.
Markus Handell6deec382020-07-07 10:17:12198 bool TryToIncreaseJitterBufferSize() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14199
stefan@webrtc.org912981f2012-10-12 07:04:52200 // Recycles oldest frames until a key frame is found. Used if jitter buffer is
201 // completely full. Returns true if a key frame was found.
Markus Handell6deec382020-07-07 10:17:12202 bool RecycleFramesUntilKeyFrame() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
niklase@google.com470e71d2011-07-07 08:21:25203
agalusza@google.comd818dcb2013-07-29 21:48:11204 // Update rolling average of packets per frame.
205 void UpdateAveragePacketsPerFrame(int current_number_packets_);
206
mikhal@webrtc.org381da4b2013-04-25 21:45:29207 // Cleans the frame list in the JB from old/empty frames.
208 // Should only be called prior to actual use.
Markus Handell6deec382020-07-07 10:17:12209 void CleanUpOldOrEmptyFrames() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
niklase@google.com470e71d2011-07-07 08:21:25210
Artem Titovdcd7fc72021-08-09 11:02:57211 // Returns true if `packet` is likely to have been retransmitted.
stefan@webrtc.org912981f2012-10-12 07:04:52212 bool IsPacketRetransmitted(const VCMPacket& packet) const;
henrik.lundin@webrtc.orgbaf6db52011-11-02 18:58:39213
stefan@webrtc.org912981f2012-10-12 07:04:52214 // The following three functions update the jitter estimate with the
215 // payload size, receive time and RTP timestamp of a frame.
216 void UpdateJitterEstimate(const VCMJitterSample& sample,
217 bool incomplete_frame);
218 void UpdateJitterEstimate(const VCMFrameBuffer& frame, bool incomplete_frame);
219 void UpdateJitterEstimate(int64_t latest_packet_time_ms,
220 uint32_t timestamp,
221 unsigned int frame_size,
222 bool incomplete_frame);
223
Markus Handell6deec382020-07-07 10:17:12224 int NonContinuousOrIncompleteDuration() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
stefan@webrtc.orgef144882013-05-07 19:16:33225
226 uint16_t EstimatedLowSequenceNumber(const VCMFrameBuffer& frame) const;
227
sprang22691e02016-07-13 17:57:07228 // Reset frame buffer and return it to free_frames_.
229 void RecycleFrameBuffer(VCMFrameBuffer* frame)
Markus Handell6deec382020-07-07 10:17:12230 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
sprang22691e02016-07-13 17:57:07231
stefan@webrtc.orga678a3b2013-01-21 07:42:11232 Clock* clock_;
stefan@webrtc.org912981f2012-10-12 07:04:52233 // If we are running (have started) or not.
234 bool running_;
Markus Handell6deec382020-07-07 10:17:12235 mutable Mutex mutex_;
stefan@webrtc.org912981f2012-10-12 07:04:52236 // Event to signal when we have a frame ready for decoder.
kwiberg3f55dea2016-02-29 13:51:59237 std::unique_ptr<EventWrapper> frame_event_;
stefan@webrtc.org912981f2012-10-12 07:04:52238 // Number of allocated frames.
239 int max_number_of_frames_;
Markus Handell6deec382020-07-07 10:17:12240 UnorderedFrameList free_frames_ RTC_GUARDED_BY(mutex_);
241 FrameList decodable_frames_ RTC_GUARDED_BY(mutex_);
242 FrameList incomplete_frames_ RTC_GUARDED_BY(mutex_);
243 VCMDecodingState last_decoded_state_ RTC_GUARDED_BY(mutex_);
stefan@webrtc.org3417eb42013-05-21 15:25:53244 bool first_packet_since_reset_;
stefan@webrtc.org912981f2012-10-12 07:04:52245
stefan@webrtc.org912981f2012-10-12 07:04:52246 // Number of packets in a row that have been too old.
247 int num_consecutive_old_packets_;
asapersson@webrtc.org96dc6852014-11-03 14:40:38248 // Number of packets received.
Markus Handell6deec382020-07-07 10:17:12249 int num_packets_ RTC_GUARDED_BY(mutex_);
asapersson@webrtc.org96dc6852014-11-03 14:40:38250 // Number of duplicated packets received.
Markus Handell6deec382020-07-07 10:17:12251 int num_duplicated_packets_ RTC_GUARDED_BY(mutex_);
stefan@webrtc.org912981f2012-10-12 07:04:52252
253 // Jitter estimation.
254 // Filter for estimating jitter.
Rasmus Brandt10944e62022-05-25 08:12:42255 JitterEstimator jitter_estimate_;
stefan@webrtc.org912981f2012-10-12 07:04:52256 // Calculates network delays used for jitter calculations.
Rasmus Brandt65a6eca2023-03-03 08:22:18257 InterFrameDelayVariationCalculator inter_frame_delay_;
stefan@webrtc.org912981f2012-10-12 07:04:52258 VCMJitterSample waiting_for_completion_;
stefan@webrtc.org912981f2012-10-12 07:04:52259
stefan@webrtc.org912981f2012-10-12 07:04:52260 // Holds the internal NACK list (the missing sequence numbers).
stefan@webrtc.orga64300a2013-03-04 15:24:40261 SequenceNumberSet missing_sequence_numbers_;
262 uint16_t latest_received_sequence_number_;
stefan@webrtc.orgbecf9c82013-02-01 15:09:57263 size_t max_nack_list_size_;
264 int max_packet_age_to_nack_; // Measured in sequence numbers.
stefan@webrtc.orgef144882013-05-07 19:16:33265 int max_incomplete_time_ms_;
stefan@webrtc.org912981f2012-10-12 07:04:52266
agalusza@google.comd818dcb2013-07-29 21:48:11267 // Estimated rolling average of packets per frame
268 float average_packets_per_frame_;
269 // average_packets_per_frame converges fast if we have fewer than this many
270 // frames.
271 int frame_counter_;
niklase@google.com470e71d2011-07-07 08:21:25272};
stefan@webrtc.org912981f2012-10-12 07:04:52273} // namespace webrtc
niklase@google.com470e71d2011-07-07 08:21:25274
Mirko Bonadei92ea95e2017-09-15 04:47:31275#endif // MODULES_VIDEO_CODING_JITTER_BUFFER_H_