| /* |
| * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. |
| * |
| * Use of this source code is governed by a BSD-style license |
| * that can be found in the LICENSE file in the root of the source |
| * tree. An additional intellectual property rights grant can be found |
| * in the file PATENTS. All contributing project authors may |
| * be found in the AUTHORS file in the root of the source tree. |
| */ |
| |
| #ifndef MODULES_VIDEO_CODING_FRAME_BUFFER2_H_ |
| #define MODULES_VIDEO_CODING_FRAME_BUFFER2_H_ |
| |
| #include <array> |
| #include <map> |
| #include <memory> |
| #include <utility> |
| #include <vector> |
| |
| #include "absl/container/inlined_vector.h" |
| #include "api/video/encoded_frame.h" |
| #include "modules/video_coding/include/video_coding_defines.h" |
| #include "modules/video_coding/inter_frame_delay.h" |
| #include "modules/video_coding/jitter_estimator.h" |
| #include "modules/video_coding/utility/decoded_frames_history.h" |
| #include "rtc_base/event.h" |
| #include "rtc_base/experiments/rtt_mult_experiment.h" |
| #include "rtc_base/numerics/sequence_number_util.h" |
| #include "rtc_base/synchronization/mutex.h" |
| #include "rtc_base/synchronization/sequence_checker.h" |
| #include "rtc_base/system/no_unique_address.h" |
| #include "rtc_base/task_queue.h" |
| #include "rtc_base/task_utils/repeating_task.h" |
| #include "rtc_base/thread_annotations.h" |
| |
| namespace webrtc { |
| |
| class Clock; |
| class VCMReceiveStatisticsCallback; |
| class VCMJitterEstimator; |
| class VCMTiming; |
| |
| namespace video_coding { |
| |
| class FrameBuffer { |
| public: |
| enum ReturnReason { kFrameFound, kTimeout, kStopped }; |
| |
| FrameBuffer(Clock* clock, |
| VCMTiming* timing, |
| VCMReceiveStatisticsCallback* stats_callback); |
| |
| FrameBuffer() = delete; |
| FrameBuffer(const FrameBuffer&) = delete; |
| FrameBuffer& operator=(const FrameBuffer&) = delete; |
| |
| virtual ~FrameBuffer(); |
| |
| // Insert a frame into the frame buffer. Returns the picture id |
| // of the last continuous frame or -1 if there is no continuous frame. |
| // TODO(philipel): Return a VideoLayerFrameId and not only the picture id. |
| int64_t InsertFrame(std::unique_ptr<EncodedFrame> frame); |
| |
| // Get the next frame for decoding. Will return at latest after |
| // |max_wait_time_ms|. |
| void NextFrame( |
| int64_t max_wait_time_ms, |
| bool keyframe_required, |
| rtc::TaskQueue* callback_queue, |
| std::function<void(std::unique_ptr<EncodedFrame>, ReturnReason)> handler); |
| |
| // Tells the FrameBuffer which protection mode that is in use. Affects |
| // the frame timing. |
| // TODO(philipel): Remove this when new timing calculations has been |
| // implemented. |
| void SetProtectionMode(VCMVideoProtection mode); |
| |
| // Stop the frame buffer, causing any sleeping thread in NextFrame to |
| // return immediately. |
| void Stop(); |
| |
| // Updates the RTT for jitter buffer estimation. |
| void UpdateRtt(int64_t rtt_ms); |
| |
| // Clears the FrameBuffer, removing all the buffered frames. |
| void Clear(); |
| |
| int Size(); |
| |
| private: |
| struct FrameInfo { |
| FrameInfo(); |
| FrameInfo(FrameInfo&&); |
| ~FrameInfo(); |
| |
| // Which other frames that have direct unfulfilled dependencies |
| // on this frame. |
| absl::InlinedVector<VideoLayerFrameId, 8> dependent_frames; |
| |
| // A frame is continiuous if it has all its referenced/indirectly |
| // referenced frames. |
| // |
| // How many unfulfilled frames this frame have until it becomes continuous. |
| size_t num_missing_continuous = 0; |
| |
| // A frame is decodable if all its referenced frames have been decoded. |
| // |
| // How many unfulfilled frames this frame have until it becomes decodable. |
| size_t num_missing_decodable = 0; |
| |
| // If this frame is continuous or not. |
| bool continuous = false; |
| |
| // The actual EncodedFrame. |
| std::unique_ptr<EncodedFrame> frame; |
| }; |
| |
| using FrameMap = std::map<VideoLayerFrameId, FrameInfo>; |
| |
| // Check that the references of |frame| are valid. |
| bool ValidReferences(const EncodedFrame& frame) const; |
| |
| int64_t FindNextFrame(int64_t now_ms) RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); |
| EncodedFrame* GetNextFrame() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); |
| |
| void StartWaitForNextFrameOnQueue() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); |
| void CancelCallback() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); |
| |
| // Update all directly dependent and indirectly dependent frames and mark |
| // them as continuous if all their references has been fulfilled. |
| void PropagateContinuity(FrameMap::iterator start) |
| RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); |
| |
| // Marks the frame as decoded and updates all directly dependent frames. |
| void PropagateDecodability(const FrameInfo& info) |
| RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); |
| |
| // Update the corresponding FrameInfo of |frame| and all FrameInfos that |
| // |frame| references. |
| // Return false if |frame| will never be decodable, true otherwise. |
| bool UpdateFrameInfoWithIncomingFrame(const EncodedFrame& frame, |
| FrameMap::iterator info) |
| RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); |
| |
| void UpdateJitterDelay() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); |
| |
| void UpdateTimingFrameInfo() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); |
| |
| void ClearFramesAndHistory() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); |
| |
| // Checks if the superframe, which current frame belongs to, is complete. |
| bool IsCompleteSuperFrame(const EncodedFrame& frame) |
| RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); |
| |
| bool HasBadRenderTiming(const EncodedFrame& frame, int64_t now_ms) |
| RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); |
| |
| // The cleaner solution would be to have the NextFrame function return a |
| // vector of frames, but until the decoding pipeline can support decoding |
| // multiple frames at the same time we combine all frames to one frame and |
| // return it. See bugs.webrtc.org/10064 |
| EncodedFrame* CombineAndDeleteFrames( |
| const std::vector<EncodedFrame*>& frames) const; |
| |
| RTC_NO_UNIQUE_ADDRESS SequenceChecker construction_checker_; |
| RTC_NO_UNIQUE_ADDRESS SequenceChecker callback_checker_; |
| |
| // Stores only undecoded frames. |
| FrameMap frames_ RTC_GUARDED_BY(mutex_); |
| DecodedFramesHistory decoded_frames_history_ RTC_GUARDED_BY(mutex_); |
| |
| Mutex mutex_; |
| Clock* const clock_; |
| |
| rtc::TaskQueue* callback_queue_ RTC_GUARDED_BY(mutex_); |
| RepeatingTaskHandle callback_task_ RTC_GUARDED_BY(mutex_); |
| std::function<void(std::unique_ptr<EncodedFrame>, ReturnReason)> |
| frame_handler_ RTC_GUARDED_BY(mutex_); |
| int64_t latest_return_time_ms_ RTC_GUARDED_BY(mutex_); |
| bool keyframe_required_ RTC_GUARDED_BY(mutex_); |
| |
| VCMJitterEstimator jitter_estimator_ RTC_GUARDED_BY(mutex_); |
| VCMTiming* const timing_ RTC_GUARDED_BY(mutex_); |
| VCMInterFrameDelay inter_frame_delay_ RTC_GUARDED_BY(mutex_); |
| absl::optional<VideoLayerFrameId> last_continuous_frame_ |
| RTC_GUARDED_BY(mutex_); |
| std::vector<FrameMap::iterator> frames_to_decode_ RTC_GUARDED_BY(mutex_); |
| bool stopped_ RTC_GUARDED_BY(mutex_); |
| VCMVideoProtection protection_mode_ RTC_GUARDED_BY(mutex_); |
| VCMReceiveStatisticsCallback* const stats_callback_; |
| int64_t last_log_non_decoded_ms_ RTC_GUARDED_BY(mutex_); |
| |
| const bool add_rtt_to_playout_delay_; |
| |
| // rtt_mult experiment settings. |
| const absl::optional<RttMultExperiment::Settings> rtt_mult_settings_; |
| }; |
| |
| } // namespace video_coding |
| } // namespace webrtc |
| |
| #endif // MODULES_VIDEO_CODING_FRAME_BUFFER2_H_ |