| /* |
| * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
| * |
| * Use of this source code is governed by a BSD-style license |
| * that can be found in the LICENSE file in the root of the source |
| * tree. An additional intellectual property rights grant can be found |
| * in the file PATENTS. All contributing project authors may |
| * be found in the AUTHORS file in the root of the source tree. |
| */ |
| |
| #ifndef API_NETEQ_NETEQ_H_ |
| #define API_NETEQ_NETEQ_H_ |
| |
| #include <stddef.h> // Provide access to size_t. |
| |
| #include <map> |
| #include <string> |
| #include <vector> |
| |
| #include "absl/types/optional.h" |
| #include "api/audio_codecs/audio_codec_pair_id.h" |
| #include "api/audio_codecs/audio_decoder.h" |
| #include "api/audio_codecs/audio_format.h" |
| #include "api/rtp_headers.h" |
| #include "api/scoped_refptr.h" |
| |
| namespace webrtc { |
| |
| // Forward declarations. |
| class AudioFrame; |
| class AudioDecoderFactory; |
| class Clock; |
| |
| struct NetEqNetworkStatistics { |
| uint16_t current_buffer_size_ms; // Current jitter buffer size in ms. |
| uint16_t preferred_buffer_size_ms; // Target buffer size in ms. |
| uint16_t jitter_peaks_found; // 1 if adding extra delay due to peaky |
| // jitter; 0 otherwise. |
| uint16_t packet_loss_rate; // Loss rate (network + late) in Q14. |
| uint16_t expand_rate; // Fraction (of original stream) of synthesized |
| // audio inserted through expansion (in Q14). |
| uint16_t speech_expand_rate; // Fraction (of original stream) of synthesized |
| // speech inserted through expansion (in Q14). |
| uint16_t preemptive_rate; // Fraction of data inserted through pre-emptive |
| // expansion (in Q14). |
| uint16_t accelerate_rate; // Fraction of data removed through acceleration |
| // (in Q14). |
| uint16_t secondary_decoded_rate; // Fraction of data coming from FEC/RED |
| // decoding (in Q14). |
| uint16_t secondary_discarded_rate; // Fraction of discarded FEC/RED data (in |
| // Q14). |
| size_t added_zero_samples; // Number of zero samples added in "off" mode. |
| // Statistics for packet waiting times, i.e., the time between a packet |
| // arrives until it is decoded. |
| int mean_waiting_time_ms; |
| int median_waiting_time_ms; |
| int min_waiting_time_ms; |
| int max_waiting_time_ms; |
| }; |
| |
| // NetEq statistics that persist over the lifetime of the class. |
| // These metrics are never reset. |
| struct NetEqLifetimeStatistics { |
| // Stats below correspond to similarly-named fields in the WebRTC stats spec. |
| // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats |
| uint64_t total_samples_received = 0; |
| uint64_t concealed_samples = 0; |
| uint64_t concealment_events = 0; |
| uint64_t jitter_buffer_delay_ms = 0; |
| uint64_t jitter_buffer_emitted_count = 0; |
| uint64_t inserted_samples_for_deceleration = 0; |
| uint64_t removed_samples_for_acceleration = 0; |
| uint64_t silent_concealed_samples = 0; |
| uint64_t fec_packets_received = 0; |
| uint64_t fec_packets_discarded = 0; |
| // Below stats are not part of the spec. |
| uint64_t delayed_packet_outage_samples = 0; |
| // This is sum of relative packet arrival delays of received packets so far. |
| // Since end-to-end delay of a packet is difficult to measure and is not |
| // necessarily useful for measuring jitter buffer performance, we report a |
| // relative packet arrival delay. The relative packet arrival delay of a |
| // packet is defined as the arrival delay compared to the first packet |
| // received, given that it had zero delay. To avoid clock drift, the "first" |
| // packet can be made dynamic. |
| uint64_t relative_packet_arrival_delay_ms = 0; |
| uint64_t jitter_buffer_packets_received = 0; |
| // An interruption is a loss-concealment event lasting at least 150 ms. The |
| // two stats below count the number os such events and the total duration of |
| // these events. |
| int32_t interruption_count = 0; |
| int32_t total_interruption_duration_ms = 0; |
| }; |
| |
| // Metrics that describe the operations performed in NetEq, and the internal |
| // state. |
| struct NetEqOperationsAndState { |
| // These sample counters are cumulative, and don't reset. As a reference, the |
| // total number of output samples can be found in |
| // NetEqLifetimeStatistics::total_samples_received. |
| uint64_t preemptive_samples = 0; |
| uint64_t accelerate_samples = 0; |
| // Count of the number of buffer flushes. |
| uint64_t packet_buffer_flushes = 0; |
| // The number of primary packets that were discarded. |
| uint64_t discarded_primary_packets = 0; |
| // The statistics below are not cumulative. |
| // The waiting time of the last decoded packet. |
| uint64_t last_waiting_time_ms = 0; |
| // The sum of the packet and jitter buffer size in ms. |
| uint64_t current_buffer_size_ms = 0; |
| // The current frame size in ms. |
| uint64_t current_frame_size_ms = 0; |
| // Flag to indicate that the next packet is available. |
| bool next_packet_available = false; |
| }; |
| |
| // This is the interface class for NetEq. |
| class NetEq { |
| public: |
| struct Config { |
| Config(); |
| Config(const Config&); |
| Config(Config&&); |
| ~Config(); |
| Config& operator=(const Config&); |
| Config& operator=(Config&&); |
| |
| std::string ToString() const; |
| |
| int sample_rate_hz = 16000; // Initial value. Will change with input data. |
| bool enable_post_decode_vad = false; |
| size_t max_packets_in_buffer = 200; |
| int max_delay_ms = 0; |
| int min_delay_ms = 0; |
| bool enable_fast_accelerate = false; |
| bool enable_muted_state = false; |
| bool enable_rtx_handling = false; |
| absl::optional<AudioCodecPairId> codec_pair_id; |
| bool for_test_no_time_stretching = false; // Use only for testing. |
| }; |
| |
| enum ReturnCodes { kOK = 0, kFail = -1 }; |
| |
| enum class Operation { |
| kNormal, |
| kMerge, |
| kExpand, |
| kAccelerate, |
| kFastAccelerate, |
| kPreemptiveExpand, |
| kRfc3389Cng, |
| kRfc3389CngNoPacket, |
| kCodecInternalCng, |
| kDtmf, |
| kUndefined, |
| }; |
| |
| enum class Mode { |
| kNormal, |
| kExpand, |
| kMerge, |
| kAccelerateSuccess, |
| kAccelerateLowEnergy, |
| kAccelerateFail, |
| kPreemptiveExpandSuccess, |
| kPreemptiveExpandLowEnergy, |
| kPreemptiveExpandFail, |
| kRfc3389Cng, |
| kCodecInternalCng, |
| kCodecPlc, |
| kDtmf, |
| kError, |
| kUndefined, |
| }; |
| |
| // Return type for GetDecoderFormat. |
| struct DecoderFormat { |
| int sample_rate_hz; |
| int num_channels; |
| SdpAudioFormat sdp_format; |
| }; |
| |
| // Creates a new NetEq object, with parameters set in |config|. The |config| |
| // object will only have to be valid for the duration of the call to this |
| // method. |
| static NetEq* Create( |
| const NetEq::Config& config, |
| Clock* clock, |
| const rtc::scoped_refptr<AudioDecoderFactory>& decoder_factory); |
| |
| virtual ~NetEq() {} |
| |
| // Inserts a new packet into NetEq. |
| // Returns 0 on success, -1 on failure. |
| virtual int InsertPacket(const RTPHeader& rtp_header, |
| rtc::ArrayView<const uint8_t> payload) = 0; |
| |
| // Deprecated. Use the version without the `receive_timestamp` argument. |
| int InsertPacket(const RTPHeader& rtp_header, |
| rtc::ArrayView<const uint8_t> payload, |
| uint32_t /*receive_timestamp*/) { |
| return InsertPacket(rtp_header, payload); |
| } |
| |
| // Lets NetEq know that a packet arrived with an empty payload. This typically |
| // happens when empty packets are used for probing the network channel, and |
| // these packets use RTP sequence numbers from the same series as the actual |
| // audio packets. |
| virtual void InsertEmptyPacket(const RTPHeader& rtp_header) = 0; |
| |
| // Instructs NetEq to deliver 10 ms of audio data. The data is written to |
| // |audio_frame|. All data in |audio_frame| is wiped; |data_|, |speech_type_|, |
| // |num_channels_|, |sample_rate_hz_|, |samples_per_channel_|, and |
| // |vad_activity_| are updated upon success. If an error is returned, some |
| // fields may not have been updated, or may contain inconsistent values. |
| // If muted state is enabled (through Config::enable_muted_state), |muted| |
| // may be set to true after a prolonged expand period. When this happens, the |
| // |data_| in |audio_frame| is not written, but should be interpreted as being |
| // all zeros. For testing purposes, an override can be supplied in the |
| // |action_override| argument, which will cause NetEq to take this action |
| // next, instead of the action it would normally choose. |
| // Returns kOK on success, or kFail in case of an error. |
| virtual int GetAudio( |
| AudioFrame* audio_frame, |
| bool* muted, |
| absl::optional<Operation> action_override = absl::nullopt) = 0; |
| |
| // Replaces the current set of decoders with the given one. |
| virtual void SetCodecs(const std::map<int, SdpAudioFormat>& codecs) = 0; |
| |
| // Associates |rtp_payload_type| with the given codec, which NetEq will |
| // instantiate when it needs it. Returns true iff successful. |
| virtual bool RegisterPayloadType(int rtp_payload_type, |
| const SdpAudioFormat& audio_format) = 0; |
| |
| // Removes |rtp_payload_type| from the codec database. Returns 0 on success, |
| // -1 on failure. Removing a payload type that is not registered is ok and |
| // will not result in an error. |
| virtual int RemovePayloadType(uint8_t rtp_payload_type) = 0; |
| |
| // Removes all payload types from the codec database. |
| virtual void RemoveAllPayloadTypes() = 0; |
| |
| // Sets a minimum delay in millisecond for packet buffer. The minimum is |
| // maintained unless a higher latency is dictated by channel condition. |
| // Returns true if the minimum is successfully applied, otherwise false is |
| // returned. |
| virtual bool SetMinimumDelay(int delay_ms) = 0; |
| |
| // Sets a maximum delay in milliseconds for packet buffer. The latency will |
| // not exceed the given value, even required delay (given the channel |
| // conditions) is higher. Calling this method has the same effect as setting |
| // the |max_delay_ms| value in the NetEq::Config struct. |
| virtual bool SetMaximumDelay(int delay_ms) = 0; |
| |
| // Sets a base minimum delay in milliseconds for packet buffer. The minimum |
| // delay which is set via |SetMinimumDelay| can't be lower than base minimum |
| // delay. Calling this method is similar to setting the |min_delay_ms| value |
| // in the NetEq::Config struct. Returns true if the base minimum is |
| // successfully applied, otherwise false is returned. |
| virtual bool SetBaseMinimumDelayMs(int delay_ms) = 0; |
| |
| // Returns current value of base minimum delay in milliseconds. |
| virtual int GetBaseMinimumDelayMs() const = 0; |
| |
| // Returns the current target delay in ms. This includes any extra delay |
| // requested through SetMinimumDelay. |
| virtual int TargetDelayMs() const = 0; |
| |
| // Returns the current total delay (packet buffer and sync buffer) in ms, |
| // with smoothing applied to even out short-time fluctuations due to jitter. |
| // The packet buffer part of the delay is not updated during DTX/CNG periods. |
| virtual int FilteredCurrentDelayMs() const = 0; |
| |
| // Writes the current network statistics to |stats|. The statistics are reset |
| // after the call. |
| virtual int NetworkStatistics(NetEqNetworkStatistics* stats) = 0; |
| |
| // Returns a copy of this class's lifetime statistics. These statistics are |
| // never reset. |
| virtual NetEqLifetimeStatistics GetLifetimeStatistics() const = 0; |
| |
| // Returns statistics about the performed operations and internal state. These |
| // statistics are never reset. |
| virtual NetEqOperationsAndState GetOperationsAndState() const = 0; |
| |
| // Enables post-decode VAD. When enabled, GetAudio() will return |
| // kOutputVADPassive when the signal contains no speech. |
| virtual void EnableVad() = 0; |
| |
| // Disables post-decode VAD. |
| virtual void DisableVad() = 0; |
| |
| // Returns the RTP timestamp for the last sample delivered by GetAudio(). |
| // The return value will be empty if no valid timestamp is available. |
| virtual absl::optional<uint32_t> GetPlayoutTimestamp() const = 0; |
| |
| // Returns the sample rate in Hz of the audio produced in the last GetAudio |
| // call. If GetAudio has not been called yet, the configured sample rate |
| // (Config::sample_rate_hz) is returned. |
| virtual int last_output_sample_rate_hz() const = 0; |
| |
| // Returns the decoder info for the given payload type. Returns empty if no |
| // such payload type was registered. |
| virtual absl::optional<DecoderFormat> GetDecoderFormat( |
| int payload_type) const = 0; |
| |
| // Flushes both the packet buffer and the sync buffer. |
| virtual void FlushBuffers() = 0; |
| |
| // Enables NACK and sets the maximum size of the NACK list, which should be |
| // positive and no larger than Nack::kNackListSizeLimit. If NACK is already |
| // enabled then the maximum NACK list size is modified accordingly. |
| virtual void EnableNack(size_t max_nack_list_size) = 0; |
| |
| virtual void DisableNack() = 0; |
| |
| // Returns a list of RTP sequence numbers corresponding to packets to be |
| // retransmitted, given an estimate of the round-trip time in milliseconds. |
| virtual std::vector<uint16_t> GetNackList( |
| int64_t round_trip_time_ms) const = 0; |
| |
| // Returns a vector containing the timestamps of the packets that were decoded |
| // in the last GetAudio call. If no packets were decoded in the last call, the |
| // vector is empty. |
| // Mainly intended for testing. |
| virtual std::vector<uint32_t> LastDecodedTimestamps() const = 0; |
| |
| // Returns the length of the audio yet to play in the sync buffer. |
| // Mainly intended for testing. |
| virtual int SyncBufferSizeMs() const = 0; |
| }; |
| |
| } // namespace webrtc |
| #endif // API_NETEQ_NETEQ_H_ |