Add accounting of actual audio bit usage
Part of a set of CL to allow video to borrow underused audio bitrate.
Bug: webrtc:35055527
Change-Id: Idb504cbbc5794c06b28bdc21b3d860c9da9df175
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/358202
Reviewed-by: Jakob Ivarsson‎ <jakobi@webrtc.org>
Commit-Queue: Dan Tan <dwtan@google.com>
Cr-Commit-Position: refs/heads/main@{#42733}
diff --git a/audio/audio_send_stream.cc b/audio/audio_send_stream.cc
index 5b3f624..fc194c9 100644
--- a/audio/audio_send_stream.cc
+++ b/audio/audio_send_stream.cc
@@ -492,8 +492,7 @@
}
absl::optional<DataRate> AudioSendStream::GetUsedRate() const {
- // TODO(bugs.webrtc.org/35055527): Implement
- return absl::nullopt;
+ return channel_send_->GetUsedRate();
}
void AudioSendStream::SetTransportOverhead(
@@ -518,6 +517,7 @@
if (registered_with_allocator_) {
ConfigureBitrateObserver();
}
+ channel_send_->RegisterPacketOverhead(overhead_per_packet_bytes);
}
size_t AudioSendStream::TestOnlyGetPerPacketOverheadBytes() const {
diff --git a/audio/audio_send_stream_unittest.cc b/audio/audio_send_stream_unittest.cc
index 666d263..6aa95c7 100644
--- a/audio/audio_send_stream_unittest.cc
+++ b/audio/audio_send_stream_unittest.cc
@@ -560,6 +560,7 @@
}));
EXPECT_CALL(*helper.rtp_rtcp(), ExpectedPerPacketOverhead)
.WillRepeatedly(Return(kOverheadPerPacket.bytes<size_t>()));
+ EXPECT_CALL(*helper.channel_send(), RegisterPacketOverhead);
auto send_stream = helper.CreateAudioSendStream();
@@ -672,6 +673,7 @@
"WebRTC-Audio-LegacyOverhead/Disabled/");
EXPECT_CALL(*helper.rtp_rtcp(), ExpectedPerPacketOverhead)
.WillRepeatedly(Return(kOverheadPerPacket.bytes<size_t>()));
+ EXPECT_CALL(*helper.channel_send(), RegisterPacketOverhead);
auto send_stream = helper.CreateAudioSendStream();
const DataRate bitrate =
DataRate::BitsPerSec(helper.config().max_bitrate_bps) +
@@ -694,6 +696,7 @@
"WebRTC-Audio-Allocation/min:6kbps,max:64kbps/");
EXPECT_CALL(*helper.rtp_rtcp(), ExpectedPerPacketOverhead)
.WillRepeatedly(Return(kOverheadPerPacket.bytes<size_t>()));
+ EXPECT_CALL(*helper.channel_send(), RegisterPacketOverhead);
auto send_stream = helper.CreateAudioSendStream();
const DataRate bitrate = DataRate::KilobitsPerSec(6) + kMinOverheadRate;
EXPECT_CALL(*helper.channel_send(),
@@ -714,6 +717,7 @@
"WebRTC-Audio-Allocation/min:6kbps,max:64kbps/");
EXPECT_CALL(*helper.rtp_rtcp(), ExpectedPerPacketOverhead)
.WillRepeatedly(Return(kOverheadPerPacket.bytes<size_t>()));
+ EXPECT_CALL(*helper.channel_send(), RegisterPacketOverhead);
auto send_stream = helper.CreateAudioSendStream();
const DataRate bitrate = DataRate::KilobitsPerSec(64) + kMaxOverheadRate;
EXPECT_CALL(*helper.channel_send(),
@@ -796,6 +800,7 @@
// CallEncoder will be called on overhead change.
EXPECT_CALL(*helper.channel_send(), CallEncoder);
+ EXPECT_CALL(*helper.channel_send(), RegisterPacketOverhead);
const size_t transport_overhead_per_packet_bytes = 333;
send_stream->SetTransportOverhead(transport_overhead_per_packet_bytes);
@@ -811,6 +816,8 @@
auto send_stream = helper.CreateAudioSendStream();
auto new_config = helper.config();
+ EXPECT_CALL(*helper.channel_send(), RegisterPacketOverhead).Times(2);
+
// CallEncoder will be called on overhead change.
EXPECT_CALL(*helper.channel_send(), CallEncoder);
const size_t transport_overhead_per_packet_bytes = 333;
@@ -832,6 +839,7 @@
const size_t audio_overhead_per_packet_bytes = 555;
EXPECT_CALL(*helper.rtp_rtcp(), ExpectedPerPacketOverhead)
.WillRepeatedly(Return(audio_overhead_per_packet_bytes));
+ EXPECT_CALL(*helper.channel_send(), RegisterPacketOverhead).Times(2);
auto send_stream = helper.CreateAudioSendStream();
BitrateAllocationUpdate update;
@@ -862,6 +870,7 @@
const size_t audio_overhead_per_packet_bytes = 555;
EXPECT_CALL(*helper.rtp_rtcp(), ExpectedPerPacketOverhead)
.WillRepeatedly(Return(audio_overhead_per_packet_bytes));
+ EXPECT_CALL(*helper.channel_send(), RegisterPacketOverhead).Times(2);
auto send_stream = helper.CreateAudioSendStream();
auto new_config = helper.config();
diff --git a/audio/channel_send.cc b/audio/channel_send.cc
index 4d59d20..b683cc3 100644
--- a/audio/channel_send.cc
+++ b/audio/channel_send.cc
@@ -57,6 +57,47 @@
class RtpPacketSenderProxy;
class TransportSequenceNumberProxy;
+class AudioBitrateAccountant {
+ public:
+ void RegisterPacketOverhead(int packet_byte_overhead) {
+ packet_overhead_ = DataSize::Bytes(packet_byte_overhead);
+ }
+
+ void Reset() {
+ rate_last_frame_ = DataRate::BitsPerSec(0);
+ next_frame_duration_ = TimeDelta::Millis(0);
+ report_rate_ = absl::nullopt;
+ }
+
+ // A new frame is formed when bytesize is nonzero.
+ void UpdateBpsEstimate(DataSize payload_size, TimeDelta frame_duration) {
+ next_frame_duration_ += frame_duration;
+ // Do not have a full frame yet.
+ if (payload_size.bytes() == 0)
+ return;
+
+ // We report the larger of the rates computed using the last frame, and
+ // second last frame. Under DTX, frame sizes sometimes alternate, it is
+ // preferable to report the upper envelop.
+ DataRate rate_cur_frame =
+ (payload_size + packet_overhead_) / next_frame_duration_;
+
+ report_rate_ =
+ (rate_cur_frame > rate_last_frame_) ? rate_cur_frame : rate_last_frame_;
+
+ rate_last_frame_ = rate_cur_frame;
+ next_frame_duration_ = TimeDelta::Millis(0);
+ }
+
+ absl::optional<DataRate> GetUsedRate() const { return report_rate_; }
+
+ private:
+ TimeDelta next_frame_duration_ = TimeDelta::Millis(0);
+ DataSize packet_overhead_ = DataSize::Bytes(72);
+ DataRate rate_last_frame_ = DataRate::BitsPerSec(0);
+ absl::optional<DataRate> report_rate_;
+};
+
class ChannelSend : public ChannelSendInterface,
public AudioPacketizationCallback, // receive encoded
// packets from the ACM
@@ -152,6 +193,17 @@
// ReportBlockDataObserver.
void OnReportBlockDataUpdated(ReportBlockData report_block) override;
+ // Reports actual bitrate used (vs allocated).
+ absl::optional<DataRate> GetUsedRate() const override {
+ MutexLock lock(&bitrate_accountant_mutex_);
+ return bitrate_accountant_.GetUsedRate();
+ }
+
+ void RegisterPacketOverhead(int packet_byte_overhead) override {
+ MutexLock lock(&bitrate_accountant_mutex_);
+ bitrate_accountant_.RegisterPacketOverhead(packet_byte_overhead);
+ }
+
private:
// From AudioPacketizationCallback in the ACM
int32_t SendData(AudioFrameType frameType,
@@ -240,6 +292,10 @@
RTC_NO_UNIQUE_ADDRESS SequenceChecker encoder_queue_checker_;
SdpAudioFormat encoder_format_;
+
+ mutable Mutex bitrate_accountant_mutex_;
+ AudioBitrateAccountant bitrate_accountant_
+ RTC_GUARDED_BY(bitrate_accountant_mutex_);
};
const int kTelephoneEventAttenuationdB = 10;
@@ -800,10 +856,15 @@
// encoding is done and payload is ready for packetization and
// transmission. Otherwise, it will return without invoking the
// callback.
- if (audio_coding_->Add10MsData(*audio_frame) < 0) {
+ int32_t encoded_bytes = audio_coding_->Add10MsData(*audio_frame);
+ MutexLock lock(&bitrate_accountant_mutex_);
+ if (encoded_bytes < 0) {
RTC_DLOG(LS_ERROR) << "ACM::Add10MsData() failed.";
+ bitrate_accountant_.Reset();
return;
}
+ bitrate_accountant_.UpdateBpsEstimate(DataSize::Bytes(encoded_bytes),
+ TimeDelta::Millis(10));
});
}
diff --git a/audio/channel_send.h b/audio/channel_send.h
index 704df8c..6082c0e 100644
--- a/audio/channel_send.h
+++ b/audio/channel_send.h
@@ -110,6 +110,12 @@
virtual void SetEncoderToPacketizerFrameTransformer(
rtc::scoped_refptr<webrtc::FrameTransformerInterface>
frame_transformer) = 0;
+
+ // Returns payload bitrate actually used.
+ virtual absl::optional<DataRate> GetUsedRate() const = 0;
+
+ // Registers per packet byte overhead.
+ virtual void RegisterPacketOverhead(int packet_byte_overhead) = 0;
};
std::unique_ptr<ChannelSendInterface> CreateChannelSend(
diff --git a/audio/channel_send_unittest.cc b/audio/channel_send_unittest.cc
index 5755776..3819859 100644
--- a/audio/channel_send_unittest.cc
+++ b/audio/channel_send_unittest.cc
@@ -306,6 +306,62 @@
EXPECT_TRUE_WAIT(sent_audio_level, 1000);
EXPECT_EQ(*sent_audio_level, audio_level);
}
+
+// Ensure that GetUsedRate returns null if no frames are coded.
+TEST_F(ChannelSendTest, NoUsedRateInitially) {
+ channel_->StartSend();
+ auto used_rate = channel_->GetUsedRate();
+ EXPECT_EQ(used_rate, absl::nullopt);
+}
+
+// Ensure that GetUsedRate returns value with one coded frame.
+TEST_F(ChannelSendTest, ValidUsedRateWithOneCodedFrame) {
+ channel_->StartSend();
+ EXPECT_CALL(transport_, SendRtp).Times(1);
+ ProcessNextFrame();
+ ProcessNextFrame();
+ auto used_rate = channel_->GetUsedRate();
+ EXPECT_GT(used_rate.value().bps(), 0);
+}
+
+// Ensure that GetUsedRate returns value with one coded frame.
+TEST_F(ChannelSendTest, UsedRateIsLargerofLastTwoFrames) {
+ channel_->StartSend();
+ channel_->CallEncoder(
+ [&](AudioEncoder* encoder) { encoder->OnReceivedOverhead(72); });
+ DataRate lowrate = DataRate::BitsPerSec(40000);
+ DataRate highrate = DataRate::BitsPerSec(80000);
+ BitrateAllocationUpdate update;
+ update.bwe_period = TimeDelta::Millis(100);
+
+ update.target_bitrate = lowrate;
+ channel_->OnBitrateAllocation(update);
+ EXPECT_CALL(transport_, SendRtp).Times(1);
+ ProcessNextFrame();
+ ProcessNextFrame();
+ // Last two frames have rates [32kbps, -], yielding 32kbps.
+ auto used_rate_1 = channel_->GetUsedRate();
+
+ update.target_bitrate = highrate;
+ channel_->OnBitrateAllocation(update);
+ EXPECT_CALL(transport_, SendRtp).Times(1);
+ ProcessNextFrame();
+ ProcessNextFrame();
+ // Last two frames have rates [54kbps, 32kbps], yielding 54kbps
+ auto used_rate_2 = channel_->GetUsedRate();
+
+ update.target_bitrate = lowrate;
+ channel_->OnBitrateAllocation(update);
+ EXPECT_CALL(transport_, SendRtp).Times(1);
+ ProcessNextFrame();
+ ProcessNextFrame();
+ // Last two frames have rates [32kbps 54kbps], yielding 54kbps
+ auto used_rate_3 = channel_->GetUsedRate();
+
+ EXPECT_GT(used_rate_2, used_rate_1);
+ EXPECT_EQ(used_rate_3, used_rate_2);
+}
+
} // namespace
} // namespace voe
} // namespace webrtc
diff --git a/audio/mock_voe_channel_proxy.h b/audio/mock_voe_channel_proxy.h
index 5836d88..607070a 100644
--- a/audio/mock_voe_channel_proxy.h
+++ b/audio/mock_voe_channel_proxy.h
@@ -182,6 +182,11 @@
SetEncoderToPacketizerFrameTransformer,
(rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer),
(override));
+ MOCK_METHOD(absl::optional<DataRate>, GetUsedRate, (), (const, override));
+ MOCK_METHOD(void,
+ RegisterPacketOverhead,
+ (int packet_byte_overhead),
+ (override));
};
} // namespace test
} // namespace webrtc
