Remove typing detection
Motivation: never used.
Follow-up CL of https://webrtc-review.googlesource.com/c/src/+/250680.
Tested on a custom chromium build by making a video call and with
keyboard activity. The expected logs from the APM transient suppressor
sub-module were shown.
Bug: webrtc:11226
Change-Id: I4186994412dd8ba2e71ed8f9dcc9cf8f8e40fbd3
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/250667
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Per Kjellander <perkj@webrtc.org>
Reviewed-by: Björn Terelius <terelius@webrtc.org>
Reviewed-by: Gustaf Ullberg <gustaf@webrtc.org>
Commit-Queue: Alessio Bazzica <alessiob@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#36300}
diff --git a/api/audio_options.cc b/api/audio_options.cc
index fad35cb..3a70bc6 100644
--- a/api/audio_options.cc
+++ b/api/audio_options.cc
@@ -54,7 +54,6 @@
change.audio_jitter_buffer_min_delay_ms);
SetFrom(&audio_jitter_buffer_enable_rtx_handling,
change.audio_jitter_buffer_enable_rtx_handling);
- SetFrom(&typing_detection, change.typing_detection);
SetFrom(&residual_echo_detector, change.residual_echo_detector);
SetFrom(&combined_audio_video_bwe, change.combined_audio_video_bwe);
SetFrom(&audio_network_adaptor, change.audio_network_adaptor);
@@ -78,7 +77,6 @@
o.audio_jitter_buffer_min_delay_ms &&
audio_jitter_buffer_enable_rtx_handling ==
o.audio_jitter_buffer_enable_rtx_handling &&
- typing_detection == o.typing_detection &&
residual_echo_detector == o.residual_echo_detector &&
combined_audio_video_bwe == o.combined_audio_video_bwe &&
audio_network_adaptor == o.audio_network_adaptor &&
@@ -107,7 +105,6 @@
audio_jitter_buffer_min_delay_ms);
ToStringIfSet(&result, "audio_jitter_buffer_enable_rtx_handling",
audio_jitter_buffer_enable_rtx_handling);
- ToStringIfSet(&result, "typing", typing_detection);
ToStringIfSet(&result, "residual_echo_detector", residual_echo_detector);
ToStringIfSet(&result, "combined_audio_video_bwe", combined_audio_video_bwe);
ToStringIfSet(&result, "audio_network_adaptor", audio_network_adaptor);
diff --git a/api/audio_options.h b/api/audio_options.h
index 3fcc38d..b9f99df 100644
--- a/api/audio_options.h
+++ b/api/audio_options.h
@@ -60,10 +60,6 @@
absl::optional<int> audio_jitter_buffer_min_delay_ms;
// Audio receiver jitter buffer (NetEq) should handle retransmitted packets.
absl::optional<bool> audio_jitter_buffer_enable_rtx_handling;
- // Deprecated.
- // TODO(bugs.webrtc.org/11226): Remove.
- // Audio processing to detect typing.
- absl::optional<bool> typing_detection;
// TODO(bugs.webrtc.org/11539): Deprecated, replaced by
// webrtc::CreateEchoDetector() and injection when creating the audio
// processing module.
diff --git a/api/stats_types.cc b/api/stats_types.cc
index b044e4a..8f69e5f 100644
--- a/api/stats_types.cc
+++ b/api/stats_types.cc
@@ -648,9 +648,6 @@
return "googTrackId";
case kStatsValueNameTimingFrameInfo:
return "googTimingFrameInfo";
- // TODO(bugs.webrtc.org/11226): Remove.
- case kStatsValueNameTypingNoiseState:
- return "googTypingNoiseState";
case kStatsValueNameWritable:
return "googWritable";
case kStatsValueNameAudioDeviceUnderrunCounter:
diff --git a/api/stats_types.h b/api/stats_types.h
index e7dd528..d75da46 100644
--- a/api/stats_types.h
+++ b/api/stats_types.h
@@ -235,8 +235,6 @@
kStatsValueNameTrackId,
kStatsValueNameTransmitBitrate,
kStatsValueNameTransportType,
- // TODO(bugs.webrtc.org/11226): Remove.
- kStatsValueNameTypingNoiseState,
kStatsValueNameWritable,
kStatsValueNameAudioDeviceUnderrunCounter,
kStatsValueNameLocalCandidateRelayProtocol,
diff --git a/audio/audio_send_stream.cc b/audio/audio_send_stream.cc
index 6aa0ec9..c291534 100644
--- a/audio/audio_send_stream.cc
+++ b/audio/audio_send_stream.cc
@@ -475,7 +475,6 @@
stats.total_input_duration = audio_level_.TotalDuration();
}
- stats.typing_noise_detected = audio_state()->typing_noise_detected();
stats.ana_statistics = channel_send_->GetANAStatistics();
AudioProcessing* ap = audio_state_->audio_processing();
diff --git a/audio/audio_send_stream_unittest.cc b/audio/audio_send_stream_unittest.cc
index 2d2e64c..0ec9964 100644
--- a/audio/audio_send_stream_unittest.cc
+++ b/audio/audio_send_stream_unittest.cc
@@ -472,7 +472,6 @@
stats.apm_statistics.residual_echo_likelihood);
EXPECT_EQ(kResidualEchoLikelihoodMax,
stats.apm_statistics.residual_echo_likelihood_recent_max);
- EXPECT_FALSE(stats.typing_noise_detected);
}
}
}
diff --git a/audio/audio_state.cc b/audio/audio_state.cc
index 9e5b63b..df8ce8a 100644
--- a/audio/audio_state.cc
+++ b/audio/audio_state.cc
@@ -50,11 +50,6 @@
return &audio_transport_;
}
-bool AudioState::typing_noise_detected() const {
- RTC_DCHECK(thread_checker_.IsCurrent());
- return audio_transport_.typing_noise_detected();
-}
-
void AudioState::AddReceivingStream(webrtc::AudioReceiveStream* stream) {
RTC_DCHECK(thread_checker_.IsCurrent());
RTC_DCHECK_EQ(0, receiving_streams_.count(stream));
diff --git a/audio/audio_state.h b/audio/audio_state.h
index 55f3551..b8ef4fd 100644
--- a/audio/audio_state.h
+++ b/audio/audio_state.h
@@ -51,8 +51,6 @@
return config_.audio_device_module.get();
}
- bool typing_noise_detected() const;
-
void AddReceivingStream(webrtc::AudioReceiveStream* stream);
void RemoveReceivingStream(webrtc::AudioReceiveStream* stream);
diff --git a/audio/audio_transport_impl.h b/audio/audio_transport_impl.h
index 8999956..ba067de 100644
--- a/audio/audio_transport_impl.h
+++ b/audio/audio_transport_impl.h
@@ -20,7 +20,6 @@
#include "modules/async_audio_processing/async_audio_processing.h"
#include "modules/audio_device/include/audio_device.h"
#include "modules/audio_processing/include/audio_processing.h"
-#include "modules/audio_processing/typing_detection.h"
#include "rtc_base/synchronization/mutex.h"
#include "rtc_base/thread_annotations.h"
@@ -86,9 +85,6 @@
int send_sample_rate_hz,
size_t send_num_channels);
void SetStereoChannelSwapping(bool enable);
- // Deprecated.
- // TODO(bugs.webrtc.org/11226): Remove.
- bool typing_noise_detected() const { return false; }
private:
void SendProcessedData(std::unique_ptr<AudioFrame> audio_frame);
@@ -107,7 +103,6 @@
size_t send_num_channels_ RTC_GUARDED_BY(capture_lock_) = 1;
bool swap_stereo_channels_ RTC_GUARDED_BY(capture_lock_) = false;
PushResampler<int16_t> capture_resampler_;
- TypingDetection typing_detection_;
// Render side.
diff --git a/audio/test/audio_stats_test.cc b/audio/test/audio_stats_test.cc
index ea33270..8f599b0 100644
--- a/audio/test/audio_stats_test.cc
+++ b/audio/test/audio_stats_test.cc
@@ -63,7 +63,6 @@
EXPECT_FALSE(send_stats.apm_statistics.echo_return_loss_enhancement);
EXPECT_FALSE(send_stats.apm_statistics.residual_echo_likelihood);
EXPECT_FALSE(send_stats.apm_statistics.residual_echo_likelihood_recent_max);
- EXPECT_EQ(false, send_stats.typing_noise_detected);
AudioReceiveStream::Stats recv_stats =
receive_stream()->GetStats(/*get_and_clear_legacy_stats=*/true);
diff --git a/call/audio_send_stream.h b/call/audio_send_stream.h
index e38a47f..25d086b 100644
--- a/call/audio_send_stream.h
+++ b/call/audio_send_stream.h
@@ -59,7 +59,6 @@
// https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalaudioenergy
double total_input_energy = 0.0;
double total_input_duration = 0.0;
- bool typing_noise_detected = false;
ANAStats ana_statistics;
AudioProcessingStats apm_statistics;
diff --git a/media/base/media_channel.h b/media/base/media_channel.h
index 2b0ef81..7da3fbe 100644
--- a/media/base/media_channel.h
+++ b/media/base/media_channel.h
@@ -465,7 +465,6 @@
// https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalaudioenergy
double total_input_energy = 0.0;
double total_input_duration = 0.0;
- bool typing_noise_detected = false;
webrtc::ANAStats ana_statistics;
webrtc::AudioProcessingStats apm_statistics;
};
diff --git a/media/engine/webrtc_voice_engine.cc b/media/engine/webrtc_voice_engine.cc
index a229658..54d1963 100644
--- a/media/engine/webrtc_voice_engine.cc
+++ b/media/engine/webrtc_voice_engine.cc
@@ -393,10 +393,8 @@
#if defined(WEBRTC_IOS)
// On iOS, VPIO provides built-in NS.
options.noise_suppression = false;
- options.typing_detection = false;
#else
options.noise_suppression = true;
- options.typing_detection = true;
#endif
options.highpass_filter = true;
options.stereo_swapping = false;
@@ -453,11 +451,6 @@
use_mobile_software_aec = true;
#endif
-// Override noise suppression options for Android.
-#if defined(WEBRTC_ANDROID)
- options.typing_detection = false;
-#endif
-
// Set and adjust gain control options.
#if defined(WEBRTC_IOS)
// On iOS, VPIO provides built-in AGC.
@@ -602,10 +595,6 @@
RTC_LOG(LS_INFO) << "NS set to " << enabled;
}
- if (options.typing_detection) {
- RTC_LOG(LS_WARNING) << "Typing detection is requested, but unsupported.";
- }
-
ap->ApplyConfig(apm_config);
return true;
}
@@ -2339,7 +2328,6 @@
sinfo.audio_level = stats.audio_level;
sinfo.total_input_energy = stats.total_input_energy;
sinfo.total_input_duration = stats.total_input_duration;
- sinfo.typing_noise_detected = (send_ ? stats.typing_noise_detected : false);
sinfo.ana_statistics = stats.ana_statistics;
sinfo.apm_statistics = stats.apm_statistics;
sinfo.report_block_datas = std::move(stats.report_block_datas);
diff --git a/media/engine/webrtc_voice_engine_unittest.cc b/media/engine/webrtc_voice_engine_unittest.cc
index 6f93829..0bf35c7 100644
--- a/media/engine/webrtc_voice_engine_unittest.cc
+++ b/media/engine/webrtc_voice_engine_unittest.cc
@@ -606,7 +606,6 @@
stats.ana_statistics.frame_length_increase_counter = 765;
stats.ana_statistics.frame_length_decrease_counter = 876;
stats.ana_statistics.uplink_packet_loss_fraction = 987.0;
- stats.typing_noise_detected = true;
return stats;
}
void SetAudioSendStreamStats() {
@@ -655,8 +654,6 @@
stats.ana_statistics.frame_length_decrease_counter);
EXPECT_EQ(info.ana_statistics.uplink_packet_loss_fraction,
stats.ana_statistics.uplink_packet_loss_fraction);
- EXPECT_EQ(info.typing_noise_detected,
- stats.typing_noise_detected && is_sending);
}
webrtc::AudioReceiveStream::Stats GetAudioReceiveStreamStats() const {
diff --git a/modules/audio_processing/BUILD.gn b/modules/audio_processing/BUILD.gn
index ee6b579..9179383 100644
--- a/modules/audio_processing/BUILD.gn
+++ b/modules/audio_processing/BUILD.gn
@@ -151,8 +151,6 @@
"gain_control_impl.cc",
"gain_control_impl.h",
"render_queue_item_verifier.h",
- "typing_detection.cc",
- "typing_detection.h",
]
defines = []
diff --git a/modules/audio_processing/typing_detection.cc b/modules/audio_processing/typing_detection.cc
deleted file mode 100644
index e725b26..0000000
--- a/modules/audio_processing/typing_detection.cc
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#include "modules/audio_processing/typing_detection.h"
-
-namespace webrtc {
-
-TypingDetection::TypingDetection()
- : time_active_(0),
- time_since_last_typing_(0),
- penalty_counter_(0),
- counter_since_last_detection_update_(0),
- detection_to_report_(false),
- new_detection_to_report_(false),
- time_window_(10),
- cost_per_typing_(100),
- reporting_threshold_(300),
- penalty_decay_(1),
- type_event_delay_(2),
- report_detection_update_period_(1) {}
-
-TypingDetection::~TypingDetection() {}
-
-bool TypingDetection::Process(bool key_pressed, bool vad_activity) {
- if (vad_activity)
- time_active_++;
- else
- time_active_ = 0;
-
- // Keep track if time since last typing event
- if (key_pressed)
- time_since_last_typing_ = 0;
- else
- ++time_since_last_typing_;
-
- if (time_since_last_typing_ < type_event_delay_ && vad_activity &&
- time_active_ < time_window_) {
- penalty_counter_ += cost_per_typing_;
- if (penalty_counter_ > reporting_threshold_)
- new_detection_to_report_ = true;
- }
-
- if (penalty_counter_ > 0)
- penalty_counter_ -= penalty_decay_;
-
- if (++counter_since_last_detection_update_ ==
- report_detection_update_period_) {
- detection_to_report_ = new_detection_to_report_;
- new_detection_to_report_ = false;
- counter_since_last_detection_update_ = 0;
- }
-
- return detection_to_report_;
-}
-
-int TypingDetection::TimeSinceLastDetectionInSeconds() {
- // Round to whole seconds.
- return (time_since_last_typing_ + 50) / 100;
-}
-
-void TypingDetection::SetParameters(int time_window,
- int cost_per_typing,
- int reporting_threshold,
- int penalty_decay,
- int type_event_delay,
- int report_detection_update_period) {
- if (time_window)
- time_window_ = time_window;
-
- if (cost_per_typing)
- cost_per_typing_ = cost_per_typing;
-
- if (reporting_threshold)
- reporting_threshold_ = reporting_threshold;
-
- if (penalty_decay)
- penalty_decay_ = penalty_decay;
-
- if (type_event_delay)
- type_event_delay_ = type_event_delay;
-
- if (report_detection_update_period)
- report_detection_update_period_ = report_detection_update_period;
-}
-
-} // namespace webrtc
diff --git a/modules/audio_processing/typing_detection.h b/modules/audio_processing/typing_detection.h
deleted file mode 100644
index 9d96583..0000000
--- a/modules/audio_processing/typing_detection.h
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#ifndef MODULES_AUDIO_PROCESSING_TYPING_DETECTION_H_
-#define MODULES_AUDIO_PROCESSING_TYPING_DETECTION_H_
-
-#include "rtc_base/system/rtc_export.h"
-
-namespace webrtc {
-
-class RTC_EXPORT TypingDetection {
- public:
- TypingDetection();
- virtual ~TypingDetection();
-
- // Run the detection algortihm. Shall be called every 10 ms. Returns true if
- // typing is detected, or false if not, based on the update period as set with
- // SetParameters(). See `report_detection_update_period_` description below.
- bool Process(bool key_pressed, bool vad_activity);
-
- // Gets the time in seconds since the last detection.
- int TimeSinceLastDetectionInSeconds();
-
- // Sets the algorithm parameters. A parameter value of 0 leaves it unchanged.
- // See the correspondning member variables below for descriptions.
- void SetParameters(int time_window,
- int cost_per_typing,
- int reporting_threshold,
- int penalty_decay,
- int type_event_delay,
- int report_detection_update_period);
-
- private:
- int time_active_;
- int time_since_last_typing_;
- int penalty_counter_;
-
- // Counter since last time the detection status reported by Process() was
- // updated. See also `report_detection_update_period_`.
- int counter_since_last_detection_update_;
-
- // The detection status to report. Updated every
- // `report_detection_update_period_` call to Process().
- bool detection_to_report_;
-
- // What `detection_to_report_` should be set to next time it is updated.
- bool new_detection_to_report_;
-
- // Settable threshold values.
-
- // Number of 10 ms slots accepted to count as a hit.
- int time_window_;
-
- // Penalty added for a typing + activity coincide.
- int cost_per_typing_;
-
- // Threshold for `penalty_counter_`.
- int reporting_threshold_;
-
- // How much we reduce `penalty_counter_` every 10 ms.
- int penalty_decay_;
-
- // How old typing events we allow.
- int type_event_delay_;
-
- // Settable update period.
-
- // Number of 10 ms slots between each update of the detection status returned
- // by Process(). This inertia added to the algorithm is usually desirable and
- // provided so that consumers of the class don't have to implement that
- // themselves if they don't wish.
- // If set to 1, each call to Process() will return the detection status for
- // that 10 ms slot.
- // If set to N (where N > 1), the detection status returned from Process()
- // will remain the same until Process() has been called N times. Then, if none
- // of the last N calls to Process() has detected typing for each respective
- // 10 ms slot, Process() will return false. If at least one of the last N
- // calls has detected typing, Process() will return true. And that returned
- // status will then remain the same until the next N calls have been done.
- int report_detection_update_period_;
-};
-
-} // namespace webrtc
-
-#endif // #ifndef MODULES_AUDIO_PROCESSING_TYPING_DETECTION_H_
diff --git a/pc/stats_collector.cc b/pc/stats_collector.cc
index 927e993..44873fa 100644
--- a/pc/stats_collector.cc
+++ b/pc/stats_collector.cc
@@ -143,10 +143,7 @@
}
void SetAudioProcessingStats(StatsReport* report,
- bool typing_noise_detected,
const AudioProcessingStats& apm_stats) {
- report->AddBoolean(StatsReport::kStatsValueNameTypingNoiseState,
- typing_noise_detected);
if (apm_stats.delay_median_ms) {
report->AddInt(StatsReport::kStatsValueNameEchoDelayMedian,
*apm_stats.delay_median_ms);
@@ -245,8 +242,7 @@
bool use_standard_bytes_stats) {
ExtractCommonSendProperties(info, report, use_standard_bytes_stats);
- SetAudioProcessingStats(report, info.typing_noise_detected,
- info.apm_statistics);
+ SetAudioProcessingStats(report, info.apm_statistics);
const FloatForAdd floats[] = {
{StatsReport::kStatsValueNameTotalAudioEnergy, info.total_input_energy},
@@ -1354,8 +1350,7 @@
AudioProcessorInterface::AudioProcessorStatistics stats =
audio_processor->GetStats(has_remote_tracks);
- SetAudioProcessingStats(report, stats.typing_noise_detected,
- stats.apm_statistics);
+ SetAudioProcessingStats(report, stats.apm_statistics);
}
}
diff --git a/pc/stats_collector_unittest.cc b/pc/stats_collector_unittest.cc
index 144ca34..61d0953 100644
--- a/pc/stats_collector_unittest.cc
+++ b/pc/stats_collector_unittest.cc
@@ -89,7 +89,6 @@
AudioProcessorInterface::AudioProcessorStatistics GetStats(
bool has_recv_streams) override {
AudioProcessorStatistics stats;
- stats.typing_noise_detected = true;
if (has_recv_streams) {
stats.apm_statistics.echo_return_loss = 2.0;
stats.apm_statistics.echo_return_loss_enhancement = 3.0;
@@ -132,7 +131,6 @@
AudioProcessorInterface::AudioProcessorStatistics GetStats(
bool /*has_recv_streams*/) override {
AudioProcessorStatistics stats;
- stats.typing_noise_detected = false;
return stats;
}
};
@@ -488,10 +486,6 @@
EXPECT_TRUE(GetValue(report, StatsReport::kStatsValueNameAudioInputLevel,
&value_in_report));
EXPECT_EQ(rtc::ToString(sinfo.audio_level), value_in_report);
- EXPECT_TRUE(GetValue(report, StatsReport::kStatsValueNameTypingNoiseState,
- &value_in_report));
- std::string typing_detected = sinfo.typing_noise_detected ? "true" : "false";
- EXPECT_EQ(typing_detected, value_in_report);
EXPECT_TRUE(GetValue(report,
StatsReport::kStatsValueNameAnaBitrateActionCounter,
&value_in_report));
@@ -551,7 +545,6 @@
voice_sender_info->apm_statistics.echo_return_loss_enhancement = 109;
voice_sender_info->apm_statistics.delay_median_ms = 110;
voice_sender_info->apm_statistics.delay_standard_deviation_ms = 111;
- voice_sender_info->typing_noise_detected = false;
voice_sender_info->ana_statistics.bitrate_action_counter = 112;
voice_sender_info->ana_statistics.channel_action_counter = 113;
voice_sender_info->ana_statistics.dtx_action_counter = 114;
@@ -568,8 +561,6 @@
audio_track->GetSignalLevel(&voice_sender_info->audio_level);
AudioProcessorInterface::AudioProcessorStatistics audio_processor_stats =
audio_track->GetAudioProcessor()->GetStats(has_remote_tracks);
- voice_sender_info->typing_noise_detected =
- audio_processor_stats.typing_noise_detected;
voice_sender_info->apm_statistics = audio_processor_stats.apm_statistics;
}
