Remove typing detection

Motivation: never used.
Follow-up CL of https://webrtc-review.googlesource.com/c/src/+/250680.

Tested on a custom chromium build by making a video call and with
keyboard activity. The expected logs from the APM transient suppressor
sub-module were shown.

Bug: webrtc:11226
Change-Id: I4186994412dd8ba2e71ed8f9dcc9cf8f8e40fbd3
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/250667
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Per Kjellander <perkj@webrtc.org>
Reviewed-by: Björn Terelius <terelius@webrtc.org>
Reviewed-by: Gustaf Ullberg <gustaf@webrtc.org>
Commit-Queue: Alessio Bazzica <alessiob@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#36300}

api/audio_options.cc

@@ -54,7 +54,6 @@ void AudioOptions::SetAll(const AudioOptions& change) {
           change.audio_jitter_buffer_min_delay_ms);
   SetFrom(&audio_jitter_buffer_enable_rtx_handling,
           change.audio_jitter_buffer_enable_rtx_handling);
-  SetFrom(&typing_detection, change.typing_detection);
   SetFrom(&residual_echo_detector, change.residual_echo_detector);
   SetFrom(&combined_audio_video_bwe, change.combined_audio_video_bwe);
   SetFrom(&audio_network_adaptor, change.audio_network_adaptor);
@@ -78,7 +77,6 @@ bool AudioOptions::operator==(const AudioOptions& o) const {
              o.audio_jitter_buffer_min_delay_ms &&
          audio_jitter_buffer_enable_rtx_handling ==
              o.audio_jitter_buffer_enable_rtx_handling &&
-         typing_detection == o.typing_detection &&
          residual_echo_detector == o.residual_echo_detector &&
          combined_audio_video_bwe == o.combined_audio_video_bwe &&
          audio_network_adaptor == o.audio_network_adaptor &&
@@ -107,7 +105,6 @@ std::string AudioOptions::ToString() const {
                 audio_jitter_buffer_min_delay_ms);
   ToStringIfSet(&result, "audio_jitter_buffer_enable_rtx_handling",
                 audio_jitter_buffer_enable_rtx_handling);
-  ToStringIfSet(&result, "typing", typing_detection);
   ToStringIfSet(&result, "residual_echo_detector", residual_echo_detector);
   ToStringIfSet(&result, "combined_audio_video_bwe", combined_audio_video_bwe);
   ToStringIfSet(&result, "audio_network_adaptor", audio_network_adaptor);

api/audio_options.h

@@ -60,10 +60,6 @@ struct RTC_EXPORT AudioOptions {
   absl::optional<int> audio_jitter_buffer_min_delay_ms;
   // Audio receiver jitter buffer (NetEq) should handle retransmitted packets.
   absl::optional<bool> audio_jitter_buffer_enable_rtx_handling;
-  // Deprecated.
-  // TODO(bugs.webrtc.org/11226): Remove.
-  // Audio processing to detect typing.
-  absl::optional<bool> typing_detection;
   // TODO(bugs.webrtc.org/11539): Deprecated, replaced by
   // webrtc::CreateEchoDetector() and injection when creating the audio
   // processing module.

api/stats_types.cc

@@ -648,9 +648,6 @@ const char* StatsReport::Value::display_name() const {
       return "googTrackId";
     case kStatsValueNameTimingFrameInfo:
       return "googTimingFrameInfo";
-    // TODO(bugs.webrtc.org/11226): Remove.
-    case kStatsValueNameTypingNoiseState:
-      return "googTypingNoiseState";
     case kStatsValueNameWritable:
       return "googWritable";
     case kStatsValueNameAudioDeviceUnderrunCounter:

api/stats_types.h

@@ -235,8 +235,6 @@ class RTC_EXPORT StatsReport {
     kStatsValueNameTrackId,
     kStatsValueNameTransmitBitrate,
     kStatsValueNameTransportType,
-    // TODO(bugs.webrtc.org/11226): Remove.
-    kStatsValueNameTypingNoiseState,
     kStatsValueNameWritable,
     kStatsValueNameAudioDeviceUnderrunCounter,
     kStatsValueNameLocalCandidateRelayProtocol,

audio/audio_send_stream.cc

@@ -475,7 +475,6 @@ webrtc::AudioSendStream::Stats AudioSendStream::GetStats(
     stats.total_input_duration = audio_level_.TotalDuration();
   }
 
-  stats.typing_noise_detected = audio_state()->typing_noise_detected();
   stats.ana_statistics = channel_send_->GetANAStatistics();
 
   AudioProcessing* ap = audio_state_->audio_processing();

audio/audio_send_stream_unittest.cc

@@ -472,7 +472,6 @@ TEST(AudioSendStreamTest, GetStats) {
                 stats.apm_statistics.residual_echo_likelihood);
       EXPECT_EQ(kResidualEchoLikelihoodMax,
                 stats.apm_statistics.residual_echo_likelihood_recent_max);
-      EXPECT_FALSE(stats.typing_noise_detected);
     }
   }
 }

audio/audio_state.cc

@@ -50,11 +50,6 @@ AudioTransport* AudioState::audio_transport() {
   return &audio_transport_;
 }
 
-bool AudioState::typing_noise_detected() const {
-  RTC_DCHECK(thread_checker_.IsCurrent());
-  return audio_transport_.typing_noise_detected();
-}
-
 void AudioState::AddReceivingStream(webrtc::AudioReceiveStream* stream) {
   RTC_DCHECK(thread_checker_.IsCurrent());
   RTC_DCHECK_EQ(0, receiving_streams_.count(stream));

audio/audio_state.h

@@ -51,8 +51,6 @@ class AudioState : public webrtc::AudioState {
     return config_.audio_device_module.get();
   }
 
-  bool typing_noise_detected() const;
-
   void AddReceivingStream(webrtc::AudioReceiveStream* stream);
   void RemoveReceivingStream(webrtc::AudioReceiveStream* stream);
 

audio/audio_transport_impl.h

@@ -20,7 +20,6 @@
 #include "modules/async_audio_processing/async_audio_processing.h"
 #include "modules/audio_device/include/audio_device.h"
 #include "modules/audio_processing/include/audio_processing.h"
-#include "modules/audio_processing/typing_detection.h"
 #include "rtc_base/synchronization/mutex.h"
 #include "rtc_base/thread_annotations.h"
 
@@ -86,9 +85,6 @@ class AudioTransportImpl : public AudioTransport {
                           int send_sample_rate_hz,
                           size_t send_num_channels);
   void SetStereoChannelSwapping(bool enable);
-  // Deprecated.
-  // TODO(bugs.webrtc.org/11226): Remove.
-  bool typing_noise_detected() const { return false; }
 
  private:
   void SendProcessedData(std::unique_ptr<AudioFrame> audio_frame);
@@ -107,7 +103,6 @@ class AudioTransportImpl : public AudioTransport {
   size_t send_num_channels_ RTC_GUARDED_BY(capture_lock_) = 1;
   bool swap_stereo_channels_ RTC_GUARDED_BY(capture_lock_) = false;
   PushResampler<int16_t> capture_resampler_;
-  TypingDetection typing_detection_;
 
   // Render side.
 

audio/test/audio_stats_test.cc

@@ -63,7 +63,6 @@ class NoLossTest : public AudioEndToEndTest {
     EXPECT_FALSE(send_stats.apm_statistics.echo_return_loss_enhancement);
     EXPECT_FALSE(send_stats.apm_statistics.residual_echo_likelihood);
     EXPECT_FALSE(send_stats.apm_statistics.residual_echo_likelihood_recent_max);
-    EXPECT_EQ(false, send_stats.typing_noise_detected);
 
     AudioReceiveStream::Stats recv_stats =
         receive_stream()->GetStats(/*get_and_clear_legacy_stats=*/true);

call/audio_send_stream.h

@@ -59,7 +59,6 @@ class AudioSendStream : public AudioSender {
     // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalaudioenergy
     double total_input_energy = 0.0;
     double total_input_duration = 0.0;
-    bool typing_noise_detected = false;
 
     ANAStats ana_statistics;
     AudioProcessingStats apm_statistics;

media/base/media_channel.h

@@ -465,7 +465,6 @@ struct VoiceSenderInfo : public MediaSenderInfo {
   // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalaudioenergy
   double total_input_energy = 0.0;
   double total_input_duration = 0.0;
-  bool typing_noise_detected = false;
   webrtc::ANAStats ana_statistics;
   webrtc::AudioProcessingStats apm_statistics;
 };

media/engine/webrtc_voice_engine.cc

@@ -393,10 +393,8 @@ void WebRtcVoiceEngine::Init() {
 #if defined(WEBRTC_IOS)
     // On iOS, VPIO provides built-in NS.
     options.noise_suppression = false;
-    options.typing_detection = false;
 #else
     options.noise_suppression = true;
-    options.typing_detection = true;
 #endif
     options.highpass_filter = true;
     options.stereo_swapping = false;
@@ -453,11 +451,6 @@ bool WebRtcVoiceEngine::ApplyOptions(const AudioOptions& options_in) {
   use_mobile_software_aec = true;
 #endif
 
-// Override noise suppression options for Android.
-#if defined(WEBRTC_ANDROID)
-  options.typing_detection = false;
-#endif
-
 // Set and adjust gain control options.
 #if defined(WEBRTC_IOS)
   // On iOS, VPIO provides built-in AGC.
@@ -602,10 +595,6 @@ bool WebRtcVoiceEngine::ApplyOptions(const AudioOptions& options_in) {
     RTC_LOG(LS_INFO) << "NS set to " << enabled;
   }
 
-  if (options.typing_detection) {
-    RTC_LOG(LS_WARNING) << "Typing detection is requested, but unsupported.";
-  }
-
   ap->ApplyConfig(apm_config);
   return true;
 }
@@ -2339,7 +2328,6 @@ bool WebRtcVoiceMediaChannel::GetStats(VoiceMediaInfo* info,
     sinfo.audio_level = stats.audio_level;
     sinfo.total_input_energy = stats.total_input_energy;
     sinfo.total_input_duration = stats.total_input_duration;
-    sinfo.typing_noise_detected = (send_ ? stats.typing_noise_detected : false);
     sinfo.ana_statistics = stats.ana_statistics;
     sinfo.apm_statistics = stats.apm_statistics;
     sinfo.report_block_datas = std::move(stats.report_block_datas);

media/engine/webrtc_voice_engine_unittest.cc

@@ -606,7 +606,6 @@ class WebRtcVoiceEngineTestFake : public ::testing::TestWithParam<bool> {
     stats.ana_statistics.frame_length_increase_counter = 765;
     stats.ana_statistics.frame_length_decrease_counter = 876;
     stats.ana_statistics.uplink_packet_loss_fraction = 987.0;
-    stats.typing_noise_detected = true;
     return stats;
   }
   void SetAudioSendStreamStats() {
@@ -655,8 +654,6 @@ class WebRtcVoiceEngineTestFake : public ::testing::TestWithParam<bool> {
               stats.ana_statistics.frame_length_decrease_counter);
     EXPECT_EQ(info.ana_statistics.uplink_packet_loss_fraction,
               stats.ana_statistics.uplink_packet_loss_fraction);
-    EXPECT_EQ(info.typing_noise_detected,
-              stats.typing_noise_detected && is_sending);
   }
 
   webrtc::AudioReceiveStream::Stats GetAudioReceiveStreamStats() const {

modules/audio_processing/BUILD.gn

@@ -151,8 +151,6 @@ rtc_library("audio_processing") {
     "gain_control_impl.cc",
     "gain_control_impl.h",
     "render_queue_item_verifier.h",
-    "typing_detection.cc",
-    "typing_detection.h",
   ]
 
   defines = []

modules/audio_processing/typing_detection.cc

@@ -1,93 +0,0 @@
-/*
- *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
- *
- *  Use of this source code is governed by a BSD-style license
- *  that can be found in the LICENSE file in the root of the source
- *  tree. An additional intellectual property rights grant can be found
- *  in the file PATENTS.  All contributing project authors may
- *  be found in the AUTHORS file in the root of the source tree.
- */
-
-#include "modules/audio_processing/typing_detection.h"
-
-namespace webrtc {
-
-TypingDetection::TypingDetection()
-    : time_active_(0),
-      time_since_last_typing_(0),
-      penalty_counter_(0),
-      counter_since_last_detection_update_(0),
-      detection_to_report_(false),
-      new_detection_to_report_(false),
-      time_window_(10),
-      cost_per_typing_(100),
-      reporting_threshold_(300),
-      penalty_decay_(1),
-      type_event_delay_(2),
-      report_detection_update_period_(1) {}
-
-TypingDetection::~TypingDetection() {}
-
-bool TypingDetection::Process(bool key_pressed, bool vad_activity) {
-  if (vad_activity)
-    time_active_++;
-  else
-    time_active_ = 0;
-
-  // Keep track if time since last typing event
-  if (key_pressed)
-    time_since_last_typing_ = 0;
-  else
-    ++time_since_last_typing_;
-
-  if (time_since_last_typing_ < type_event_delay_ && vad_activity &&
-      time_active_ < time_window_) {
-    penalty_counter_ += cost_per_typing_;
-    if (penalty_counter_ > reporting_threshold_)
-      new_detection_to_report_ = true;
-  }
-
-  if (penalty_counter_ > 0)
-    penalty_counter_ -= penalty_decay_;
-
-  if (++counter_since_last_detection_update_ ==
-      report_detection_update_period_) {
-    detection_to_report_ = new_detection_to_report_;
-    new_detection_to_report_ = false;
-    counter_since_last_detection_update_ = 0;
-  }
-
-  return detection_to_report_;
-}
-
-int TypingDetection::TimeSinceLastDetectionInSeconds() {
-  // Round to whole seconds.
-  return (time_since_last_typing_ + 50) / 100;
-}
-
-void TypingDetection::SetParameters(int time_window,
-                                    int cost_per_typing,
-                                    int reporting_threshold,
-                                    int penalty_decay,
-                                    int type_event_delay,
-                                    int report_detection_update_period) {
-  if (time_window)
-    time_window_ = time_window;
-
-  if (cost_per_typing)
-    cost_per_typing_ = cost_per_typing;
-
-  if (reporting_threshold)
-    reporting_threshold_ = reporting_threshold;
-
-  if (penalty_decay)
-    penalty_decay_ = penalty_decay;
-
-  if (type_event_delay)
-    type_event_delay_ = type_event_delay;
-
-  if (report_detection_update_period)
-    report_detection_update_period_ = report_detection_update_period;
-}
-
-}  // namespace webrtc

modules/audio_processing/typing_detection.h

@@ -1,92 +0,0 @@
-/*
- *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
- *
- *  Use of this source code is governed by a BSD-style license
- *  that can be found in the LICENSE file in the root of the source
- *  tree. An additional intellectual property rights grant can be found
- *  in the file PATENTS.  All contributing project authors may
- *  be found in the AUTHORS file in the root of the source tree.
- */
-
-#ifndef MODULES_AUDIO_PROCESSING_TYPING_DETECTION_H_
-#define MODULES_AUDIO_PROCESSING_TYPING_DETECTION_H_
-
-#include "rtc_base/system/rtc_export.h"
-
-namespace webrtc {
-
-class RTC_EXPORT TypingDetection {
- public:
-  TypingDetection();
-  virtual ~TypingDetection();
-
-  // Run the detection algortihm. Shall be called every 10 ms. Returns true if
-  // typing is detected, or false if not, based on the update period as set with
-  // SetParameters(). See `report_detection_update_period_` description below.
-  bool Process(bool key_pressed, bool vad_activity);
-
-  // Gets the time in seconds since the last detection.
-  int TimeSinceLastDetectionInSeconds();
-
-  // Sets the algorithm parameters. A parameter value of 0 leaves it unchanged.
-  // See the correspondning member variables below for descriptions.
-  void SetParameters(int time_window,
-                     int cost_per_typing,
-                     int reporting_threshold,
-                     int penalty_decay,
-                     int type_event_delay,
-                     int report_detection_update_period);
-
- private:
-  int time_active_;
-  int time_since_last_typing_;
-  int penalty_counter_;
-
-  // Counter since last time the detection status reported by Process() was
-  // updated. See also `report_detection_update_period_`.
-  int counter_since_last_detection_update_;
-
-  // The detection status to report. Updated every
-  // `report_detection_update_period_` call to Process().
-  bool detection_to_report_;
-
-  // What `detection_to_report_` should be set to next time it is updated.
-  bool new_detection_to_report_;
-
-  // Settable threshold values.
-
-  // Number of 10 ms slots accepted to count as a hit.
-  int time_window_;
-
-  // Penalty added for a typing + activity coincide.
-  int cost_per_typing_;
-
-  // Threshold for `penalty_counter_`.
-  int reporting_threshold_;
-
-  // How much we reduce `penalty_counter_` every 10 ms.
-  int penalty_decay_;
-
-  // How old typing events we allow.
-  int type_event_delay_;
-
-  // Settable update period.
-
-  // Number of 10 ms slots between each update of the detection status returned
-  // by Process(). This inertia added to the algorithm is usually desirable and
-  // provided so that consumers of the class don't have to implement that
-  // themselves if they don't wish.
-  // If set to 1, each call to Process() will return the detection status for
-  // that 10 ms slot.
-  // If set to N (where N > 1), the detection status returned from Process()
-  // will remain the same until Process() has been called N times. Then, if none
-  // of the last N calls to Process() has detected typing for each respective
-  // 10 ms slot, Process() will return false. If at least one of the last N
-  // calls has detected typing, Process() will return true. And that returned
-  // status will then remain the same until the next N calls have been done.
-  int report_detection_update_period_;
-};
-
-}  // namespace webrtc
-
-#endif  // #ifndef MODULES_AUDIO_PROCESSING_TYPING_DETECTION_H_

pc/stats_collector.cc

@@ -143,10 +143,7 @@ void ExtractCommonReceiveProperties(const cricket::MediaReceiverInfo& info,
 }
 
 void SetAudioProcessingStats(StatsReport* report,
-                             bool typing_noise_detected,
                              const AudioProcessingStats& apm_stats) {
-  report->AddBoolean(StatsReport::kStatsValueNameTypingNoiseState,
-                     typing_noise_detected);
   if (apm_stats.delay_median_ms) {
     report->AddInt(StatsReport::kStatsValueNameEchoDelayMedian,
                    *apm_stats.delay_median_ms);
@@ -245,8 +242,7 @@ void ExtractStats(const cricket::VoiceSenderInfo& info,
                   bool use_standard_bytes_stats) {
   ExtractCommonSendProperties(info, report, use_standard_bytes_stats);
 
-  SetAudioProcessingStats(report, info.typing_noise_detected,
+  SetAudioProcessingStats(report, info.apm_statistics);
-                          info.apm_statistics);
 
   const FloatForAdd floats[] = {
       {StatsReport::kStatsValueNameTotalAudioEnergy, info.total_input_energy},
@@ -1354,8 +1350,7 @@ void StatsCollector::UpdateReportFromAudioTrack(AudioTrackInterface* track,
     AudioProcessorInterface::AudioProcessorStatistics stats =
         audio_processor->GetStats(has_remote_tracks);
 
-    SetAudioProcessingStats(report, stats.typing_noise_detected,
+    SetAudioProcessingStats(report, stats.apm_statistics);
-                            stats.apm_statistics);
   }
 }
 

pc/stats_collector_unittest.cc

@@ -89,7 +89,6 @@ class FakeAudioProcessor : public AudioProcessorInterface {
   AudioProcessorInterface::AudioProcessorStatistics GetStats(
       bool has_recv_streams) override {
     AudioProcessorStatistics stats;
-    stats.typing_noise_detected = true;
     if (has_recv_streams) {
       stats.apm_statistics.echo_return_loss = 2.0;
       stats.apm_statistics.echo_return_loss_enhancement = 3.0;
@@ -132,7 +131,6 @@ class FakeAudioProcessorWithInitValue : public AudioProcessorInterface {
   AudioProcessorInterface::AudioProcessorStatistics GetStats(
       bool /*has_recv_streams*/) override {
     AudioProcessorStatistics stats;
-    stats.typing_noise_detected = false;
     return stats;
   }
 };
@@ -488,10 +486,6 @@ void VerifyVoiceSenderInfoReport(const StatsReport* report,
   EXPECT_TRUE(GetValue(report, StatsReport::kStatsValueNameAudioInputLevel,
                        &value_in_report));
   EXPECT_EQ(rtc::ToString(sinfo.audio_level), value_in_report);
-  EXPECT_TRUE(GetValue(report, StatsReport::kStatsValueNameTypingNoiseState,
-                       &value_in_report));
-  std::string typing_detected = sinfo.typing_noise_detected ? "true" : "false";
-  EXPECT_EQ(typing_detected, value_in_report);
   EXPECT_TRUE(GetValue(report,
                        StatsReport::kStatsValueNameAnaBitrateActionCounter,
                        &value_in_report));
@@ -551,7 +545,6 @@ void InitVoiceSenderInfo(cricket::VoiceSenderInfo* voice_sender_info,
   voice_sender_info->apm_statistics.echo_return_loss_enhancement = 109;
   voice_sender_info->apm_statistics.delay_median_ms = 110;
   voice_sender_info->apm_statistics.delay_standard_deviation_ms = 111;
-  voice_sender_info->typing_noise_detected = false;
   voice_sender_info->ana_statistics.bitrate_action_counter = 112;
   voice_sender_info->ana_statistics.channel_action_counter = 113;
   voice_sender_info->ana_statistics.dtx_action_counter = 114;
@@ -568,8 +561,6 @@ void UpdateVoiceSenderInfoFromAudioTrack(
   audio_track->GetSignalLevel(&voice_sender_info->audio_level);
   AudioProcessorInterface::AudioProcessorStatistics audio_processor_stats =
       audio_track->GetAudioProcessor()->GetStats(has_remote_tracks);
-  voice_sender_info->typing_noise_detected =
-      audio_processor_stats.typing_noise_detected;
   voice_sender_info->apm_statistics = audio_processor_stats.apm_statistics;
 }