From 3be565b502850f073fbfba2137a3d798464634b9 Mon Sep 17 00:00:00 2001 From: "niklas.enbom@webrtc.org" Date: Tue, 7 May 2013 21:04:24 +0000 Subject: [PATCH] Refactoring for typing detection R=henrika@webrtc.org Review URL: https://webrtc-codereview.appspot.com/1370004 git-svn-id: http://webrtc.googlecode.com/svn/trunk@3976 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../audio_device/audio_device_buffer.cc | 8 +++++ .../audio_device/audio_device_buffer.h | 4 +++ .../include/audio_device_defines.h | 1 + .../audio_device/mac/audio_device_mac.cc | 22 +++++++++++--- .../audio_device/mac/audio_device_mac.h | 3 ++ .../test/audio_device_test_api.cc | 1 + .../audio_device/test/func_test_manager.cc | 1 + .../audio_device/test/func_test_manager.h | 1 + .../audio_device/win/audio_device_core_win.cc | 12 ++++++++ .../audio_device/win/audio_device_core_win.h | 3 ++ .../audio_device/win/audio_device_wave_win.cc | 12 ++++++++ .../audio_device/win/audio_device_wave_win.h | 3 ++ .../system_wrappers/interface/event_wrapper.h | 6 ---- webrtc/system_wrappers/source/event.cc | 29 ------------------- webrtc/voice_engine/transmit_mixer.cc | 14 +++------ webrtc/voice_engine/transmit_mixer.h | 5 ++-- webrtc/voice_engine/voe_base_impl.cc | 3 +- webrtc/voice_engine/voe_base_impl.h | 1 + .../voice_engine/voe_external_media_impl.cc | 3 +- 19 files changed, 79 insertions(+), 53 deletions(-) diff --git a/webrtc/modules/audio_device/audio_device_buffer.cc b/webrtc/modules/audio_device/audio_device_buffer.cc index 12c41a811f..779641dc81 100644 --- a/webrtc/modules/audio_device/audio_device_buffer.cc +++ b/webrtc/modules/audio_device/audio_device_buffer.cc @@ -46,6 +46,7 @@ AudioDeviceBuffer::AudioDeviceBuffer() : _playFile(*FileWrapper::Create()), _currentMicLevel(0), _newMicLevel(0), + _typingStatus(false), _playDelayMS(0), _recDelayMS(0), _clockDrift(0) { @@ -266,6 +267,12 @@ int32_t AudioDeviceBuffer::SetCurrentMicLevel(uint32_t level) return 0; } +int32_t AudioDeviceBuffer::SetTypingStatus(bool typingStatus) +{ + _typingStatus = typingStatus; + return 0; +} + // ---------------------------------------------------------------------------- // NewMicLevel // ---------------------------------------------------------------------------- @@ -469,6 +476,7 @@ int32_t AudioDeviceBuffer::DeliverRecordedData() totalDelayMS, _clockDrift, _currentMicLevel, + _typingStatus, newMicLevel); if (res != -1) { diff --git a/webrtc/modules/audio_device/audio_device_buffer.h b/webrtc/modules/audio_device/audio_device_buffer.h index 67962d69a5..6a02c2be24 100644 --- a/webrtc/modules/audio_device/audio_device_buffer.h +++ b/webrtc/modules/audio_device/audio_device_buffer.h @@ -67,6 +67,8 @@ public: const char fileName[kAdmMaxFileNameSize]); int32_t StopOutputFileRecording(); + int32_t SetTypingStatus(bool typingStatus); + AudioDeviceBuffer(); ~AudioDeviceBuffer(); @@ -110,6 +112,8 @@ private: uint32_t _currentMicLevel; uint32_t _newMicLevel; + bool _typingStatus; + uint32_t _playDelayMS; uint32_t _recDelayMS; diff --git a/webrtc/modules/audio_device/include/audio_device_defines.h b/webrtc/modules/audio_device/include/audio_device_defines.h index ab7ed603c7..8f375e8338 100644 --- a/webrtc/modules/audio_device/include/audio_device_defines.h +++ b/webrtc/modules/audio_device/include/audio_device_defines.h @@ -62,6 +62,7 @@ public: const uint32_t totalDelayMS, const int32_t clockDrift, const uint32_t currentMicLevel, + const bool keyPressed, uint32_t& newMicLevel) = 0; virtual int32_t NeedMorePlayData(const uint32_t nSamples, diff --git a/webrtc/modules/audio_device/mac/audio_device_mac.cc b/webrtc/modules/audio_device/mac/audio_device_mac.cc index 25514c451e..78555bd742 100644 --- a/webrtc/modules/audio_device/mac/audio_device_mac.cc +++ b/webrtc/modules/audio_device/mac/audio_device_mac.cc @@ -13,15 +13,17 @@ #include "audio_device_config.h" #include "event_wrapper.h" +#include "portaudio/pa_ringbuffer.h" #include "trace.h" #include "thread_wrapper.h" +#include #include - -#include // sysctlbyname() -#include // mach_task_self() #include // OSAtomicCompareAndSwap() -#include "portaudio/pa_ringbuffer.h" +#include // mach_task_self() +#include // sysctlbyname() + + namespace webrtc { @@ -3207,6 +3209,8 @@ bool AudioDeviceMac::CaptureWorkerThread() _ptrAudioBuffer->SetVQEData(msecOnPlaySide, msecOnRecordSide, 0); + _ptrAudioBuffer->SetTypingStatus(KeyPressed()); + // deliver recorded samples at specified sample rate, mic level etc. // to the observer using callback _ptrAudioBuffer->DeliverRecordedData(); @@ -3236,4 +3240,14 @@ bool AudioDeviceMac::CaptureWorkerThread() return true; } +bool AudioDeviceMac::KeyPressed() const{ + + bool key_down = false; + // loop through all Mac virtual key constant values + for (int key_index = 0; key_index <= 0x5C; key_index++) { + key_down |= CGEventSourceKeyState(kCGEventSourceStateHIDSystemState, + key_index); + } + return(key_down); +} } // namespace webrtc diff --git a/webrtc/modules/audio_device/mac/audio_device_mac.h b/webrtc/modules/audio_device/mac/audio_device_mac.h index 9a05e8d38a..258216c442 100644 --- a/webrtc/modules/audio_device/mac/audio_device_mac.h +++ b/webrtc/modules/audio_device/mac/audio_device_mac.h @@ -293,6 +293,9 @@ private: bool CaptureWorkerThread(); bool RenderWorkerThread(); +private: + bool KeyPressed() const; + private: AudioDeviceBuffer* _ptrAudioBuffer; diff --git a/webrtc/modules/audio_device/test/audio_device_test_api.cc b/webrtc/modules/audio_device/test/audio_device_test_api.cc index a1b841d239..4705f0a530 100644 --- a/webrtc/modules/audio_device/test/audio_device_test_api.cc +++ b/webrtc/modules/audio_device/test/audio_device_test_api.cc @@ -92,6 +92,7 @@ class AudioTransportAPI: public AudioTransport { const uint32_t totalDelay, const int32_t clockSkew, const uint32_t currentMicLevel, + const bool keyPressed, uint32_t& newMicLevel) { rec_count_++; if (rec_count_ % 100 == 0) { diff --git a/webrtc/modules/audio_device/test/func_test_manager.cc b/webrtc/modules/audio_device/test/func_test_manager.cc index bc0dd78485..5eccbe6889 100644 --- a/webrtc/modules/audio_device/test/func_test_manager.cc +++ b/webrtc/modules/audio_device/test/func_test_manager.cc @@ -176,6 +176,7 @@ int32_t AudioTransportImpl::RecordedDataIsAvailable( const uint32_t totalDelayMS, const int32_t clockDrift, const uint32_t currentMicLevel, + const bool keyPressed, uint32_t& newMicLevel) { if (_fullDuplex && _audioList.GetSize() < 15) diff --git a/webrtc/modules/audio_device/test/func_test_manager.h b/webrtc/modules/audio_device/test/func_test_manager.h index aafd4d8199..0036b5c9b8 100644 --- a/webrtc/modules/audio_device/test/func_test_manager.h +++ b/webrtc/modules/audio_device/test/func_test_manager.h @@ -101,6 +101,7 @@ public: const uint32_t totalDelayMS, const int32_t clockDrift, const uint32_t currentMicLevel, + const bool keyPressed, uint32_t& newMicLevel); virtual int32_t NeedMorePlayData(const uint32_t nSamples, diff --git a/webrtc/modules/audio_device/win/audio_device_core_win.cc b/webrtc/modules/audio_device/win/audio_device_core_win.cc index 83189712ce..5783113ecf 100644 --- a/webrtc/modules/audio_device/win/audio_device_core_win.cc +++ b/webrtc/modules/audio_device/win/audio_device_core_win.cc @@ -4092,6 +4092,8 @@ DWORD AudioDeviceWindowsCore::DoCaptureThread() sndCardRecDelay, 0); + _ptrAudioBuffer->SetTypingStatus(KeyPressed()); + QueryPerformanceCounter(&t1); // measure time: START _UnLock(); // release lock while making the callback @@ -5145,6 +5147,16 @@ char* AudioDeviceWindowsCore::WideToUTF8(const TCHAR* src) const { #endif } + +bool AudioDeviceWindowsCore::KeyPressed() const{ + + int key_down = 0; + for (int key = VK_SPACE; key < VK_NUMLOCK; key++) { + short res = GetAsyncKeyState(key); + key_down |= res & 0x1; // Get the LSB + } + return (key_down > 0); +} } // namespace webrtc #endif // WEBRTC_WINDOWS_CORE_AUDIO_BUILD diff --git a/webrtc/modules/audio_device/win/audio_device_core_win.h b/webrtc/modules/audio_device/win/audio_device_core_win.h index 2ed999d5d4..24a136fc4a 100644 --- a/webrtc/modules/audio_device/win/audio_device_core_win.h +++ b/webrtc/modules/audio_device/win/audio_device_core_win.h @@ -210,6 +210,9 @@ public: public: virtual void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer); +private: + bool KeyPressed() const; + private: // avrt function pointers PAvRevertMmThreadCharacteristics _PAvRevertMmThreadCharacteristics; PAvSetMmThreadCharacteristicsA _PAvSetMmThreadCharacteristicsA; diff --git a/webrtc/modules/audio_device/win/audio_device_wave_win.cc b/webrtc/modules/audio_device/win/audio_device_wave_win.cc index 9368748b91..7a5cd69fa3 100644 --- a/webrtc/modules/audio_device/win/audio_device_wave_win.cc +++ b/webrtc/modules/audio_device/win/audio_device_wave_win.cc @@ -3359,6 +3359,8 @@ int32_t AudioDeviceWindowsWave::RecProc(LONGLONG& consumedTime) _ptrAudioBuffer->SetVQEData(msecOnPlaySide, msecOnRecordSide, drift); + _ptrAudioBuffer->SetTypingStatus(KeyPressed()); + // Store the play and rec delay values for video synchronization _sndCardPlayDelay = msecOnPlaySide; _sndCardRecDelay = msecOnRecordSide; @@ -3820,5 +3822,15 @@ int32_t AudioDeviceWindowsWave::RestartTimerIfNeeded(const uint32_t time) return 0; } + +bool AudioDeviceWindowsWave::KeyPressed() const{ + + int key_down = 0; + for (int key = VK_SPACE; key < VK_NUMLOCK; key++) { + short res = GetAsyncKeyState(key); + key_down |= res & 0x1; // Get the LSB + } + return (key_down > 0); +} } // namespace webrtc diff --git a/webrtc/modules/audio_device/win/audio_device_wave_win.h b/webrtc/modules/audio_device/win/audio_device_wave_win.h index 665f4712d4..161a30690e 100644 --- a/webrtc/modules/audio_device/win/audio_device_wave_win.h +++ b/webrtc/modules/audio_device/win/audio_device_wave_win.h @@ -177,6 +177,9 @@ private: inline int32_t InputSanityCheckAfterUnlockedPeriod() const; inline int32_t OutputSanityCheckAfterUnlockedPeriod() const; +private: + bool KeyPressed() const; + private: int32_t EnumeratePlayoutDevices(); int32_t EnumerateRecordingDevices(); diff --git a/webrtc/system_wrappers/interface/event_wrapper.h b/webrtc/system_wrappers/interface/event_wrapper.h index af34ed9702..4ee18a6f7b 100644 --- a/webrtc/system_wrappers/interface/event_wrapper.h +++ b/webrtc/system_wrappers/interface/event_wrapper.h @@ -54,12 +54,6 @@ class EventWrapper { virtual bool StopTimer() = 0; - // Only implemented on Windows - // Returns 1 if a key has been pressed since last call to this function. - // -1 indicates failure - // 0 indicates no key has been pressed since last call - // TODO(hellner) this function does not seem to belong here - static int KeyPressed(); }; } // namespace webrtc diff --git a/webrtc/system_wrappers/source/event.cc b/webrtc/system_wrappers/source/event.cc index a72edfdc0d..18ac4f04e9 100644 --- a/webrtc/system_wrappers/source/event.cc +++ b/webrtc/system_wrappers/source/event.cc @@ -30,33 +30,4 @@ EventWrapper* EventWrapper::Create() { return EventPosix::Create(); #endif } - -int EventWrapper::KeyPressed() { -#if defined(_WIN32) - int key_down = 0; - for (int key = 0x20; key < 0x90; key++) { - short res = GetAsyncKeyState(key); - key_down |= res % 2; // Get the LSB - } - if (key_down) { - return 1; - } else { - return 0; - } -#elif defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) - bool key_down = false; - // loop through all Mac virtual key constant values - for (int key_index = 0; key_index <= 0x5C; key_index++) { - key_down |= CGEventSourceKeyState(kCGEventSourceStateHIDSystemState, - key_index); - } - if (key_down) { - return 1; - } else { - return 0; - } -#else - return -1; -#endif -} } // namespace webrtc diff --git a/webrtc/voice_engine/transmit_mixer.cc b/webrtc/voice_engine/transmit_mixer.cc index ea35afb9c8..3af2603e34 100644 --- a/webrtc/voice_engine/transmit_mixer.cc +++ b/webrtc/voice_engine/transmit_mixer.cc @@ -332,7 +332,8 @@ TransmitMixer::PrepareDemux(const void* audioSamples, const uint32_t samplesPerSec, const uint16_t totalDelayMS, const int32_t clockDrift, - const uint16_t currentMicLevel) + const uint16_t currentMicLevel, + const bool keyPressed) { WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, -1), "TransmitMixer::PrepareDemux(nSamples=%u, nChannels=%u," @@ -369,7 +370,7 @@ TransmitMixer::PrepareDemux(const void* audioSamples, // --- Annoying typing detection (utilizes the APM/VAD decision) #ifdef WEBRTC_VOICE_ENGINE_TYPING_DETECTION - TypingDetection(); + TypingDetection(keyPressed); #endif // --- Mute during DTMF tone if direct feedback is enabled @@ -1327,7 +1328,7 @@ void TransmitMixer::ProcessAudio(int delay_ms, int clock_drift, } #ifdef WEBRTC_VOICE_ENGINE_TYPING_DETECTION -int TransmitMixer::TypingDetection() +int TransmitMixer::TypingDetection(const bool keyPressed) { // We let the VAD determine if we're using this feature or not. @@ -1336,13 +1337,6 @@ int TransmitMixer::TypingDetection() return (0); } - int keyPressed = EventWrapper::KeyPressed(); - - if (keyPressed < 0) - { - return (-1); - } - if (_audioFrame.vad_activity_ == AudioFrame::kVadActive) _timeActive++; else diff --git a/webrtc/voice_engine/transmit_mixer.h b/webrtc/voice_engine/transmit_mixer.h index c98f640699..44153b156f 100644 --- a/webrtc/voice_engine/transmit_mixer.h +++ b/webrtc/voice_engine/transmit_mixer.h @@ -56,7 +56,8 @@ public: const uint32_t samplesPerSec, const uint16_t totalDelayMS, const int32_t clockDrift, - const uint16_t currentMicLevel); + const uint16_t currentMicLevel, + const bool keyPressed); int32_t DemuxAndMix(); @@ -178,7 +179,7 @@ private: void ProcessAudio(int delay_ms, int clock_drift, int current_mic_level); #ifdef WEBRTC_VOICE_ENGINE_TYPING_DETECTION - int TypingDetection(); + int TypingDetection(const bool keyPressed); #endif // uses diff --git a/webrtc/voice_engine/voe_base_impl.cc b/webrtc/voice_engine/voe_base_impl.cc index 0aed543d20..aeec9bcc9f 100644 --- a/webrtc/voice_engine/voe_base_impl.cc +++ b/webrtc/voice_engine/voe_base_impl.cc @@ -133,6 +133,7 @@ int32_t VoEBaseImpl::RecordedDataIsAvailable( const uint32_t totalDelayMS, const int32_t clockDrift, const uint32_t currentMicLevel, + const bool keyPressed, uint32_t& newMicLevel) { WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_shared->instance_id(), -1), @@ -195,7 +196,7 @@ int32_t VoEBaseImpl::RecordedDataIsAvailable( // (APM, mix with file, record to file, mute, etc.) _shared->transmit_mixer()->PrepareDemux(audioSamples, nSamples, nChannels, samplesPerSec, static_cast(totalDelayMS), clockDrift, - currentVoEMicLevel); + currentVoEMicLevel, keyPressed); // Copy the audio frame to each sending channel and perform // channel-dependent operations (file mixing, mute, etc.) to prepare diff --git a/webrtc/voice_engine/voe_base_impl.h b/webrtc/voice_engine/voe_base_impl.h index 42369c317e..9d89733e32 100644 --- a/webrtc/voice_engine/voe_base_impl.h +++ b/webrtc/voice_engine/voe_base_impl.h @@ -80,6 +80,7 @@ public: const uint32_t totalDelayMS, const int32_t clockDrift, const uint32_t currentMicLevel, + const bool keyPressed, uint32_t& newMicLevel); virtual int32_t NeedMorePlayData(const uint32_t nSamples, diff --git a/webrtc/voice_engine/voe_external_media_impl.cc b/webrtc/voice_engine/voe_external_media_impl.cc index 324dfec526..13e1aa1f6b 100644 --- a/webrtc/voice_engine/voe_external_media_impl.cc +++ b/webrtc/voice_engine/voe_external_media_impl.cc @@ -244,7 +244,8 @@ int VoEExternalMediaImpl::ExternalRecordingInsertData( samplingFreqHz, totalDelayMS, 0, - 0); + 0, + false); // Typing detection not supported shared_->transmit_mixer()->DemuxAndMix(); shared_->transmit_mixer()->EncodeAndSend();