From 301fc4a712fa507fe14a3bd4688856441bd60011 Mon Sep 17 00:00:00 2001 From: minyue Date: Tue, 13 Dec 2016 06:52:56 -0800 Subject: [PATCH] Update common_audio/smoothing_filter. The improvement is mainly to extrapolate missing samples so that when querying the output, it assumes the input to continue even if no actual new samples are added. The new implementation does not rely on base/exp_filter any longer. This is because it would be a bit cumbersome. base/exp_filter does pre-extrapolate, i.e., it assumes the all missing samples since the last sample equals the new sample. BUG=webrtc:6443 Review-Url: https://codereview.webrtc.org/2551363002 Cr-Commit-Position: refs/heads/master@{#15575} --- webrtc/common_audio/BUILD.gn | 1 - .../mocks/mock_smoothing_filter.h | 4 +- webrtc/common_audio/smoothing_filter.cc | 140 +++++++++++++----- webrtc/common_audio/smoothing_filter.h | 45 ++++-- .../common_audio/smoothing_filter_unittest.cc | 138 +++++++++++------ 5 files changed, 221 insertions(+), 107 deletions(-) diff --git a/webrtc/common_audio/BUILD.gn b/webrtc/common_audio/BUILD.gn index 5d7a9c6123..2d46f67ef2 100644 --- a/webrtc/common_audio/BUILD.gn +++ b/webrtc/common_audio/BUILD.gn @@ -62,7 +62,6 @@ rtc_static_library("common_audio") { ] deps = [ - "../base:rtc_analytics", "../system_wrappers", ] public_deps = [ diff --git a/webrtc/common_audio/mocks/mock_smoothing_filter.h b/webrtc/common_audio/mocks/mock_smoothing_filter.h index 171a8ca574..82cea55159 100644 --- a/webrtc/common_audio/mocks/mock_smoothing_filter.h +++ b/webrtc/common_audio/mocks/mock_smoothing_filter.h @@ -19,8 +19,8 @@ namespace webrtc { class MockSmoothingFilter : public SmoothingFilter { public: MOCK_METHOD1(AddSample, void(float)); - MOCK_CONST_METHOD0(GetAverage, rtc::Optional()); - MOCK_METHOD1(SetTimeConstantMs, void(int)); + MOCK_METHOD0(GetAverage, rtc::Optional()); + MOCK_METHOD1(SetTimeConstantMs, bool(int)); }; } // namespace webrtc diff --git a/webrtc/common_audio/smoothing_filter.cc b/webrtc/common_audio/smoothing_filter.cc index ff79ab8799..d2dde9d854 100644 --- a/webrtc/common_audio/smoothing_filter.cc +++ b/webrtc/common_audio/smoothing_filter.cc @@ -8,60 +8,118 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include - #include "webrtc/common_audio/smoothing_filter.h" +#include + namespace webrtc { -SmoothingFilterImpl::SmoothingFilterImpl(int time_constant_ms, - const Clock* clock) - : time_constant_ms_(time_constant_ms), - clock_(clock), - first_sample_received_(false), - initialized_(false), - first_sample_time_ms_(0), - last_sample_time_ms_(0), - filter_(0.0) {} +SmoothingFilterImpl::SmoothingFilterImpl(int init_time_ms_, const Clock* clock) + : init_time_ms_(init_time_ms_), + // Duing the initalization time, we use an increasing alpha. Specifically, + // alpha(n) = exp(pow(init_factor_, n)), + // where |init_factor_| is chosen such that + // alpha(init_time_ms_) = exp(-1.0f / init_time_ms_), + init_factor_(pow(init_time_ms_, 1.0f / init_time_ms_)), + // |init_const_| is to a factor to help the calculation during + // initialization phase. + init_const_(1.0f / (init_time_ms_ - + pow(init_time_ms_, 1.0f - 1.0f / init_time_ms_))), + clock_(clock) { + UpdateAlpha(init_time_ms_); +} + +SmoothingFilterImpl::~SmoothingFilterImpl() = default; void SmoothingFilterImpl::AddSample(float sample) { - if (!first_sample_received_) { - last_sample_time_ms_ = first_sample_time_ms_ = clock_->TimeInMilliseconds(); - first_sample_received_ = true; - RTC_DCHECK_EQ(rtc::ExpFilter::kValueUndefined, filter_.filtered()); + const int64_t now_ms = clock_->TimeInMilliseconds(); - // Since this is first sample, any value for argument 1 should work. - filter_.Apply(0.0f, sample); + if (!first_sample_time_ms_) { + // This is equivalent to assuming the filter has been receiving the same + // value as the first sample since time -infinity. + state_ = last_sample_ = sample; + first_sample_time_ms_ = rtc::Optional(now_ms); + last_state_time_ms_ = now_ms; return; } - int64_t now_ms = clock_->TimeInMilliseconds(); - if (!initialized_) { - float duration = now_ms - first_sample_time_ms_; - if (duration < static_cast(time_constant_ms_)) { - filter_.UpdateBase(exp(1.0f / duration)); - } else { - initialized_ = true; - filter_.UpdateBase(exp(1.0f / time_constant_ms_)); + ExtrapolateLastSample(now_ms); + last_sample_ = sample; +} + +rtc::Optional SmoothingFilterImpl::GetAverage() { + if (!first_sample_time_ms_) + return rtc::Optional(); + ExtrapolateLastSample(clock_->TimeInMilliseconds()); + return rtc::Optional(state_); +} + +bool SmoothingFilterImpl::SetTimeConstantMs(int time_constant_ms) { + if (!first_sample_time_ms_ || + last_state_time_ms_ < *first_sample_time_ms_ + init_time_ms_) { + return false; + } + UpdateAlpha(time_constant_ms); + return true; +} + +void SmoothingFilterImpl::UpdateAlpha(int time_constant_ms) { + alpha_ = exp(-1.0f / time_constant_ms); +} + +void SmoothingFilterImpl::ExtrapolateLastSample(int64_t time_ms) { + RTC_DCHECK_GE(time_ms, last_state_time_ms_); + RTC_DCHECK(first_sample_time_ms_); + + float multiplier = 0.0f; + if (time_ms <= *first_sample_time_ms_ + init_time_ms_) { + // Current update is to be made during initialization phase. + // We update the state as if the |alpha| has been increased according + // alpha(n) = exp(pow(init_factor_, n)), + // where n is the time (in millisecond) since the first sample received. + // With algebraic derivation as shown in the Appendix, we can find that the + // state can be updated in a similar manner as if alpha is a constant, + // except for a different multiplier. + multiplier = exp(-init_const_ * + (pow(init_factor_, + *first_sample_time_ms_ + init_time_ms_ - last_state_time_ms_) - + pow(init_factor_, *first_sample_time_ms_ + init_time_ms_ - time_ms))); + } else { + if (last_state_time_ms_ < *first_sample_time_ms_ + init_time_ms_) { + // The latest state update was made during initialization phase. + // We first extrapolate to the initialization time. + ExtrapolateLastSample(*first_sample_time_ms_ + init_time_ms_); + // Then extrapolate the rest by the following. } + multiplier = pow(alpha_, time_ms - last_state_time_ms_); } - // The filter will do the following: - // float alpha = pow(base, last_update_time_ms_ - now_ms); - // filtered_ = alpha * filtered_ + (1 - alpha) * sample; - filter_.Apply(static_cast(last_sample_time_ms_ - now_ms), sample); - last_sample_time_ms_ = now_ms; -} - -rtc::Optional SmoothingFilterImpl::GetAverage() const { - float value = filter_.filtered(); - return value == rtc::ExpFilter::kValueUndefined ? rtc::Optional() - : rtc::Optional(value); -} - -void SmoothingFilterImpl::SetTimeConstantMs(int time_constant_ms) { - time_constant_ms_ = time_constant_ms; - filter_.UpdateBase(exp(1.0f / time_constant_ms_)); + state_ = multiplier * state_ + (1.0f - multiplier) * last_sample_; + last_state_time_ms_ = time_ms; } } // namespace webrtc + +// Appendix: derivation of extrapolation during initialization phase. +// (LaTeX syntax) +// Assuming +// \begin{align} +// y(n) &= \alpha_{n-1} y(n-1) + \left(1 - \alpha_{n-1}\right) x(m) \\* +// &= \left(\prod_{i=m}^{n-1} \alpha_i\right) y(m) + +// \left(1 - \prod_{i=m}^{n-1} \alpha_i \right) x(m) +// \end{align} +// Taking $\alpha_{n} = \exp{\gamma^n}$, $\gamma$ denotes init\_factor\_, the +// multiplier becomes +// \begin{align} +// \prod_{i=m}^{n-1} \alpha_i +// &= \exp\left(\prod_{i=m}^{n-1} \gamma^i \right) \\* +// &= \exp\left(\frac{\gamma^m - \gamma^n}{1 - \gamma} \right) +// \end{align} +// We know $\gamma = T^\frac{1}{T}$, where $T$ denotes init\_time\_ms\_. Then +// $1 - \gamma$ approaches zero when $T$ increases. This can cause numerical +// difficulties. We multiply $T$ to both numerator and denominator in the +// fraction. See. +// \begin{align} +// \frac{\gamma^m - \gamma^n}{1 - \gamma} +// &= \frac{T^\frac{T-m}{T} - T^\frac{T-n}{T}}{T - T^{1-\frac{1}{T}}} +// \end{align} diff --git a/webrtc/common_audio/smoothing_filter.h b/webrtc/common_audio/smoothing_filter.h index 03eb718d54..4967a34442 100644 --- a/webrtc/common_audio/smoothing_filter.h +++ b/webrtc/common_audio/smoothing_filter.h @@ -11,7 +11,6 @@ #ifndef WEBRTC_COMMON_AUDIO_SMOOTHING_FILTER_H_ #define WEBRTC_COMMON_AUDIO_SMOOTHING_FILTER_H_ -#include "webrtc/base/analytics/exp_filter.h" #include "webrtc/base/constructormagic.h" #include "webrtc/base/optional.h" #include "webrtc/system_wrappers/include/clock.h" @@ -22,31 +21,49 @@ class SmoothingFilter { public: virtual ~SmoothingFilter() = default; virtual void AddSample(float sample) = 0; - virtual rtc::Optional GetAverage() const = 0; - virtual void SetTimeConstantMs(int time_constant_ms) = 0; + virtual rtc::Optional GetAverage() = 0; + virtual bool SetTimeConstantMs(int time_constant_ms) = 0; }; // SmoothingFilterImpl applies an exponential filter -// alpha = exp(-sample_interval / time_constant); +// alpha = exp(-1.0 / time_constant_ms); // y[t] = alpha * y[t-1] + (1 - alpha) * sample; +// This implies a sample rate of 1000 Hz, i.e., 1 sample / ms. +// But SmoothingFilterImpl allows sparse samples. All missing samples will be +// assumed to equal the last received sample. class SmoothingFilterImpl final : public SmoothingFilter { public: - // |time_constant_ms| is the time constant for the exponential filter. - SmoothingFilterImpl(int time_constant_ms, const Clock* clock); + // |init_time_ms| is initialization time. It defines a period starting from + // the arriving time of the first sample. During this period, the exponential + // filter uses a varying time constant so that a smaller time constant will be + // applied to the earlier samples. This is to allow the the filter to adapt to + // earlier samples quickly. After the initialization period, the time constant + // will be set to |init_time_ms| first and can be changed through + // |SetTimeConstantMs|. + SmoothingFilterImpl(int init_time_ms, const Clock* clock); + ~SmoothingFilterImpl() override; void AddSample(float sample) override; - rtc::Optional GetAverage() const override; - void SetTimeConstantMs(int time_constant_ms) override; + rtc::Optional GetAverage() override; + bool SetTimeConstantMs(int time_constant_ms) override; + + // Methods used for unittests. + float alpha() const { return alpha_; } private: - int time_constant_ms_; + void UpdateAlpha(int time_constant_ms); + void ExtrapolateLastSample(int64_t time_ms); + + const int init_time_ms_; + const float init_factor_; + const float init_const_; const Clock* const clock_; - bool first_sample_received_; - bool initialized_; - int64_t first_sample_time_ms_; - int64_t last_sample_time_ms_; - rtc::ExpFilter filter_; + rtc::Optional first_sample_time_ms_; + float last_sample_; + float alpha_; + float state_; + int64_t last_state_time_ms_; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(SmoothingFilterImpl); }; diff --git a/webrtc/common_audio/smoothing_filter_unittest.cc b/webrtc/common_audio/smoothing_filter_unittest.cc index c80ffb672c..ff06d50d8c 100644 --- a/webrtc/common_audio/smoothing_filter_unittest.cc +++ b/webrtc/common_audio/smoothing_filter_unittest.cc @@ -8,6 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include #include #include "webrtc/common_audio/smoothing_filter.h" @@ -17,29 +18,34 @@ namespace webrtc { namespace { -constexpr int kTimeConstantMs = 1000; -constexpr float kMaxAbsError = 0.0001f; +constexpr int kInitTimeMs = 795; +constexpr float kMaxAbsError = 1e-5f; constexpr int64_t kClockInitialTime = 123456; struct SmoothingFilterStates { std::unique_ptr simulated_clock; - std::unique_ptr smoothing_filter; + std::unique_ptr smoothing_filter; }; SmoothingFilterStates CreateSmoothingFilter() { SmoothingFilterStates states; states.simulated_clock.reset(new SimulatedClock(kClockInitialTime)); states.smoothing_filter.reset( - new SmoothingFilterImpl(kTimeConstantMs, states.simulated_clock.get())); + new SmoothingFilterImpl(kInitTimeMs, states.simulated_clock.get())); return states; } +// This function does the following: +// 1. Add a sample to filter at current clock, +// 2. Advance the clock by |advance_time_ms|, +// 3. Get the output of both SmoothingFilter and verify that it equals to an +// expected value. void CheckOutput(SmoothingFilterStates* states, - int advance_time_ms, float sample, + int advance_time_ms, float expected_ouput) { - states->simulated_clock->AdvanceTimeMilliseconds(advance_time_ms); states->smoothing_filter->AddSample(sample); + states->simulated_clock->AdvanceTimeMilliseconds(advance_time_ms); auto output = states->smoothing_filter->GetAverage(); EXPECT_TRUE(output); EXPECT_NEAR(expected_ouput, *output, kMaxAbsError); @@ -53,56 +59,90 @@ TEST(SmoothingFilterTest, NoOutputWhenNoSampleAdded) { } // Python script to calculate the reference values used in this test. -// import math +// import math // -// class ExpFilter: -// alpha = 0.0 -// old_value = 0.0 -// def calc(self, new_value): -// self.old_value = self.old_value * self.alpha -// + (1.0 - self.alpha) * new_value -// return self.old_value +// class ExpFilter: +// def add_sample(self, new_value): +// self.state = self.state * self.alpha + (1.0 - self.alpha) * new_value // -// delta_t = 100.0 -// filter = ExpFilter() -// total_t = 100.0 -// filter.alpha = math.exp(-delta_t/ total_t) -// print filter.calc(1.0) -// total_t = 200.0 -// filter.alpha = math.exp(-delta_t/ total_t) -// print filter.calc(0.0) -// total_t = 300.0 -// filter.alpha = math.exp(-delta_t/ total_t) -// print filter.calc(1.0) -TEST(SmoothingFilterTest, CheckBehaviorBeforeInitialized) { - // Adding three samples, all added before |kTimeConstantMs| is reached. - constexpr int kTimeIntervalMs = 100; +// filter = ExpFilter() +// init_time = 795 +// init_factor = (1.0 / init_time) ** (1.0 / init_time) +// +// filter.state = 1.0 +// +// for time_now in range(1, 500): +// filter.alpha = math.exp(-init_factor ** time_now) +// filter.add_sample(1.0) +// print filter.state +// +// for time_now in range(500, 600): +// filter.alpha = math.exp(-init_factor ** time_now) +// filter.add_sample(0.5) +// print filter.state +// +// for time_now in range(600, 700): +// filter.alpha = math.exp(-init_factor ** time_now) +// filter.add_sample(1.0) +// print filter.state +// +// for time_now in range(700, init_time): +// filter.alpha = math.exp(-init_factor ** time_now) +// filter.add_sample(1.0) +// +// filter.alpha = math.exp(-1.0 / init_time) +// for time_now in range(init_time, 800): +// filter.add_sample(1.0) +// print filter.state +// +// for i in range(800, 900): +// filter.add_sample(0.5) +// print filter.state +// +// for i in range(900, 1000): +// filter.add_sample(1.0) +// print filter.state +TEST(SmoothingFilterTest, CheckBehaviorAroundInitTime) { auto states = CreateSmoothingFilter(); - states.smoothing_filter->AddSample(0.0); - CheckOutput(&states, kTimeIntervalMs, 1.0, 0.63212f); - CheckOutput(&states, kTimeIntervalMs, 0.0, 0.38340f); - CheckOutput(&states, kTimeIntervalMs, 1.0, 0.55818f); + CheckOutput(&states, 1.0f, 500, 1.0f); + CheckOutput(&states, 0.5f, 100, 0.680562264029f); + CheckOutput(&states, 1.0f, 100, 0.794207139813f); + // Next step will go across initialization time. + CheckOutput(&states, 1.0f, 100, 0.829803409752f); + CheckOutput(&states, 0.5f, 100, 0.790821764210f); + CheckOutput(&states, 1.0f, 100, 0.815545922911f); } -// Python script to calculate the reference value used in this test. -// (after defining ExpFilter as for CheckBehaviorBeforeInitialized) -// time_constant_ms = 1000.0 -// filter = ExpFilter() -// delta_t = 1100.0 -// filter.alpha = math.exp(-delta_t/ time_constant_ms) -// print filter.calc(1.0) -// delta_t = 100.0 -// filter.alpha = math.exp(-delta_t/ time_constant_ms) -// print filter.calc(0.0) -// print filter.calc(1.0) -TEST(SmoothingFilterTest, CheckBehaviorAfterInitialized) { - constexpr int kTimeIntervalMs = 100; +TEST(SmoothingFilterTest, GetAverageOutputsEmptyBeforeFirstSample) { + auto states = CreateSmoothingFilter(); + EXPECT_FALSE(states.smoothing_filter->GetAverage()); + constexpr float kFirstSample = 1.2345f; + states.smoothing_filter->AddSample(kFirstSample); + EXPECT_EQ(rtc::Optional(kFirstSample), + states.smoothing_filter->GetAverage()); +} + +TEST(SmoothingFilterTest, CannotChangeTimeConstantDuringInitialization) { auto states = CreateSmoothingFilter(); states.smoothing_filter->AddSample(0.0); - states.simulated_clock->AdvanceTimeMilliseconds(kTimeConstantMs); - CheckOutput(&states, kTimeIntervalMs, 1.0, 0.66713f); - CheckOutput(&states, kTimeIntervalMs, 0.0, 0.60364f); - CheckOutput(&states, kTimeIntervalMs, 1.0, 0.64136f); + + // During initialization, |SetTimeConstantMs| does not take effect. + states.simulated_clock->AdvanceTimeMilliseconds(kInitTimeMs - 1); + states.smoothing_filter->AddSample(0.0); + + EXPECT_FALSE(states.smoothing_filter->SetTimeConstantMs(kInitTimeMs * 2)); + EXPECT_NE(exp(-1.0f / (kInitTimeMs * 2)), states.smoothing_filter->alpha()); + + states.simulated_clock->AdvanceTimeMilliseconds(1); + states.smoothing_filter->AddSample(0.0); + // When initialization finishes, the time constant should be come + // |kInitTimeConstantMs|. + EXPECT_FLOAT_EQ(exp(-1.0f / kInitTimeMs), states.smoothing_filter->alpha()); + + // After initialization, |SetTimeConstantMs| takes effect. + EXPECT_TRUE(states.smoothing_filter->SetTimeConstantMs(kInitTimeMs * 2)); + EXPECT_FLOAT_EQ(exp(-1.0f / (kInitTimeMs * 2)), + states.smoothing_filter->alpha()); } } // namespace webrtc