diff --git a/webrtc/common_audio/BUILD.gn b/webrtc/common_audio/BUILD.gn index 5d7a9c6123..2d46f67ef2 100644 --- a/webrtc/common_audio/BUILD.gn +++ b/webrtc/common_audio/BUILD.gn @@ -62,7 +62,6 @@ rtc_static_library("common_audio") { ] deps = [ - "../base:rtc_analytics", "../system_wrappers", ] public_deps = [ diff --git a/webrtc/common_audio/mocks/mock_smoothing_filter.h b/webrtc/common_audio/mocks/mock_smoothing_filter.h index 171a8ca574..82cea55159 100644 --- a/webrtc/common_audio/mocks/mock_smoothing_filter.h +++ b/webrtc/common_audio/mocks/mock_smoothing_filter.h @@ -19,8 +19,8 @@ namespace webrtc { class MockSmoothingFilter : public SmoothingFilter { public: MOCK_METHOD1(AddSample, void(float)); - MOCK_CONST_METHOD0(GetAverage, rtc::Optional()); - MOCK_METHOD1(SetTimeConstantMs, void(int)); + MOCK_METHOD0(GetAverage, rtc::Optional()); + MOCK_METHOD1(SetTimeConstantMs, bool(int)); }; } // namespace webrtc diff --git a/webrtc/common_audio/smoothing_filter.cc b/webrtc/common_audio/smoothing_filter.cc index ff79ab8799..d2dde9d854 100644 --- a/webrtc/common_audio/smoothing_filter.cc +++ b/webrtc/common_audio/smoothing_filter.cc @@ -8,60 +8,118 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include - #include "webrtc/common_audio/smoothing_filter.h" +#include + namespace webrtc { -SmoothingFilterImpl::SmoothingFilterImpl(int time_constant_ms, - const Clock* clock) - : time_constant_ms_(time_constant_ms), - clock_(clock), - first_sample_received_(false), - initialized_(false), - first_sample_time_ms_(0), - last_sample_time_ms_(0), - filter_(0.0) {} +SmoothingFilterImpl::SmoothingFilterImpl(int init_time_ms_, const Clock* clock) + : init_time_ms_(init_time_ms_), + // Duing the initalization time, we use an increasing alpha. Specifically, + // alpha(n) = exp(pow(init_factor_, n)), + // where |init_factor_| is chosen such that + // alpha(init_time_ms_) = exp(-1.0f / init_time_ms_), + init_factor_(pow(init_time_ms_, 1.0f / init_time_ms_)), + // |init_const_| is to a factor to help the calculation during + // initialization phase. + init_const_(1.0f / (init_time_ms_ - + pow(init_time_ms_, 1.0f - 1.0f / init_time_ms_))), + clock_(clock) { + UpdateAlpha(init_time_ms_); +} + +SmoothingFilterImpl::~SmoothingFilterImpl() = default; void SmoothingFilterImpl::AddSample(float sample) { - if (!first_sample_received_) { - last_sample_time_ms_ = first_sample_time_ms_ = clock_->TimeInMilliseconds(); - first_sample_received_ = true; - RTC_DCHECK_EQ(rtc::ExpFilter::kValueUndefined, filter_.filtered()); + const int64_t now_ms = clock_->TimeInMilliseconds(); - // Since this is first sample, any value for argument 1 should work. - filter_.Apply(0.0f, sample); + if (!first_sample_time_ms_) { + // This is equivalent to assuming the filter has been receiving the same + // value as the first sample since time -infinity. + state_ = last_sample_ = sample; + first_sample_time_ms_ = rtc::Optional(now_ms); + last_state_time_ms_ = now_ms; return; } - int64_t now_ms = clock_->TimeInMilliseconds(); - if (!initialized_) { - float duration = now_ms - first_sample_time_ms_; - if (duration < static_cast(time_constant_ms_)) { - filter_.UpdateBase(exp(1.0f / duration)); - } else { - initialized_ = true; - filter_.UpdateBase(exp(1.0f / time_constant_ms_)); + ExtrapolateLastSample(now_ms); + last_sample_ = sample; +} + +rtc::Optional SmoothingFilterImpl::GetAverage() { + if (!first_sample_time_ms_) + return rtc::Optional(); + ExtrapolateLastSample(clock_->TimeInMilliseconds()); + return rtc::Optional(state_); +} + +bool SmoothingFilterImpl::SetTimeConstantMs(int time_constant_ms) { + if (!first_sample_time_ms_ || + last_state_time_ms_ < *first_sample_time_ms_ + init_time_ms_) { + return false; + } + UpdateAlpha(time_constant_ms); + return true; +} + +void SmoothingFilterImpl::UpdateAlpha(int time_constant_ms) { + alpha_ = exp(-1.0f / time_constant_ms); +} + +void SmoothingFilterImpl::ExtrapolateLastSample(int64_t time_ms) { + RTC_DCHECK_GE(time_ms, last_state_time_ms_); + RTC_DCHECK(first_sample_time_ms_); + + float multiplier = 0.0f; + if (time_ms <= *first_sample_time_ms_ + init_time_ms_) { + // Current update is to be made during initialization phase. + // We update the state as if the |alpha| has been increased according + // alpha(n) = exp(pow(init_factor_, n)), + // where n is the time (in millisecond) since the first sample received. + // With algebraic derivation as shown in the Appendix, we can find that the + // state can be updated in a similar manner as if alpha is a constant, + // except for a different multiplier. + multiplier = exp(-init_const_ * + (pow(init_factor_, + *first_sample_time_ms_ + init_time_ms_ - last_state_time_ms_) - + pow(init_factor_, *first_sample_time_ms_ + init_time_ms_ - time_ms))); + } else { + if (last_state_time_ms_ < *first_sample_time_ms_ + init_time_ms_) { + // The latest state update was made during initialization phase. + // We first extrapolate to the initialization time. + ExtrapolateLastSample(*first_sample_time_ms_ + init_time_ms_); + // Then extrapolate the rest by the following. } + multiplier = pow(alpha_, time_ms - last_state_time_ms_); } - // The filter will do the following: - // float alpha = pow(base, last_update_time_ms_ - now_ms); - // filtered_ = alpha * filtered_ + (1 - alpha) * sample; - filter_.Apply(static_cast(last_sample_time_ms_ - now_ms), sample); - last_sample_time_ms_ = now_ms; -} - -rtc::Optional SmoothingFilterImpl::GetAverage() const { - float value = filter_.filtered(); - return value == rtc::ExpFilter::kValueUndefined ? rtc::Optional() - : rtc::Optional(value); -} - -void SmoothingFilterImpl::SetTimeConstantMs(int time_constant_ms) { - time_constant_ms_ = time_constant_ms; - filter_.UpdateBase(exp(1.0f / time_constant_ms_)); + state_ = multiplier * state_ + (1.0f - multiplier) * last_sample_; + last_state_time_ms_ = time_ms; } } // namespace webrtc + +// Appendix: derivation of extrapolation during initialization phase. +// (LaTeX syntax) +// Assuming +// \begin{align} +// y(n) &= \alpha_{n-1} y(n-1) + \left(1 - \alpha_{n-1}\right) x(m) \\* +// &= \left(\prod_{i=m}^{n-1} \alpha_i\right) y(m) + +// \left(1 - \prod_{i=m}^{n-1} \alpha_i \right) x(m) +// \end{align} +// Taking $\alpha_{n} = \exp{\gamma^n}$, $\gamma$ denotes init\_factor\_, the +// multiplier becomes +// \begin{align} +// \prod_{i=m}^{n-1} \alpha_i +// &= \exp\left(\prod_{i=m}^{n-1} \gamma^i \right) \\* +// &= \exp\left(\frac{\gamma^m - \gamma^n}{1 - \gamma} \right) +// \end{align} +// We know $\gamma = T^\frac{1}{T}$, where $T$ denotes init\_time\_ms\_. Then +// $1 - \gamma$ approaches zero when $T$ increases. This can cause numerical +// difficulties. We multiply $T$ to both numerator and denominator in the +// fraction. See. +// \begin{align} +// \frac{\gamma^m - \gamma^n}{1 - \gamma} +// &= \frac{T^\frac{T-m}{T} - T^\frac{T-n}{T}}{T - T^{1-\frac{1}{T}}} +// \end{align} diff --git a/webrtc/common_audio/smoothing_filter.h b/webrtc/common_audio/smoothing_filter.h index 03eb718d54..4967a34442 100644 --- a/webrtc/common_audio/smoothing_filter.h +++ b/webrtc/common_audio/smoothing_filter.h @@ -11,7 +11,6 @@ #ifndef WEBRTC_COMMON_AUDIO_SMOOTHING_FILTER_H_ #define WEBRTC_COMMON_AUDIO_SMOOTHING_FILTER_H_ -#include "webrtc/base/analytics/exp_filter.h" #include "webrtc/base/constructormagic.h" #include "webrtc/base/optional.h" #include "webrtc/system_wrappers/include/clock.h" @@ -22,31 +21,49 @@ class SmoothingFilter { public: virtual ~SmoothingFilter() = default; virtual void AddSample(float sample) = 0; - virtual rtc::Optional GetAverage() const = 0; - virtual void SetTimeConstantMs(int time_constant_ms) = 0; + virtual rtc::Optional GetAverage() = 0; + virtual bool SetTimeConstantMs(int time_constant_ms) = 0; }; // SmoothingFilterImpl applies an exponential filter -// alpha = exp(-sample_interval / time_constant); +// alpha = exp(-1.0 / time_constant_ms); // y[t] = alpha * y[t-1] + (1 - alpha) * sample; +// This implies a sample rate of 1000 Hz, i.e., 1 sample / ms. +// But SmoothingFilterImpl allows sparse samples. All missing samples will be +// assumed to equal the last received sample. class SmoothingFilterImpl final : public SmoothingFilter { public: - // |time_constant_ms| is the time constant for the exponential filter. - SmoothingFilterImpl(int time_constant_ms, const Clock* clock); + // |init_time_ms| is initialization time. It defines a period starting from + // the arriving time of the first sample. During this period, the exponential + // filter uses a varying time constant so that a smaller time constant will be + // applied to the earlier samples. This is to allow the the filter to adapt to + // earlier samples quickly. After the initialization period, the time constant + // will be set to |init_time_ms| first and can be changed through + // |SetTimeConstantMs|. + SmoothingFilterImpl(int init_time_ms, const Clock* clock); + ~SmoothingFilterImpl() override; void AddSample(float sample) override; - rtc::Optional GetAverage() const override; - void SetTimeConstantMs(int time_constant_ms) override; + rtc::Optional GetAverage() override; + bool SetTimeConstantMs(int time_constant_ms) override; + + // Methods used for unittests. + float alpha() const { return alpha_; } private: - int time_constant_ms_; + void UpdateAlpha(int time_constant_ms); + void ExtrapolateLastSample(int64_t time_ms); + + const int init_time_ms_; + const float init_factor_; + const float init_const_; const Clock* const clock_; - bool first_sample_received_; - bool initialized_; - int64_t first_sample_time_ms_; - int64_t last_sample_time_ms_; - rtc::ExpFilter filter_; + rtc::Optional first_sample_time_ms_; + float last_sample_; + float alpha_; + float state_; + int64_t last_state_time_ms_; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(SmoothingFilterImpl); }; diff --git a/webrtc/common_audio/smoothing_filter_unittest.cc b/webrtc/common_audio/smoothing_filter_unittest.cc index c80ffb672c..ff06d50d8c 100644 --- a/webrtc/common_audio/smoothing_filter_unittest.cc +++ b/webrtc/common_audio/smoothing_filter_unittest.cc @@ -8,6 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include #include #include "webrtc/common_audio/smoothing_filter.h" @@ -17,29 +18,34 @@ namespace webrtc { namespace { -constexpr int kTimeConstantMs = 1000; -constexpr float kMaxAbsError = 0.0001f; +constexpr int kInitTimeMs = 795; +constexpr float kMaxAbsError = 1e-5f; constexpr int64_t kClockInitialTime = 123456; struct SmoothingFilterStates { std::unique_ptr simulated_clock; - std::unique_ptr smoothing_filter; + std::unique_ptr smoothing_filter; }; SmoothingFilterStates CreateSmoothingFilter() { SmoothingFilterStates states; states.simulated_clock.reset(new SimulatedClock(kClockInitialTime)); states.smoothing_filter.reset( - new SmoothingFilterImpl(kTimeConstantMs, states.simulated_clock.get())); + new SmoothingFilterImpl(kInitTimeMs, states.simulated_clock.get())); return states; } +// This function does the following: +// 1. Add a sample to filter at current clock, +// 2. Advance the clock by |advance_time_ms|, +// 3. Get the output of both SmoothingFilter and verify that it equals to an +// expected value. void CheckOutput(SmoothingFilterStates* states, - int advance_time_ms, float sample, + int advance_time_ms, float expected_ouput) { - states->simulated_clock->AdvanceTimeMilliseconds(advance_time_ms); states->smoothing_filter->AddSample(sample); + states->simulated_clock->AdvanceTimeMilliseconds(advance_time_ms); auto output = states->smoothing_filter->GetAverage(); EXPECT_TRUE(output); EXPECT_NEAR(expected_ouput, *output, kMaxAbsError); @@ -53,56 +59,90 @@ TEST(SmoothingFilterTest, NoOutputWhenNoSampleAdded) { } // Python script to calculate the reference values used in this test. -// import math +// import math // -// class ExpFilter: -// alpha = 0.0 -// old_value = 0.0 -// def calc(self, new_value): -// self.old_value = self.old_value * self.alpha -// + (1.0 - self.alpha) * new_value -// return self.old_value +// class ExpFilter: +// def add_sample(self, new_value): +// self.state = self.state * self.alpha + (1.0 - self.alpha) * new_value // -// delta_t = 100.0 -// filter = ExpFilter() -// total_t = 100.0 -// filter.alpha = math.exp(-delta_t/ total_t) -// print filter.calc(1.0) -// total_t = 200.0 -// filter.alpha = math.exp(-delta_t/ total_t) -// print filter.calc(0.0) -// total_t = 300.0 -// filter.alpha = math.exp(-delta_t/ total_t) -// print filter.calc(1.0) -TEST(SmoothingFilterTest, CheckBehaviorBeforeInitialized) { - // Adding three samples, all added before |kTimeConstantMs| is reached. - constexpr int kTimeIntervalMs = 100; +// filter = ExpFilter() +// init_time = 795 +// init_factor = (1.0 / init_time) ** (1.0 / init_time) +// +// filter.state = 1.0 +// +// for time_now in range(1, 500): +// filter.alpha = math.exp(-init_factor ** time_now) +// filter.add_sample(1.0) +// print filter.state +// +// for time_now in range(500, 600): +// filter.alpha = math.exp(-init_factor ** time_now) +// filter.add_sample(0.5) +// print filter.state +// +// for time_now in range(600, 700): +// filter.alpha = math.exp(-init_factor ** time_now) +// filter.add_sample(1.0) +// print filter.state +// +// for time_now in range(700, init_time): +// filter.alpha = math.exp(-init_factor ** time_now) +// filter.add_sample(1.0) +// +// filter.alpha = math.exp(-1.0 / init_time) +// for time_now in range(init_time, 800): +// filter.add_sample(1.0) +// print filter.state +// +// for i in range(800, 900): +// filter.add_sample(0.5) +// print filter.state +// +// for i in range(900, 1000): +// filter.add_sample(1.0) +// print filter.state +TEST(SmoothingFilterTest, CheckBehaviorAroundInitTime) { auto states = CreateSmoothingFilter(); - states.smoothing_filter->AddSample(0.0); - CheckOutput(&states, kTimeIntervalMs, 1.0, 0.63212f); - CheckOutput(&states, kTimeIntervalMs, 0.0, 0.38340f); - CheckOutput(&states, kTimeIntervalMs, 1.0, 0.55818f); + CheckOutput(&states, 1.0f, 500, 1.0f); + CheckOutput(&states, 0.5f, 100, 0.680562264029f); + CheckOutput(&states, 1.0f, 100, 0.794207139813f); + // Next step will go across initialization time. + CheckOutput(&states, 1.0f, 100, 0.829803409752f); + CheckOutput(&states, 0.5f, 100, 0.790821764210f); + CheckOutput(&states, 1.0f, 100, 0.815545922911f); } -// Python script to calculate the reference value used in this test. -// (after defining ExpFilter as for CheckBehaviorBeforeInitialized) -// time_constant_ms = 1000.0 -// filter = ExpFilter() -// delta_t = 1100.0 -// filter.alpha = math.exp(-delta_t/ time_constant_ms) -// print filter.calc(1.0) -// delta_t = 100.0 -// filter.alpha = math.exp(-delta_t/ time_constant_ms) -// print filter.calc(0.0) -// print filter.calc(1.0) -TEST(SmoothingFilterTest, CheckBehaviorAfterInitialized) { - constexpr int kTimeIntervalMs = 100; +TEST(SmoothingFilterTest, GetAverageOutputsEmptyBeforeFirstSample) { + auto states = CreateSmoothingFilter(); + EXPECT_FALSE(states.smoothing_filter->GetAverage()); + constexpr float kFirstSample = 1.2345f; + states.smoothing_filter->AddSample(kFirstSample); + EXPECT_EQ(rtc::Optional(kFirstSample), + states.smoothing_filter->GetAverage()); +} + +TEST(SmoothingFilterTest, CannotChangeTimeConstantDuringInitialization) { auto states = CreateSmoothingFilter(); states.smoothing_filter->AddSample(0.0); - states.simulated_clock->AdvanceTimeMilliseconds(kTimeConstantMs); - CheckOutput(&states, kTimeIntervalMs, 1.0, 0.66713f); - CheckOutput(&states, kTimeIntervalMs, 0.0, 0.60364f); - CheckOutput(&states, kTimeIntervalMs, 1.0, 0.64136f); + + // During initialization, |SetTimeConstantMs| does not take effect. + states.simulated_clock->AdvanceTimeMilliseconds(kInitTimeMs - 1); + states.smoothing_filter->AddSample(0.0); + + EXPECT_FALSE(states.smoothing_filter->SetTimeConstantMs(kInitTimeMs * 2)); + EXPECT_NE(exp(-1.0f / (kInitTimeMs * 2)), states.smoothing_filter->alpha()); + + states.simulated_clock->AdvanceTimeMilliseconds(1); + states.smoothing_filter->AddSample(0.0); + // When initialization finishes, the time constant should be come + // |kInitTimeConstantMs|. + EXPECT_FLOAT_EQ(exp(-1.0f / kInitTimeMs), states.smoothing_filter->alpha()); + + // After initialization, |SetTimeConstantMs| takes effect. + EXPECT_TRUE(states.smoothing_filter->SetTimeConstantMs(kInitTimeMs * 2)); + EXPECT_FLOAT_EQ(exp(-1.0f / (kInitTimeMs * 2)), + states.smoothing_filter->alpha()); } } // namespace webrtc