diff --git a/webrtc/modules/audio_processing/aecm/aecm_core_neon.c b/webrtc/modules/audio_processing/aecm/aecm_core_neon.c index 48d63f7d63..ff733b7f79 100644 --- a/webrtc/modules/audio_processing/aecm/aecm_core_neon.c +++ b/webrtc/modules/audio_processing/aecm/aecm_core_neon.c @@ -227,72 +227,85 @@ void WebRtcAecm_InverseFFTAndWindowNeon(AecmCore_t* aecm, } } +static inline void AddLanes(uint32_t* ptr, uint32x4_t v) { +#if defined(__aarch64__) + *(ptr) = vaddvq_u32(v); +#else + uint32x2_t tmp_v; + tmp_v = vadd_u32(vget_low_u32(v), vget_high_u32(v)); + tmp_v = vpadd_u32(tmp_v, tmp_v); + *(ptr) = vget_lane_u32(tmp_v, 0); +#endif +} + void WebRtcAecm_CalcLinearEnergiesNeon(AecmCore_t* aecm, const uint16_t* far_spectrum, int32_t* echo_est, uint32_t* far_energy, uint32_t* echo_energy_adapt, uint32_t* echo_energy_stored) { - int i; + int16_t* start_stored_p = aecm->channelStored; + int16_t* start_adapt_p = aecm->channelAdapt16; + int32_t* echo_est_p = echo_est; + const int16_t* end_stored_p = aecm->channelStored + PART_LEN; + const uint16_t* far_spectrum_p = far_spectrum; + int16x8_t store_v, adapt_v, spectrum_v; + uint32x4_t echo_est_v_low, echo_est_v_high; + uint32x4_t far_energy_v, echo_energy_stored_v, echo_energy_adapt_v; - register uint32_t far_energy_r; - register uint32_t echo_energy_stored_r; - register uint32_t echo_energy_adapt_r; + far_energy_v = vdupq_n_u32(0); + echo_energy_adapt_v = vdupq_n_u32(0); + echo_energy_stored_v = vdupq_n_u32(0); - assert((uintptr_t)echo_est % 32 == 0); - assert((uintptr_t)(aecm->channelStored) % 16 == 0); - assert((uintptr_t)(aecm->channelAdapt16) % 16 == 0); - assert((uintptr_t)(aecm->channelStored) % 16 == 0); - assert((uintptr_t)(aecm->channelStored) % 16 == 0); + // Get energy for the delayed far end signal and estimated + // echo using both stored and adapted channels. + // The C code: + // for (i = 0; i < PART_LEN1; i++) { + // echo_est[i] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i], + // far_spectrum[i]); + // (*far_energy) += (uint32_t)(far_spectrum[i]); + // *echo_energy_adapt += aecm->channelAdapt16[i] * far_spectrum[i]; + // (*echo_energy_stored) += (uint32_t)echo_est[i]; + // } + while (start_stored_p < end_stored_p) { + spectrum_v = vld1q_u16(far_spectrum_p); + adapt_v = vld1q_s16(start_adapt_p); + store_v = vld1q_s16(start_stored_p); - __asm __volatile("vmov.i32 q14, #0" : : : "q14"); // far_energy - __asm __volatile("vmov.i32 q8, #0" : : : "q8"); // echo_energy_stored - __asm __volatile("vmov.i32 q9, #0" : : : "q9"); // echo_energy_adapt + far_energy_v = vaddw_u16(far_energy_v, vget_low_s16(spectrum_v)); + far_energy_v = vaddw_u16(far_energy_v, vget_high_s16(spectrum_v)); - for (i = 0; i < PART_LEN - 7; i += 8) { - // far_energy += (uint32_t)(far_spectrum[i]); - __asm __volatile("vld1.16 {d26, d27}, [%0]" : : "r"(&far_spectrum[i]) : "q13"); - __asm __volatile("vaddw.u16 q14, q14, d26" : : : "q14", "q13"); - __asm __volatile("vaddw.u16 q14, q14, d27" : : : "q14", "q13"); + echo_est_v_low = vmull_u16(vget_low_s16(store_v), vget_low_s16(spectrum_v)); + echo_est_v_high = vmull_u16(vget_high_s16(store_v), + vget_high_s16(spectrum_v)); + vst1q_s32(echo_est_p, echo_est_v_low); + vst1q_s32(echo_est_p + 4, echo_est_v_high); - // Get estimated echo energies for adaptive channel and stored channel. - // echoEst[i] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i], far_spectrum[i]); - __asm __volatile("vld1.16 {d24, d25}, [%0, :128]" : : "r"(&aecm->channelStored[i]) : "q12"); - __asm __volatile("vmull.u16 q10, d26, d24" : : : "q12", "q13", "q10"); - __asm __volatile("vmull.u16 q11, d27, d25" : : : "q12", "q13", "q11"); - __asm __volatile("vst1.32 {d20, d21, d22, d23}, [%0, :256]" : : "r"(&echo_est[i]): - "q10", "q11"); + echo_energy_stored_v = vaddq_s32(echo_est_v_low, echo_energy_stored_v); + echo_energy_stored_v = vaddq_s32(echo_est_v_high, echo_energy_stored_v); - // echo_energy_stored += (uint32_t)echoEst[i]; - __asm __volatile("vadd.u32 q8, q10" : : : "q10", "q8"); - __asm __volatile("vadd.u32 q8, q11" : : : "q11", "q8"); + echo_energy_adapt_v = vmlal_u16(echo_energy_adapt_v, + vget_low_s16(adapt_v), + vget_low_s16(spectrum_v)); + echo_energy_adapt_v = vmlal_u16(echo_energy_adapt_v, + vget_high_s16(adapt_v), + vget_high_s16(spectrum_v)); - // echo_energy_adapt += aecm->channelAdapt16[i] * far_spectrum[i]; - __asm __volatile("vld1.16 {d24, d25}, [%0, :128]" : : "r"(&aecm->channelAdapt16[i]) : "q12"); - __asm __volatile("vmull.u16 q10, d26, d24" : : : "q12", "q13", "q10"); - __asm __volatile("vmull.u16 q11, d27, d25" : : : "q12", "q13", "q11"); - __asm __volatile("vadd.u32 q9, q10" : : : "q9", "q15"); - __asm __volatile("vadd.u32 q9, q11" : : : "q9", "q11"); + start_stored_p += 8; + start_adapt_p += 8; + far_spectrum_p += 8; + echo_est_p += 8; } - __asm __volatile("vadd.u32 d28, d29" : : : "q14"); - __asm __volatile("vpadd.u32 d28, d28" : : : "q14"); - __asm __volatile("vmov.32 %0, d28[0]" : "=r"(far_energy_r): : "q14"); + AddLanes(far_energy, far_energy_v); + AddLanes(echo_energy_stored, echo_energy_stored_v); + AddLanes(echo_energy_adapt, echo_energy_adapt_v); - __asm __volatile("vadd.u32 d18, d19" : : : "q9"); - __asm __volatile("vpadd.u32 d18, d18" : : : "q9"); - __asm __volatile("vmov.32 %0, d18[0]" : "=r"(echo_energy_adapt_r): : "q9"); - - __asm __volatile("vadd.u32 d16, d17" : : : "q8"); - __asm __volatile("vpadd.u32 d16, d16" : : : "q8"); - __asm __volatile("vmov.32 %0, d16[0]" : "=r"(echo_energy_stored_r): : "q8"); - - // Get estimated echo energies for adaptive channel and stored channel. - echo_est[i] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[i], far_spectrum[i]); - *echo_energy_stored = echo_energy_stored_r + (uint32_t)echo_est[i]; - *far_energy = far_energy_r + (uint32_t)(far_spectrum[i]); - *echo_energy_adapt = echo_energy_adapt_r + - aecm->channelAdapt16[i] * far_spectrum[i]; + echo_est[PART_LEN] = WEBRTC_SPL_MUL_16_U16(aecm->channelStored[PART_LEN], + far_spectrum[PART_LEN]); + *echo_energy_stored += (uint32_t)echo_est[PART_LEN]; + *far_energy += (uint32_t)far_spectrum[PART_LEN]; + *echo_energy_adapt += aecm->channelAdapt16[PART_LEN] * far_spectrum[PART_LEN]; } void WebRtcAecm_StoreAdaptiveChannelNeon(AecmCore_t* aecm,