From 751a36590a401ccbbe148adeb2c83efb3a53da4d Mon Sep 17 00:00:00 2001 From: "henrik.lundin@webrtc.org" Date: Fri, 6 Feb 2015 14:03:29 +0000 Subject: [PATCH] Switch to using AudioEncoderPcmU/A instead of ACMPCMU/A This change switches from the old codec wrappers ACMPCMU and ACMPCMA to the new AudioEncoderPcmU and AudioEncoderPcmA wrapped in an ACMGenericCodecWrapper. RED and CNG is also switched to using their AudioEncoder implementations (AudioEncoderCopyRed and AudioEncoderCng, respectively), when RED and/or CNG is combined with PCM u/A. This is the first in a series of changes that will switch all codecs to use the new AudioEncoder interface. BUG=4228 COAUTHOR=kwiberg@webrtc.org R=minyue@webrtc.org Review URL: https://webrtc-codereview.appspot.com/33209004 Cr-Commit-Position: refs/heads/master@{#8268} git-svn-id: http://webrtc.googlecode.com/svn/trunk@8268 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../main/acm2/acm_codec_database.cc | 59 +++++++++--------- .../main/acm2/acm_codec_database.h | 20 ++++-- .../main/acm2/audio_coding_module_impl.cc | 62 +++++++++++++++++-- .../main/acm2/audio_coding_module_impl.h | 6 ++ 4 files changed, 108 insertions(+), 39 deletions(-) diff --git a/webrtc/modules/audio_coding/main/acm2/acm_codec_database.cc b/webrtc/modules/audio_coding/main/acm2/acm_codec_database.cc index 18caa61ff2..1b268dcb7c 100644 --- a/webrtc/modules/audio_coding/main/acm2/acm_codec_database.cc +++ b/webrtc/modules/audio_coding/main/acm2/acm_codec_database.cc @@ -568,42 +568,41 @@ int ACMCodecDB::MirrorID(int codec_id) { } // Creates memory/instance for storing codec state. -ACMGenericCodec* ACMCodecDB::CreateCodecInstance(const CodecInst& codec_inst) { +ACMGenericCodec* ACMCodecDB::CreateCodecInstance(const CodecInst& codec_inst, + int cng_pt_nb, + int cng_pt_wb, + int cng_pt_swb, + int cng_pt_fb, + bool enable_red, + int red_payload_type) { // All we have support for right now. if (!STR_CASE_CMP(codec_inst.plname, "ISAC")) { #if (defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX)) - return new ACMISAC(kISAC, false); + return new ACMISAC(kISAC, enable_red); #endif - } else if (!STR_CASE_CMP(codec_inst.plname, "PCMU")) { - if (codec_inst.channels == 1) { - return new ACMPCMU(kPCMU, false); - } else { - return new ACMPCMU(kPCMU_2ch, false); - } - } else if (!STR_CASE_CMP(codec_inst.plname, "PCMA")) { - if (codec_inst.channels == 1) { - return new ACMPCMA(kPCMA, false); - } else { - return new ACMPCMA(kPCMA_2ch, false); - } + } else if (!STR_CASE_CMP(codec_inst.plname, "PCMU") || + !STR_CASE_CMP(codec_inst.plname, "PCMA")) { + return new ACMGenericCodecWrapper(codec_inst, cng_pt_nb, cng_pt_wb, + cng_pt_swb, cng_pt_fb, enable_red, + red_payload_type); } else if (!STR_CASE_CMP(codec_inst.plname, "ILBC")) { #ifdef WEBRTC_CODEC_ILBC - return new ACMILBC(kILBC, false); + return new ACMILBC(kILBC, enable_red); #endif } else if (!STR_CASE_CMP(codec_inst.plname, "AMR")) { #ifdef WEBRTC_CODEC_AMR - return new ACMAMR(kGSMAMR, false); + return new ACMAMR(kGSMAMR, enable_red); #endif } else if (!STR_CASE_CMP(codec_inst.plname, "AMR-WB")) { #ifdef WEBRTC_CODEC_AMRWB - return new ACMAMRwb(kGSMAMRWB, false); + return new ACMAMRwb(kGSMAMRWB, enable_red); #endif } else if (!STR_CASE_CMP(codec_inst.plname, "G722")) { #ifdef WEBRTC_CODEC_G722 if (codec_inst.channels == 1) { - return new ACMG722(kG722, false); + return new ACMG722(kG722, enable_red); } else { - return new ACMG722(kG722_2ch, false); + return new ACMG722(kG722_2ch, enable_red); } #endif } else if (!STR_CASE_CMP(codec_inst.plname, "G7221")) { @@ -628,7 +627,7 @@ ACMGenericCodec* ACMCodecDB::CreateCodecInstance(const CodecInst& codec_inst) { return NULL; } } - return new ACMG722_1(codec_id, false); + return new ACMG722_1(codec_id, enable_red); #endif FALLTHROUGH(); } @@ -652,7 +651,7 @@ ACMGenericCodec* ACMCodecDB::CreateCodecInstance(const CodecInst& codec_inst) { return NULL; } } - return new ACMG722_1C(codec_id, false); + return new ACMG722_1C(codec_id, enable_red); #endif FALLTHROUGH(); } @@ -685,18 +684,18 @@ ACMGenericCodec* ACMCodecDB::CreateCodecInstance(const CodecInst& codec_inst) { return NULL; } } - return new ACMCNG(codec_id, false); + return new ACMCNG(codec_id, enable_red); } else if (!STR_CASE_CMP(codec_inst.plname, "G729")) { #ifdef WEBRTC_CODEC_G729 - return new ACMG729(kG729, false); + return new ACMG729(kG729, enable_red); #endif } else if (!STR_CASE_CMP(codec_inst.plname, "G7291")) { #ifdef WEBRTC_CODEC_G729_1 - return new ACMG729_1(kG729_1, false); + return new ACMG729_1(kG729_1, enable_red); #endif } else if (!STR_CASE_CMP(codec_inst.plname, "opus")) { #ifdef WEBRTC_CODEC_OPUS - return new ACMOpus(kOpus, false); + return new ACMOpus(kOpus, enable_red); #endif } else if (!STR_CASE_CMP(codec_inst.plname, "speex")) { #ifdef WEBRTC_CODEC_SPEEX @@ -714,7 +713,7 @@ ACMGenericCodec* ACMCodecDB::CreateCodecInstance(const CodecInst& codec_inst) { return NULL; } } - return new ACMSPEEX(codec_id, false); + return new ACMSPEEX(codec_id, enable_red); #endif } else if (!STR_CASE_CMP(codec_inst.plname, "CN")) { // For CN we need to check sampling frequency to know what codec to create. @@ -742,7 +741,7 @@ ACMGenericCodec* ACMCodecDB::CreateCodecInstance(const CodecInst& codec_inst) { return NULL; } } - return new ACMCNG(codec_id, false); + return new ACMCNG(codec_id, enable_red); } else if (!STR_CASE_CMP(codec_inst.plname, "L16")) { #ifdef WEBRTC_CODEC_PCM16 // For L16 we need to check sampling frequency to know what codec to create. @@ -784,15 +783,15 @@ ACMGenericCodec* ACMCodecDB::CreateCodecInstance(const CodecInst& codec_inst) { } } } - return new ACMPCM16B(codec_id, false); + return new ACMPCM16B(codec_id, enable_red); #endif } else if (!STR_CASE_CMP(codec_inst.plname, "telephone-event")) { #ifdef WEBRTC_CODEC_AVT - return new ACMDTMFPlayout(kAVT, false); + return new ACMDTMFPlayout(kAVT, enable_red); #endif } else if (!STR_CASE_CMP(codec_inst.plname, "red")) { #ifdef WEBRTC_CODEC_RED - return new ACMRED(kRED, false); + return new ACMRED(kRED, enable_red); #endif } return NULL; diff --git a/webrtc/modules/audio_coding/main/acm2/acm_codec_database.h b/webrtc/modules/audio_coding/main/acm2/acm_codec_database.h index bf4fe241ae..608eb6b64f 100644 --- a/webrtc/modules/audio_coding/main/acm2/acm_codec_database.h +++ b/webrtc/modules/audio_coding/main/acm2/acm_codec_database.h @@ -292,11 +292,21 @@ class ACMCodecDB { // Mirror id on success, otherwise -1. static int MirrorID(int codec_id); - // Create memory/instance for storing codec state. - // Input: - // [codec_inst] - information about codec. Only name of codec, "plname", is - // used in this function. - static ACMGenericCodec* CreateCodecInstance(const CodecInst& codec_inst); + // Creates a codec wrapper containing an AudioEncoder object (or an + // ACMGenericCodec subclass during the refactoring time). The type of + // AudioEncoder is decided by looking at the information in |codec_inst|. + // The |cng_pt_*| parameters should contain the RTP payload type used for each + // type of comfort noise; if not used (or not know when this function is + // called), -1 can be set. The parameter |enable_red| indicates that RED + // is enabled, and that |red_payload_type| should be used as RTP payload type + // for RED encodings. + static ACMGenericCodec* CreateCodecInstance(const CodecInst& codec_inst, + int cng_pt_nb, + int cng_pt_wb, + int cng_pt_swb, + int cng_pt_fb, + bool enable_red, + int red_payload_type); // Specifies if the codec specified by |codec_id| MUST own its own decoder. // This is the case for codecs which *should* share a single codec instance diff --git a/webrtc/modules/audio_coding/main/acm2/audio_coding_module_impl.cc b/webrtc/modules/audio_coding/main/acm2/audio_coding_module_impl.cc index 4098889adb..56cdd2ed28 100644 --- a/webrtc/modules/audio_coding/main/acm2/audio_coding_module_impl.cc +++ b/webrtc/modules/audio_coding/main/acm2/audio_coding_module_impl.cc @@ -15,6 +15,7 @@ #include #include "webrtc/base/checks.h" +#include "webrtc/base/safe_conversions.h" #include "webrtc/engine_configurations.h" #include "webrtc/modules/audio_coding/main/interface/audio_coding_module_typedefs.h" #include "webrtc/modules/audio_coding/main/acm2/acm_codec_database.h" @@ -96,6 +97,27 @@ int UpMix(const AudioFrame& frame, int length_out_buff, int16_t* out_buff) { return 0; } +void ConvertEncodedInfoToFragmentationHeader( + const AudioEncoder::EncodedInfo& info, + RTPFragmentationHeader* frag) { + if (info.redundant.empty()) { + frag->fragmentationVectorSize = 0; + return; + } + + frag->VerifyAndAllocateFragmentationHeader( + static_cast(info.redundant.size())); + frag->fragmentationVectorSize = static_cast(info.redundant.size()); + size_t offset = 0; + for (size_t i = 0; i < info.redundant.size(); ++i) { + frag->fragmentationOffset[i] = offset; + offset += info.redundant[i].encoded_bytes; + frag->fragmentationLength[i] = info.redundant[i].encoded_bytes; + frag->fragmentationTimeDiff[i] = rtc::checked_cast( + info.encoded_timestamp - info.redundant[i].encoded_timestamp); + frag->fragmentationPlType[i] = info.redundant[i].payload_type; + } +} } // namespace AudioCodingModuleImpl::AudioCodingModuleImpl( @@ -312,12 +334,22 @@ int32_t AudioCodingModuleImpl::Process() { has_data_to_send = true; previous_pltype_ = current_payload_type; + ConvertEncodedInfoToFragmentationHeader(encoded_info, &my_fragmentation); + // If RED is produced by the AudioEncoder object, the payload type for + // RED must be set. + if (!encoded_info.redundant.empty()) + current_payload_type = encoded_info.payload_type; + // Redundancy encode is done here. The two bitstreams packetized into // one RTP packet and the fragmentation points are set. // Only apply RED on speech data. - if ((red_enabled_) && + // Note: This will only happen if |encoded_info| did not contain any + // redundancy data. The if statement below will be removed once all codecs + // have been switched to the new AudioEncoder interface. + if ((codecs_[current_send_codec_idx_]->ExternalRedNeeded()) && ((encoding_type == kActiveNormalEncoded) || (encoding_type == kPassiveNormalEncoded))) { + DCHECK(encoded_info.redundant.empty()); // RED is enabled within this scope. // // Note that, a special solution exists for iSAC since it is the only @@ -425,7 +457,7 @@ int32_t AudioCodingModuleImpl::Process() { CriticalSectionScoped lock(callback_crit_sect_); if (packetization_callback_ != NULL) { - if (red_active) { + if (red_active || my_fragmentation.fragmentationVectorSize > 0) { // Callback with payload data, including redundant data (RED). packetization_callback_->SendData(frame_type, current_payload_type, rtp_timestamp, stream, length_bytes, @@ -488,8 +520,10 @@ int AudioCodingModuleImpl::ResetEncoder() { ACMGenericCodec* AudioCodingModuleImpl::CreateCodec(const CodecInst& codec) { ACMGenericCodec* my_codec = NULL; - - my_codec = ACMCodecDB::CreateCodecInstance(codec); + CriticalSectionScoped lock(acm_crit_sect_); + my_codec = ACMCodecDB::CreateCodecInstance( + codec, cng_nb_pltype_, cng_wb_pltype_, cng_swb_pltype_, cng_fb_pltype_, + red_enabled_, red_pltype_); if (my_codec == NULL) { // Error, could not create the codec. WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_, @@ -629,6 +663,7 @@ int AudioCodingModuleImpl::RegisterSendCodec(const CodecInst& send_codec) { return -1; } } + SetCngPayloadType(send_codec.plfreq, send_codec.pltype); return 0; } @@ -1075,6 +1110,8 @@ int AudioCodingModuleImpl::SetREDStatus( return -1; } + EnableCopyRedForAllCodecs(enable_red); + if (red_enabled_ != enable_red) { // Reset the RED buffer. memset(red_buffer_, 0, MAX_PAYLOAD_SIZE_BYTE); @@ -1643,6 +1680,23 @@ int AudioCodingModuleImpl::GetAudioDecoder(const CodecInst& codec, int codec_id, return 0; } +void AudioCodingModuleImpl::SetCngPayloadType(int sample_rate_hz, + int payload_type) { + for (auto codec : codecs_) { + if (codec) { + codec->SetCngPt(sample_rate_hz, payload_type); + } + } +} + +void AudioCodingModuleImpl::EnableCopyRedForAllCodecs(bool enable) { + for (auto codec : codecs_) { + if (codec) { + codec->EnableCopyRed(enable, red_pltype_); + } + } +} + int AudioCodingModuleImpl::SetInitialPlayoutDelay(int delay_ms) { { CriticalSectionScoped lock(acm_crit_sect_); diff --git a/webrtc/modules/audio_coding/main/acm2/audio_coding_module_impl.h b/webrtc/modules/audio_coding/main/acm2/audio_coding_module_impl.h index bbcb0b76be..12d4bdf657 100644 --- a/webrtc/modules/audio_coding/main/acm2/audio_coding_module_impl.h +++ b/webrtc/modules/audio_coding/main/acm2/audio_coding_module_impl.h @@ -290,6 +290,12 @@ class AudioCodingModuleImpl : public AudioCodingModule { int mirror_id, AudioDecoder** decoder) EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); + void SetCngPayloadType(int sample_rate_hz, int payload_type) + EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); + + void EnableCopyRedForAllCodecs(bool enable) + EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_); + CriticalSectionWrapper* acm_crit_sect_; int id_; // TODO(henrik.lundin) Make const. uint32_t expected_codec_ts_ GUARDED_BY(acm_crit_sect_);