From 0d526d558b0f7b677c43f83e64e67ad65a9639c0 Mon Sep 17 00:00:00 2001 From: ossu Date: Wed, 21 Sep 2016 01:57:31 -0700 Subject: [PATCH] Moved codec-specific audio packet splitting into decoders. There's still some code run specifically for Opus w/ FEC. It will be addressed in a separate CL. BUG=webrtc:5805 Review-Url: https://codereview.webrtc.org/2326003002 Cr-Commit-Position: refs/heads/master@{#14319} --- webrtc/modules/BUILD.gn | 5 +- webrtc/modules/audio_coding/BUILD.gn | 10 + webrtc/modules/audio_coding/audio_coding.gypi | 20 + .../audio_coding/codecs/audio_decoder.cc | 52 +-- .../audio_coding/codecs/audio_decoder.h | 3 +- .../codecs/g711/audio_decoder_pcm.cc | 17 + .../codecs/g711/audio_decoder_pcm.h | 6 + .../audio_coding/codecs/g711/g711.gypi | 2 + .../codecs/g722/audio_decoder_g722.cc | 17 + .../codecs/g722/audio_decoder_g722.h | 6 + .../audio_coding/codecs/g722/g722.gypi | 2 + .../codecs/ilbc/audio_decoder_ilbc.cc | 49 +++ .../codecs/ilbc/audio_decoder_ilbc.h | 3 + .../audio_coding/codecs/ilbc/ilbc_unittest.cc | 82 ++++ .../codecs/legacy_encoded_audio_frame.cc | 105 +++++ .../codecs/legacy_encoded_audio_frame.h | 52 +++ .../legacy_encoded_audio_frame_unittest.cc | 169 ++++++++ .../codecs/pcm16b/audio_decoder_pcm16b.cc | 11 + .../codecs/pcm16b/audio_decoder_pcm16b.h | 3 + .../audio_coding/codecs/pcm16b/pcm16b.gypi | 2 + .../neteq/mock/mock_payload_splitter.h | 8 - .../modules/audio_coding/neteq/neteq_impl.cc | 17 +- .../audio_coding/neteq/neteq_impl_unittest.cc | 3 - .../audio_coding/neteq/payload_splitter.cc | 210 ---------- .../audio_coding/neteq/payload_splitter.h | 30 +- .../neteq/payload_splitter_unittest.cc | 372 +----------------- 26 files changed, 571 insertions(+), 685 deletions(-) create mode 100644 webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc create mode 100644 webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h create mode 100644 webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc diff --git a/webrtc/modules/BUILD.gn b/webrtc/modules/BUILD.gn index 72af5792c1..09983c23e4 100644 --- a/webrtc/modules/BUILD.gn +++ b/webrtc/modules/BUILD.gn @@ -262,7 +262,9 @@ if (rtc_include_tests) { testonly = true defines = audio_coding_defines - deps = [] + deps = [ + "audio_coding:legacy_encoded_audio_frame", + ] sources = [ "audio_coding/acm2/acm_receiver_unittest_oldapi.cc", "audio_coding/acm2/audio_coding_module_unittest_oldapi.cc", @@ -281,6 +283,7 @@ if (rtc_include_tests) { "audio_coding/codecs/isac/main/source/audio_encoder_isac_unittest.cc", "audio_coding/codecs/isac/main/source/isac_unittest.cc", "audio_coding/codecs/isac/unittest.cc", + "audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc", "audio_coding/codecs/mock/mock_audio_encoder.cc", "audio_coding/codecs/opus/audio_encoder_opus_unittest.cc", "audio_coding/codecs/opus/opus_unittest.cc", diff --git a/webrtc/modules/audio_coding/BUILD.gn b/webrtc/modules/audio_coding/BUILD.gn index 2c90c7a758..a7b50a0882 100644 --- a/webrtc/modules/audio_coding/BUILD.gn +++ b/webrtc/modules/audio_coding/BUILD.gn @@ -130,6 +130,7 @@ rtc_source_set("audio_decoder_interface") { "codecs/audio_decoder.h", ] deps = [ + ":legacy_encoded_audio_frame", "../..:webrtc_common", "../../base:rtc_base_approved", ] @@ -187,6 +188,13 @@ rtc_source_set("red") { ] } +rtc_source_set("legacy_encoded_audio_frame") { + sources = [ + "codecs/legacy_encoded_audio_frame.cc", + "codecs/legacy_encoded_audio_frame.h", + ] +} + config("g711_config") { include_dirs = [ "../../..", @@ -211,6 +219,7 @@ rtc_source_set("g711") { deps = [ ":audio_decoder_interface", ":audio_encoder_interface", + ":legacy_encoded_audio_frame", ] } @@ -239,6 +248,7 @@ rtc_source_set("g722") { deps = [ ":audio_decoder_interface", ":audio_encoder_interface", + ":legacy_encoded_audio_frame", ] } diff --git a/webrtc/modules/audio_coding/audio_coding.gypi b/webrtc/modules/audio_coding/audio_coding.gypi index cb318ec777..2901855464 100644 --- a/webrtc/modules/audio_coding/audio_coding.gypi +++ b/webrtc/modules/audio_coding/audio_coding.gypi @@ -186,6 +186,26 @@ 'include/audio_coding_module_typedefs.h', ], }, + { + 'target_name': 'legacy_encoded_audio_frame', + 'type': 'static_library', + 'dependencies': [ + '<(webrtc_root)/common.gyp:webrtc_common', + 'audio_decoder_interface', + ], + 'include_dirs': [ + '<(webrtc_root)', + ], + 'direct_dependent_settings': { + 'include_dirs': [ + '<(webrtc_root)', + ], + }, + 'sources': [ + 'codecs/legacy_encoded_audio_frame.cc', + 'codecs/legacy_encoded_audio_frame.h', + ], + }, ], 'conditions': [ ['include_opus==1', { diff --git a/webrtc/modules/audio_coding/codecs/audio_decoder.cc b/webrtc/modules/audio_coding/codecs/audio_decoder.cc index 9dbecd5e0a..6c67924260 100644 --- a/webrtc/modules/audio_coding/codecs/audio_decoder.cc +++ b/webrtc/modules/audio_coding/codecs/audio_decoder.cc @@ -11,6 +11,8 @@ #include "webrtc/modules/audio_coding/codecs/audio_decoder.h" #include +#include +#include #include @@ -18,56 +20,10 @@ #include "webrtc/base/checks.h" #include "webrtc/base/sanitizer.h" #include "webrtc/base/trace_event.h" +#include "webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h" namespace webrtc { -namespace { -class LegacyFrame final : public AudioDecoder::EncodedAudioFrame { - public: - LegacyFrame(AudioDecoder* decoder, - rtc::Buffer&& payload, - bool is_primary_payload) - : decoder_(decoder), - payload_(std::move(payload)), - is_primary_payload_(is_primary_payload) {} - - size_t Duration() const override { - int ret; - if (is_primary_payload_) { - ret = decoder_->PacketDuration(payload_.data(), payload_.size()); - } else { - ret = decoder_->PacketDurationRedundant(payload_.data(), payload_.size()); - } - return (ret < 0) ? 0 : static_cast(ret); - } - - rtc::Optional Decode( - rtc::ArrayView decoded) const override { - AudioDecoder::SpeechType speech_type = AudioDecoder::kSpeech; - int ret; - if (is_primary_payload_) { - ret = decoder_->Decode( - payload_.data(), payload_.size(), decoder_->SampleRateHz(), - decoded.size() * sizeof(int16_t), decoded.data(), &speech_type); - } else { - ret = decoder_->DecodeRedundant( - payload_.data(), payload_.size(), decoder_->SampleRateHz(), - decoded.size() * sizeof(int16_t), decoded.data(), &speech_type); - } - - if (ret < 0) - return rtc::Optional(); - - return rtc::Optional({static_cast(ret), speech_type}); - } - - private: - AudioDecoder* const decoder_; - const rtc::Buffer payload_; - const bool is_primary_payload_; -}; -} // namespace - AudioDecoder::ParseResult::ParseResult() = default; AudioDecoder::ParseResult::ParseResult(ParseResult&& b) = default; AudioDecoder::ParseResult::ParseResult(uint32_t timestamp, @@ -86,7 +42,7 @@ std::vector AudioDecoder::ParsePayload( bool is_primary) { std::vector results; std::unique_ptr frame( - new LegacyFrame(this, std::move(payload), is_primary)); + new LegacyEncodedAudioFrame(this, std::move(payload), is_primary)); results.emplace_back(timestamp, is_primary, std::move(frame)); return results; } diff --git a/webrtc/modules/audio_coding/codecs/audio_decoder.h b/webrtc/modules/audio_coding/codecs/audio_decoder.h index 9608c321db..b6338d2102 100644 --- a/webrtc/modules/audio_coding/codecs/audio_decoder.h +++ b/webrtc/modules/audio_coding/codecs/audio_decoder.h @@ -11,7 +11,8 @@ #ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_INCLUDE_AUDIO_DECODER_H_ #define WEBRTC_MODULES_AUDIO_CODING_NETEQ_INCLUDE_AUDIO_DECODER_H_ -#include // NULL +#include +#include #include #include diff --git a/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc b/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc index af164c4bb7..f2fdb1f15c 100644 --- a/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc +++ b/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc @@ -10,12 +10,21 @@ #include "webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h" +#include "webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h" #include "webrtc/modules/audio_coding/codecs/g711/g711_interface.h" namespace webrtc { void AudioDecoderPcmU::Reset() {} +std::vector AudioDecoderPcmU::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp, + bool is_primary) { + return LegacyEncodedAudioFrame::SplitBySamples( + this, std::move(payload), timestamp, is_primary, 8 * num_channels_, 8); +} + int AudioDecoderPcmU::SampleRateHz() const { return 8000; } @@ -44,6 +53,14 @@ int AudioDecoderPcmU::PacketDuration(const uint8_t* encoded, void AudioDecoderPcmA::Reset() {} +std::vector AudioDecoderPcmA::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp, + bool is_primary) { + return LegacyEncodedAudioFrame::SplitBySamples( + this, std::move(payload), timestamp, is_primary, 8 * num_channels_, 8); +} + int AudioDecoderPcmA::SampleRateHz() const { return 8000; } diff --git a/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h b/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h index 7fdc3591b3..ed390217a3 100644 --- a/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h +++ b/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h @@ -23,6 +23,9 @@ class AudioDecoderPcmU final : public AudioDecoder { RTC_DCHECK_GE(num_channels, 1u); } void Reset() override; + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp, + bool is_primary) override; int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; int SampleRateHz() const override; size_t Channels() const override; @@ -45,6 +48,9 @@ class AudioDecoderPcmA final : public AudioDecoder { RTC_DCHECK_GE(num_channels, 1u); } void Reset() override; + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp, + bool is_primary) override; int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; int SampleRateHz() const override; size_t Channels() const override; diff --git a/webrtc/modules/audio_coding/codecs/g711/g711.gypi b/webrtc/modules/audio_coding/codecs/g711/g711.gypi index d209628528..207790fe4e 100644 --- a/webrtc/modules/audio_coding/codecs/g711/g711.gypi +++ b/webrtc/modules/audio_coding/codecs/g711/g711.gypi @@ -13,6 +13,8 @@ 'type': 'static_library', 'dependencies': [ 'audio_encoder_interface', + 'audio_decoder_interface', + 'legacy_encoded_audio_frame', ], 'sources': [ 'audio_decoder_pcm.cc', diff --git a/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc b/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc index 379293b748..93b24bdf2c 100644 --- a/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc +++ b/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc @@ -13,6 +13,7 @@ #include #include "webrtc/base/checks.h" +#include "webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h" #include "webrtc/modules/audio_coding/codecs/g722/g722_interface.h" namespace webrtc { @@ -47,6 +48,14 @@ void AudioDecoderG722::Reset() { WebRtcG722_DecoderInit(dec_state_); } +std::vector AudioDecoderG722::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp, + bool is_primary) { + return LegacyEncodedAudioFrame::SplitBySamples(this, std::move(payload), + timestamp, is_primary, 8, 16); +} + int AudioDecoderG722::PacketDuration(const uint8_t* encoded, size_t encoded_len) const { // 1/2 encoded byte per sample per channel. @@ -117,6 +126,14 @@ void AudioDecoderG722Stereo::Reset() { WebRtcG722_DecoderInit(dec_state_right_); } +std::vector AudioDecoderG722Stereo::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp, + bool is_primary) { + return LegacyEncodedAudioFrame::SplitBySamples( + this, std::move(payload), timestamp, is_primary, 2 * 8, 16); +} + // Split the stereo packet and place left and right channel after each other // in the output array. void AudioDecoderG722Stereo::SplitStereoPacket(const uint8_t* encoded, diff --git a/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h b/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h index ccca73d1a1..ad39619d3c 100644 --- a/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h +++ b/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h @@ -24,6 +24,9 @@ class AudioDecoderG722 final : public AudioDecoder { ~AudioDecoderG722() override; bool HasDecodePlc() const override; void Reset() override; + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp, + bool is_primary) override; int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; int SampleRateHz() const override; size_t Channels() const override; @@ -45,6 +48,9 @@ class AudioDecoderG722Stereo final : public AudioDecoder { AudioDecoderG722Stereo(); ~AudioDecoderG722Stereo() override; void Reset() override; + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp, + bool is_primary) override; int SampleRateHz() const override; size_t Channels() const override; diff --git a/webrtc/modules/audio_coding/codecs/g722/g722.gypi b/webrtc/modules/audio_coding/codecs/g722/g722.gypi index 13c6f2dcb7..a661a75cf3 100644 --- a/webrtc/modules/audio_coding/codecs/g722/g722.gypi +++ b/webrtc/modules/audio_coding/codecs/g722/g722.gypi @@ -12,6 +12,8 @@ 'type': 'static_library', 'dependencies': [ 'audio_encoder_interface', + 'audio_decoder_interface', + 'legacy_encoded_audio_frame', ], 'sources': [ 'audio_decoder_g722.cc', diff --git a/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc b/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc index dab5805f98..b4bd59911f 100644 --- a/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc +++ b/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc @@ -11,7 +11,9 @@ #include "webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h" #include "webrtc/base/checks.h" +#include "webrtc/base/logging.h" #include "webrtc/modules/audio_coding/codecs/ilbc/ilbc.h" +#include "webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h" namespace webrtc { @@ -49,6 +51,53 @@ void AudioDecoderIlbc::Reset() { WebRtcIlbcfix_Decoderinit30Ms(dec_state_); } +std::vector AudioDecoderIlbc::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp, + bool is_primary) { + std::vector results; + size_t bytes_per_frame; + int timestamps_per_frame; + if (payload.size() >= 950) { + LOG(LS_WARNING) << "AudioDecoderIlbc::ParsePayload: Payload too large"; + return results; + } + if (payload.size() % 38 == 0) { + // 20 ms frames. + bytes_per_frame = 38; + timestamps_per_frame = 160; + } else if (payload.size() % 50 == 0) { + // 30 ms frames. + bytes_per_frame = 50; + timestamps_per_frame = 240; + } else { + LOG(LS_WARNING) << "AudioDecoderIlbc::ParsePayload: Invalid payload"; + return results; + } + + RTC_DCHECK_EQ(0u, payload.size() % bytes_per_frame); + if (payload.size() == bytes_per_frame) { + std::unique_ptr frame( + new LegacyEncodedAudioFrame(this, std::move(payload), is_primary)); + results.emplace_back(timestamp, is_primary, std::move(frame)); + } else { + size_t byte_offset; + uint32_t timestamp_offset; + for (byte_offset = 0, timestamp_offset = 0; + byte_offset < payload.size(); + byte_offset += bytes_per_frame, + timestamp_offset += timestamps_per_frame) { + rtc::Buffer new_payload(payload.data() + byte_offset, bytes_per_frame); + std::unique_ptr frame(new LegacyEncodedAudioFrame( + this, std::move(new_payload), is_primary)); + results.emplace_back(timestamp + timestamp_offset, is_primary, + std::move(frame)); + } + } + + return results; +} + int AudioDecoderIlbc::SampleRateHz() const { return 8000; } diff --git a/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h b/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h index 1083479ad8..fdc856e985 100644 --- a/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h +++ b/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h @@ -25,6 +25,9 @@ class AudioDecoderIlbc final : public AudioDecoder { bool HasDecodePlc() const override; size_t DecodePlc(size_t num_frames, int16_t* decoded) override; void Reset() override; + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp, + bool is_primary) override; int SampleRateHz() const override; size_t Channels() const override; diff --git a/webrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc b/webrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc index c23cbc44e8..a8b76a5bc9 100644 --- a/webrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc +++ b/webrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc @@ -11,6 +11,7 @@ #include "testing/gtest/include/gtest/gtest.h" #include "webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h" #include "webrtc/modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.h" +#include "webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h" namespace webrtc { @@ -54,4 +55,85 @@ TEST(IlbcTest, BadPacket) { decoded_samples.data(), &speech_type)); } +class SplitIlbcTest : public ::testing::TestWithParam > { + protected: + virtual void SetUp() { + const std::pair parameters = GetParam(); + num_frames_ = parameters.first; + frame_length_ms_ = parameters.second; + frame_length_bytes_ = (frame_length_ms_ == 20) ? 38 : 50; + } + size_t num_frames_; + int frame_length_ms_; + size_t frame_length_bytes_; +}; + +TEST_P(SplitIlbcTest, NumFrames) { + AudioDecoderIlbc decoder; + const size_t frame_length_samples = frame_length_ms_ * 8; + const auto generate_payload = [] (size_t payload_length_bytes) { + rtc::Buffer payload(payload_length_bytes); + // Fill payload with increasing integers {0, 1, 2, ...}. + for (size_t i = 0; i < payload.size(); ++i) { + payload[i] = static_cast(i); + } + return payload; + }; + + const auto results = decoder.ParsePayload( + generate_payload(frame_length_bytes_ * num_frames_), 0, true); + EXPECT_EQ(num_frames_, results.size()); + + size_t frame_num = 0; + uint8_t payload_value = 0; + for (const auto& result : results) { + EXPECT_EQ(frame_length_samples * frame_num, result.timestamp); + const LegacyEncodedAudioFrame* frame = + static_cast(result.frame.get()); + const rtc::Buffer& payload = frame->payload(); + EXPECT_EQ(frame_length_bytes_, payload.size()); + for (size_t i = 0; i < payload.size(); ++i, ++payload_value) { + EXPECT_EQ(payload_value, payload[i]); + } + ++frame_num; + } +} + +// Test 1 through 5 frames of 20 and 30 ms size. +// Also test the maximum number of frames in one packet for 20 and 30 ms. +// The maximum is defined by the largest payload length that can be uniquely +// resolved to a frame size of either 38 bytes (20 ms) or 50 bytes (30 ms). +INSTANTIATE_TEST_CASE_P( + IlbcTest, SplitIlbcTest, + ::testing::Values(std::pair(1, 20), // 1 frame, 20 ms. + std::pair(2, 20), // 2 frames, 20 ms. + std::pair(3, 20), // And so on. + std::pair(4, 20), + std::pair(5, 20), + std::pair(24, 20), + std::pair(1, 30), + std::pair(2, 30), + std::pair(3, 30), + std::pair(4, 30), + std::pair(5, 30), + std::pair(18, 30))); + +// Test too large payload size. +TEST(IlbcTest, SplitTooLargePayload) { + AudioDecoderIlbc decoder; + constexpr size_t kPayloadLengthBytes = 950; + const auto results = + decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0, true); + EXPECT_TRUE(results.empty()); +} + +// Payload not an integer number of frames. +TEST(IlbcTest, SplitUnevenPayload) { + AudioDecoderIlbc decoder; + constexpr size_t kPayloadLengthBytes = 39; // Not an even number of frames. + const auto results = + decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0, true); + EXPECT_TRUE(results.empty()); +} + } // namespace webrtc diff --git a/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc new file mode 100644 index 0000000000..5e6ff01f22 --- /dev/null +++ b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h" + +#include +#include +#include + +namespace webrtc { + +LegacyEncodedAudioFrame::LegacyEncodedAudioFrame(AudioDecoder* decoder, + rtc::Buffer&& payload, + bool is_primary_payload) + : decoder_(decoder), + payload_(std::move(payload)), + is_primary_payload_(is_primary_payload) {} + +LegacyEncodedAudioFrame::~LegacyEncodedAudioFrame() = default; + +size_t LegacyEncodedAudioFrame::Duration() const { + int ret; + if (is_primary_payload_) { + ret = decoder_->PacketDuration(payload_.data(), payload_.size()); + } else { + ret = decoder_->PacketDurationRedundant(payload_.data(), payload_.size()); + } + return (ret < 0) ? 0 : static_cast(ret); +} + +rtc::Optional +LegacyEncodedAudioFrame::Decode(rtc::ArrayView decoded) const { + AudioDecoder::SpeechType speech_type = AudioDecoder::kSpeech; + int ret; + if (is_primary_payload_) { + ret = decoder_->Decode( + payload_.data(), payload_.size(), decoder_->SampleRateHz(), + decoded.size() * sizeof(int16_t), decoded.data(), &speech_type); + } else { + ret = decoder_->DecodeRedundant( + payload_.data(), payload_.size(), decoder_->SampleRateHz(), + decoded.size() * sizeof(int16_t), decoded.data(), &speech_type); + } + + if (ret < 0) + return rtc::Optional(); + + return rtc::Optional({static_cast(ret), speech_type}); +} + +std::vector LegacyEncodedAudioFrame::SplitBySamples( + AudioDecoder* decoder, + rtc::Buffer&& payload, + uint32_t timestamp, + bool is_primary, + size_t bytes_per_ms, + uint32_t timestamps_per_ms) { + RTC_DCHECK(payload.data()); + std::vector results; + size_t split_size_bytes = payload.size(); + + // Find a "chunk size" >= 20 ms and < 40 ms. + const size_t min_chunk_size = bytes_per_ms * 20; + if (min_chunk_size >= payload.size()) { + std::unique_ptr frame( + new LegacyEncodedAudioFrame(decoder, std::move(payload), is_primary)); + results.emplace_back(timestamp, is_primary, std::move(frame)); + } else { + // Reduce the split size by half as long as |split_size_bytes| is at least + // twice the minimum chunk size (so that the resulting size is at least as + // large as the minimum chunk size). + while (split_size_bytes >= 2 * min_chunk_size) { + split_size_bytes /= 2; + } + + const uint32_t timestamps_per_chunk = static_cast( + split_size_bytes * timestamps_per_ms / bytes_per_ms); + size_t byte_offset; + uint32_t timestamp_offset; + for (byte_offset = 0, timestamp_offset = 0; + byte_offset < payload.size(); + byte_offset += split_size_bytes, + timestamp_offset += timestamps_per_chunk) { + split_size_bytes = + std::min(split_size_bytes, payload.size() - byte_offset); + rtc::Buffer new_payload(payload.data() + byte_offset, split_size_bytes); + std::unique_ptr frame( + new LegacyEncodedAudioFrame(decoder, std::move(new_payload), + is_primary)); + results.emplace_back(timestamp + timestamp_offset, is_primary, + std::move(frame)); + } + } + + return results; +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h new file mode 100644 index 0000000000..1c9c3f542a --- /dev/null +++ b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_CODING_CODECS_LEGACY_ENCODED_AUDIO_FRAME_H_ +#define WEBRTC_MODULES_AUDIO_CODING_CODECS_LEGACY_ENCODED_AUDIO_FRAME_H_ + +#include + +#include "webrtc/base/array_view.h" +#include "webrtc/modules/audio_coding/codecs/audio_decoder.h" + +namespace webrtc { + +class LegacyEncodedAudioFrame final : public AudioDecoder::EncodedAudioFrame { + public: + LegacyEncodedAudioFrame(AudioDecoder* decoder, + rtc::Buffer&& payload, + bool is_primary_payload); + ~LegacyEncodedAudioFrame() override; + + static std::vector SplitBySamples( + AudioDecoder* decoder, + rtc::Buffer&& payload, + uint32_t timestamp, + bool is_primary, + size_t bytes_per_ms, + uint32_t timestamps_per_ms); + + size_t Duration() const override; + + rtc::Optional Decode( + rtc::ArrayView decoded) const override; + + // For testing: + const rtc::Buffer& payload() const { return payload_; } + + private: + AudioDecoder* const decoder_; + const rtc::Buffer payload_; + const bool is_primary_payload_; +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_AUDIO_CODING_CODECS_LEGACY_ENCODED_AUDIO_FRAME_H_ diff --git a/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc new file mode 100644 index 0000000000..2a4d9ed9f2 --- /dev/null +++ b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "testing/gtest/include/gtest/gtest.h" +#include "webrtc/modules/audio_coding/acm2/rent_a_codec.h" +#include "webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h" + +namespace webrtc { + +using NetEqDecoder = acm2::RentACodec::NetEqDecoder; + +class SplitBySamplesTest : public ::testing::TestWithParam { + protected: + virtual void SetUp() { + decoder_type_ = GetParam(); + switch (decoder_type_) { + case NetEqDecoder::kDecoderPCMu: + case NetEqDecoder::kDecoderPCMa: + bytes_per_ms_ = 8; + samples_per_ms_ = 8; + break; + case NetEqDecoder::kDecoderPCMu_2ch: + case NetEqDecoder::kDecoderPCMa_2ch: + bytes_per_ms_ = 2 * 8; + samples_per_ms_ = 8; + break; + case NetEqDecoder::kDecoderG722: + bytes_per_ms_ = 8; + samples_per_ms_ = 16; + break; + case NetEqDecoder::kDecoderPCM16B: + bytes_per_ms_ = 16; + samples_per_ms_ = 8; + break; + case NetEqDecoder::kDecoderPCM16Bwb: + bytes_per_ms_ = 32; + samples_per_ms_ = 16; + break; + case NetEqDecoder::kDecoderPCM16Bswb32kHz: + bytes_per_ms_ = 64; + samples_per_ms_ = 32; + break; + case NetEqDecoder::kDecoderPCM16Bswb48kHz: + bytes_per_ms_ = 96; + samples_per_ms_ = 48; + break; + case NetEqDecoder::kDecoderPCM16B_2ch: + bytes_per_ms_ = 2 * 16; + samples_per_ms_ = 8; + break; + case NetEqDecoder::kDecoderPCM16Bwb_2ch: + bytes_per_ms_ = 2 * 32; + samples_per_ms_ = 16; + break; + case NetEqDecoder::kDecoderPCM16Bswb32kHz_2ch: + bytes_per_ms_ = 2 * 64; + samples_per_ms_ = 32; + break; + case NetEqDecoder::kDecoderPCM16Bswb48kHz_2ch: + bytes_per_ms_ = 2 * 96; + samples_per_ms_ = 48; + break; + case NetEqDecoder::kDecoderPCM16B_5ch: + bytes_per_ms_ = 5 * 16; + samples_per_ms_ = 8; + break; + default: + assert(false); + break; + } + } + size_t bytes_per_ms_; + int samples_per_ms_; + NetEqDecoder decoder_type_; +}; + +// Test splitting sample-based payloads. +TEST_P(SplitBySamplesTest, PayloadSizes) { + constexpr uint32_t kBaseTimestamp = 0x12345678; + struct ExpectedSplit { + size_t payload_size_ms; + size_t num_frames; + // For simplicity. We only expect up to two packets per split. + size_t frame_sizes[2]; + }; + // The payloads are expected to be split as follows: + // 10 ms -> 10 ms + // 20 ms -> 20 ms + // 30 ms -> 30 ms + // 40 ms -> 20 + 20 ms + // 50 ms -> 25 + 25 ms + // 60 ms -> 30 + 30 ms + ExpectedSplit expected_splits[] = { + {10, 1, {10}}, + {20, 1, {20}}, + {30, 1, {30}}, + {40, 2, {20, 20}}, + {50, 2, {25, 25}}, + {60, 2, {30, 30}} + }; + + for (const auto& expected_split : expected_splits) { + // The payload values are set to steadily increase (modulo 256), so that the + // resulting frames can be checked and we can be reasonably certain no + // sample was missed or repeated. + const auto generate_payload = [] (size_t num_bytes) { + rtc::Buffer payload(num_bytes); + uint8_t value = 0; + // Allow wrap-around of value in counter below. + for (size_t i = 0; i != payload.size(); ++i, ++value) { + payload[i] = value; + } + return payload; + }; + + const auto results = LegacyEncodedAudioFrame::SplitBySamples( + nullptr, + generate_payload(expected_split.payload_size_ms * bytes_per_ms_), + kBaseTimestamp, true, bytes_per_ms_, samples_per_ms_); + + EXPECT_EQ(expected_split.num_frames, results.size()); + uint32_t expected_timestamp = kBaseTimestamp; + uint32_t expected_byte_offset = 0; + uint8_t value = 0; + for (size_t i = 0; i != expected_split.num_frames; ++i) { + const auto& result = results[i]; + const LegacyEncodedAudioFrame* frame = + static_cast(result.frame.get()); + const size_t length_bytes = expected_split.frame_sizes[i] * bytes_per_ms_; + EXPECT_EQ(length_bytes, frame->payload().size()); + EXPECT_EQ(expected_timestamp, result.timestamp); + const rtc::Buffer& payload = frame->payload(); + // Allow wrap-around of value in counter below. + for (size_t i = 0; i != payload.size(); ++i, ++value) { + ASSERT_EQ(value, payload[i]); + } + + expected_timestamp += expected_split.frame_sizes[i] * samples_per_ms_; + expected_byte_offset += length_bytes; + } + } +} + +INSTANTIATE_TEST_CASE_P( + LegacyEncodedAudioFrame, + SplitBySamplesTest, + ::testing::Values(NetEqDecoder::kDecoderPCMu, + NetEqDecoder::kDecoderPCMa, + NetEqDecoder::kDecoderPCMu_2ch, + NetEqDecoder::kDecoderPCMa_2ch, + NetEqDecoder::kDecoderG722, + NetEqDecoder::kDecoderPCM16B, + NetEqDecoder::kDecoderPCM16Bwb, + NetEqDecoder::kDecoderPCM16Bswb32kHz, + NetEqDecoder::kDecoderPCM16Bswb48kHz, + NetEqDecoder::kDecoderPCM16B_2ch, + NetEqDecoder::kDecoderPCM16Bwb_2ch, + NetEqDecoder::kDecoderPCM16Bswb32kHz_2ch, + NetEqDecoder::kDecoderPCM16Bswb48kHz_2ch, + NetEqDecoder::kDecoderPCM16B_5ch)); + +} // namespace webrtc diff --git a/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc b/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc index dce5f4c516..e600d2d56c 100644 --- a/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc +++ b/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc @@ -11,6 +11,7 @@ #include "webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h" #include "webrtc/base/checks.h" +#include "webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h" #include "webrtc/modules/audio_coding/codecs/pcm16b/pcm16b.h" namespace webrtc { @@ -44,6 +45,16 @@ int AudioDecoderPcm16B::DecodeInternal(const uint8_t* encoded, return static_cast(ret); } +std::vector AudioDecoderPcm16B::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp, + bool is_primary) { + const int samples_per_ms = rtc::CheckedDivExact(sample_rate_hz_, 1000); + return LegacyEncodedAudioFrame::SplitBySamples( + this, std::move(payload), timestamp, is_primary, + samples_per_ms * 2 * num_channels_, samples_per_ms); +} + int AudioDecoderPcm16B::PacketDuration(const uint8_t* encoded, size_t encoded_len) const { // Two encoded byte per sample per channel. diff --git a/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h b/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h index df94a6a6bd..1d2da4a718 100644 --- a/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h +++ b/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h @@ -20,6 +20,9 @@ class AudioDecoderPcm16B final : public AudioDecoder { public: AudioDecoderPcm16B(int sample_rate_hz, size_t num_channels); void Reset() override; + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp, + bool is_primary) override; int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; int SampleRateHz() const override; size_t Channels() const override; diff --git a/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b.gypi b/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b.gypi index d0dd21bb60..ae40793324 100644 --- a/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b.gypi +++ b/webrtc/modules/audio_coding/codecs/pcm16b/pcm16b.gypi @@ -13,6 +13,8 @@ 'type': 'static_library', 'dependencies': [ 'audio_encoder_interface', + 'audio_decoder_interface', + 'legacy_encoded_audio_frame', 'g711', ], 'sources': [ diff --git a/webrtc/modules/audio_coding/neteq/mock/mock_payload_splitter.h b/webrtc/modules/audio_coding/neteq/mock/mock_payload_splitter.h index 9fa05e9cce..cf0613403b 100644 --- a/webrtc/modules/audio_coding/neteq/mock/mock_payload_splitter.h +++ b/webrtc/modules/audio_coding/neteq/mock/mock_payload_splitter.h @@ -25,14 +25,6 @@ class MockPayloadSplitter : public PayloadSplitter { int(PacketList* packet_list, DecoderDatabase* decoder_database)); MOCK_METHOD2(CheckRedPayloads, int(PacketList* packet_list, const DecoderDatabase& decoder_database)); - MOCK_METHOD2(SplitAudio, - int(PacketList* packet_list, const DecoderDatabase& decoder_database)); - MOCK_METHOD4(SplitBySamples, - void(const Packet* packet, size_t bytes_per_ms, - uint32_t timestamps_per_ms, PacketList* new_packets)); - MOCK_METHOD4(SplitByFrames, - int(const Packet* packet, size_t bytes_per_frame, - uint32_t timestamps_per_frame, PacketList* new_packets)); }; } // namespace webrtc diff --git a/webrtc/modules/audio_coding/neteq/neteq_impl.cc b/webrtc/modules/audio_coding/neteq/neteq_impl.cc index ca20e5ba6d..98588f49cb 100644 --- a/webrtc/modules/audio_coding/neteq/neteq_impl.cc +++ b/webrtc/modules/audio_coding/neteq/neteq_impl.cc @@ -658,21 +658,6 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header, } } - // Split payloads into smaller chunks. This also verifies that all payloads - // are of a known payload type. - ret = payload_splitter_->SplitAudio(&packet_list, *decoder_database_); - if (ret != PayloadSplitter::kOK) { - PacketBuffer::DeleteAllPackets(&packet_list); - switch (ret) { - case PayloadSplitter::kUnknownPayloadType: - return kUnknownRtpPayloadType; - case PayloadSplitter::kFrameSplitError: - return kFrameSplitError; - default: - return kOtherError; - } - } - // Update bandwidth estimate, if the packet is not comfort noise. if (!packet_list.empty() && !decoder_database_->IsComfortNoise(main_header.payloadType)) { @@ -710,7 +695,7 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header, const RTPHeader& original_header = packet->header; for (auto& result : results) { RTC_DCHECK(result.frame); - // Reuse the packet if possible + // Reuse the packet if possible. if (!packet) { packet.reset(new Packet); packet->header = original_header; diff --git a/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc b/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc index 4f98005cbb..5b1015d576 100644 --- a/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc +++ b/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc @@ -336,9 +336,6 @@ TEST_F(NetEqImplTest, InsertPacket) { EXPECT_CALL(*mock_payload_splitter_, SplitFec(_, _)) .Times(2) .WillRepeatedly(Return(PayloadSplitter::kOK)); - EXPECT_CALL(*mock_payload_splitter_, SplitAudio(_, _)) - .Times(2) - .WillRepeatedly(Return(PayloadSplitter::kOK)); // Insert first packet. neteq_->InsertPacket(rtp_header, payload, kFirstReceiveTime); diff --git a/webrtc/modules/audio_coding/neteq/payload_splitter.cc b/webrtc/modules/audio_coding/neteq/payload_splitter.cc index 074d1a2f0c..28e561a65f 100644 --- a/webrtc/modules/audio_coding/neteq/payload_splitter.cc +++ b/webrtc/modules/audio_coding/neteq/payload_splitter.cc @@ -212,214 +212,4 @@ int PayloadSplitter::CheckRedPayloads(PacketList* packet_list, return num_deleted_packets; } -int PayloadSplitter::SplitAudio(PacketList* packet_list, - const DecoderDatabase& decoder_database) { - PacketList::iterator it = packet_list->begin(); - // Iterate through all packets in |packet_list|. - while (it != packet_list->end()) { - Packet* packet = (*it); // Just to make the notation more intuitive. - // Get codec type for this payload. - const DecoderDatabase::DecoderInfo* info = - decoder_database.GetDecoderInfo(packet->header.payloadType); - if (!info) { - LOG(LS_WARNING) << "SplitAudio unknown payload type"; - return kUnknownPayloadType; - } - PacketList new_packets; - switch (info->codec_type) { - case NetEqDecoder::kDecoderPCMu: - case NetEqDecoder::kDecoderPCMa: { - // 8 bytes per ms; 8 timestamps per ms. - SplitBySamples(packet, 8, 8, &new_packets); - break; - } - case NetEqDecoder::kDecoderPCMu_2ch: - case NetEqDecoder::kDecoderPCMa_2ch: { - // 2 * 8 bytes per ms; 8 timestamps per ms. - SplitBySamples(packet, 2 * 8, 8, &new_packets); - break; - } - case NetEqDecoder::kDecoderG722: { - // 8 bytes per ms; 16 timestamps per ms. - SplitBySamples(packet, 8, 16, &new_packets); - break; - } - case NetEqDecoder::kDecoderPCM16B: { - // 16 bytes per ms; 8 timestamps per ms. - SplitBySamples(packet, 16, 8, &new_packets); - break; - } - case NetEqDecoder::kDecoderPCM16Bwb: { - // 32 bytes per ms; 16 timestamps per ms. - SplitBySamples(packet, 32, 16, &new_packets); - break; - } - case NetEqDecoder::kDecoderPCM16Bswb32kHz: { - // 64 bytes per ms; 32 timestamps per ms. - SplitBySamples(packet, 64, 32, &new_packets); - break; - } - case NetEqDecoder::kDecoderPCM16Bswb48kHz: { - // 96 bytes per ms; 48 timestamps per ms. - SplitBySamples(packet, 96, 48, &new_packets); - break; - } - case NetEqDecoder::kDecoderPCM16B_2ch: { - // 2 * 16 bytes per ms; 8 timestamps per ms. - SplitBySamples(packet, 2 * 16, 8, &new_packets); - break; - } - case NetEqDecoder::kDecoderPCM16Bwb_2ch: { - // 2 * 32 bytes per ms; 16 timestamps per ms. - SplitBySamples(packet, 2 * 32, 16, &new_packets); - break; - } - case NetEqDecoder::kDecoderPCM16Bswb32kHz_2ch: { - // 2 * 64 bytes per ms; 32 timestamps per ms. - SplitBySamples(packet, 2 * 64, 32, &new_packets); - break; - } - case NetEqDecoder::kDecoderPCM16Bswb48kHz_2ch: { - // 2 * 96 bytes per ms; 48 timestamps per ms. - SplitBySamples(packet, 2 * 96, 48, &new_packets); - break; - } - case NetEqDecoder::kDecoderPCM16B_5ch: { - // 5 * 16 bytes per ms; 8 timestamps per ms. - SplitBySamples(packet, 5 * 16, 8, &new_packets); - break; - } - case NetEqDecoder::kDecoderILBC: { - size_t bytes_per_frame; - int timestamps_per_frame; - if (packet->payload.size() >= 950) { - LOG(LS_WARNING) << "SplitAudio too large iLBC payload"; - return kTooLargePayload; - } - if (packet->payload.size() % 38 == 0) { - // 20 ms frames. - bytes_per_frame = 38; - timestamps_per_frame = 160; - } else if (packet->payload.size() % 50 == 0) { - // 30 ms frames. - bytes_per_frame = 50; - timestamps_per_frame = 240; - } else { - LOG(LS_WARNING) << "SplitAudio invalid iLBC payload"; - return kFrameSplitError; - } - int ret = SplitByFrames(packet, bytes_per_frame, timestamps_per_frame, - &new_packets); - if (ret < 0) { - return ret; - } else if (ret == kNoSplit) { - // Do not split at all. Simply advance to the next packet in the list. - ++it; - // We do not have any new packets to insert, and should not delete the - // old one. Skip the code after the switch case, and jump straight to - // the next packet in the while loop. - continue; - } - break; - } - default: { - // Do not split at all. Simply advance to the next packet in the list. - ++it; - // We do not have any new packets to insert, and should not delete the - // old one. Skip the code after the switch case, and jump straight to - // the next packet in the while loop. - continue; - } - } - // Insert new packets into original list, before the element pointed to by - // iterator |it|. - packet_list->splice(it, new_packets, new_packets.begin(), - new_packets.end()); - // Delete old packet payload. - delete (*it); - // Remove |it| from the packet list. This operation effectively moves the - // iterator |it| to the next packet in the list. Thus, we do not have to - // increment it manually. - it = packet_list->erase(it); - } - return kOK; -} - -void PayloadSplitter::SplitBySamples(const Packet* packet, - size_t bytes_per_ms, - uint32_t timestamps_per_ms, - PacketList* new_packets) { - assert(packet); - assert(new_packets); - - size_t split_size_bytes = packet->payload.size(); - - // Find a "chunk size" >= 20 ms and < 40 ms. - size_t min_chunk_size = bytes_per_ms * 20; - // Reduce the split size by half as long as |split_size_bytes| is at least - // twice the minimum chunk size (so that the resulting size is at least as - // large as the minimum chunk size). - while (split_size_bytes >= 2 * min_chunk_size) { - split_size_bytes >>= 1; - } - uint32_t timestamps_per_chunk = static_cast( - split_size_bytes * timestamps_per_ms / bytes_per_ms); - uint32_t timestamp = packet->header.timestamp; - - const uint8_t* payload_ptr = packet->payload.data(); - size_t len = packet->payload.size(); - while (len >= (2 * split_size_bytes)) { - Packet* new_packet = new Packet; - new_packet->header = packet->header; - new_packet->header.timestamp = timestamp; - timestamp += timestamps_per_chunk; - new_packet->primary = packet->primary; - new_packet->payload.SetData(payload_ptr, split_size_bytes); - payload_ptr += split_size_bytes; - new_packets->push_back(new_packet); - len -= split_size_bytes; - } - - if (len > 0) { - Packet* new_packet = new Packet; - new_packet->header = packet->header; - new_packet->header.timestamp = timestamp; - new_packet->primary = packet->primary; - new_packet->payload.SetData(payload_ptr, len); - new_packets->push_back(new_packet); - } -} - -int PayloadSplitter::SplitByFrames(const Packet* packet, - size_t bytes_per_frame, - uint32_t timestamps_per_frame, - PacketList* new_packets) { - if (packet->payload.size() % bytes_per_frame != 0) { - LOG(LS_WARNING) << "SplitByFrames length mismatch"; - return kFrameSplitError; - } - - if (packet->payload.size() == bytes_per_frame) { - // Special case. Do not split the payload. - return kNoSplit; - } - - uint32_t timestamp = packet->header.timestamp; - const uint8_t* payload_ptr = packet->payload.data(); - size_t len = packet->payload.size(); - while (len > 0) { - assert(len >= bytes_per_frame); - Packet* new_packet = new Packet; - new_packet->header = packet->header; - new_packet->header.timestamp = timestamp; - timestamp += timestamps_per_frame; - new_packet->primary = packet->primary; - new_packet->payload.SetData(payload_ptr, bytes_per_frame); - payload_ptr += bytes_per_frame; - new_packets->push_back(new_packet); - len -= bytes_per_frame; - } - return kOK; -} - } // namespace webrtc diff --git a/webrtc/modules/audio_coding/neteq/payload_splitter.h b/webrtc/modules/audio_coding/neteq/payload_splitter.h index b0c4b5fe5c..a3e1b1d91c 100644 --- a/webrtc/modules/audio_coding/neteq/payload_splitter.h +++ b/webrtc/modules/audio_coding/neteq/payload_splitter.h @@ -20,16 +20,14 @@ namespace webrtc { class DecoderDatabase; // This class handles splitting of payloads into smaller parts. -// The class does not have any member variables, and the methods could have -// been made static. The reason for not making them static is testability. -// With this design, the splitting functionality can be mocked during testing -// of the NetEqImpl class. + +// For RED and FEC the splitting is done internally. Other codecs' packets are +// split by calling AudioDecoder::SplitPacket. class PayloadSplitter { public: enum SplitterReturnCodes { kOK = 0, kNoSplit = 1, - kTooLargePayload = -1, kFrameSplitError = -2, kUnknownPayloadType = -3, kRedLengthMismatch = -4, @@ -60,29 +58,7 @@ class PayloadSplitter { virtual int CheckRedPayloads(PacketList* packet_list, const DecoderDatabase& decoder_database); - // Iterates through |packet_list| and, if possible, splits each audio payload - // into suitable size chunks. The result is written back to |packet_list| as - // new packets. The decoder database is needed to get information about which - // payload type each packet contains. - virtual int SplitAudio(PacketList* packet_list, - const DecoderDatabase& decoder_database); - private: - // Splits the payload in |packet|. The payload is assumed to be from a - // sample-based codec. - virtual void SplitBySamples(const Packet* packet, - size_t bytes_per_ms, - uint32_t timestamps_per_ms, - PacketList* new_packets); - - // Splits the payload in |packet|. The payload will be split into chunks of - // size |bytes_per_frame|, corresponding to a |timestamps_per_frame| - // RTP timestamps. - virtual int SplitByFrames(const Packet* packet, - size_t bytes_per_frame, - uint32_t timestamps_per_frame, - PacketList* new_packets); - RTC_DISALLOW_COPY_AND_ASSIGN(PayloadSplitter); }; diff --git a/webrtc/modules/audio_coding/neteq/payload_splitter_unittest.cc b/webrtc/modules/audio_coding/neteq/payload_splitter_unittest.cc index bd9d4d4659..0a4b2157b5 100644 --- a/webrtc/modules/audio_coding/neteq/payload_splitter_unittest.cc +++ b/webrtc/modules/audio_coding/neteq/payload_splitter_unittest.cc @@ -152,7 +152,7 @@ void VerifyPacket(const Packet* packet, EXPECT_EQ(primary, packet->primary); ASSERT_FALSE(packet->payload.empty()); for (size_t i = 0; i < packet->payload.size(); ++i) { - EXPECT_EQ(payload_value, packet->payload[i]); + ASSERT_EQ(payload_value, packet->payload.data()[i]); } } @@ -344,376 +344,6 @@ TEST(RedPayloadSplitter, WrongPayloadLength) { packet_list.pop_front(); } -// Test that iSAC, iSAC-swb, RED, DTMF, CNG, and "Arbitrary" payloads do not -// get split. -TEST(AudioPayloadSplitter, NonSplittable) { - // Set up packets with different RTP payload types. The actual values do not - // matter, since we are mocking the decoder database anyway. - PacketList packet_list; - for (uint8_t i = 0; i < 6; ++i) { - // Let the payload type be |i|, and the payload value 10 * |i|. - packet_list.push_back(CreatePacket(i, kPayloadLength, 10 * i)); - } - - MockDecoderDatabase decoder_database; - // Tell the mock decoder database to return DecoderInfo structs with different - // codec types. - // Use scoped pointers to avoid having to delete them later. - std::unique_ptr info0( - new DecoderDatabase::DecoderInfo(NetEqDecoder::kDecoderISAC, "")); - EXPECT_CALL(decoder_database, GetDecoderInfo(0)) - .WillRepeatedly(Return(info0.get())); - std::unique_ptr info1( - new DecoderDatabase::DecoderInfo(NetEqDecoder::kDecoderISACswb, "")); - EXPECT_CALL(decoder_database, GetDecoderInfo(1)) - .WillRepeatedly(Return(info1.get())); - std::unique_ptr info2( - new DecoderDatabase::DecoderInfo(NetEqDecoder::kDecoderRED, "")); - EXPECT_CALL(decoder_database, GetDecoderInfo(2)) - .WillRepeatedly(Return(info2.get())); - std::unique_ptr info3( - new DecoderDatabase::DecoderInfo(NetEqDecoder::kDecoderAVT, "")); - EXPECT_CALL(decoder_database, GetDecoderInfo(3)) - .WillRepeatedly(Return(info3.get())); - std::unique_ptr info4( - new DecoderDatabase::DecoderInfo(NetEqDecoder::kDecoderCNGnb, "")); - EXPECT_CALL(decoder_database, GetDecoderInfo(4)) - .WillRepeatedly(Return(info4.get())); - std::unique_ptr info5( - new DecoderDatabase::DecoderInfo(NetEqDecoder::kDecoderArbitrary, "")); - EXPECT_CALL(decoder_database, GetDecoderInfo(5)) - .WillRepeatedly(Return(info5.get())); - - PayloadSplitter splitter; - EXPECT_EQ(0, splitter.SplitAudio(&packet_list, decoder_database)); - EXPECT_EQ(6u, packet_list.size()); - - // Check that all payloads are intact. - uint8_t payload_type = 0; - PacketList::iterator it = packet_list.begin(); - while (it != packet_list.end()) { - VerifyPacket((*it), kPayloadLength, payload_type, kSequenceNumber, - kBaseTimestamp, 10 * payload_type); - ++payload_type; - delete (*it); - it = packet_list.erase(it); - } - - // The destructor is called when decoder_database goes out of scope. - EXPECT_CALL(decoder_database, Die()); -} - -// Test unknown payload type. -TEST(AudioPayloadSplitter, UnknownPayloadType) { - PacketList packet_list; - static const uint8_t kPayloadType = 17; // Just a random number. - size_t kPayloadLengthBytes = 4711; // Random number. - packet_list.push_back(CreatePacket(kPayloadType, kPayloadLengthBytes, 0)); - - MockDecoderDatabase decoder_database; - // Tell the mock decoder database to return NULL when asked for decoder info. - // This signals that the decoder database does not recognize the payload type. - EXPECT_CALL(decoder_database, GetDecoderInfo(kPayloadType)) - .WillRepeatedly(ReturnNull()); - - PayloadSplitter splitter; - EXPECT_EQ(PayloadSplitter::kUnknownPayloadType, - splitter.SplitAudio(&packet_list, decoder_database)); - EXPECT_EQ(1u, packet_list.size()); - - - // Delete the packets and payloads to avoid having the test leak memory. - PacketList::iterator it = packet_list.begin(); - while (it != packet_list.end()) { - delete (*it); - it = packet_list.erase(it); - } - - // The destructor is called when decoder_database goes out of scope. - EXPECT_CALL(decoder_database, Die()); -} - -class SplitBySamplesTest : public ::testing::TestWithParam { - protected: - virtual void SetUp() { - decoder_type_ = GetParam(); - switch (decoder_type_) { - case NetEqDecoder::kDecoderPCMu: - case NetEqDecoder::kDecoderPCMa: - bytes_per_ms_ = 8; - samples_per_ms_ = 8; - break; - case NetEqDecoder::kDecoderPCMu_2ch: - case NetEqDecoder::kDecoderPCMa_2ch: - bytes_per_ms_ = 2 * 8; - samples_per_ms_ = 8; - break; - case NetEqDecoder::kDecoderG722: - bytes_per_ms_ = 8; - samples_per_ms_ = 16; - break; - case NetEqDecoder::kDecoderPCM16B: - bytes_per_ms_ = 16; - samples_per_ms_ = 8; - break; - case NetEqDecoder::kDecoderPCM16Bwb: - bytes_per_ms_ = 32; - samples_per_ms_ = 16; - break; - case NetEqDecoder::kDecoderPCM16Bswb32kHz: - bytes_per_ms_ = 64; - samples_per_ms_ = 32; - break; - case NetEqDecoder::kDecoderPCM16Bswb48kHz: - bytes_per_ms_ = 96; - samples_per_ms_ = 48; - break; - case NetEqDecoder::kDecoderPCM16B_2ch: - bytes_per_ms_ = 2 * 16; - samples_per_ms_ = 8; - break; - case NetEqDecoder::kDecoderPCM16Bwb_2ch: - bytes_per_ms_ = 2 * 32; - samples_per_ms_ = 16; - break; - case NetEqDecoder::kDecoderPCM16Bswb32kHz_2ch: - bytes_per_ms_ = 2 * 64; - samples_per_ms_ = 32; - break; - case NetEqDecoder::kDecoderPCM16Bswb48kHz_2ch: - bytes_per_ms_ = 2 * 96; - samples_per_ms_ = 48; - break; - case NetEqDecoder::kDecoderPCM16B_5ch: - bytes_per_ms_ = 5 * 16; - samples_per_ms_ = 8; - break; - default: - assert(false); - break; - } - } - size_t bytes_per_ms_; - int samples_per_ms_; - NetEqDecoder decoder_type_; -}; - -// Test splitting sample-based payloads. -TEST_P(SplitBySamplesTest, PayloadSizes) { - PacketList packet_list; - static const uint8_t kPayloadType = 17; // Just a random number. - for (int payload_size_ms = 10; payload_size_ms <= 60; payload_size_ms += 10) { - // The payload values are set to be the same as the payload_size, so that - // one can distinguish from which packet the split payloads come from. - size_t payload_size_bytes = payload_size_ms * bytes_per_ms_; - packet_list.push_back(CreatePacket(kPayloadType, payload_size_bytes, - payload_size_ms)); - } - - MockDecoderDatabase decoder_database; - // Tell the mock decoder database to return DecoderInfo structs with different - // codec types. - // Use scoped pointers to avoid having to delete them later. - // (Sample rate is set to 8000 Hz, but does not matter.) - std::unique_ptr info( - new DecoderDatabase::DecoderInfo(decoder_type_, "")); - EXPECT_CALL(decoder_database, GetDecoderInfo(kPayloadType)) - .WillRepeatedly(Return(info.get())); - - PayloadSplitter splitter; - EXPECT_EQ(0, splitter.SplitAudio(&packet_list, decoder_database)); - // The payloads are expected to be split as follows: - // 10 ms -> 10 ms - // 20 ms -> 20 ms - // 30 ms -> 30 ms - // 40 ms -> 20 + 20 ms - // 50 ms -> 25 + 25 ms - // 60 ms -> 30 + 30 ms - int expected_size_ms[] = {10, 20, 30, 20, 20, 25, 25, 30, 30}; - int expected_payload_value[] = {10, 20, 30, 40, 40, 50, 50, 60, 60}; - int expected_timestamp_offset_ms[] = {0, 0, 0, 0, 20, 0, 25, 0, 30}; - size_t expected_num_packets = - sizeof(expected_size_ms) / sizeof(expected_size_ms[0]); - EXPECT_EQ(expected_num_packets, packet_list.size()); - - PacketList::iterator it = packet_list.begin(); - int i = 0; - while (it != packet_list.end()) { - size_t length_bytes = expected_size_ms[i] * bytes_per_ms_; - uint32_t expected_timestamp = kBaseTimestamp + - expected_timestamp_offset_ms[i] * samples_per_ms_; - VerifyPacket((*it), length_bytes, kPayloadType, kSequenceNumber, - expected_timestamp, expected_payload_value[i]); - delete (*it); - it = packet_list.erase(it); - ++i; - } - - // The destructor is called when decoder_database goes out of scope. - EXPECT_CALL(decoder_database, Die()); -} - -INSTANTIATE_TEST_CASE_P( - PayloadSplitter, - SplitBySamplesTest, - ::testing::Values(NetEqDecoder::kDecoderPCMu, - NetEqDecoder::kDecoderPCMa, - NetEqDecoder::kDecoderPCMu_2ch, - NetEqDecoder::kDecoderPCMa_2ch, - NetEqDecoder::kDecoderG722, - NetEqDecoder::kDecoderPCM16B, - NetEqDecoder::kDecoderPCM16Bwb, - NetEqDecoder::kDecoderPCM16Bswb32kHz, - NetEqDecoder::kDecoderPCM16Bswb48kHz, - NetEqDecoder::kDecoderPCM16B_2ch, - NetEqDecoder::kDecoderPCM16Bwb_2ch, - NetEqDecoder::kDecoderPCM16Bswb32kHz_2ch, - NetEqDecoder::kDecoderPCM16Bswb48kHz_2ch, - NetEqDecoder::kDecoderPCM16B_5ch)); - -class SplitIlbcTest : public ::testing::TestWithParam > { - protected: - virtual void SetUp() { - const std::pair parameters = GetParam(); - num_frames_ = parameters.first; - frame_length_ms_ = parameters.second; - frame_length_bytes_ = (frame_length_ms_ == 20) ? 38 : 50; - } - size_t num_frames_; - int frame_length_ms_; - size_t frame_length_bytes_; -}; - -// Test splitting sample-based payloads. -TEST_P(SplitIlbcTest, NumFrames) { - PacketList packet_list; - static const uint8_t kPayloadType = 17; // Just a random number. - const int frame_length_samples = frame_length_ms_ * 8; - size_t payload_length_bytes = frame_length_bytes_ * num_frames_; - Packet* packet = CreatePacket(kPayloadType, payload_length_bytes, 0); - // Fill payload with increasing integers {0, 1, 2, ...}. - for (size_t i = 0; i < packet->payload.size(); ++i) { - packet->payload[i] = static_cast(i); - } - packet_list.push_back(packet); - - MockDecoderDatabase decoder_database; - // Tell the mock decoder database to return DecoderInfo structs with different - // codec types. - // Use scoped pointers to avoid having to delete them later. - std::unique_ptr info( - new DecoderDatabase::DecoderInfo(NetEqDecoder::kDecoderILBC, "")); - EXPECT_CALL(decoder_database, GetDecoderInfo(kPayloadType)) - .WillRepeatedly(Return(info.get())); - - PayloadSplitter splitter; - EXPECT_EQ(0, splitter.SplitAudio(&packet_list, decoder_database)); - EXPECT_EQ(num_frames_, packet_list.size()); - - PacketList::iterator it = packet_list.begin(); - int frame_num = 0; - uint8_t payload_value = 0; - while (it != packet_list.end()) { - Packet* packet = (*it); - EXPECT_EQ(kBaseTimestamp + frame_length_samples * frame_num, - packet->header.timestamp); - EXPECT_EQ(frame_length_bytes_, packet->payload.size()); - EXPECT_EQ(kPayloadType, packet->header.payloadType); - EXPECT_EQ(kSequenceNumber, packet->header.sequenceNumber); - EXPECT_EQ(true, packet->primary); - ASSERT_FALSE(packet->payload.empty()); - for (size_t i = 0; i < packet->payload.size(); ++i) { - EXPECT_EQ(payload_value, packet->payload[i]); - ++payload_value; - } - delete (*it); - it = packet_list.erase(it); - ++frame_num; - } - - // The destructor is called when decoder_database goes out of scope. - EXPECT_CALL(decoder_database, Die()); -} - -// Test 1 through 5 frames of 20 and 30 ms size. -// Also test the maximum number of frames in one packet for 20 and 30 ms. -// The maximum is defined by the largest payload length that can be uniquely -// resolved to a frame size of either 38 bytes (20 ms) or 50 bytes (30 ms). -INSTANTIATE_TEST_CASE_P( - PayloadSplitter, SplitIlbcTest, - ::testing::Values(std::pair(1, 20), // 1 frame, 20 ms. - std::pair(2, 20), // 2 frames, 20 ms. - std::pair(3, 20), // And so on. - std::pair(4, 20), - std::pair(5, 20), - std::pair(24, 20), - std::pair(1, 30), - std::pair(2, 30), - std::pair(3, 30), - std::pair(4, 30), - std::pair(5, 30), - std::pair(18, 30))); - -// Test too large payload size. -TEST(IlbcPayloadSplitter, TooLargePayload) { - PacketList packet_list; - static const uint8_t kPayloadType = 17; // Just a random number. - size_t kPayloadLengthBytes = 950; - Packet* packet = CreatePacket(kPayloadType, kPayloadLengthBytes, 0); - packet_list.push_back(packet); - - MockDecoderDatabase decoder_database; - std::unique_ptr info( - new DecoderDatabase::DecoderInfo(NetEqDecoder::kDecoderILBC, "")); - EXPECT_CALL(decoder_database, GetDecoderInfo(kPayloadType)) - .WillRepeatedly(Return(info.get())); - - PayloadSplitter splitter; - EXPECT_EQ(PayloadSplitter::kTooLargePayload, - splitter.SplitAudio(&packet_list, decoder_database)); - EXPECT_EQ(1u, packet_list.size()); - - // Delete the packets and payloads to avoid having the test leak memory. - PacketList::iterator it = packet_list.begin(); - while (it != packet_list.end()) { - delete (*it); - it = packet_list.erase(it); - } - - // The destructor is called when decoder_database goes out of scope. - EXPECT_CALL(decoder_database, Die()); -} - -// Payload not an integer number of frames. -TEST(IlbcPayloadSplitter, UnevenPayload) { - PacketList packet_list; - static const uint8_t kPayloadType = 17; // Just a random number. - size_t kPayloadLengthBytes = 39; // Not an even number of frames. - Packet* packet = CreatePacket(kPayloadType, kPayloadLengthBytes, 0); - packet_list.push_back(packet); - - MockDecoderDatabase decoder_database; - std::unique_ptr info( - new DecoderDatabase::DecoderInfo(NetEqDecoder::kDecoderILBC, "")); - EXPECT_CALL(decoder_database, GetDecoderInfo(kPayloadType)) - .WillRepeatedly(Return(info.get())); - - PayloadSplitter splitter; - EXPECT_EQ(PayloadSplitter::kFrameSplitError, - splitter.SplitAudio(&packet_list, decoder_database)); - EXPECT_EQ(1u, packet_list.size()); - - // Delete the packets and payloads to avoid having the test leak memory. - PacketList::iterator it = packet_list.begin(); - while (it != packet_list.end()) { - delete (*it); - it = packet_list.erase(it); - } - - // The destructor is called when decoder_database goes out of scope. - EXPECT_CALL(decoder_database, Die()); -} - TEST(FecPayloadSplitter, MixedPayload) { PacketList packet_list; DecoderDatabase decoder_database(CreateBuiltinAudioDecoderFactory());