From a70695a3e1e7ead375d4bbc7997cd630d1ac1731 Mon Sep 17 00:00:00 2001 From: ossu Date: Thu, 22 Sep 2016 02:06:28 -0700 Subject: [PATCH] Moved Opus-specific payload splitting into AudioDecoderOpus. The biggest change to NetEq is the move from a primary flag, to a Priority with two separate levels: one set by RED splitting and one set by the codec itself. This allows us to unambigously prioritize "fallback" packets from these two sources. I've chosen what I believe is the sensible ordering: packets that the codec prioritizes are chosen first, regardless of if they are secondary RED packets or not. So if we were to use Opus w/ FEC in RED, we'd only do Opus FEC decoding if there was no RED packet that could cover the time slot. With this change, PayloadSplitter now only deals with RED packets. Maybe it should be renamed RedPayloadSplitter? BUG=webrtc:5805 Review-Url: https://codereview.webrtc.org/2342443005 Cr-Commit-Position: refs/heads/master@{#14347} --- webrtc/modules/BUILD.gn | 4 +- webrtc/modules/audio_coding/BUILD.gn | 4 +- .../audio_coding/codecs/audio_decoder.cc | 15 +- .../audio_coding/codecs/audio_decoder.h | 19 +- .../codecs/g711/audio_decoder_pcm.cc | 10 +- .../codecs/g711/audio_decoder_pcm.h | 6 +- .../codecs/g722/audio_decoder_g722.cc | 12 +- .../codecs/g722/audio_decoder_g722.h | 6 +- .../codecs/ilbc/audio_decoder_ilbc.cc | 15 +- .../codecs/ilbc/audio_decoder_ilbc.h | 3 +- .../audio_coding/codecs/ilbc/ilbc_unittest.cc | 6 +- .../codecs/legacy_encoded_audio_frame.cc | 38 +--- .../codecs/legacy_encoded_audio_frame.h | 6 +- .../legacy_encoded_audio_frame_unittest.cc | 2 +- .../codecs/opus/audio_decoder_opus.cc | 70 +++++++ .../codecs/opus/audio_decoder_opus.h | 2 + .../codecs/pcm16b/audio_decoder_pcm16b.cc | 7 +- .../codecs/pcm16b/audio_decoder_pcm16b.h | 3 +- .../audio_coding/neteq/include/neteq.h | 1 - .../neteq/mock/mock_payload_splitter.h | 31 --- .../neteq/mock/mock_red_payload_splitter.h | 29 +++ webrtc/modules/audio_coding/neteq/neteq.gypi | 4 +- .../modules/audio_coding/neteq/neteq_impl.cc | 39 ++-- .../modules/audio_coding/neteq/neteq_impl.h | 6 +- .../audio_coding/neteq/neteq_impl_unittest.cc | 17 +- .../neteq/neteq_network_stats_unittest.cc | 67 +++++-- webrtc/modules/audio_coding/neteq/packet.h | 55 +++++- .../audio_coding/neteq/packet_buffer.cc | 9 +- .../neteq/packet_buffer_unittest.cc | 63 ++++++- ...ad_splitter.cc => red_payload_splitter.cc} | 139 +++++--------- ...load_splitter.h => red_payload_splitter.h} | 41 ++-- ...st.cc => red_payload_splitter_unittest.cc} | 176 ++++-------------- 32 files changed, 441 insertions(+), 464 deletions(-) delete mode 100644 webrtc/modules/audio_coding/neteq/mock/mock_payload_splitter.h create mode 100644 webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h rename webrtc/modules/audio_coding/neteq/{payload_splitter.cc => red_payload_splitter.cc} (54%) rename webrtc/modules/audio_coding/neteq/{payload_splitter.h => red_payload_splitter.h} (56%) rename webrtc/modules/audio_coding/neteq/{payload_splitter_unittest.cc => red_payload_splitter_unittest.cc} (70%) diff --git a/webrtc/modules/BUILD.gn b/webrtc/modules/BUILD.gn index 8fb9f86d14..11a5ef0973 100644 --- a/webrtc/modules/BUILD.gn +++ b/webrtc/modules/BUILD.gn @@ -313,7 +313,7 @@ if (rtc_include_tests) { "audio_coding/neteq/mock/mock_expand.h", "audio_coding/neteq/mock/mock_external_decoder_pcm16b.h", "audio_coding/neteq/mock/mock_packet_buffer.h", - "audio_coding/neteq/mock/mock_payload_splitter.h", + "audio_coding/neteq/mock/mock_red_payload_splitter.h", "audio_coding/neteq/nack_tracker_unittest.cc", "audio_coding/neteq/neteq_external_decoder_unittest.cc", "audio_coding/neteq/neteq_impl_unittest.cc", @@ -322,9 +322,9 @@ if (rtc_include_tests) { "audio_coding/neteq/neteq_unittest.cc", "audio_coding/neteq/normal_unittest.cc", "audio_coding/neteq/packet_buffer_unittest.cc", - "audio_coding/neteq/payload_splitter_unittest.cc", "audio_coding/neteq/post_decode_vad_unittest.cc", "audio_coding/neteq/random_vector_unittest.cc", + "audio_coding/neteq/red_payload_splitter_unittest.cc", "audio_coding/neteq/sync_buffer_unittest.cc", "audio_coding/neteq/tick_timer_unittest.cc", "audio_coding/neteq/time_stretch_unittest.cc", diff --git a/webrtc/modules/audio_coding/BUILD.gn b/webrtc/modules/audio_coding/BUILD.gn index a60baba5da..b55829d786 100644 --- a/webrtc/modules/audio_coding/BUILD.gn +++ b/webrtc/modules/audio_coding/BUILD.gn @@ -784,14 +784,14 @@ rtc_source_set("neteq") { "neteq/packet.h", "neteq/packet_buffer.cc", "neteq/packet_buffer.h", - "neteq/payload_splitter.cc", - "neteq/payload_splitter.h", "neteq/post_decode_vad.cc", "neteq/post_decode_vad.h", "neteq/preemptive_expand.cc", "neteq/preemptive_expand.h", "neteq/random_vector.cc", "neteq/random_vector.h", + "neteq/red_payload_splitter.cc", + "neteq/red_payload_splitter.h", "neteq/rtcp.cc", "neteq/rtcp.h", "neteq/statistics_calculator.cc", diff --git a/webrtc/modules/audio_coding/codecs/audio_decoder.cc b/webrtc/modules/audio_coding/codecs/audio_decoder.cc index 6c67924260..afa5115d5a 100644 --- a/webrtc/modules/audio_coding/codecs/audio_decoder.cc +++ b/webrtc/modules/audio_coding/codecs/audio_decoder.cc @@ -14,8 +14,6 @@ #include #include -#include - #include "webrtc/base/array_view.h" #include "webrtc/base/checks.h" #include "webrtc/base/sanitizer.h" @@ -27,9 +25,11 @@ namespace webrtc { AudioDecoder::ParseResult::ParseResult() = default; AudioDecoder::ParseResult::ParseResult(ParseResult&& b) = default; AudioDecoder::ParseResult::ParseResult(uint32_t timestamp, - bool primary, + int priority, std::unique_ptr frame) - : timestamp(timestamp), primary(primary), frame(std::move(frame)) {} + : timestamp(timestamp), priority(priority), frame(std::move(frame)) { + RTC_DCHECK_GE(priority, 0); +} AudioDecoder::ParseResult::~ParseResult() = default; @@ -38,12 +38,11 @@ AudioDecoder::ParseResult& AudioDecoder::ParseResult::operator=( std::vector AudioDecoder::ParsePayload( rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) { + uint32_t timestamp) { std::vector results; std::unique_ptr frame( - new LegacyEncodedAudioFrame(this, std::move(payload), is_primary)); - results.emplace_back(timestamp, is_primary, std::move(frame)); + new LegacyEncodedAudioFrame(this, std::move(payload))); + results.emplace_back(timestamp, 0, std::move(frame)); return results; } diff --git a/webrtc/modules/audio_coding/codecs/audio_decoder.h b/webrtc/modules/audio_coding/codecs/audio_decoder.h index b6338d2102..8468da20f2 100644 --- a/webrtc/modules/audio_coding/codecs/audio_decoder.h +++ b/webrtc/modules/audio_coding/codecs/audio_decoder.h @@ -8,11 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_INCLUDE_AUDIO_DECODER_H_ -#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_INCLUDE_AUDIO_DECODER_H_ - -#include -#include +#ifndef WEBRTC_MODULES_AUDIO_CODING_CODECS_AUDIO_DECODER_H_ +#define WEBRTC_MODULES_AUDIO_CODING_CODECS_AUDIO_DECODER_H_ #include #include @@ -66,7 +63,7 @@ class AudioDecoder { struct ParseResult { ParseResult(); ParseResult(uint32_t timestamp, - bool primary, + int priority, std::unique_ptr frame); ParseResult(ParseResult&& b); ~ParseResult(); @@ -75,7 +72,10 @@ class AudioDecoder { // The timestamp of the frame is in samples per channel. uint32_t timestamp; - bool primary; + // The relative priority of the frame compared to other frames of the same + // payload and the same timeframe. A higher value means a lower priority. + // The highest priority is zero - negative values are not allowed. + int priority; std::unique_ptr frame; }; @@ -86,8 +86,7 @@ class AudioDecoder { // buffer. |timestamp| is the input timestamp, in samples, corresponding to // the start of the payload. virtual std::vector ParsePayload(rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary); + uint32_t timestamp); // Decodes |encode_len| bytes from |encoded| and writes the result in // |decoded|. The maximum bytes allowed to be written into |decoded| is @@ -177,4 +176,4 @@ class AudioDecoder { }; } // namespace webrtc -#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_INCLUDE_AUDIO_DECODER_H_ +#endif // WEBRTC_MODULES_AUDIO_CODING_CODECS_AUDIO_DECODER_H_ diff --git a/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc b/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc index f2fdb1f15c..fd285a75b2 100644 --- a/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc +++ b/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc @@ -19,10 +19,9 @@ void AudioDecoderPcmU::Reset() {} std::vector AudioDecoderPcmU::ParsePayload( rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) { + uint32_t timestamp) { return LegacyEncodedAudioFrame::SplitBySamples( - this, std::move(payload), timestamp, is_primary, 8 * num_channels_, 8); + this, std::move(payload), timestamp, 8 * num_channels_, 8); } int AudioDecoderPcmU::SampleRateHz() const { @@ -55,10 +54,9 @@ void AudioDecoderPcmA::Reset() {} std::vector AudioDecoderPcmA::ParsePayload( rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) { + uint32_t timestamp) { return LegacyEncodedAudioFrame::SplitBySamples( - this, std::move(payload), timestamp, is_primary, 8 * num_channels_, 8); + this, std::move(payload), timestamp, 8 * num_channels_, 8); } int AudioDecoderPcmA::SampleRateHz() const { diff --git a/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h b/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h index ed390217a3..483311b61b 100644 --- a/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h +++ b/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h @@ -24,8 +24,7 @@ class AudioDecoderPcmU final : public AudioDecoder { } void Reset() override; std::vector ParsePayload(rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) override; + uint32_t timestamp) override; int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; int SampleRateHz() const override; size_t Channels() const override; @@ -49,8 +48,7 @@ class AudioDecoderPcmA final : public AudioDecoder { } void Reset() override; std::vector ParsePayload(rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) override; + uint32_t timestamp) override; int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; int SampleRateHz() const override; size_t Channels() const override; diff --git a/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc b/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc index 93b24bdf2c..3a0f6ede66 100644 --- a/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc +++ b/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc @@ -50,10 +50,9 @@ void AudioDecoderG722::Reset() { std::vector AudioDecoderG722::ParsePayload( rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) { + uint32_t timestamp) { return LegacyEncodedAudioFrame::SplitBySamples(this, std::move(payload), - timestamp, is_primary, 8, 16); + timestamp, 8, 16); } int AudioDecoderG722::PacketDuration(const uint8_t* encoded, @@ -128,10 +127,9 @@ void AudioDecoderG722Stereo::Reset() { std::vector AudioDecoderG722Stereo::ParsePayload( rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) { - return LegacyEncodedAudioFrame::SplitBySamples( - this, std::move(payload), timestamp, is_primary, 2 * 8, 16); + uint32_t timestamp) { + return LegacyEncodedAudioFrame::SplitBySamples(this, std::move(payload), + timestamp, 2 * 8, 16); } // Split the stereo packet and place left and right channel after each other diff --git a/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h b/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h index ad39619d3c..9e7c5d9d05 100644 --- a/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h +++ b/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h @@ -25,8 +25,7 @@ class AudioDecoderG722 final : public AudioDecoder { bool HasDecodePlc() const override; void Reset() override; std::vector ParsePayload(rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) override; + uint32_t timestamp) override; int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; int SampleRateHz() const override; size_t Channels() const override; @@ -49,8 +48,7 @@ class AudioDecoderG722Stereo final : public AudioDecoder { ~AudioDecoderG722Stereo() override; void Reset() override; std::vector ParsePayload(rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) override; + uint32_t timestamp) override; int SampleRateHz() const override; size_t Channels() const override; diff --git a/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc b/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc index b4bd59911f..354f8194d8 100644 --- a/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc +++ b/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc @@ -10,6 +10,8 @@ #include "webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h" +#include + #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" #include "webrtc/modules/audio_coding/codecs/ilbc/ilbc.h" @@ -53,8 +55,7 @@ void AudioDecoderIlbc::Reset() { std::vector AudioDecoderIlbc::ParsePayload( rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) { + uint32_t timestamp) { std::vector results; size_t bytes_per_frame; int timestamps_per_frame; @@ -78,8 +79,8 @@ std::vector AudioDecoderIlbc::ParsePayload( RTC_DCHECK_EQ(0u, payload.size() % bytes_per_frame); if (payload.size() == bytes_per_frame) { std::unique_ptr frame( - new LegacyEncodedAudioFrame(this, std::move(payload), is_primary)); - results.emplace_back(timestamp, is_primary, std::move(frame)); + new LegacyEncodedAudioFrame(this, std::move(payload))); + results.emplace_back(timestamp, 0, std::move(frame)); } else { size_t byte_offset; uint32_t timestamp_offset; @@ -87,11 +88,9 @@ std::vector AudioDecoderIlbc::ParsePayload( byte_offset < payload.size(); byte_offset += bytes_per_frame, timestamp_offset += timestamps_per_frame) { - rtc::Buffer new_payload(payload.data() + byte_offset, bytes_per_frame); std::unique_ptr frame(new LegacyEncodedAudioFrame( - this, std::move(new_payload), is_primary)); - results.emplace_back(timestamp + timestamp_offset, is_primary, - std::move(frame)); + this, rtc::Buffer(payload.data() + byte_offset, bytes_per_frame))); + results.emplace_back(timestamp + timestamp_offset, 0, std::move(frame)); } } diff --git a/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h b/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h index fdc856e985..48edfede76 100644 --- a/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h +++ b/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h @@ -26,8 +26,7 @@ class AudioDecoderIlbc final : public AudioDecoder { size_t DecodePlc(size_t num_frames, int16_t* decoded) override; void Reset() override; std::vector ParsePayload(rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) override; + uint32_t timestamp) override; int SampleRateHz() const override; size_t Channels() const override; diff --git a/webrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc b/webrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc index a8b76a5bc9..2abdacae97 100644 --- a/webrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc +++ b/webrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc @@ -81,7 +81,7 @@ TEST_P(SplitIlbcTest, NumFrames) { }; const auto results = decoder.ParsePayload( - generate_payload(frame_length_bytes_ * num_frames_), 0, true); + generate_payload(frame_length_bytes_ * num_frames_), 0); EXPECT_EQ(num_frames_, results.size()); size_t frame_num = 0; @@ -123,7 +123,7 @@ TEST(IlbcTest, SplitTooLargePayload) { AudioDecoderIlbc decoder; constexpr size_t kPayloadLengthBytes = 950; const auto results = - decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0, true); + decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0); EXPECT_TRUE(results.empty()); } @@ -132,7 +132,7 @@ TEST(IlbcTest, SplitUnevenPayload) { AudioDecoderIlbc decoder; constexpr size_t kPayloadLengthBytes = 39; // Not an even number of frames. const auto results = - decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0, true); + decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0); EXPECT_TRUE(results.empty()); } diff --git a/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc index 5e6ff01f22..e0f1fafcaa 100644 --- a/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc +++ b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc @@ -17,37 +17,22 @@ namespace webrtc { LegacyEncodedAudioFrame::LegacyEncodedAudioFrame(AudioDecoder* decoder, - rtc::Buffer&& payload, - bool is_primary_payload) - : decoder_(decoder), - payload_(std::move(payload)), - is_primary_payload_(is_primary_payload) {} + rtc::Buffer&& payload) + : decoder_(decoder), payload_(std::move(payload)) {} LegacyEncodedAudioFrame::~LegacyEncodedAudioFrame() = default; size_t LegacyEncodedAudioFrame::Duration() const { - int ret; - if (is_primary_payload_) { - ret = decoder_->PacketDuration(payload_.data(), payload_.size()); - } else { - ret = decoder_->PacketDurationRedundant(payload_.data(), payload_.size()); - } + const int ret = decoder_->PacketDuration(payload_.data(), payload_.size()); return (ret < 0) ? 0 : static_cast(ret); } rtc::Optional LegacyEncodedAudioFrame::Decode(rtc::ArrayView decoded) const { AudioDecoder::SpeechType speech_type = AudioDecoder::kSpeech; - int ret; - if (is_primary_payload_) { - ret = decoder_->Decode( - payload_.data(), payload_.size(), decoder_->SampleRateHz(), - decoded.size() * sizeof(int16_t), decoded.data(), &speech_type); - } else { - ret = decoder_->DecodeRedundant( - payload_.data(), payload_.size(), decoder_->SampleRateHz(), - decoded.size() * sizeof(int16_t), decoded.data(), &speech_type); - } + const int ret = decoder_->Decode( + payload_.data(), payload_.size(), decoder_->SampleRateHz(), + decoded.size() * sizeof(int16_t), decoded.data(), &speech_type); if (ret < 0) return rtc::Optional(); @@ -59,7 +44,6 @@ std::vector LegacyEncodedAudioFrame::SplitBySamples( AudioDecoder* decoder, rtc::Buffer&& payload, uint32_t timestamp, - bool is_primary, size_t bytes_per_ms, uint32_t timestamps_per_ms) { RTC_DCHECK(payload.data()); @@ -70,8 +54,8 @@ std::vector LegacyEncodedAudioFrame::SplitBySamples( const size_t min_chunk_size = bytes_per_ms * 20; if (min_chunk_size >= payload.size()) { std::unique_ptr frame( - new LegacyEncodedAudioFrame(decoder, std::move(payload), is_primary)); - results.emplace_back(timestamp, is_primary, std::move(frame)); + new LegacyEncodedAudioFrame(decoder, std::move(payload))); + results.emplace_back(timestamp, 0, std::move(frame)); } else { // Reduce the split size by half as long as |split_size_bytes| is at least // twice the minimum chunk size (so that the resulting size is at least as @@ -92,10 +76,8 @@ std::vector LegacyEncodedAudioFrame::SplitBySamples( std::min(split_size_bytes, payload.size() - byte_offset); rtc::Buffer new_payload(payload.data() + byte_offset, split_size_bytes); std::unique_ptr frame( - new LegacyEncodedAudioFrame(decoder, std::move(new_payload), - is_primary)); - results.emplace_back(timestamp + timestamp_offset, is_primary, - std::move(frame)); + new LegacyEncodedAudioFrame(decoder, std::move(new_payload))); + results.emplace_back(timestamp + timestamp_offset, 0, std::move(frame)); } } diff --git a/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h index 1c9c3f542a..466c8b34be 100644 --- a/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h +++ b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h @@ -20,16 +20,13 @@ namespace webrtc { class LegacyEncodedAudioFrame final : public AudioDecoder::EncodedAudioFrame { public: - LegacyEncodedAudioFrame(AudioDecoder* decoder, - rtc::Buffer&& payload, - bool is_primary_payload); + LegacyEncodedAudioFrame(AudioDecoder* decoder, rtc::Buffer&& payload); ~LegacyEncodedAudioFrame() override; static std::vector SplitBySamples( AudioDecoder* decoder, rtc::Buffer&& payload, uint32_t timestamp, - bool is_primary, size_t bytes_per_ms, uint32_t timestamps_per_ms); @@ -44,7 +41,6 @@ class LegacyEncodedAudioFrame final : public AudioDecoder::EncodedAudioFrame { private: AudioDecoder* const decoder_; const rtc::Buffer payload_; - const bool is_primary_payload_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc index 2a4d9ed9f2..9a075639af 100644 --- a/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc +++ b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc @@ -123,7 +123,7 @@ TEST_P(SplitBySamplesTest, PayloadSizes) { const auto results = LegacyEncodedAudioFrame::SplitBySamples( nullptr, generate_payload(expected_split.payload_size_ms * bytes_per_ms_), - kBaseTimestamp, true, bytes_per_ms_, samples_per_ms_); + kBaseTimestamp, bytes_per_ms_, samples_per_ms_); EXPECT_EQ(expected_split.num_frames, results.size()); uint32_t expected_timestamp = kBaseTimestamp; diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc b/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc index 42abd0a0d6..b6d8a3a1db 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc +++ b/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc @@ -10,10 +10,60 @@ #include "webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h" +#include + #include "webrtc/base/checks.h" namespace webrtc { +namespace { +class OpusFrame : public AudioDecoder::EncodedAudioFrame { + public: + OpusFrame(AudioDecoderOpus* decoder, + rtc::Buffer&& payload, + bool is_primary_payload) + : decoder_(decoder), + payload_(std::move(payload)), + is_primary_payload_(is_primary_payload) {} + + size_t Duration() const override { + int ret; + if (is_primary_payload_) { + ret = decoder_->PacketDuration(payload_.data(), payload_.size()); + } else { + ret = decoder_->PacketDurationRedundant(payload_.data(), payload_.size()); + } + return (ret < 0) ? 0 : static_cast(ret); + } + + rtc::Optional Decode( + rtc::ArrayView decoded) const override { + AudioDecoder::SpeechType speech_type = AudioDecoder::kSpeech; + int ret; + if (is_primary_payload_) { + ret = decoder_->Decode( + payload_.data(), payload_.size(), decoder_->SampleRateHz(), + decoded.size() * sizeof(int16_t), decoded.data(), &speech_type); + } else { + ret = decoder_->DecodeRedundant( + payload_.data(), payload_.size(), decoder_->SampleRateHz(), + decoded.size() * sizeof(int16_t), decoded.data(), &speech_type); + } + + if (ret < 0) + return rtc::Optional(); + + return rtc::Optional({static_cast(ret), speech_type}); + } + + private: + AudioDecoderOpus* const decoder_; + const rtc::Buffer payload_; + const bool is_primary_payload_; +}; + +} // namespace + AudioDecoderOpus::AudioDecoderOpus(size_t num_channels) : channels_(num_channels) { RTC_DCHECK(num_channels == 1 || num_channels == 2); @@ -25,6 +75,26 @@ AudioDecoderOpus::~AudioDecoderOpus() { WebRtcOpus_DecoderFree(dec_state_); } +std::vector AudioDecoderOpus::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp) { + std::vector results; + + if (PacketHasFec(payload.data(), payload.size())) { + const int duration = + PacketDurationRedundant(payload.data(), payload.size()); + RTC_DCHECK_GE(duration, 0); + rtc::Buffer payload_copy(payload.data(), payload.size()); + std::unique_ptr fec_frame( + new OpusFrame(this, std::move(payload_copy), false)); + results.emplace_back(timestamp - duration, 1, std::move(fec_frame)); + } + std::unique_ptr frame( + new OpusFrame(this, std::move(payload), true)); + results.emplace_back(timestamp, 0, std::move(frame)); + return results; +} + int AudioDecoderOpus::DecodeInternal(const uint8_t* encoded, size_t encoded_len, int sample_rate_hz, diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h b/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h index c22204156b..a0fb34c442 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h +++ b/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h @@ -22,6 +22,8 @@ class AudioDecoderOpus final : public AudioDecoder { explicit AudioDecoderOpus(size_t num_channels); ~AudioDecoderOpus() override; + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) override; void Reset() override; int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; int PacketDurationRedundant(const uint8_t* encoded, diff --git a/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc b/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc index e600d2d56c..43d2dacf9b 100644 --- a/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc +++ b/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc @@ -47,12 +47,11 @@ int AudioDecoderPcm16B::DecodeInternal(const uint8_t* encoded, std::vector AudioDecoderPcm16B::ParsePayload( rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) { + uint32_t timestamp) { const int samples_per_ms = rtc::CheckedDivExact(sample_rate_hz_, 1000); return LegacyEncodedAudioFrame::SplitBySamples( - this, std::move(payload), timestamp, is_primary, - samples_per_ms * 2 * num_channels_, samples_per_ms); + this, std::move(payload), timestamp, samples_per_ms * 2 * num_channels_, + samples_per_ms); } int AudioDecoderPcm16B::PacketDuration(const uint8_t* encoded, diff --git a/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h b/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h index 1d2da4a718..577eecc535 100644 --- a/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h +++ b/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h @@ -21,8 +21,7 @@ class AudioDecoderPcm16B final : public AudioDecoder { AudioDecoderPcm16B(int sample_rate_hz, size_t num_channels); void Reset() override; std::vector ParsePayload(rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) override; + uint32_t timestamp) override; int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; int SampleRateHz() const override; size_t Channels() const override; diff --git a/webrtc/modules/audio_coding/neteq/include/neteq.h b/webrtc/modules/audio_coding/neteq/include/neteq.h index 98bb37cf1f..c07143c062 100644 --- a/webrtc/modules/audio_coding/neteq/include/neteq.h +++ b/webrtc/modules/audio_coding/neteq/include/neteq.h @@ -126,7 +126,6 @@ class NetEq { kStereoNotSupported, kSampleUnderrun, kDecodedTooMuch, - kFrameSplitError, kRedundancySplitError, kPacketBufferCorruption }; diff --git a/webrtc/modules/audio_coding/neteq/mock/mock_payload_splitter.h b/webrtc/modules/audio_coding/neteq/mock/mock_payload_splitter.h deleted file mode 100644 index cf0613403b..0000000000 --- a/webrtc/modules/audio_coding/neteq/mock/mock_payload_splitter.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_PAYLOAD_SPLITTER_H_ -#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_PAYLOAD_SPLITTER_H_ - -#include "webrtc/modules/audio_coding/neteq/payload_splitter.h" - -#include "testing/gmock/include/gmock/gmock.h" - -namespace webrtc { - -class MockPayloadSplitter : public PayloadSplitter { - public: - MOCK_METHOD1(SplitRed, - int(PacketList* packet_list)); - MOCK_METHOD2(SplitFec, - int(PacketList* packet_list, DecoderDatabase* decoder_database)); - MOCK_METHOD2(CheckRedPayloads, - int(PacketList* packet_list, const DecoderDatabase& decoder_database)); -}; - -} // namespace webrtc -#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_PAYLOAD_SPLITTER_H_ diff --git a/webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h b/webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h new file mode 100644 index 0000000000..e1f4e04cfd --- /dev/null +++ b/webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_RED_PAYLOAD_SPLITTER_H_ +#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_RED_PAYLOAD_SPLITTER_H_ + +#include "webrtc/modules/audio_coding/neteq/red_payload_splitter.h" + +#include "testing/gmock/include/gmock/gmock.h" + +namespace webrtc { + +class MockRedPayloadSplitter : public RedPayloadSplitter { + public: + MOCK_METHOD1(SplitRed, bool(PacketList* packet_list)); + MOCK_METHOD2(CheckRedPayloads, + int(PacketList* packet_list, + const DecoderDatabase& decoder_database)); +}; + +} // namespace webrtc +#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_RED_PAYLOAD_SPLITTER_H_ diff --git a/webrtc/modules/audio_coding/neteq/neteq.gypi b/webrtc/modules/audio_coding/neteq/neteq.gypi index 630076ec8d..90aaaed9c2 100644 --- a/webrtc/modules/audio_coding/neteq/neteq.gypi +++ b/webrtc/modules/audio_coding/neteq/neteq.gypi @@ -113,8 +113,8 @@ 'packet.h', 'packet_buffer.cc', 'packet_buffer.h', - 'payload_splitter.cc', - 'payload_splitter.h', + 'red_payload_splitter.cc', + 'red_payload_splitter.h', 'post_decode_vad.cc', 'post_decode_vad.h', 'preemptive_expand.cc', diff --git a/webrtc/modules/audio_coding/neteq/neteq_impl.cc b/webrtc/modules/audio_coding/neteq/neteq_impl.cc index 221b07c969..6b2fa4c9ec 100644 --- a/webrtc/modules/audio_coding/neteq/neteq_impl.cc +++ b/webrtc/modules/audio_coding/neteq/neteq_impl.cc @@ -41,7 +41,7 @@ #include "webrtc/modules/audio_coding/neteq/normal.h" #include "webrtc/modules/audio_coding/neteq/packet_buffer.h" #include "webrtc/modules/audio_coding/neteq/packet.h" -#include "webrtc/modules/audio_coding/neteq/payload_splitter.h" +#include "webrtc/modules/audio_coding/neteq/red_payload_splitter.h" #include "webrtc/modules/audio_coding/neteq/post_decode_vad.h" #include "webrtc/modules/audio_coding/neteq/preemptive_expand.h" #include "webrtc/modules/audio_coding/neteq/sync_buffer.h" @@ -65,7 +65,7 @@ NetEqImpl::Dependencies::Dependencies( dtmf_tone_generator(new DtmfToneGenerator), packet_buffer( new PacketBuffer(config.max_packets_in_buffer, tick_timer.get())), - payload_splitter(new PayloadSplitter), + red_payload_splitter(new RedPayloadSplitter), timestamp_scaler(new TimestampScaler(*decoder_database)), accelerate_factory(new AccelerateFactory), expand_factory(new ExpandFactory), @@ -84,7 +84,7 @@ NetEqImpl::NetEqImpl(const NetEq::Config& config, dtmf_buffer_(std::move(deps.dtmf_buffer)), dtmf_tone_generator_(std::move(deps.dtmf_tone_generator)), packet_buffer_(std::move(deps.packet_buffer)), - payload_splitter_(std::move(deps.payload_splitter)), + red_payload_splitter_(std::move(deps.red_payload_splitter)), timestamp_scaler_(std::move(deps.timestamp_scaler)), vad_(new PostDecodeVad()), expand_factory_(std::move(deps.expand_factory)), @@ -567,7 +567,6 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header, packet->header.ssrc = rtp_header.header.ssrc; packet->header.numCSRCs = 0; packet->payload.SetData(payload.data(), payload.size()); - packet->primary = true; // Waiting time will be set upon inserting the packet in the buffer. RTC_DCHECK(!packet->waiting_time); // Insert packet in a packet list. @@ -609,13 +608,13 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header, // Check for RED payload type, and separate payloads into several packets. if (decoder_database_->IsRed(main_header.payloadType)) { - if (payload_splitter_->SplitRed(&packet_list) != PayloadSplitter::kOK) { + if (!red_payload_splitter_->SplitRed(&packet_list)) { PacketBuffer::DeleteAllPackets(&packet_list); return kRedundancySplitError; } // Only accept a few RED payloads of the same type as the main data, // DTMF events and CNG. - payload_splitter_->CheckRedPayloads(&packet_list, *decoder_database_); + red_payload_splitter_->CheckRedPayloads(&packet_list, *decoder_database_); // Update the stored main payload header since the main payload has now // changed. memcpy(&main_header, &packet_list.front()->header, sizeof(main_header)); @@ -658,18 +657,6 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header, } } - // Check for FEC in packets, and separate payloads into several packets. - int ret = payload_splitter_->SplitFec(&packet_list, decoder_database_.get()); - if (ret != PayloadSplitter::kOK) { - PacketBuffer::DeleteAllPackets(&packet_list); - switch (ret) { - case PayloadSplitter::kUnknownPayloadType: - return kUnknownRtpPayloadType; - default: - return kOtherError; - } - } - // Update bandwidth estimate, if the packet is not comfort noise. if (!packet_list.empty() && !decoder_database_->IsComfortNoise(main_header.payloadType)) { @@ -702,9 +689,9 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header, } else { std::vector results = info->GetDecoder()->ParsePayload(std::move(packet->payload), - packet->header.timestamp, - packet->primary); + packet->header.timestamp); const RTPHeader& original_header = packet->header; + const Packet::Priority original_priority = packet->priority; for (auto& result : results) { RTC_DCHECK(result.frame); // Reuse the packet if possible. @@ -713,8 +700,9 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header, packet->header = original_header; } packet->header.timestamp = result.timestamp; - // TODO(ossu): Move from primary to some sort of priority level. - packet->primary = result.primary; + RTC_DCHECK_GE(result.priority, 0); + packet->priority.codec_level = result.priority; + packet->priority.red_level = original_priority.red_level; packet->frame = std::move(result.frame); parsed_packet_list.push_back(packet.release()); } @@ -734,7 +722,7 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header, // Insert packets in buffer. const size_t buffer_length_before_insert = packet_buffer_->NumPacketsInBuffer(); - ret = packet_buffer_->InsertPacketList( + const int ret = packet_buffer_->InsertPacketList( &parsed_packet_list, *decoder_database_, ¤t_rtp_payload_type_, ¤t_cng_rtp_payload_type_); if (ret == PacketBuffer::kFlushed) { @@ -1979,9 +1967,8 @@ int NetEqImpl::ExtractPackets(size_t required_samples, size_t packet_duration = 0; if (packet->frame) { packet_duration = packet->frame->Duration(); - // TODO(ossu): Is this the correct way to track samples decoded from a - // redundant packet? - if (packet_duration > 0 && !packet->primary) { + // TODO(ossu): Is this the correct way to track Opus FEC packets? + if (packet->priority.codec_level > 0) { stats_.SecondaryDecodedSamples(rtc::checked_cast(packet_duration)); } } else if (!decoder_database_->IsComfortNoise(packet->header.payloadType)) { diff --git a/webrtc/modules/audio_coding/neteq/neteq_impl.h b/webrtc/modules/audio_coding/neteq/neteq_impl.h index dd35301d7a..dda3d64494 100644 --- a/webrtc/modules/audio_coding/neteq/neteq_impl.h +++ b/webrtc/modules/audio_coding/neteq/neteq_impl.h @@ -46,7 +46,7 @@ class Merge; class NackTracker; class Normal; class PacketBuffer; -class PayloadSplitter; +class RedPayloadSplitter; class PostDecodeVad; class PreemptiveExpand; class RandomVector; @@ -86,7 +86,7 @@ class NetEqImpl : public webrtc::NetEq { std::unique_ptr dtmf_buffer; std::unique_ptr dtmf_tone_generator; std::unique_ptr packet_buffer; - std::unique_ptr payload_splitter; + std::unique_ptr red_payload_splitter; std::unique_ptr timestamp_scaler; std::unique_ptr accelerate_factory; std::unique_ptr expand_factory; @@ -353,7 +353,7 @@ class NetEqImpl : public webrtc::NetEq { const std::unique_ptr dtmf_tone_generator_ GUARDED_BY(crit_sect_); const std::unique_ptr packet_buffer_ GUARDED_BY(crit_sect_); - const std::unique_ptr payload_splitter_ + const std::unique_ptr red_payload_splitter_ GUARDED_BY(crit_sect_); const std::unique_ptr timestamp_scaler_ GUARDED_BY(crit_sect_); diff --git a/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc b/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc index 5b1015d576..e3029dba8d 100644 --- a/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc +++ b/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc @@ -28,7 +28,7 @@ #include "webrtc/modules/audio_coding/neteq/mock/mock_dtmf_buffer.h" #include "webrtc/modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h" #include "webrtc/modules/audio_coding/neteq/mock/mock_packet_buffer.h" -#include "webrtc/modules/audio_coding/neteq/mock/mock_payload_splitter.h" +#include "webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h" #include "webrtc/modules/audio_coding/neteq/preemptive_expand.h" #include "webrtc/modules/audio_coding/neteq/sync_buffer.h" #include "webrtc/modules/audio_coding/neteq/timestamp_scaler.h" @@ -124,11 +124,11 @@ class NetEqImplTest : public ::testing::Test { packet_buffer_ = deps.packet_buffer.get(); if (use_mock_payload_splitter_) { - std::unique_ptr mock(new MockPayloadSplitter); + std::unique_ptr mock(new MockRedPayloadSplitter); mock_payload_splitter_ = mock.get(); - deps.payload_splitter = std::move(mock); + deps.red_payload_splitter = std::move(mock); } - payload_splitter_ = deps.payload_splitter.get(); + red_payload_splitter_ = deps.red_payload_splitter.get(); deps.timestamp_scaler = std::unique_ptr( new TimestampScaler(*deps.decoder_database.get())); @@ -197,8 +197,8 @@ class NetEqImplTest : public ::testing::Test { MockPacketBuffer* mock_packet_buffer_ = nullptr; PacketBuffer* packet_buffer_ = nullptr; bool use_mock_packet_buffer_ = true; - MockPayloadSplitter* mock_payload_splitter_ = nullptr; - PayloadSplitter* payload_splitter_ = nullptr; + MockRedPayloadSplitter* mock_payload_splitter_ = nullptr; + RedPayloadSplitter* red_payload_splitter_ = nullptr; bool use_mock_payload_splitter_ = true; }; @@ -332,11 +332,6 @@ TEST_F(NetEqImplTest, InsertPacket) { .WillOnce(Return(0)); } - // Expectations for payload splitter. - EXPECT_CALL(*mock_payload_splitter_, SplitFec(_, _)) - .Times(2) - .WillRepeatedly(Return(PayloadSplitter::kOK)); - // Insert first packet. neteq_->InsertPacket(rtp_header, payload, kFirstReceiveTime); diff --git a/webrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc b/webrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc index 15c60a83b5..70424fde79 100644 --- a/webrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc +++ b/webrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc @@ -39,18 +39,57 @@ class MockAudioDecoder final : public AudioDecoder { MOCK_METHOD0(Reset, void()); - int PacketDuration(const uint8_t* encoded, - size_t encoded_len) const /* override */ { - return kPacketDuration; + class MockFrame : public AudioDecoder::EncodedAudioFrame { + public: + MockFrame(size_t num_channels) : num_channels_(num_channels) {} + + size_t Duration() const override { return kPacketDuration; } + + rtc::Optional Decode( + rtc::ArrayView decoded) const override { + const size_t output_size = + sizeof(int16_t) * kPacketDuration * num_channels_; + if (decoded.size() >= output_size) { + memset(decoded.data(), 0, + sizeof(int16_t) * kPacketDuration * num_channels_); + return rtc::Optional( + {kPacketDuration * num_channels_, kSpeech}); + } else { + ADD_FAILURE() << "Expected decoded.size() to be >= output_size (" + << decoded.size() << " vs. " << output_size << ")"; + return rtc::Optional(); + } + } + + private: + const size_t num_channels_; + }; + + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) /* override */ { + std::vector results; + if (fec_enabled_) { + std::unique_ptr fec_frame(new MockFrame(num_channels_)); + results.emplace_back(timestamp - kPacketDuration, 1, + std::move(fec_frame)); + } + + std::unique_ptr frame(new MockFrame(num_channels_)); + results.emplace_back(timestamp, 0, std::move(frame)); + return results; } - int PacketDurationRedundant(const uint8_t* encoded, - size_t encoded_len) const /* override */ { + int PacketDuration(const uint8_t* encoded, size_t encoded_len) const + /* override */ { + ADD_FAILURE() << "Since going through ParsePayload, PacketDuration should " + "never get called."; return kPacketDuration; } bool PacketHasFec( const uint8_t* encoded, size_t encoded_len) const /* override */ { + ADD_FAILURE() << "Since going through ParsePayload, PacketHasFec should " + "never get called."; return fec_enabled_; } @@ -63,24 +102,14 @@ class MockAudioDecoder final : public AudioDecoder { bool fec_enabled() const { return fec_enabled_; } protected: - // Override the following methods such that no actual payload is needed. int DecodeInternal(const uint8_t* encoded, size_t encoded_len, - int /*sample_rate_hz*/, + int sample_rate_hz, int16_t* decoded, SpeechType* speech_type) /* override */ { - *speech_type = kSpeech; - memset(decoded, 0, sizeof(int16_t) * kPacketDuration * Channels()); - return kPacketDuration * Channels(); - } - - int DecodeRedundantInternal(const uint8_t* encoded, - size_t encoded_len, - int sample_rate_hz, - int16_t* decoded, - SpeechType* speech_type) /* override */ { - return DecodeInternal(encoded, encoded_len, sample_rate_hz, decoded, - speech_type); + ADD_FAILURE() << "Since going through ParsePayload, DecodeInternal should " + "never get called."; + return -1; } private: diff --git a/webrtc/modules/audio_coding/neteq/packet.h b/webrtc/modules/audio_coding/neteq/packet.h index 4e17a88dcd..cf590caf4b 100644 --- a/webrtc/modules/audio_coding/neteq/packet.h +++ b/webrtc/modules/audio_coding/neteq/packet.h @@ -24,10 +24,52 @@ namespace webrtc { // Struct for holding RTP packets. struct Packet { + struct Priority { + Priority() : codec_level(0), red_level(0) {} + Priority(int codec_level, int red_level) + : codec_level(codec_level), red_level(red_level) { + CheckInvariant(); + } + + int codec_level; + int red_level; + + // Priorities are sorted low-to-high, first on the level the codec + // prioritizes it, then on the level of RED packet it is; i.e. if it is a + // primary or secondary payload of a RED packet. For example: with Opus, an + // Fec packet (which the decoder prioritizes lower than a regular packet) + // will not be used if there is _any_ RED payload for the same + // timeframe. The highest priority packet will have levels {0, 0}. Negative + // priorities are not allowed. + bool operator<(const Priority& b) const { + CheckInvariant(); + b.CheckInvariant(); + if (codec_level == b.codec_level) + return red_level < b.red_level; + + return codec_level < b.codec_level; + } + bool operator==(const Priority& b) const { + CheckInvariant(); + b.CheckInvariant(); + return codec_level == b.codec_level && red_level == b.red_level; + } + bool operator!=(const Priority& b) const { return !(*this == b); } + bool operator>(const Priority& b) const { return b < *this; } + bool operator<=(const Priority& b) const { return !(b > *this); } + bool operator>=(const Priority& b) const { return !(b < *this); } + + private: + void CheckInvariant() const { + RTC_DCHECK_GE(codec_level, 0); + RTC_DCHECK_GE(red_level, 0); + } + }; + RTPHeader header; // Datagram excluding RTP header and header extension. rtc::Buffer payload; - bool primary = true; // Primary, i.e., not redundant payload. + Priority priority; std::unique_ptr waiting_time; std::unique_ptr frame; @@ -41,17 +83,16 @@ struct Packet { // primary payload is considered "smaller" than a secondary. bool operator==(const Packet& rhs) const { return (this->header.timestamp == rhs.header.timestamp && - this->header.sequenceNumber == rhs.header.sequenceNumber && - this->primary == rhs.primary); + this->header.sequenceNumber == rhs.header.sequenceNumber && + this->priority == rhs.priority); } bool operator!=(const Packet& rhs) const { return !operator==(rhs); } bool operator<(const Packet& rhs) const { if (this->header.timestamp == rhs.header.timestamp) { if (this->header.sequenceNumber == rhs.header.sequenceNumber) { - // Timestamp and sequence numbers are identical - deem the left - // hand side to be "smaller" (i.e., "earlier") if it is primary, and - // right hand side is not. - return (this->primary && !rhs.primary); + // Timestamp and sequence numbers are identical - deem the left hand + // side to be "smaller" (i.e., "earlier") if it has higher priority. + return this->priority < rhs.priority; } return (static_cast(rhs.header.sequenceNumber - this->header.sequenceNumber) < 0xFFFF / 2); diff --git a/webrtc/modules/audio_coding/neteq/packet_buffer.cc b/webrtc/modules/audio_coding/neteq/packet_buffer.cc index c5b23dce06..eeb1d272b9 100644 --- a/webrtc/modules/audio_coding/neteq/packet_buffer.cc +++ b/webrtc/modules/audio_coding/neteq/packet_buffer.cc @@ -76,6 +76,9 @@ int PacketBuffer::InsertPacket(Packet* packet) { return kInvalidPacket; } + RTC_DCHECK_GE(packet->priority.codec_level, 0); + RTC_DCHECK_GE(packet->priority.red_level, 0); + int return_val = kOK; packet->waiting_time = tick_timer_->GetNewStopwatch(); @@ -262,7 +265,7 @@ int PacketBuffer::DiscardAllOldPackets(uint32_t timestamp_limit) { void PacketBuffer::DiscardPacketsWithPayloadType(uint8_t payload_type) { for (auto it = buffer_.begin(); it != buffer_.end(); /* */) { - Packet *packet = *it; + Packet* packet = *it; if (packet->header.payloadType == payload_type) { delete packet; it = buffer_.erase(it); @@ -281,7 +284,9 @@ size_t PacketBuffer::NumSamplesInBuffer(size_t last_decoded_length) const { size_t last_duration = last_decoded_length; for (Packet* packet : buffer_) { if (packet->frame) { - if (!packet->primary) { + // TODO(hlundin): Verify that it's fine to count all packets and remove + // this check. + if (packet->priority != Packet::Priority(0, 0)) { continue; } size_t duration = packet->frame->Duration(); diff --git a/webrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc b/webrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc index 1b86d8326b..b39169f909 100644 --- a/webrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc +++ b/webrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc @@ -60,7 +60,6 @@ Packet* PacketGenerator::NextPacket(int payload_size_bytes) { packet->header.ssrc = 0x12345678; packet->header.numCSRCs = 0; packet->header.paddingLength = 0; - packet->primary = true; packet->payload.SetSize(payload_size_bytes); ++seq_no_; ts_ += frame_size_; @@ -284,7 +283,7 @@ TEST(PacketBuffer, ExtractOrderRedundancy) { packet_facts[i].payload_type, kFrameSize); Packet* packet = gen.NextPacket(kPayloadLength); - packet->primary = packet_facts[i].primary; + packet->priority.red_level = packet_facts[i].primary ? 0 : 1; EXPECT_EQ(PacketBuffer::kOK, buffer.InsertPacket(packet)); if (packet_facts[i].extract_order >= 0) { expect_order[packet_facts[i].extract_order] = packet; @@ -512,8 +511,8 @@ TEST(PacketBuffer, Failures) { // The function should return true if the first packet "goes before" the second. TEST(PacketBuffer, ComparePackets) { PacketGenerator gen(0, 0, 0, 10); - Packet* a = gen.NextPacket(10); // SN = 0, TS = 0. - Packet* b = gen.NextPacket(10); // SN = 1, TS = 10. + std::unique_ptr a(gen.NextPacket(10)); // SN = 0, TS = 0. + std::unique_ptr b(gen.NextPacket(10)); // SN = 1, TS = 10. EXPECT_FALSE(*a == *b); EXPECT_TRUE(*a != *b); EXPECT_TRUE(*a < *b); @@ -556,10 +555,11 @@ TEST(PacketBuffer, ComparePackets) { EXPECT_TRUE(*a <= *b); EXPECT_FALSE(*a >= *b); - // Test equal timestamps and sequence numbers, but only 'b' is primary. + // Test equal timestamps and sequence numbers, but differing priorities. a->header.sequenceNumber = b->header.sequenceNumber; - a->primary = false; - b->primary = true; + a->priority = {1, 0}; + b->priority = {0, 0}; + // a after b EXPECT_FALSE(*a == *b); EXPECT_TRUE(*a != *b); EXPECT_FALSE(*a < *b); @@ -567,8 +567,53 @@ TEST(PacketBuffer, ComparePackets) { EXPECT_FALSE(*a <= *b); EXPECT_TRUE(*a >= *b); - delete a; - delete b; + std::unique_ptr c(gen.NextPacket(0)); // SN = 2, TS = 20. + std::unique_ptr d(gen.NextPacket(0)); // SN = 3, TS = 20. + c->header.timestamp = b->header.timestamp; + d->header.timestamp = b->header.timestamp; + c->header.sequenceNumber = b->header.sequenceNumber; + d->header.sequenceNumber = b->header.sequenceNumber; + c->priority = {1, 1}; + d->priority = {0, 1}; + // c after d + EXPECT_FALSE(*c == *d); + EXPECT_TRUE(*c != *d); + EXPECT_FALSE(*c < *d); + EXPECT_TRUE(*c > *d); + EXPECT_FALSE(*c <= *d); + EXPECT_TRUE(*c >= *d); + + // c after a + EXPECT_FALSE(*c == *a); + EXPECT_TRUE(*c != *a); + EXPECT_FALSE(*c < *a); + EXPECT_TRUE(*c > *a); + EXPECT_FALSE(*c <= *a); + EXPECT_TRUE(*c >= *a); + + // c after b + EXPECT_FALSE(*c == *b); + EXPECT_TRUE(*c != *b); + EXPECT_FALSE(*c < *b); + EXPECT_TRUE(*c > *b); + EXPECT_FALSE(*c <= *b); + EXPECT_TRUE(*c >= *b); + + // a after d + EXPECT_FALSE(*a == *d); + EXPECT_TRUE(*a != *d); + EXPECT_FALSE(*a < *d); + EXPECT_TRUE(*a > *d); + EXPECT_FALSE(*a <= *d); + EXPECT_TRUE(*a >= *d); + + // d after b + EXPECT_FALSE(*d == *b); + EXPECT_TRUE(*d != *b); + EXPECT_FALSE(*d < *b); + EXPECT_TRUE(*d > *b); + EXPECT_FALSE(*d <= *b); + EXPECT_TRUE(*d >= *b); } // Test the DeleteFirstPacket DeleteAllPackets methods. diff --git a/webrtc/modules/audio_coding/neteq/payload_splitter.cc b/webrtc/modules/audio_coding/neteq/red_payload_splitter.cc similarity index 54% rename from webrtc/modules/audio_coding/neteq/payload_splitter.cc rename to webrtc/modules/audio_coding/neteq/red_payload_splitter.cc index 28e561a65f..a2f109832a 100644 --- a/webrtc/modules/audio_coding/neteq/payload_splitter.cc +++ b/webrtc/modules/audio_coding/neteq/red_payload_splitter.cc @@ -8,12 +8,14 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "webrtc/modules/audio_coding/neteq/payload_splitter.h" +#include "webrtc/modules/audio_coding/neteq/red_payload_splitter.h" #include +#include #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" +#include "webrtc/base/safe_conversions.h" #include "webrtc/modules/audio_coding/neteq/decoder_database.h" namespace webrtc { @@ -25,8 +27,11 @@ namespace webrtc { // is replaced by the new ones in |new_packets|, so that |packet_list| becomes: // {A1, A2, ..., B, C, ...}. The method then continues with B, and C, until all // the original packets have been replaced by their split payloads. -int PayloadSplitter::SplitRed(PacketList* packet_list) { - int ret = kOK; +bool RedPayloadSplitter::SplitRed(PacketList* packet_list) { + // Too many RED blocks indicates that something is wrong. Clamp it at some + // reasonable value. + const size_t kMaxRedBlocks = 32; + bool ret = true; PacketList::iterator it = packet_list->begin(); while (it != packet_list->end()) { const Packet* red_packet = (*it); @@ -50,7 +55,6 @@ int PayloadSplitter::SplitRed(PacketList* packet_list) { uint8_t payload_type; uint32_t timestamp; size_t payload_length; - bool primary; }; std::vector new_headers; @@ -67,17 +71,15 @@ int PayloadSplitter::SplitRed(PacketList* packet_list) { ++sum_length; // Account for RED header size of 1 byte. new_header.timestamp = red_packet->header.timestamp; new_header.payload_length = red_packet->payload.size() - sum_length; - new_header.primary = true; // Last block is always primary. payload_ptr += 1; // Advance to first payload byte. } else { // Bits 8 through 21 are timestamp offset. - int timestamp_offset = (payload_ptr[1] << 6) + - ((payload_ptr[2] & 0xFC) >> 2); + int timestamp_offset = + (payload_ptr[1] << 6) + ((payload_ptr[2] & 0xFC) >> 2); new_header.timestamp = red_packet->header.timestamp - timestamp_offset; // Bits 22 through 31 are payload length. new_header.payload_length = ((payload_ptr[2] & 0x03) << 8) + payload_ptr[3]; - new_header.primary = false; payload_ptr += 4; // Advance to next RED header. } sum_length += new_header.payload_length; @@ -86,33 +88,41 @@ int PayloadSplitter::SplitRed(PacketList* packet_list) { new_headers.push_back(new_header); } - // Populate the new packets with payload data. - // |payload_ptr| now points at the first payload byte. - PacketList new_packets; // An empty list to store the split packets in. - for (const auto& new_header : new_headers) { - size_t payload_length = new_header.payload_length; - if (payload_ptr + payload_length > - red_packet->payload.data() + red_packet->payload.size()) { - // The block lengths in the RED headers do not match the overall packet - // length. Something is corrupt. Discard this and the remaining - // payloads from this packet. - LOG(LS_WARNING) << "SplitRed length mismatch"; - ret = kRedLengthMismatch; - break; + if (new_headers.size() <= kMaxRedBlocks) { + // Populate the new packets with payload data. + // |payload_ptr| now points at the first payload byte. + PacketList new_packets; // An empty list to store the split packets in. + for (size_t i = 0; i != new_headers.size(); ++i) { + const auto& new_header = new_headers[i]; + size_t payload_length = new_header.payload_length; + if (payload_ptr + payload_length > + red_packet->payload.data() + red_packet->payload.size()) { + // The block lengths in the RED headers do not match the overall + // packet length. Something is corrupt. Discard this and the remaining + // payloads from this packet. + LOG(LS_WARNING) << "SplitRed length mismatch"; + ret = false; + break; + } + + Packet* new_packet = new Packet; + new_packet->header = red_packet->header; + new_packet->header.timestamp = new_header.timestamp; + new_packet->header.payloadType = new_header.payload_type; + new_packet->priority.red_level = + rtc::checked_cast((new_headers.size() - 1) - i); + new_packet->payload.SetData(payload_ptr, payload_length); + new_packets.push_front(new_packet); + payload_ptr += payload_length; } - Packet* new_packet = new Packet; - new_packet->header = red_packet->header; - new_packet->header.timestamp = new_header.timestamp; - new_packet->header.payloadType = new_header.payload_type; - new_packet->primary = new_header.primary; - new_packet->payload.SetData(payload_ptr, payload_length); - new_packets.push_front(new_packet); - payload_ptr += payload_length; + // Insert new packets into original list, before the element pointed to by + // iterator |it|. + packet_list->splice(it, new_packets, new_packets.begin(), + new_packets.end()); + } else { + LOG(LS_WARNING) << "SplitRed too many blocks: " << new_headers.size(); + ret = false; } - // Insert new packets into original list, before the element pointed to by - // iterator |it|. - packet_list->splice(it, new_packets, new_packets.begin(), - new_packets.end()); // Delete old packet payload. delete (*it); // Remove |it| from the packet list. This operation effectively moves the @@ -123,66 +133,9 @@ int PayloadSplitter::SplitRed(PacketList* packet_list) { return ret; } -int PayloadSplitter::SplitFec(PacketList* packet_list, - DecoderDatabase* decoder_database) { - PacketList::iterator it = packet_list->begin(); - // Iterate through all packets in |packet_list|. - while (it != packet_list->end()) { - Packet* packet = (*it); // Just to make the notation more intuitive. - // Get codec type for this payload. - uint8_t payload_type = packet->header.payloadType; - const DecoderDatabase::DecoderInfo* info = - decoder_database->GetDecoderInfo(payload_type); - if (!info) { - LOG(LS_WARNING) << "SplitFec unknown payload type"; - return kUnknownPayloadType; - } - - // Not an FEC packet. - AudioDecoder* decoder = decoder_database->GetDecoder(payload_type); - // decoder should not return NULL, except for comfort noise payloads which - // are handled separately. - assert(decoder != NULL || decoder_database->IsComfortNoise(payload_type)); - if (!decoder || - !decoder->PacketHasFec(packet->payload.data(), - packet->payload.size())) { - ++it; - continue; - } - - switch (info->codec_type) { - case NetEqDecoder::kDecoderOpus: - case NetEqDecoder::kDecoderOpus_2ch: { - // The main payload of this packet should be decoded as a primary - // payload, even if it comes as a secondary payload in a RED packet. - packet->primary = true; - - Packet* new_packet = new Packet; - new_packet->header = packet->header; - int duration = decoder->PacketDurationRedundant(packet->payload.data(), - packet->payload.size()); - new_packet->header.timestamp -= duration; - new_packet->payload.SetData(packet->payload); - new_packet->primary = false; - // Waiting time should not be set here. - RTC_DCHECK(!packet->waiting_time); - - packet_list->insert(it, new_packet); - break; - } - default: { - LOG(LS_WARNING) << "SplitFec wrong payload type"; - return kFecSplitError; - } - } - - ++it; - } - return kOK; -} - -int PayloadSplitter::CheckRedPayloads(PacketList* packet_list, - const DecoderDatabase& decoder_database) { +int RedPayloadSplitter::CheckRedPayloads( + PacketList* packet_list, + const DecoderDatabase& decoder_database) { PacketList::iterator it = packet_list->begin(); int main_payload_type = -1; int num_deleted_packets = 0; diff --git a/webrtc/modules/audio_coding/neteq/payload_splitter.h b/webrtc/modules/audio_coding/neteq/red_payload_splitter.h similarity index 56% rename from webrtc/modules/audio_coding/neteq/payload_splitter.h rename to webrtc/modules/audio_coding/neteq/red_payload_splitter.h index a3e1b1d91c..deb86eb166 100644 --- a/webrtc/modules/audio_coding/neteq/payload_splitter.h +++ b/webrtc/modules/audio_coding/neteq/red_payload_splitter.h @@ -8,8 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_PAYLOAD_SPLITTER_H_ -#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_PAYLOAD_SPLITTER_H_ +#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_RED_PAYLOAD_SPLITTER_H_ +#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_RED_PAYLOAD_SPLITTER_H_ #include "webrtc/base/constructormagic.h" #include "webrtc/modules/audio_coding/neteq/packet.h" @@ -19,24 +19,14 @@ namespace webrtc { // Forward declarations. class DecoderDatabase; -// This class handles splitting of payloads into smaller parts. - -// For RED and FEC the splitting is done internally. Other codecs' packets are -// split by calling AudioDecoder::SplitPacket. -class PayloadSplitter { +// This class handles splitting of RED payloads into smaller parts. +// Codec-specific packet splitting can be performed by +// AudioDecoder::ParsePayload. +class RedPayloadSplitter { public: - enum SplitterReturnCodes { - kOK = 0, - kNoSplit = 1, - kFrameSplitError = -2, - kUnknownPayloadType = -3, - kRedLengthMismatch = -4, - kFecSplitError = -5, - }; + RedPayloadSplitter() {} - PayloadSplitter() {} - - virtual ~PayloadSplitter() {} + virtual ~RedPayloadSplitter() {} // Splits each packet in |packet_list| into its separate RED payloads. Each // RED payload is packetized into a Packet. The original elements in @@ -44,23 +34,18 @@ class PayloadSplitter { // Note that all packets in |packet_list| must be RED payloads, i.e., have // RED headers according to RFC 2198 at the very beginning of the payload. // Returns kOK or an error. - virtual int SplitRed(PacketList* packet_list); - - // Iterates through |packet_list| and, duplicate each audio payload that has - // FEC as new packet for redundant decoding. The decoder database is needed to - // get information about which payload type each packet contains. - virtual int SplitFec(PacketList* packet_list, - DecoderDatabase* decoder_database); + virtual bool SplitRed(PacketList* packet_list); // Checks all packets in |packet_list|. Packets that are DTMF events or // comfort noise payloads are kept. Except that, only one single payload type - // is accepted. Any packet with another payload type is discarded. + // is accepted. Any packet with another payload type is discarded. Returns + // the number of discarded packets. virtual int CheckRedPayloads(PacketList* packet_list, const DecoderDatabase& decoder_database); private: - RTC_DISALLOW_COPY_AND_ASSIGN(PayloadSplitter); + RTC_DISALLOW_COPY_AND_ASSIGN(RedPayloadSplitter); }; } // namespace webrtc -#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_PAYLOAD_SPLITTER_H_ +#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_RED_PAYLOAD_SPLITTER_H_ diff --git a/webrtc/modules/audio_coding/neteq/payload_splitter_unittest.cc b/webrtc/modules/audio_coding/neteq/red_payload_splitter_unittest.cc similarity index 70% rename from webrtc/modules/audio_coding/neteq/payload_splitter_unittest.cc rename to webrtc/modules/audio_coding/neteq/red_payload_splitter_unittest.cc index 0a4b2157b5..5396ff4d0a 100644 --- a/webrtc/modules/audio_coding/neteq/payload_splitter_unittest.cc +++ b/webrtc/modules/audio_coding/neteq/red_payload_splitter_unittest.cc @@ -8,9 +8,9 @@ * be found in the AUTHORS file in the root of the source tree. */ -// Unit tests for PayloadSplitter class. +// Unit tests for RedPayloadSplitter class. -#include "webrtc/modules/audio_coding/neteq/payload_splitter.h" +#include "webrtc/modules/audio_coding/neteq/red_payload_splitter.h" #include @@ -46,7 +46,8 @@ static const uint32_t kBaseTimestamp = 0x12345678; // : | // | | // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -void CreateOpusFecPayload(uint8_t* payload, size_t payload_length, +void CreateOpusFecPayload(uint8_t* payload, + size_t payload_length, uint8_t payload_value) { if (payload_length < 2) { return; @@ -121,8 +122,10 @@ Packet* CreateRedPayload(size_t num_payloads, } // Create a packet with all payload bytes set to |payload_value|. -Packet* CreatePacket(uint8_t payload_type, size_t payload_length, - uint8_t payload_value, bool opus_fec = false) { +Packet* CreatePacket(uint8_t payload_type, + size_t payload_length, + uint8_t payload_value, + bool opus_fec = false) { Packet* packet = new Packet; packet->header.payloadType = payload_type; packet->header.timestamp = kBaseTimestamp; @@ -144,22 +147,34 @@ void VerifyPacket(const Packet* packet, uint16_t sequence_number, uint32_t timestamp, uint8_t payload_value, - bool primary = true) { + Packet::Priority priority) { EXPECT_EQ(payload_length, packet->payload.size()); EXPECT_EQ(payload_type, packet->header.payloadType); EXPECT_EQ(sequence_number, packet->header.sequenceNumber); EXPECT_EQ(timestamp, packet->header.timestamp); - EXPECT_EQ(primary, packet->primary); + EXPECT_EQ(priority, packet->priority); ASSERT_FALSE(packet->payload.empty()); for (size_t i = 0; i < packet->payload.size(); ++i) { ASSERT_EQ(payload_value, packet->payload.data()[i]); } } +void VerifyPacket(const Packet* packet, + size_t payload_length, + uint8_t payload_type, + uint16_t sequence_number, + uint32_t timestamp, + uint8_t payload_value, + bool primary) { + return VerifyPacket(packet, payload_length, payload_type, sequence_number, + timestamp, payload_value, + Packet::Priority{0, primary ? 0 : 1}); +} + // Start of test definitions. -TEST(PayloadSplitter, CreateAndDestroy) { - PayloadSplitter* splitter = new PayloadSplitter; +TEST(RedPayloadSplitter, CreateAndDestroy) { + RedPayloadSplitter* splitter = new RedPayloadSplitter; delete splitter; } @@ -170,8 +185,8 @@ TEST(RedPayloadSplitter, OnePacketTwoPayloads) { Packet* packet = CreateRedPayload(2, payload_types, kTimestampOffset); PacketList packet_list; packet_list.push_back(packet); - PayloadSplitter splitter; - EXPECT_EQ(PayloadSplitter::kOK, splitter.SplitRed(&packet_list)); + RedPayloadSplitter splitter; + EXPECT_TRUE(splitter.SplitRed(&packet_list)); ASSERT_EQ(2u, packet_list.size()); // Check first packet. The first in list should always be the primary payload. packet = packet_list.front(); @@ -201,8 +216,8 @@ TEST(RedPayloadSplitter, TwoPacketsOnePayload) { packet->header.timestamp += kTimestampOffset; packet->header.sequenceNumber++; packet_list.push_back(packet); - PayloadSplitter splitter; - EXPECT_EQ(PayloadSplitter::kOK, splitter.SplitRed(&packet_list)); + RedPayloadSplitter splitter; + EXPECT_TRUE(splitter.SplitRed(&packet_list)); ASSERT_EQ(2u, packet_list.size()); // Check first packet. packet = packet_list.front(); @@ -239,43 +254,43 @@ TEST(RedPayloadSplitter, TwoPacketsThreePayloads) { packet->header.timestamp += kTimestampOffset; packet->header.sequenceNumber++; packet_list.push_back(packet); - PayloadSplitter splitter; - EXPECT_EQ(PayloadSplitter::kOK, splitter.SplitRed(&packet_list)); + RedPayloadSplitter splitter; + EXPECT_TRUE(splitter.SplitRed(&packet_list)); ASSERT_EQ(6u, packet_list.size()); // Check first packet, A1. packet = packet_list.front(); VerifyPacket(packet, kPayloadLength, payload_types[2], kSequenceNumber, - kBaseTimestamp, 2, true); + kBaseTimestamp, 2, {0, 0}); delete packet; packet_list.pop_front(); // Check second packet, A2. packet = packet_list.front(); VerifyPacket(packet, kPayloadLength, payload_types[1], kSequenceNumber, - kBaseTimestamp - kTimestampOffset, 1, false); + kBaseTimestamp - kTimestampOffset, 1, {0, 1}); delete packet; packet_list.pop_front(); // Check third packet, A3. packet = packet_list.front(); VerifyPacket(packet, kPayloadLength, payload_types[0], kSequenceNumber, - kBaseTimestamp - 2 * kTimestampOffset, 0, false); + kBaseTimestamp - 2 * kTimestampOffset, 0, {0, 2}); delete packet; packet_list.pop_front(); // Check fourth packet, B1. packet = packet_list.front(); VerifyPacket(packet, kPayloadLength, payload_types[2], kSequenceNumber + 1, - kBaseTimestamp + kTimestampOffset, 2, true); + kBaseTimestamp + kTimestampOffset, 2, {0, 0}); delete packet; packet_list.pop_front(); // Check fifth packet, B2. packet = packet_list.front(); VerifyPacket(packet, kPayloadLength, payload_types[1], kSequenceNumber + 1, - kBaseTimestamp, 1, false); + kBaseTimestamp, 1, {0, 1}); delete packet; packet_list.pop_front(); // Check sixth packet, B3. packet = packet_list.front(); VerifyPacket(packet, kPayloadLength, payload_types[0], kSequenceNumber + 1, - kBaseTimestamp - kTimestampOffset, 0, false); + kBaseTimestamp - kTimestampOffset, 0, {0, 2}); delete packet; } @@ -305,7 +320,7 @@ TEST(RedPayloadSplitter, CheckRedPayloads) { decoder_database.RegisterPayload(2, NetEqDecoder::kDecoderAVT, "avt"); decoder_database.RegisterPayload(3, NetEqDecoder::kDecoderILBC, "ilbc"); - PayloadSplitter splitter; + RedPayloadSplitter splitter; splitter.CheckRedPayloads(&packet_list, decoder_database); ASSERT_EQ(3u, packet_list.size()); // Should have dropped the last packet. @@ -332,124 +347,13 @@ TEST(RedPayloadSplitter, WrongPayloadLength) { packet->payload.SetSize(packet->payload.size() - (kPayloadLength + 1)); PacketList packet_list; packet_list.push_back(packet); - PayloadSplitter splitter; - EXPECT_EQ(PayloadSplitter::kRedLengthMismatch, - splitter.SplitRed(&packet_list)); + RedPayloadSplitter splitter; + EXPECT_FALSE(splitter.SplitRed(&packet_list)); ASSERT_EQ(1u, packet_list.size()); // Check first packet. packet = packet_list.front(); VerifyPacket(packet, kPayloadLength, payload_types[0], kSequenceNumber, - kBaseTimestamp - 2 * kTimestampOffset, 0, false); - delete packet; - packet_list.pop_front(); -} - -TEST(FecPayloadSplitter, MixedPayload) { - PacketList packet_list; - DecoderDatabase decoder_database(CreateBuiltinAudioDecoderFactory()); - - decoder_database.RegisterPayload(0, NetEqDecoder::kDecoderOpus, "opus"); - decoder_database.RegisterPayload(1, NetEqDecoder::kDecoderPCMu, "pcmu"); - - Packet* packet = CreatePacket(0, 10, 0xFF, true); - packet_list.push_back(packet); - - packet = CreatePacket(0, 10, 0); // Non-FEC Opus payload. - packet_list.push_back(packet); - - packet = CreatePacket(1, 10, 0); // Non-Opus payload. - packet_list.push_back(packet); - - PayloadSplitter splitter; - EXPECT_EQ(PayloadSplitter::kOK, - splitter.SplitFec(&packet_list, &decoder_database)); - EXPECT_EQ(4u, packet_list.size()); - - // Check first packet. - packet = packet_list.front(); - EXPECT_EQ(0, packet->header.payloadType); - EXPECT_EQ(kBaseTimestamp - 20 * 48, packet->header.timestamp); - EXPECT_EQ(10U, packet->payload.size()); - EXPECT_FALSE(packet->primary); - delete packet; - packet_list.pop_front(); - - // Check second packet. - packet = packet_list.front(); - EXPECT_EQ(0, packet->header.payloadType); - EXPECT_EQ(kBaseTimestamp, packet->header.timestamp); - EXPECT_EQ(10U, packet->payload.size()); - EXPECT_TRUE(packet->primary); - delete packet; - packet_list.pop_front(); - - // Check third packet. - packet = packet_list.front(); - VerifyPacket(packet, 10, 0, kSequenceNumber, kBaseTimestamp, 0, true); - delete packet; - packet_list.pop_front(); - - // Check fourth packet. - packet = packet_list.front(); - VerifyPacket(packet, 10, 1, kSequenceNumber, kBaseTimestamp, 0, true); - delete packet; -} - -TEST(FecPayloadSplitter, EmbedFecInRed) { - PacketList packet_list; - DecoderDatabase decoder_database(CreateBuiltinAudioDecoderFactory()); - - const int kTimestampOffset = 20 * 48; // 20 ms * 48 kHz. - uint8_t payload_types[] = {0, 0}; - decoder_database.RegisterPayload(0, NetEqDecoder::kDecoderOpus, "opus"); - Packet* packet = CreateRedPayload(2, payload_types, kTimestampOffset, true); - packet_list.push_back(packet); - - PayloadSplitter splitter; - EXPECT_EQ(PayloadSplitter::kOK, - splitter.SplitRed(&packet_list)); - EXPECT_EQ(PayloadSplitter::kOK, - splitter.SplitFec(&packet_list, &decoder_database)); - - EXPECT_EQ(4u, packet_list.size()); - - // Check first packet. FEC packet copied from primary payload in RED. - packet = packet_list.front(); - EXPECT_EQ(0, packet->header.payloadType); - EXPECT_EQ(kBaseTimestamp - kTimestampOffset, packet->header.timestamp); - EXPECT_EQ(kPayloadLength, packet->payload.size()); - EXPECT_FALSE(packet->primary); - EXPECT_EQ(packet->payload[3], 1); - delete packet; - packet_list.pop_front(); - - // Check second packet. Normal packet copied from primary payload in RED. - packet = packet_list.front(); - EXPECT_EQ(0, packet->header.payloadType); - EXPECT_EQ(kBaseTimestamp, packet->header.timestamp); - EXPECT_EQ(kPayloadLength, packet->payload.size()); - EXPECT_TRUE(packet->primary); - EXPECT_EQ(packet->payload[3], 1); - delete packet; - packet_list.pop_front(); - - // Check third packet. FEC packet copied from secondary payload in RED. - packet = packet_list.front(); - EXPECT_EQ(0, packet->header.payloadType); - EXPECT_EQ(kBaseTimestamp - 2 * kTimestampOffset, packet->header.timestamp); - EXPECT_EQ(kPayloadLength, packet->payload.size()); - EXPECT_FALSE(packet->primary); - EXPECT_EQ(packet->payload[3], 0); - delete packet; - packet_list.pop_front(); - - // Check fourth packet. Normal packet copied from primary payload in RED. - packet = packet_list.front(); - EXPECT_EQ(0, packet->header.payloadType); - EXPECT_EQ(kBaseTimestamp - kTimestampOffset, packet->header.timestamp); - EXPECT_EQ(kPayloadLength, packet->payload.size()); - EXPECT_TRUE(packet->primary); - EXPECT_EQ(packet->payload[3], 0); + kBaseTimestamp - 2 * kTimestampOffset, 0, {0, 2}); delete packet; packet_list.pop_front(); }