diff --git a/webrtc/modules/BUILD.gn b/webrtc/modules/BUILD.gn index 8fb9f86d14..11a5ef0973 100644 --- a/webrtc/modules/BUILD.gn +++ b/webrtc/modules/BUILD.gn @@ -313,7 +313,7 @@ if (rtc_include_tests) { "audio_coding/neteq/mock/mock_expand.h", "audio_coding/neteq/mock/mock_external_decoder_pcm16b.h", "audio_coding/neteq/mock/mock_packet_buffer.h", - "audio_coding/neteq/mock/mock_payload_splitter.h", + "audio_coding/neteq/mock/mock_red_payload_splitter.h", "audio_coding/neteq/nack_tracker_unittest.cc", "audio_coding/neteq/neteq_external_decoder_unittest.cc", "audio_coding/neteq/neteq_impl_unittest.cc", @@ -322,9 +322,9 @@ if (rtc_include_tests) { "audio_coding/neteq/neteq_unittest.cc", "audio_coding/neteq/normal_unittest.cc", "audio_coding/neteq/packet_buffer_unittest.cc", - "audio_coding/neteq/payload_splitter_unittest.cc", "audio_coding/neteq/post_decode_vad_unittest.cc", "audio_coding/neteq/random_vector_unittest.cc", + "audio_coding/neteq/red_payload_splitter_unittest.cc", "audio_coding/neteq/sync_buffer_unittest.cc", "audio_coding/neteq/tick_timer_unittest.cc", "audio_coding/neteq/time_stretch_unittest.cc", diff --git a/webrtc/modules/audio_coding/BUILD.gn b/webrtc/modules/audio_coding/BUILD.gn index a60baba5da..b55829d786 100644 --- a/webrtc/modules/audio_coding/BUILD.gn +++ b/webrtc/modules/audio_coding/BUILD.gn @@ -784,14 +784,14 @@ rtc_source_set("neteq") { "neteq/packet.h", "neteq/packet_buffer.cc", "neteq/packet_buffer.h", - "neteq/payload_splitter.cc", - "neteq/payload_splitter.h", "neteq/post_decode_vad.cc", "neteq/post_decode_vad.h", "neteq/preemptive_expand.cc", "neteq/preemptive_expand.h", "neteq/random_vector.cc", "neteq/random_vector.h", + "neteq/red_payload_splitter.cc", + "neteq/red_payload_splitter.h", "neteq/rtcp.cc", "neteq/rtcp.h", "neteq/statistics_calculator.cc", diff --git a/webrtc/modules/audio_coding/codecs/audio_decoder.cc b/webrtc/modules/audio_coding/codecs/audio_decoder.cc index 6c67924260..afa5115d5a 100644 --- a/webrtc/modules/audio_coding/codecs/audio_decoder.cc +++ b/webrtc/modules/audio_coding/codecs/audio_decoder.cc @@ -14,8 +14,6 @@ #include #include -#include - #include "webrtc/base/array_view.h" #include "webrtc/base/checks.h" #include "webrtc/base/sanitizer.h" @@ -27,9 +25,11 @@ namespace webrtc { AudioDecoder::ParseResult::ParseResult() = default; AudioDecoder::ParseResult::ParseResult(ParseResult&& b) = default; AudioDecoder::ParseResult::ParseResult(uint32_t timestamp, - bool primary, + int priority, std::unique_ptr frame) - : timestamp(timestamp), primary(primary), frame(std::move(frame)) {} + : timestamp(timestamp), priority(priority), frame(std::move(frame)) { + RTC_DCHECK_GE(priority, 0); +} AudioDecoder::ParseResult::~ParseResult() = default; @@ -38,12 +38,11 @@ AudioDecoder::ParseResult& AudioDecoder::ParseResult::operator=( std::vector AudioDecoder::ParsePayload( rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) { + uint32_t timestamp) { std::vector results; std::unique_ptr frame( - new LegacyEncodedAudioFrame(this, std::move(payload), is_primary)); - results.emplace_back(timestamp, is_primary, std::move(frame)); + new LegacyEncodedAudioFrame(this, std::move(payload))); + results.emplace_back(timestamp, 0, std::move(frame)); return results; } diff --git a/webrtc/modules/audio_coding/codecs/audio_decoder.h b/webrtc/modules/audio_coding/codecs/audio_decoder.h index b6338d2102..8468da20f2 100644 --- a/webrtc/modules/audio_coding/codecs/audio_decoder.h +++ b/webrtc/modules/audio_coding/codecs/audio_decoder.h @@ -8,11 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_INCLUDE_AUDIO_DECODER_H_ -#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_INCLUDE_AUDIO_DECODER_H_ - -#include -#include +#ifndef WEBRTC_MODULES_AUDIO_CODING_CODECS_AUDIO_DECODER_H_ +#define WEBRTC_MODULES_AUDIO_CODING_CODECS_AUDIO_DECODER_H_ #include #include @@ -66,7 +63,7 @@ class AudioDecoder { struct ParseResult { ParseResult(); ParseResult(uint32_t timestamp, - bool primary, + int priority, std::unique_ptr frame); ParseResult(ParseResult&& b); ~ParseResult(); @@ -75,7 +72,10 @@ class AudioDecoder { // The timestamp of the frame is in samples per channel. uint32_t timestamp; - bool primary; + // The relative priority of the frame compared to other frames of the same + // payload and the same timeframe. A higher value means a lower priority. + // The highest priority is zero - negative values are not allowed. + int priority; std::unique_ptr frame; }; @@ -86,8 +86,7 @@ class AudioDecoder { // buffer. |timestamp| is the input timestamp, in samples, corresponding to // the start of the payload. virtual std::vector ParsePayload(rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary); + uint32_t timestamp); // Decodes |encode_len| bytes from |encoded| and writes the result in // |decoded|. The maximum bytes allowed to be written into |decoded| is @@ -177,4 +176,4 @@ class AudioDecoder { }; } // namespace webrtc -#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_INCLUDE_AUDIO_DECODER_H_ +#endif // WEBRTC_MODULES_AUDIO_CODING_CODECS_AUDIO_DECODER_H_ diff --git a/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc b/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc index f2fdb1f15c..fd285a75b2 100644 --- a/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc +++ b/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc @@ -19,10 +19,9 @@ void AudioDecoderPcmU::Reset() {} std::vector AudioDecoderPcmU::ParsePayload( rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) { + uint32_t timestamp) { return LegacyEncodedAudioFrame::SplitBySamples( - this, std::move(payload), timestamp, is_primary, 8 * num_channels_, 8); + this, std::move(payload), timestamp, 8 * num_channels_, 8); } int AudioDecoderPcmU::SampleRateHz() const { @@ -55,10 +54,9 @@ void AudioDecoderPcmA::Reset() {} std::vector AudioDecoderPcmA::ParsePayload( rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) { + uint32_t timestamp) { return LegacyEncodedAudioFrame::SplitBySamples( - this, std::move(payload), timestamp, is_primary, 8 * num_channels_, 8); + this, std::move(payload), timestamp, 8 * num_channels_, 8); } int AudioDecoderPcmA::SampleRateHz() const { diff --git a/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h b/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h index ed390217a3..483311b61b 100644 --- a/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h +++ b/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h @@ -24,8 +24,7 @@ class AudioDecoderPcmU final : public AudioDecoder { } void Reset() override; std::vector ParsePayload(rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) override; + uint32_t timestamp) override; int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; int SampleRateHz() const override; size_t Channels() const override; @@ -49,8 +48,7 @@ class AudioDecoderPcmA final : public AudioDecoder { } void Reset() override; std::vector ParsePayload(rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) override; + uint32_t timestamp) override; int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; int SampleRateHz() const override; size_t Channels() const override; diff --git a/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc b/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc index 93b24bdf2c..3a0f6ede66 100644 --- a/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc +++ b/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc @@ -50,10 +50,9 @@ void AudioDecoderG722::Reset() { std::vector AudioDecoderG722::ParsePayload( rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) { + uint32_t timestamp) { return LegacyEncodedAudioFrame::SplitBySamples(this, std::move(payload), - timestamp, is_primary, 8, 16); + timestamp, 8, 16); } int AudioDecoderG722::PacketDuration(const uint8_t* encoded, @@ -128,10 +127,9 @@ void AudioDecoderG722Stereo::Reset() { std::vector AudioDecoderG722Stereo::ParsePayload( rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) { - return LegacyEncodedAudioFrame::SplitBySamples( - this, std::move(payload), timestamp, is_primary, 2 * 8, 16); + uint32_t timestamp) { + return LegacyEncodedAudioFrame::SplitBySamples(this, std::move(payload), + timestamp, 2 * 8, 16); } // Split the stereo packet and place left and right channel after each other diff --git a/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h b/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h index ad39619d3c..9e7c5d9d05 100644 --- a/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h +++ b/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h @@ -25,8 +25,7 @@ class AudioDecoderG722 final : public AudioDecoder { bool HasDecodePlc() const override; void Reset() override; std::vector ParsePayload(rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) override; + uint32_t timestamp) override; int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; int SampleRateHz() const override; size_t Channels() const override; @@ -49,8 +48,7 @@ class AudioDecoderG722Stereo final : public AudioDecoder { ~AudioDecoderG722Stereo() override; void Reset() override; std::vector ParsePayload(rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) override; + uint32_t timestamp) override; int SampleRateHz() const override; size_t Channels() const override; diff --git a/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc b/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc index b4bd59911f..354f8194d8 100644 --- a/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc +++ b/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc @@ -10,6 +10,8 @@ #include "webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h" +#include + #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" #include "webrtc/modules/audio_coding/codecs/ilbc/ilbc.h" @@ -53,8 +55,7 @@ void AudioDecoderIlbc::Reset() { std::vector AudioDecoderIlbc::ParsePayload( rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) { + uint32_t timestamp) { std::vector results; size_t bytes_per_frame; int timestamps_per_frame; @@ -78,8 +79,8 @@ std::vector AudioDecoderIlbc::ParsePayload( RTC_DCHECK_EQ(0u, payload.size() % bytes_per_frame); if (payload.size() == bytes_per_frame) { std::unique_ptr frame( - new LegacyEncodedAudioFrame(this, std::move(payload), is_primary)); - results.emplace_back(timestamp, is_primary, std::move(frame)); + new LegacyEncodedAudioFrame(this, std::move(payload))); + results.emplace_back(timestamp, 0, std::move(frame)); } else { size_t byte_offset; uint32_t timestamp_offset; @@ -87,11 +88,9 @@ std::vector AudioDecoderIlbc::ParsePayload( byte_offset < payload.size(); byte_offset += bytes_per_frame, timestamp_offset += timestamps_per_frame) { - rtc::Buffer new_payload(payload.data() + byte_offset, bytes_per_frame); std::unique_ptr frame(new LegacyEncodedAudioFrame( - this, std::move(new_payload), is_primary)); - results.emplace_back(timestamp + timestamp_offset, is_primary, - std::move(frame)); + this, rtc::Buffer(payload.data() + byte_offset, bytes_per_frame))); + results.emplace_back(timestamp + timestamp_offset, 0, std::move(frame)); } } diff --git a/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h b/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h index fdc856e985..48edfede76 100644 --- a/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h +++ b/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h @@ -26,8 +26,7 @@ class AudioDecoderIlbc final : public AudioDecoder { size_t DecodePlc(size_t num_frames, int16_t* decoded) override; void Reset() override; std::vector ParsePayload(rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) override; + uint32_t timestamp) override; int SampleRateHz() const override; size_t Channels() const override; diff --git a/webrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc b/webrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc index a8b76a5bc9..2abdacae97 100644 --- a/webrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc +++ b/webrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc @@ -81,7 +81,7 @@ TEST_P(SplitIlbcTest, NumFrames) { }; const auto results = decoder.ParsePayload( - generate_payload(frame_length_bytes_ * num_frames_), 0, true); + generate_payload(frame_length_bytes_ * num_frames_), 0); EXPECT_EQ(num_frames_, results.size()); size_t frame_num = 0; @@ -123,7 +123,7 @@ TEST(IlbcTest, SplitTooLargePayload) { AudioDecoderIlbc decoder; constexpr size_t kPayloadLengthBytes = 950; const auto results = - decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0, true); + decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0); EXPECT_TRUE(results.empty()); } @@ -132,7 +132,7 @@ TEST(IlbcTest, SplitUnevenPayload) { AudioDecoderIlbc decoder; constexpr size_t kPayloadLengthBytes = 39; // Not an even number of frames. const auto results = - decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0, true); + decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0); EXPECT_TRUE(results.empty()); } diff --git a/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc index 5e6ff01f22..e0f1fafcaa 100644 --- a/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc +++ b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc @@ -17,37 +17,22 @@ namespace webrtc { LegacyEncodedAudioFrame::LegacyEncodedAudioFrame(AudioDecoder* decoder, - rtc::Buffer&& payload, - bool is_primary_payload) - : decoder_(decoder), - payload_(std::move(payload)), - is_primary_payload_(is_primary_payload) {} + rtc::Buffer&& payload) + : decoder_(decoder), payload_(std::move(payload)) {} LegacyEncodedAudioFrame::~LegacyEncodedAudioFrame() = default; size_t LegacyEncodedAudioFrame::Duration() const { - int ret; - if (is_primary_payload_) { - ret = decoder_->PacketDuration(payload_.data(), payload_.size()); - } else { - ret = decoder_->PacketDurationRedundant(payload_.data(), payload_.size()); - } + const int ret = decoder_->PacketDuration(payload_.data(), payload_.size()); return (ret < 0) ? 0 : static_cast(ret); } rtc::Optional LegacyEncodedAudioFrame::Decode(rtc::ArrayView decoded) const { AudioDecoder::SpeechType speech_type = AudioDecoder::kSpeech; - int ret; - if (is_primary_payload_) { - ret = decoder_->Decode( - payload_.data(), payload_.size(), decoder_->SampleRateHz(), - decoded.size() * sizeof(int16_t), decoded.data(), &speech_type); - } else { - ret = decoder_->DecodeRedundant( - payload_.data(), payload_.size(), decoder_->SampleRateHz(), - decoded.size() * sizeof(int16_t), decoded.data(), &speech_type); - } + const int ret = decoder_->Decode( + payload_.data(), payload_.size(), decoder_->SampleRateHz(), + decoded.size() * sizeof(int16_t), decoded.data(), &speech_type); if (ret < 0) return rtc::Optional(); @@ -59,7 +44,6 @@ std::vector LegacyEncodedAudioFrame::SplitBySamples( AudioDecoder* decoder, rtc::Buffer&& payload, uint32_t timestamp, - bool is_primary, size_t bytes_per_ms, uint32_t timestamps_per_ms) { RTC_DCHECK(payload.data()); @@ -70,8 +54,8 @@ std::vector LegacyEncodedAudioFrame::SplitBySamples( const size_t min_chunk_size = bytes_per_ms * 20; if (min_chunk_size >= payload.size()) { std::unique_ptr frame( - new LegacyEncodedAudioFrame(decoder, std::move(payload), is_primary)); - results.emplace_back(timestamp, is_primary, std::move(frame)); + new LegacyEncodedAudioFrame(decoder, std::move(payload))); + results.emplace_back(timestamp, 0, std::move(frame)); } else { // Reduce the split size by half as long as |split_size_bytes| is at least // twice the minimum chunk size (so that the resulting size is at least as @@ -92,10 +76,8 @@ std::vector LegacyEncodedAudioFrame::SplitBySamples( std::min(split_size_bytes, payload.size() - byte_offset); rtc::Buffer new_payload(payload.data() + byte_offset, split_size_bytes); std::unique_ptr frame( - new LegacyEncodedAudioFrame(decoder, std::move(new_payload), - is_primary)); - results.emplace_back(timestamp + timestamp_offset, is_primary, - std::move(frame)); + new LegacyEncodedAudioFrame(decoder, std::move(new_payload))); + results.emplace_back(timestamp + timestamp_offset, 0, std::move(frame)); } } diff --git a/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h index 1c9c3f542a..466c8b34be 100644 --- a/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h +++ b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h @@ -20,16 +20,13 @@ namespace webrtc { class LegacyEncodedAudioFrame final : public AudioDecoder::EncodedAudioFrame { public: - LegacyEncodedAudioFrame(AudioDecoder* decoder, - rtc::Buffer&& payload, - bool is_primary_payload); + LegacyEncodedAudioFrame(AudioDecoder* decoder, rtc::Buffer&& payload); ~LegacyEncodedAudioFrame() override; static std::vector SplitBySamples( AudioDecoder* decoder, rtc::Buffer&& payload, uint32_t timestamp, - bool is_primary, size_t bytes_per_ms, uint32_t timestamps_per_ms); @@ -44,7 +41,6 @@ class LegacyEncodedAudioFrame final : public AudioDecoder::EncodedAudioFrame { private: AudioDecoder* const decoder_; const rtc::Buffer payload_; - const bool is_primary_payload_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc index 2a4d9ed9f2..9a075639af 100644 --- a/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc +++ b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc @@ -123,7 +123,7 @@ TEST_P(SplitBySamplesTest, PayloadSizes) { const auto results = LegacyEncodedAudioFrame::SplitBySamples( nullptr, generate_payload(expected_split.payload_size_ms * bytes_per_ms_), - kBaseTimestamp, true, bytes_per_ms_, samples_per_ms_); + kBaseTimestamp, bytes_per_ms_, samples_per_ms_); EXPECT_EQ(expected_split.num_frames, results.size()); uint32_t expected_timestamp = kBaseTimestamp; diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc b/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc index 42abd0a0d6..b6d8a3a1db 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc +++ b/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc @@ -10,10 +10,60 @@ #include "webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h" +#include + #include "webrtc/base/checks.h" namespace webrtc { +namespace { +class OpusFrame : public AudioDecoder::EncodedAudioFrame { + public: + OpusFrame(AudioDecoderOpus* decoder, + rtc::Buffer&& payload, + bool is_primary_payload) + : decoder_(decoder), + payload_(std::move(payload)), + is_primary_payload_(is_primary_payload) {} + + size_t Duration() const override { + int ret; + if (is_primary_payload_) { + ret = decoder_->PacketDuration(payload_.data(), payload_.size()); + } else { + ret = decoder_->PacketDurationRedundant(payload_.data(), payload_.size()); + } + return (ret < 0) ? 0 : static_cast(ret); + } + + rtc::Optional Decode( + rtc::ArrayView decoded) const override { + AudioDecoder::SpeechType speech_type = AudioDecoder::kSpeech; + int ret; + if (is_primary_payload_) { + ret = decoder_->Decode( + payload_.data(), payload_.size(), decoder_->SampleRateHz(), + decoded.size() * sizeof(int16_t), decoded.data(), &speech_type); + } else { + ret = decoder_->DecodeRedundant( + payload_.data(), payload_.size(), decoder_->SampleRateHz(), + decoded.size() * sizeof(int16_t), decoded.data(), &speech_type); + } + + if (ret < 0) + return rtc::Optional(); + + return rtc::Optional({static_cast(ret), speech_type}); + } + + private: + AudioDecoderOpus* const decoder_; + const rtc::Buffer payload_; + const bool is_primary_payload_; +}; + +} // namespace + AudioDecoderOpus::AudioDecoderOpus(size_t num_channels) : channels_(num_channels) { RTC_DCHECK(num_channels == 1 || num_channels == 2); @@ -25,6 +75,26 @@ AudioDecoderOpus::~AudioDecoderOpus() { WebRtcOpus_DecoderFree(dec_state_); } +std::vector AudioDecoderOpus::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp) { + std::vector results; + + if (PacketHasFec(payload.data(), payload.size())) { + const int duration = + PacketDurationRedundant(payload.data(), payload.size()); + RTC_DCHECK_GE(duration, 0); + rtc::Buffer payload_copy(payload.data(), payload.size()); + std::unique_ptr fec_frame( + new OpusFrame(this, std::move(payload_copy), false)); + results.emplace_back(timestamp - duration, 1, std::move(fec_frame)); + } + std::unique_ptr frame( + new OpusFrame(this, std::move(payload), true)); + results.emplace_back(timestamp, 0, std::move(frame)); + return results; +} + int AudioDecoderOpus::DecodeInternal(const uint8_t* encoded, size_t encoded_len, int sample_rate_hz, diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h b/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h index c22204156b..a0fb34c442 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h +++ b/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h @@ -22,6 +22,8 @@ class AudioDecoderOpus final : public AudioDecoder { explicit AudioDecoderOpus(size_t num_channels); ~AudioDecoderOpus() override; + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) override; void Reset() override; int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; int PacketDurationRedundant(const uint8_t* encoded, diff --git a/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc b/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc index e600d2d56c..43d2dacf9b 100644 --- a/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc +++ b/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc @@ -47,12 +47,11 @@ int AudioDecoderPcm16B::DecodeInternal(const uint8_t* encoded, std::vector AudioDecoderPcm16B::ParsePayload( rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) { + uint32_t timestamp) { const int samples_per_ms = rtc::CheckedDivExact(sample_rate_hz_, 1000); return LegacyEncodedAudioFrame::SplitBySamples( - this, std::move(payload), timestamp, is_primary, - samples_per_ms * 2 * num_channels_, samples_per_ms); + this, std::move(payload), timestamp, samples_per_ms * 2 * num_channels_, + samples_per_ms); } int AudioDecoderPcm16B::PacketDuration(const uint8_t* encoded, diff --git a/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h b/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h index 1d2da4a718..577eecc535 100644 --- a/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h +++ b/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h @@ -21,8 +21,7 @@ class AudioDecoderPcm16B final : public AudioDecoder { AudioDecoderPcm16B(int sample_rate_hz, size_t num_channels); void Reset() override; std::vector ParsePayload(rtc::Buffer&& payload, - uint32_t timestamp, - bool is_primary) override; + uint32_t timestamp) override; int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; int SampleRateHz() const override; size_t Channels() const override; diff --git a/webrtc/modules/audio_coding/neteq/include/neteq.h b/webrtc/modules/audio_coding/neteq/include/neteq.h index 98bb37cf1f..c07143c062 100644 --- a/webrtc/modules/audio_coding/neteq/include/neteq.h +++ b/webrtc/modules/audio_coding/neteq/include/neteq.h @@ -126,7 +126,6 @@ class NetEq { kStereoNotSupported, kSampleUnderrun, kDecodedTooMuch, - kFrameSplitError, kRedundancySplitError, kPacketBufferCorruption }; diff --git a/webrtc/modules/audio_coding/neteq/mock/mock_payload_splitter.h b/webrtc/modules/audio_coding/neteq/mock/mock_payload_splitter.h deleted file mode 100644 index cf0613403b..0000000000 --- a/webrtc/modules/audio_coding/neteq/mock/mock_payload_splitter.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_PAYLOAD_SPLITTER_H_ -#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_PAYLOAD_SPLITTER_H_ - -#include "webrtc/modules/audio_coding/neteq/payload_splitter.h" - -#include "testing/gmock/include/gmock/gmock.h" - -namespace webrtc { - -class MockPayloadSplitter : public PayloadSplitter { - public: - MOCK_METHOD1(SplitRed, - int(PacketList* packet_list)); - MOCK_METHOD2(SplitFec, - int(PacketList* packet_list, DecoderDatabase* decoder_database)); - MOCK_METHOD2(CheckRedPayloads, - int(PacketList* packet_list, const DecoderDatabase& decoder_database)); -}; - -} // namespace webrtc -#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_PAYLOAD_SPLITTER_H_ diff --git a/webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h b/webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h new file mode 100644 index 0000000000..e1f4e04cfd --- /dev/null +++ b/webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_RED_PAYLOAD_SPLITTER_H_ +#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_RED_PAYLOAD_SPLITTER_H_ + +#include "webrtc/modules/audio_coding/neteq/red_payload_splitter.h" + +#include "testing/gmock/include/gmock/gmock.h" + +namespace webrtc { + +class MockRedPayloadSplitter : public RedPayloadSplitter { + public: + MOCK_METHOD1(SplitRed, bool(PacketList* packet_list)); + MOCK_METHOD2(CheckRedPayloads, + int(PacketList* packet_list, + const DecoderDatabase& decoder_database)); +}; + +} // namespace webrtc +#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_RED_PAYLOAD_SPLITTER_H_ diff --git a/webrtc/modules/audio_coding/neteq/neteq.gypi b/webrtc/modules/audio_coding/neteq/neteq.gypi index 630076ec8d..90aaaed9c2 100644 --- a/webrtc/modules/audio_coding/neteq/neteq.gypi +++ b/webrtc/modules/audio_coding/neteq/neteq.gypi @@ -113,8 +113,8 @@ 'packet.h', 'packet_buffer.cc', 'packet_buffer.h', - 'payload_splitter.cc', - 'payload_splitter.h', + 'red_payload_splitter.cc', + 'red_payload_splitter.h', 'post_decode_vad.cc', 'post_decode_vad.h', 'preemptive_expand.cc', diff --git a/webrtc/modules/audio_coding/neteq/neteq_impl.cc b/webrtc/modules/audio_coding/neteq/neteq_impl.cc index 221b07c969..6b2fa4c9ec 100644 --- a/webrtc/modules/audio_coding/neteq/neteq_impl.cc +++ b/webrtc/modules/audio_coding/neteq/neteq_impl.cc @@ -41,7 +41,7 @@ #include "webrtc/modules/audio_coding/neteq/normal.h" #include "webrtc/modules/audio_coding/neteq/packet_buffer.h" #include "webrtc/modules/audio_coding/neteq/packet.h" -#include "webrtc/modules/audio_coding/neteq/payload_splitter.h" +#include "webrtc/modules/audio_coding/neteq/red_payload_splitter.h" #include "webrtc/modules/audio_coding/neteq/post_decode_vad.h" #include "webrtc/modules/audio_coding/neteq/preemptive_expand.h" #include "webrtc/modules/audio_coding/neteq/sync_buffer.h" @@ -65,7 +65,7 @@ NetEqImpl::Dependencies::Dependencies( dtmf_tone_generator(new DtmfToneGenerator), packet_buffer( new PacketBuffer(config.max_packets_in_buffer, tick_timer.get())), - payload_splitter(new PayloadSplitter), + red_payload_splitter(new RedPayloadSplitter), timestamp_scaler(new TimestampScaler(*decoder_database)), accelerate_factory(new AccelerateFactory), expand_factory(new ExpandFactory), @@ -84,7 +84,7 @@ NetEqImpl::NetEqImpl(const NetEq::Config& config, dtmf_buffer_(std::move(deps.dtmf_buffer)), dtmf_tone_generator_(std::move(deps.dtmf_tone_generator)), packet_buffer_(std::move(deps.packet_buffer)), - payload_splitter_(std::move(deps.payload_splitter)), + red_payload_splitter_(std::move(deps.red_payload_splitter)), timestamp_scaler_(std::move(deps.timestamp_scaler)), vad_(new PostDecodeVad()), expand_factory_(std::move(deps.expand_factory)), @@ -567,7 +567,6 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header, packet->header.ssrc = rtp_header.header.ssrc; packet->header.numCSRCs = 0; packet->payload.SetData(payload.data(), payload.size()); - packet->primary = true; // Waiting time will be set upon inserting the packet in the buffer. RTC_DCHECK(!packet->waiting_time); // Insert packet in a packet list. @@ -609,13 +608,13 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header, // Check for RED payload type, and separate payloads into several packets. if (decoder_database_->IsRed(main_header.payloadType)) { - if (payload_splitter_->SplitRed(&packet_list) != PayloadSplitter::kOK) { + if (!red_payload_splitter_->SplitRed(&packet_list)) { PacketBuffer::DeleteAllPackets(&packet_list); return kRedundancySplitError; } // Only accept a few RED payloads of the same type as the main data, // DTMF events and CNG. - payload_splitter_->CheckRedPayloads(&packet_list, *decoder_database_); + red_payload_splitter_->CheckRedPayloads(&packet_list, *decoder_database_); // Update the stored main payload header since the main payload has now // changed. memcpy(&main_header, &packet_list.front()->header, sizeof(main_header)); @@ -658,18 +657,6 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header, } } - // Check for FEC in packets, and separate payloads into several packets. - int ret = payload_splitter_->SplitFec(&packet_list, decoder_database_.get()); - if (ret != PayloadSplitter::kOK) { - PacketBuffer::DeleteAllPackets(&packet_list); - switch (ret) { - case PayloadSplitter::kUnknownPayloadType: - return kUnknownRtpPayloadType; - default: - return kOtherError; - } - } - // Update bandwidth estimate, if the packet is not comfort noise. if (!packet_list.empty() && !decoder_database_->IsComfortNoise(main_header.payloadType)) { @@ -702,9 +689,9 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header, } else { std::vector results = info->GetDecoder()->ParsePayload(std::move(packet->payload), - packet->header.timestamp, - packet->primary); + packet->header.timestamp); const RTPHeader& original_header = packet->header; + const Packet::Priority original_priority = packet->priority; for (auto& result : results) { RTC_DCHECK(result.frame); // Reuse the packet if possible. @@ -713,8 +700,9 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header, packet->header = original_header; } packet->header.timestamp = result.timestamp; - // TODO(ossu): Move from primary to some sort of priority level. - packet->primary = result.primary; + RTC_DCHECK_GE(result.priority, 0); + packet->priority.codec_level = result.priority; + packet->priority.red_level = original_priority.red_level; packet->frame = std::move(result.frame); parsed_packet_list.push_back(packet.release()); } @@ -734,7 +722,7 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header, // Insert packets in buffer. const size_t buffer_length_before_insert = packet_buffer_->NumPacketsInBuffer(); - ret = packet_buffer_->InsertPacketList( + const int ret = packet_buffer_->InsertPacketList( &parsed_packet_list, *decoder_database_, ¤t_rtp_payload_type_, ¤t_cng_rtp_payload_type_); if (ret == PacketBuffer::kFlushed) { @@ -1979,9 +1967,8 @@ int NetEqImpl::ExtractPackets(size_t required_samples, size_t packet_duration = 0; if (packet->frame) { packet_duration = packet->frame->Duration(); - // TODO(ossu): Is this the correct way to track samples decoded from a - // redundant packet? - if (packet_duration > 0 && !packet->primary) { + // TODO(ossu): Is this the correct way to track Opus FEC packets? + if (packet->priority.codec_level > 0) { stats_.SecondaryDecodedSamples(rtc::checked_cast(packet_duration)); } } else if (!decoder_database_->IsComfortNoise(packet->header.payloadType)) { diff --git a/webrtc/modules/audio_coding/neteq/neteq_impl.h b/webrtc/modules/audio_coding/neteq/neteq_impl.h index dd35301d7a..dda3d64494 100644 --- a/webrtc/modules/audio_coding/neteq/neteq_impl.h +++ b/webrtc/modules/audio_coding/neteq/neteq_impl.h @@ -46,7 +46,7 @@ class Merge; class NackTracker; class Normal; class PacketBuffer; -class PayloadSplitter; +class RedPayloadSplitter; class PostDecodeVad; class PreemptiveExpand; class RandomVector; @@ -86,7 +86,7 @@ class NetEqImpl : public webrtc::NetEq { std::unique_ptr dtmf_buffer; std::unique_ptr dtmf_tone_generator; std::unique_ptr packet_buffer; - std::unique_ptr payload_splitter; + std::unique_ptr red_payload_splitter; std::unique_ptr timestamp_scaler; std::unique_ptr accelerate_factory; std::unique_ptr expand_factory; @@ -353,7 +353,7 @@ class NetEqImpl : public webrtc::NetEq { const std::unique_ptr dtmf_tone_generator_ GUARDED_BY(crit_sect_); const std::unique_ptr packet_buffer_ GUARDED_BY(crit_sect_); - const std::unique_ptr payload_splitter_ + const std::unique_ptr red_payload_splitter_ GUARDED_BY(crit_sect_); const std::unique_ptr timestamp_scaler_ GUARDED_BY(crit_sect_); diff --git a/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc b/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc index 5b1015d576..e3029dba8d 100644 --- a/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc +++ b/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc @@ -28,7 +28,7 @@ #include "webrtc/modules/audio_coding/neteq/mock/mock_dtmf_buffer.h" #include "webrtc/modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h" #include "webrtc/modules/audio_coding/neteq/mock/mock_packet_buffer.h" -#include "webrtc/modules/audio_coding/neteq/mock/mock_payload_splitter.h" +#include "webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h" #include "webrtc/modules/audio_coding/neteq/preemptive_expand.h" #include "webrtc/modules/audio_coding/neteq/sync_buffer.h" #include "webrtc/modules/audio_coding/neteq/timestamp_scaler.h" @@ -124,11 +124,11 @@ class NetEqImplTest : public ::testing::Test { packet_buffer_ = deps.packet_buffer.get(); if (use_mock_payload_splitter_) { - std::unique_ptr mock(new MockPayloadSplitter); + std::unique_ptr mock(new MockRedPayloadSplitter); mock_payload_splitter_ = mock.get(); - deps.payload_splitter = std::move(mock); + deps.red_payload_splitter = std::move(mock); } - payload_splitter_ = deps.payload_splitter.get(); + red_payload_splitter_ = deps.red_payload_splitter.get(); deps.timestamp_scaler = std::unique_ptr( new TimestampScaler(*deps.decoder_database.get())); @@ -197,8 +197,8 @@ class NetEqImplTest : public ::testing::Test { MockPacketBuffer* mock_packet_buffer_ = nullptr; PacketBuffer* packet_buffer_ = nullptr; bool use_mock_packet_buffer_ = true; - MockPayloadSplitter* mock_payload_splitter_ = nullptr; - PayloadSplitter* payload_splitter_ = nullptr; + MockRedPayloadSplitter* mock_payload_splitter_ = nullptr; + RedPayloadSplitter* red_payload_splitter_ = nullptr; bool use_mock_payload_splitter_ = true; }; @@ -332,11 +332,6 @@ TEST_F(NetEqImplTest, InsertPacket) { .WillOnce(Return(0)); } - // Expectations for payload splitter. - EXPECT_CALL(*mock_payload_splitter_, SplitFec(_, _)) - .Times(2) - .WillRepeatedly(Return(PayloadSplitter::kOK)); - // Insert first packet. neteq_->InsertPacket(rtp_header, payload, kFirstReceiveTime); diff --git a/webrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc b/webrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc index 15c60a83b5..70424fde79 100644 --- a/webrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc +++ b/webrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc @@ -39,18 +39,57 @@ class MockAudioDecoder final : public AudioDecoder { MOCK_METHOD0(Reset, void()); - int PacketDuration(const uint8_t* encoded, - size_t encoded_len) const /* override */ { - return kPacketDuration; + class MockFrame : public AudioDecoder::EncodedAudioFrame { + public: + MockFrame(size_t num_channels) : num_channels_(num_channels) {} + + size_t Duration() const override { return kPacketDuration; } + + rtc::Optional Decode( + rtc::ArrayView decoded) const override { + const size_t output_size = + sizeof(int16_t) * kPacketDuration * num_channels_; + if (decoded.size() >= output_size) { + memset(decoded.data(), 0, + sizeof(int16_t) * kPacketDuration * num_channels_); + return rtc::Optional( + {kPacketDuration * num_channels_, kSpeech}); + } else { + ADD_FAILURE() << "Expected decoded.size() to be >= output_size (" + << decoded.size() << " vs. " << output_size << ")"; + return rtc::Optional(); + } + } + + private: + const size_t num_channels_; + }; + + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) /* override */ { + std::vector results; + if (fec_enabled_) { + std::unique_ptr fec_frame(new MockFrame(num_channels_)); + results.emplace_back(timestamp - kPacketDuration, 1, + std::move(fec_frame)); + } + + std::unique_ptr frame(new MockFrame(num_channels_)); + results.emplace_back(timestamp, 0, std::move(frame)); + return results; } - int PacketDurationRedundant(const uint8_t* encoded, - size_t encoded_len) const /* override */ { + int PacketDuration(const uint8_t* encoded, size_t encoded_len) const + /* override */ { + ADD_FAILURE() << "Since going through ParsePayload, PacketDuration should " + "never get called."; return kPacketDuration; } bool PacketHasFec( const uint8_t* encoded, size_t encoded_len) const /* override */ { + ADD_FAILURE() << "Since going through ParsePayload, PacketHasFec should " + "never get called."; return fec_enabled_; } @@ -63,24 +102,14 @@ class MockAudioDecoder final : public AudioDecoder { bool fec_enabled() const { return fec_enabled_; } protected: - // Override the following methods such that no actual payload is needed. int DecodeInternal(const uint8_t* encoded, size_t encoded_len, - int /*sample_rate_hz*/, + int sample_rate_hz, int16_t* decoded, SpeechType* speech_type) /* override */ { - *speech_type = kSpeech; - memset(decoded, 0, sizeof(int16_t) * kPacketDuration * Channels()); - return kPacketDuration * Channels(); - } - - int DecodeRedundantInternal(const uint8_t* encoded, - size_t encoded_len, - int sample_rate_hz, - int16_t* decoded, - SpeechType* speech_type) /* override */ { - return DecodeInternal(encoded, encoded_len, sample_rate_hz, decoded, - speech_type); + ADD_FAILURE() << "Since going through ParsePayload, DecodeInternal should " + "never get called."; + return -1; } private: diff --git a/webrtc/modules/audio_coding/neteq/packet.h b/webrtc/modules/audio_coding/neteq/packet.h index 4e17a88dcd..cf590caf4b 100644 --- a/webrtc/modules/audio_coding/neteq/packet.h +++ b/webrtc/modules/audio_coding/neteq/packet.h @@ -24,10 +24,52 @@ namespace webrtc { // Struct for holding RTP packets. struct Packet { + struct Priority { + Priority() : codec_level(0), red_level(0) {} + Priority(int codec_level, int red_level) + : codec_level(codec_level), red_level(red_level) { + CheckInvariant(); + } + + int codec_level; + int red_level; + + // Priorities are sorted low-to-high, first on the level the codec + // prioritizes it, then on the level of RED packet it is; i.e. if it is a + // primary or secondary payload of a RED packet. For example: with Opus, an + // Fec packet (which the decoder prioritizes lower than a regular packet) + // will not be used if there is _any_ RED payload for the same + // timeframe. The highest priority packet will have levels {0, 0}. Negative + // priorities are not allowed. + bool operator<(const Priority& b) const { + CheckInvariant(); + b.CheckInvariant(); + if (codec_level == b.codec_level) + return red_level < b.red_level; + + return codec_level < b.codec_level; + } + bool operator==(const Priority& b) const { + CheckInvariant(); + b.CheckInvariant(); + return codec_level == b.codec_level && red_level == b.red_level; + } + bool operator!=(const Priority& b) const { return !(*this == b); } + bool operator>(const Priority& b) const { return b < *this; } + bool operator<=(const Priority& b) const { return !(b > *this); } + bool operator>=(const Priority& b) const { return !(b < *this); } + + private: + void CheckInvariant() const { + RTC_DCHECK_GE(codec_level, 0); + RTC_DCHECK_GE(red_level, 0); + } + }; + RTPHeader header; // Datagram excluding RTP header and header extension. rtc::Buffer payload; - bool primary = true; // Primary, i.e., not redundant payload. + Priority priority; std::unique_ptr waiting_time; std::unique_ptr frame; @@ -41,17 +83,16 @@ struct Packet { // primary payload is considered "smaller" than a secondary. bool operator==(const Packet& rhs) const { return (this->header.timestamp == rhs.header.timestamp && - this->header.sequenceNumber == rhs.header.sequenceNumber && - this->primary == rhs.primary); + this->header.sequenceNumber == rhs.header.sequenceNumber && + this->priority == rhs.priority); } bool operator!=(const Packet& rhs) const { return !operator==(rhs); } bool operator<(const Packet& rhs) const { if (this->header.timestamp == rhs.header.timestamp) { if (this->header.sequenceNumber == rhs.header.sequenceNumber) { - // Timestamp and sequence numbers are identical - deem the left - // hand side to be "smaller" (i.e., "earlier") if it is primary, and - // right hand side is not. - return (this->primary && !rhs.primary); + // Timestamp and sequence numbers are identical - deem the left hand + // side to be "smaller" (i.e., "earlier") if it has higher priority. + return this->priority < rhs.priority; } return (static_cast(rhs.header.sequenceNumber - this->header.sequenceNumber) < 0xFFFF / 2); diff --git a/webrtc/modules/audio_coding/neteq/packet_buffer.cc b/webrtc/modules/audio_coding/neteq/packet_buffer.cc index c5b23dce06..eeb1d272b9 100644 --- a/webrtc/modules/audio_coding/neteq/packet_buffer.cc +++ b/webrtc/modules/audio_coding/neteq/packet_buffer.cc @@ -76,6 +76,9 @@ int PacketBuffer::InsertPacket(Packet* packet) { return kInvalidPacket; } + RTC_DCHECK_GE(packet->priority.codec_level, 0); + RTC_DCHECK_GE(packet->priority.red_level, 0); + int return_val = kOK; packet->waiting_time = tick_timer_->GetNewStopwatch(); @@ -262,7 +265,7 @@ int PacketBuffer::DiscardAllOldPackets(uint32_t timestamp_limit) { void PacketBuffer::DiscardPacketsWithPayloadType(uint8_t payload_type) { for (auto it = buffer_.begin(); it != buffer_.end(); /* */) { - Packet *packet = *it; + Packet* packet = *it; if (packet->header.payloadType == payload_type) { delete packet; it = buffer_.erase(it); @@ -281,7 +284,9 @@ size_t PacketBuffer::NumSamplesInBuffer(size_t last_decoded_length) const { size_t last_duration = last_decoded_length; for (Packet* packet : buffer_) { if (packet->frame) { - if (!packet->primary) { + // TODO(hlundin): Verify that it's fine to count all packets and remove + // this check. + if (packet->priority != Packet::Priority(0, 0)) { continue; } size_t duration = packet->frame->Duration(); diff --git a/webrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc b/webrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc index 1b86d8326b..b39169f909 100644 --- a/webrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc +++ b/webrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc @@ -60,7 +60,6 @@ Packet* PacketGenerator::NextPacket(int payload_size_bytes) { packet->header.ssrc = 0x12345678; packet->header.numCSRCs = 0; packet->header.paddingLength = 0; - packet->primary = true; packet->payload.SetSize(payload_size_bytes); ++seq_no_; ts_ += frame_size_; @@ -284,7 +283,7 @@ TEST(PacketBuffer, ExtractOrderRedundancy) { packet_facts[i].payload_type, kFrameSize); Packet* packet = gen.NextPacket(kPayloadLength); - packet->primary = packet_facts[i].primary; + packet->priority.red_level = packet_facts[i].primary ? 0 : 1; EXPECT_EQ(PacketBuffer::kOK, buffer.InsertPacket(packet)); if (packet_facts[i].extract_order >= 0) { expect_order[packet_facts[i].extract_order] = packet; @@ -512,8 +511,8 @@ TEST(PacketBuffer, Failures) { // The function should return true if the first packet "goes before" the second. TEST(PacketBuffer, ComparePackets) { PacketGenerator gen(0, 0, 0, 10); - Packet* a = gen.NextPacket(10); // SN = 0, TS = 0. - Packet* b = gen.NextPacket(10); // SN = 1, TS = 10. + std::unique_ptr a(gen.NextPacket(10)); // SN = 0, TS = 0. + std::unique_ptr b(gen.NextPacket(10)); // SN = 1, TS = 10. EXPECT_FALSE(*a == *b); EXPECT_TRUE(*a != *b); EXPECT_TRUE(*a < *b); @@ -556,10 +555,11 @@ TEST(PacketBuffer, ComparePackets) { EXPECT_TRUE(*a <= *b); EXPECT_FALSE(*a >= *b); - // Test equal timestamps and sequence numbers, but only 'b' is primary. + // Test equal timestamps and sequence numbers, but differing priorities. a->header.sequenceNumber = b->header.sequenceNumber; - a->primary = false; - b->primary = true; + a->priority = {1, 0}; + b->priority = {0, 0}; + // a after b EXPECT_FALSE(*a == *b); EXPECT_TRUE(*a != *b); EXPECT_FALSE(*a < *b); @@ -567,8 +567,53 @@ TEST(PacketBuffer, ComparePackets) { EXPECT_FALSE(*a <= *b); EXPECT_TRUE(*a >= *b); - delete a; - delete b; + std::unique_ptr c(gen.NextPacket(0)); // SN = 2, TS = 20. + std::unique_ptr d(gen.NextPacket(0)); // SN = 3, TS = 20. + c->header.timestamp = b->header.timestamp; + d->header.timestamp = b->header.timestamp; + c->header.sequenceNumber = b->header.sequenceNumber; + d->header.sequenceNumber = b->header.sequenceNumber; + c->priority = {1, 1}; + d->priority = {0, 1}; + // c after d + EXPECT_FALSE(*c == *d); + EXPECT_TRUE(*c != *d); + EXPECT_FALSE(*c < *d); + EXPECT_TRUE(*c > *d); + EXPECT_FALSE(*c <= *d); + EXPECT_TRUE(*c >= *d); + + // c after a + EXPECT_FALSE(*c == *a); + EXPECT_TRUE(*c != *a); + EXPECT_FALSE(*c < *a); + EXPECT_TRUE(*c > *a); + EXPECT_FALSE(*c <= *a); + EXPECT_TRUE(*c >= *a); + + // c after b + EXPECT_FALSE(*c == *b); + EXPECT_TRUE(*c != *b); + EXPECT_FALSE(*c < *b); + EXPECT_TRUE(*c > *b); + EXPECT_FALSE(*c <= *b); + EXPECT_TRUE(*c >= *b); + + // a after d + EXPECT_FALSE(*a == *d); + EXPECT_TRUE(*a != *d); + EXPECT_FALSE(*a < *d); + EXPECT_TRUE(*a > *d); + EXPECT_FALSE(*a <= *d); + EXPECT_TRUE(*a >= *d); + + // d after b + EXPECT_FALSE(*d == *b); + EXPECT_TRUE(*d != *b); + EXPECT_FALSE(*d < *b); + EXPECT_TRUE(*d > *b); + EXPECT_FALSE(*d <= *b); + EXPECT_TRUE(*d >= *b); } // Test the DeleteFirstPacket DeleteAllPackets methods. diff --git a/webrtc/modules/audio_coding/neteq/payload_splitter.cc b/webrtc/modules/audio_coding/neteq/red_payload_splitter.cc similarity index 54% rename from webrtc/modules/audio_coding/neteq/payload_splitter.cc rename to webrtc/modules/audio_coding/neteq/red_payload_splitter.cc index 28e561a65f..a2f109832a 100644 --- a/webrtc/modules/audio_coding/neteq/payload_splitter.cc +++ b/webrtc/modules/audio_coding/neteq/red_payload_splitter.cc @@ -8,12 +8,14 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "webrtc/modules/audio_coding/neteq/payload_splitter.h" +#include "webrtc/modules/audio_coding/neteq/red_payload_splitter.h" #include +#include #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" +#include "webrtc/base/safe_conversions.h" #include "webrtc/modules/audio_coding/neteq/decoder_database.h" namespace webrtc { @@ -25,8 +27,11 @@ namespace webrtc { // is replaced by the new ones in |new_packets|, so that |packet_list| becomes: // {A1, A2, ..., B, C, ...}. The method then continues with B, and C, until all // the original packets have been replaced by their split payloads. -int PayloadSplitter::SplitRed(PacketList* packet_list) { - int ret = kOK; +bool RedPayloadSplitter::SplitRed(PacketList* packet_list) { + // Too many RED blocks indicates that something is wrong. Clamp it at some + // reasonable value. + const size_t kMaxRedBlocks = 32; + bool ret = true; PacketList::iterator it = packet_list->begin(); while (it != packet_list->end()) { const Packet* red_packet = (*it); @@ -50,7 +55,6 @@ int PayloadSplitter::SplitRed(PacketList* packet_list) { uint8_t payload_type; uint32_t timestamp; size_t payload_length; - bool primary; }; std::vector new_headers; @@ -67,17 +71,15 @@ int PayloadSplitter::SplitRed(PacketList* packet_list) { ++sum_length; // Account for RED header size of 1 byte. new_header.timestamp = red_packet->header.timestamp; new_header.payload_length = red_packet->payload.size() - sum_length; - new_header.primary = true; // Last block is always primary. payload_ptr += 1; // Advance to first payload byte. } else { // Bits 8 through 21 are timestamp offset. - int timestamp_offset = (payload_ptr[1] << 6) + - ((payload_ptr[2] & 0xFC) >> 2); + int timestamp_offset = + (payload_ptr[1] << 6) + ((payload_ptr[2] & 0xFC) >> 2); new_header.timestamp = red_packet->header.timestamp - timestamp_offset; // Bits 22 through 31 are payload length. new_header.payload_length = ((payload_ptr[2] & 0x03) << 8) + payload_ptr[3]; - new_header.primary = false; payload_ptr += 4; // Advance to next RED header. } sum_length += new_header.payload_length; @@ -86,33 +88,41 @@ int PayloadSplitter::SplitRed(PacketList* packet_list) { new_headers.push_back(new_header); } - // Populate the new packets with payload data. - // |payload_ptr| now points at the first payload byte. - PacketList new_packets; // An empty list to store the split packets in. - for (const auto& new_header : new_headers) { - size_t payload_length = new_header.payload_length; - if (payload_ptr + payload_length > - red_packet->payload.data() + red_packet->payload.size()) { - // The block lengths in the RED headers do not match the overall packet - // length. Something is corrupt. Discard this and the remaining - // payloads from this packet. - LOG(LS_WARNING) << "SplitRed length mismatch"; - ret = kRedLengthMismatch; - break; + if (new_headers.size() <= kMaxRedBlocks) { + // Populate the new packets with payload data. + // |payload_ptr| now points at the first payload byte. + PacketList new_packets; // An empty list to store the split packets in. + for (size_t i = 0; i != new_headers.size(); ++i) { + const auto& new_header = new_headers[i]; + size_t payload_length = new_header.payload_length; + if (payload_ptr + payload_length > + red_packet->payload.data() + red_packet->payload.size()) { + // The block lengths in the RED headers do not match the overall + // packet length. Something is corrupt. Discard this and the remaining + // payloads from this packet. + LOG(LS_WARNING) << "SplitRed length mismatch"; + ret = false; + break; + } + + Packet* new_packet = new Packet; + new_packet->header = red_packet->header; + new_packet->header.timestamp = new_header.timestamp; + new_packet->header.payloadType = new_header.payload_type; + new_packet->priority.red_level = + rtc::checked_cast((new_headers.size() - 1) - i); + new_packet->payload.SetData(payload_ptr, payload_length); + new_packets.push_front(new_packet); + payload_ptr += payload_length; } - Packet* new_packet = new Packet; - new_packet->header = red_packet->header; - new_packet->header.timestamp = new_header.timestamp; - new_packet->header.payloadType = new_header.payload_type; - new_packet->primary = new_header.primary; - new_packet->payload.SetData(payload_ptr, payload_length); - new_packets.push_front(new_packet); - payload_ptr += payload_length; + // Insert new packets into original list, before the element pointed to by + // iterator |it|. + packet_list->splice(it, new_packets, new_packets.begin(), + new_packets.end()); + } else { + LOG(LS_WARNING) << "SplitRed too many blocks: " << new_headers.size(); + ret = false; } - // Insert new packets into original list, before the element pointed to by - // iterator |it|. - packet_list->splice(it, new_packets, new_packets.begin(), - new_packets.end()); // Delete old packet payload. delete (*it); // Remove |it| from the packet list. This operation effectively moves the @@ -123,66 +133,9 @@ int PayloadSplitter::SplitRed(PacketList* packet_list) { return ret; } -int PayloadSplitter::SplitFec(PacketList* packet_list, - DecoderDatabase* decoder_database) { - PacketList::iterator it = packet_list->begin(); - // Iterate through all packets in |packet_list|. - while (it != packet_list->end()) { - Packet* packet = (*it); // Just to make the notation more intuitive. - // Get codec type for this payload. - uint8_t payload_type = packet->header.payloadType; - const DecoderDatabase::DecoderInfo* info = - decoder_database->GetDecoderInfo(payload_type); - if (!info) { - LOG(LS_WARNING) << "SplitFec unknown payload type"; - return kUnknownPayloadType; - } - - // Not an FEC packet. - AudioDecoder* decoder = decoder_database->GetDecoder(payload_type); - // decoder should not return NULL, except for comfort noise payloads which - // are handled separately. - assert(decoder != NULL || decoder_database->IsComfortNoise(payload_type)); - if (!decoder || - !decoder->PacketHasFec(packet->payload.data(), - packet->payload.size())) { - ++it; - continue; - } - - switch (info->codec_type) { - case NetEqDecoder::kDecoderOpus: - case NetEqDecoder::kDecoderOpus_2ch: { - // The main payload of this packet should be decoded as a primary - // payload, even if it comes as a secondary payload in a RED packet. - packet->primary = true; - - Packet* new_packet = new Packet; - new_packet->header = packet->header; - int duration = decoder->PacketDurationRedundant(packet->payload.data(), - packet->payload.size()); - new_packet->header.timestamp -= duration; - new_packet->payload.SetData(packet->payload); - new_packet->primary = false; - // Waiting time should not be set here. - RTC_DCHECK(!packet->waiting_time); - - packet_list->insert(it, new_packet); - break; - } - default: { - LOG(LS_WARNING) << "SplitFec wrong payload type"; - return kFecSplitError; - } - } - - ++it; - } - return kOK; -} - -int PayloadSplitter::CheckRedPayloads(PacketList* packet_list, - const DecoderDatabase& decoder_database) { +int RedPayloadSplitter::CheckRedPayloads( + PacketList* packet_list, + const DecoderDatabase& decoder_database) { PacketList::iterator it = packet_list->begin(); int main_payload_type = -1; int num_deleted_packets = 0; diff --git a/webrtc/modules/audio_coding/neteq/payload_splitter.h b/webrtc/modules/audio_coding/neteq/red_payload_splitter.h similarity index 56% rename from webrtc/modules/audio_coding/neteq/payload_splitter.h rename to webrtc/modules/audio_coding/neteq/red_payload_splitter.h index a3e1b1d91c..deb86eb166 100644 --- a/webrtc/modules/audio_coding/neteq/payload_splitter.h +++ b/webrtc/modules/audio_coding/neteq/red_payload_splitter.h @@ -8,8 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_PAYLOAD_SPLITTER_H_ -#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_PAYLOAD_SPLITTER_H_ +#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_RED_PAYLOAD_SPLITTER_H_ +#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_RED_PAYLOAD_SPLITTER_H_ #include "webrtc/base/constructormagic.h" #include "webrtc/modules/audio_coding/neteq/packet.h" @@ -19,24 +19,14 @@ namespace webrtc { // Forward declarations. class DecoderDatabase; -// This class handles splitting of payloads into smaller parts. - -// For RED and FEC the splitting is done internally. Other codecs' packets are -// split by calling AudioDecoder::SplitPacket. -class PayloadSplitter { +// This class handles splitting of RED payloads into smaller parts. +// Codec-specific packet splitting can be performed by +// AudioDecoder::ParsePayload. +class RedPayloadSplitter { public: - enum SplitterReturnCodes { - kOK = 0, - kNoSplit = 1, - kFrameSplitError = -2, - kUnknownPayloadType = -3, - kRedLengthMismatch = -4, - kFecSplitError = -5, - }; + RedPayloadSplitter() {} - PayloadSplitter() {} - - virtual ~PayloadSplitter() {} + virtual ~RedPayloadSplitter() {} // Splits each packet in |packet_list| into its separate RED payloads. Each // RED payload is packetized into a Packet. The original elements in @@ -44,23 +34,18 @@ class PayloadSplitter { // Note that all packets in |packet_list| must be RED payloads, i.e., have // RED headers according to RFC 2198 at the very beginning of the payload. // Returns kOK or an error. - virtual int SplitRed(PacketList* packet_list); - - // Iterates through |packet_list| and, duplicate each audio payload that has - // FEC as new packet for redundant decoding. The decoder database is needed to - // get information about which payload type each packet contains. - virtual int SplitFec(PacketList* packet_list, - DecoderDatabase* decoder_database); + virtual bool SplitRed(PacketList* packet_list); // Checks all packets in |packet_list|. Packets that are DTMF events or // comfort noise payloads are kept. Except that, only one single payload type - // is accepted. Any packet with another payload type is discarded. + // is accepted. Any packet with another payload type is discarded. Returns + // the number of discarded packets. virtual int CheckRedPayloads(PacketList* packet_list, const DecoderDatabase& decoder_database); private: - RTC_DISALLOW_COPY_AND_ASSIGN(PayloadSplitter); + RTC_DISALLOW_COPY_AND_ASSIGN(RedPayloadSplitter); }; } // namespace webrtc -#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_PAYLOAD_SPLITTER_H_ +#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_RED_PAYLOAD_SPLITTER_H_ diff --git a/webrtc/modules/audio_coding/neteq/payload_splitter_unittest.cc b/webrtc/modules/audio_coding/neteq/red_payload_splitter_unittest.cc similarity index 70% rename from webrtc/modules/audio_coding/neteq/payload_splitter_unittest.cc rename to webrtc/modules/audio_coding/neteq/red_payload_splitter_unittest.cc index 0a4b2157b5..5396ff4d0a 100644 --- a/webrtc/modules/audio_coding/neteq/payload_splitter_unittest.cc +++ b/webrtc/modules/audio_coding/neteq/red_payload_splitter_unittest.cc @@ -8,9 +8,9 @@ * be found in the AUTHORS file in the root of the source tree. */ -// Unit tests for PayloadSplitter class. +// Unit tests for RedPayloadSplitter class. -#include "webrtc/modules/audio_coding/neteq/payload_splitter.h" +#include "webrtc/modules/audio_coding/neteq/red_payload_splitter.h" #include @@ -46,7 +46,8 @@ static const uint32_t kBaseTimestamp = 0x12345678; // : | // | | // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -void CreateOpusFecPayload(uint8_t* payload, size_t payload_length, +void CreateOpusFecPayload(uint8_t* payload, + size_t payload_length, uint8_t payload_value) { if (payload_length < 2) { return; @@ -121,8 +122,10 @@ Packet* CreateRedPayload(size_t num_payloads, } // Create a packet with all payload bytes set to |payload_value|. -Packet* CreatePacket(uint8_t payload_type, size_t payload_length, - uint8_t payload_value, bool opus_fec = false) { +Packet* CreatePacket(uint8_t payload_type, + size_t payload_length, + uint8_t payload_value, + bool opus_fec = false) { Packet* packet = new Packet; packet->header.payloadType = payload_type; packet->header.timestamp = kBaseTimestamp; @@ -144,22 +147,34 @@ void VerifyPacket(const Packet* packet, uint16_t sequence_number, uint32_t timestamp, uint8_t payload_value, - bool primary = true) { + Packet::Priority priority) { EXPECT_EQ(payload_length, packet->payload.size()); EXPECT_EQ(payload_type, packet->header.payloadType); EXPECT_EQ(sequence_number, packet->header.sequenceNumber); EXPECT_EQ(timestamp, packet->header.timestamp); - EXPECT_EQ(primary, packet->primary); + EXPECT_EQ(priority, packet->priority); ASSERT_FALSE(packet->payload.empty()); for (size_t i = 0; i < packet->payload.size(); ++i) { ASSERT_EQ(payload_value, packet->payload.data()[i]); } } +void VerifyPacket(const Packet* packet, + size_t payload_length, + uint8_t payload_type, + uint16_t sequence_number, + uint32_t timestamp, + uint8_t payload_value, + bool primary) { + return VerifyPacket(packet, payload_length, payload_type, sequence_number, + timestamp, payload_value, + Packet::Priority{0, primary ? 0 : 1}); +} + // Start of test definitions. -TEST(PayloadSplitter, CreateAndDestroy) { - PayloadSplitter* splitter = new PayloadSplitter; +TEST(RedPayloadSplitter, CreateAndDestroy) { + RedPayloadSplitter* splitter = new RedPayloadSplitter; delete splitter; } @@ -170,8 +185,8 @@ TEST(RedPayloadSplitter, OnePacketTwoPayloads) { Packet* packet = CreateRedPayload(2, payload_types, kTimestampOffset); PacketList packet_list; packet_list.push_back(packet); - PayloadSplitter splitter; - EXPECT_EQ(PayloadSplitter::kOK, splitter.SplitRed(&packet_list)); + RedPayloadSplitter splitter; + EXPECT_TRUE(splitter.SplitRed(&packet_list)); ASSERT_EQ(2u, packet_list.size()); // Check first packet. The first in list should always be the primary payload. packet = packet_list.front(); @@ -201,8 +216,8 @@ TEST(RedPayloadSplitter, TwoPacketsOnePayload) { packet->header.timestamp += kTimestampOffset; packet->header.sequenceNumber++; packet_list.push_back(packet); - PayloadSplitter splitter; - EXPECT_EQ(PayloadSplitter::kOK, splitter.SplitRed(&packet_list)); + RedPayloadSplitter splitter; + EXPECT_TRUE(splitter.SplitRed(&packet_list)); ASSERT_EQ(2u, packet_list.size()); // Check first packet. packet = packet_list.front(); @@ -239,43 +254,43 @@ TEST(RedPayloadSplitter, TwoPacketsThreePayloads) { packet->header.timestamp += kTimestampOffset; packet->header.sequenceNumber++; packet_list.push_back(packet); - PayloadSplitter splitter; - EXPECT_EQ(PayloadSplitter::kOK, splitter.SplitRed(&packet_list)); + RedPayloadSplitter splitter; + EXPECT_TRUE(splitter.SplitRed(&packet_list)); ASSERT_EQ(6u, packet_list.size()); // Check first packet, A1. packet = packet_list.front(); VerifyPacket(packet, kPayloadLength, payload_types[2], kSequenceNumber, - kBaseTimestamp, 2, true); + kBaseTimestamp, 2, {0, 0}); delete packet; packet_list.pop_front(); // Check second packet, A2. packet = packet_list.front(); VerifyPacket(packet, kPayloadLength, payload_types[1], kSequenceNumber, - kBaseTimestamp - kTimestampOffset, 1, false); + kBaseTimestamp - kTimestampOffset, 1, {0, 1}); delete packet; packet_list.pop_front(); // Check third packet, A3. packet = packet_list.front(); VerifyPacket(packet, kPayloadLength, payload_types[0], kSequenceNumber, - kBaseTimestamp - 2 * kTimestampOffset, 0, false); + kBaseTimestamp - 2 * kTimestampOffset, 0, {0, 2}); delete packet; packet_list.pop_front(); // Check fourth packet, B1. packet = packet_list.front(); VerifyPacket(packet, kPayloadLength, payload_types[2], kSequenceNumber + 1, - kBaseTimestamp + kTimestampOffset, 2, true); + kBaseTimestamp + kTimestampOffset, 2, {0, 0}); delete packet; packet_list.pop_front(); // Check fifth packet, B2. packet = packet_list.front(); VerifyPacket(packet, kPayloadLength, payload_types[1], kSequenceNumber + 1, - kBaseTimestamp, 1, false); + kBaseTimestamp, 1, {0, 1}); delete packet; packet_list.pop_front(); // Check sixth packet, B3. packet = packet_list.front(); VerifyPacket(packet, kPayloadLength, payload_types[0], kSequenceNumber + 1, - kBaseTimestamp - kTimestampOffset, 0, false); + kBaseTimestamp - kTimestampOffset, 0, {0, 2}); delete packet; } @@ -305,7 +320,7 @@ TEST(RedPayloadSplitter, CheckRedPayloads) { decoder_database.RegisterPayload(2, NetEqDecoder::kDecoderAVT, "avt"); decoder_database.RegisterPayload(3, NetEqDecoder::kDecoderILBC, "ilbc"); - PayloadSplitter splitter; + RedPayloadSplitter splitter; splitter.CheckRedPayloads(&packet_list, decoder_database); ASSERT_EQ(3u, packet_list.size()); // Should have dropped the last packet. @@ -332,124 +347,13 @@ TEST(RedPayloadSplitter, WrongPayloadLength) { packet->payload.SetSize(packet->payload.size() - (kPayloadLength + 1)); PacketList packet_list; packet_list.push_back(packet); - PayloadSplitter splitter; - EXPECT_EQ(PayloadSplitter::kRedLengthMismatch, - splitter.SplitRed(&packet_list)); + RedPayloadSplitter splitter; + EXPECT_FALSE(splitter.SplitRed(&packet_list)); ASSERT_EQ(1u, packet_list.size()); // Check first packet. packet = packet_list.front(); VerifyPacket(packet, kPayloadLength, payload_types[0], kSequenceNumber, - kBaseTimestamp - 2 * kTimestampOffset, 0, false); - delete packet; - packet_list.pop_front(); -} - -TEST(FecPayloadSplitter, MixedPayload) { - PacketList packet_list; - DecoderDatabase decoder_database(CreateBuiltinAudioDecoderFactory()); - - decoder_database.RegisterPayload(0, NetEqDecoder::kDecoderOpus, "opus"); - decoder_database.RegisterPayload(1, NetEqDecoder::kDecoderPCMu, "pcmu"); - - Packet* packet = CreatePacket(0, 10, 0xFF, true); - packet_list.push_back(packet); - - packet = CreatePacket(0, 10, 0); // Non-FEC Opus payload. - packet_list.push_back(packet); - - packet = CreatePacket(1, 10, 0); // Non-Opus payload. - packet_list.push_back(packet); - - PayloadSplitter splitter; - EXPECT_EQ(PayloadSplitter::kOK, - splitter.SplitFec(&packet_list, &decoder_database)); - EXPECT_EQ(4u, packet_list.size()); - - // Check first packet. - packet = packet_list.front(); - EXPECT_EQ(0, packet->header.payloadType); - EXPECT_EQ(kBaseTimestamp - 20 * 48, packet->header.timestamp); - EXPECT_EQ(10U, packet->payload.size()); - EXPECT_FALSE(packet->primary); - delete packet; - packet_list.pop_front(); - - // Check second packet. - packet = packet_list.front(); - EXPECT_EQ(0, packet->header.payloadType); - EXPECT_EQ(kBaseTimestamp, packet->header.timestamp); - EXPECT_EQ(10U, packet->payload.size()); - EXPECT_TRUE(packet->primary); - delete packet; - packet_list.pop_front(); - - // Check third packet. - packet = packet_list.front(); - VerifyPacket(packet, 10, 0, kSequenceNumber, kBaseTimestamp, 0, true); - delete packet; - packet_list.pop_front(); - - // Check fourth packet. - packet = packet_list.front(); - VerifyPacket(packet, 10, 1, kSequenceNumber, kBaseTimestamp, 0, true); - delete packet; -} - -TEST(FecPayloadSplitter, EmbedFecInRed) { - PacketList packet_list; - DecoderDatabase decoder_database(CreateBuiltinAudioDecoderFactory()); - - const int kTimestampOffset = 20 * 48; // 20 ms * 48 kHz. - uint8_t payload_types[] = {0, 0}; - decoder_database.RegisterPayload(0, NetEqDecoder::kDecoderOpus, "opus"); - Packet* packet = CreateRedPayload(2, payload_types, kTimestampOffset, true); - packet_list.push_back(packet); - - PayloadSplitter splitter; - EXPECT_EQ(PayloadSplitter::kOK, - splitter.SplitRed(&packet_list)); - EXPECT_EQ(PayloadSplitter::kOK, - splitter.SplitFec(&packet_list, &decoder_database)); - - EXPECT_EQ(4u, packet_list.size()); - - // Check first packet. FEC packet copied from primary payload in RED. - packet = packet_list.front(); - EXPECT_EQ(0, packet->header.payloadType); - EXPECT_EQ(kBaseTimestamp - kTimestampOffset, packet->header.timestamp); - EXPECT_EQ(kPayloadLength, packet->payload.size()); - EXPECT_FALSE(packet->primary); - EXPECT_EQ(packet->payload[3], 1); - delete packet; - packet_list.pop_front(); - - // Check second packet. Normal packet copied from primary payload in RED. - packet = packet_list.front(); - EXPECT_EQ(0, packet->header.payloadType); - EXPECT_EQ(kBaseTimestamp, packet->header.timestamp); - EXPECT_EQ(kPayloadLength, packet->payload.size()); - EXPECT_TRUE(packet->primary); - EXPECT_EQ(packet->payload[3], 1); - delete packet; - packet_list.pop_front(); - - // Check third packet. FEC packet copied from secondary payload in RED. - packet = packet_list.front(); - EXPECT_EQ(0, packet->header.payloadType); - EXPECT_EQ(kBaseTimestamp - 2 * kTimestampOffset, packet->header.timestamp); - EXPECT_EQ(kPayloadLength, packet->payload.size()); - EXPECT_FALSE(packet->primary); - EXPECT_EQ(packet->payload[3], 0); - delete packet; - packet_list.pop_front(); - - // Check fourth packet. Normal packet copied from primary payload in RED. - packet = packet_list.front(); - EXPECT_EQ(0, packet->header.payloadType); - EXPECT_EQ(kBaseTimestamp - kTimestampOffset, packet->header.timestamp); - EXPECT_EQ(kPayloadLength, packet->payload.size()); - EXPECT_TRUE(packet->primary); - EXPECT_EQ(packet->payload[3], 0); + kBaseTimestamp - 2 * kTimestampOffset, 0, {0, 2}); delete packet; packet_list.pop_front(); }