admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Moved Opus-specific payload splitting into AudioDecoderOpus.

Browse Source 
The biggest change to NetEq is the move from a primary flag, to a
Priority with two separate levels: one set by RED splitting and one
set by the codec itself. This allows us to unambigously prioritize
"fallback" packets from these two sources. I've chosen what I believe
is the sensible ordering: packets that the codec prioritizes are
chosen first, regardless of if they are secondary RED packets or
not. So if we were to use Opus w/ FEC in RED, we'd only do Opus FEC
decoding if there was no RED packet that could cover the time slot.

With this change, PayloadSplitter now only deals with RED
packets. Maybe it should be renamed RedPayloadSplitter?

BUG=webrtc:5805

Review-Url: https://codereview.webrtc.org/2342443005
Cr-Commit-Position: refs/heads/master@{#14347}
This commit is contained in:
ossu 2016-09-22 02:06:28 -07:00 committed by Commit bot
parent 2beb42983c
commit a70695a3e1
32 changed files with 441 additions and 464 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
webrtc/modules/BUILD.gn
View File

@ -313,7 +313,7 @@ if (rtc_include_tests) {
"audio_coding/neteq/mock/mock_expand.h",
"audio_coding/neteq/mock/mock_external_decoder_pcm16b.h",
"audio_coding/neteq/mock/mock_packet_buffer.h",
"audio_coding/neteq/mock/mock_payload_splitter.h",
"audio_coding/neteq/mock/mock_red_payload_splitter.h",
"audio_coding/neteq/nack_tracker_unittest.cc",
"audio_coding/neteq/neteq_external_decoder_unittest.cc",
"audio_coding/neteq/neteq_impl_unittest.cc",
@ -322,9 +322,9 @@ if (rtc_include_tests) {
"audio_coding/neteq/neteq_unittest.cc",
"audio_coding/neteq/normal_unittest.cc",
"audio_coding/neteq/packet_buffer_unittest.cc",
"audio_coding/neteq/payload_splitter_unittest.cc",
"audio_coding/neteq/post_decode_vad_unittest.cc",
"audio_coding/neteq/random_vector_unittest.cc",
"audio_coding/neteq/red_payload_splitter_unittest.cc",
"audio_coding/neteq/sync_buffer_unittest.cc",
"audio_coding/neteq/tick_timer_unittest.cc",
"audio_coding/neteq/time_stretch_unittest.cc",

4
webrtc/modules/audio_coding/BUILD.gn
View File

@ -784,14 +784,14 @@ rtc_source_set("neteq") {
"neteq/packet.h",
"neteq/packet_buffer.cc",
"neteq/packet_buffer.h",
"neteq/payload_splitter.cc",
"neteq/payload_splitter.h",
"neteq/post_decode_vad.cc",
"neteq/post_decode_vad.h",
"neteq/preemptive_expand.cc",
"neteq/preemptive_expand.h",
"neteq/random_vector.cc",
"neteq/random_vector.h",
"neteq/red_payload_splitter.cc",
"neteq/red_payload_splitter.h",
"neteq/rtcp.cc",
"neteq/rtcp.h",
"neteq/statistics_calculator.cc",

15
webrtc/modules/audio_coding/codecs/audio_decoder.cc
View File

@ -14,8 +14,6 @@
#include <memory>
#include <utility>
#include <utility>
#include "webrtc/base/array_view.h"
#include "webrtc/base/checks.h"
#include "webrtc/base/sanitizer.h"
@ -27,9 +25,11 @@ namespace webrtc {
AudioDecoder::ParseResult::ParseResult() = default;
AudioDecoder::ParseResult::ParseResult(ParseResult&& b) = default;
AudioDecoder::ParseResult::ParseResult(uint32_t timestamp,
bool primary,
int priority,
std::unique_ptr<EncodedAudioFrame> frame)
: timestamp(timestamp), primary(primary), frame(std::move(frame)) {}
: timestamp(timestamp), priority(priority), frame(std::move(frame)) {
RTC_DCHECK_GE(priority, 0);
}
AudioDecoder::ParseResult::~ParseResult() = default;
@ -38,12 +38,11 @@ AudioDecoder::ParseResult& AudioDecoder::ParseResult::operator=(
std::vector<AudioDecoder::ParseResult> AudioDecoder::ParsePayload(
rtc::Buffer&& payload,
uint32_t timestamp,
bool is_primary) {
uint32_t timestamp) {
std::vector<ParseResult> results;
std::unique_ptr<EncodedAudioFrame> frame(
new LegacyEncodedAudioFrame(this, std::move(payload), is_primary));
results.emplace_back(timestamp, is_primary, std::move(frame));
new LegacyEncodedAudioFrame(this, std::move(payload)));
results.emplace_back(timestamp, 0, std::move(frame));
return results;
}

19
webrtc/modules/audio_coding/codecs/audio_decoder.h
View File

@ -8,11 +8,8 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_INCLUDE_AUDIO_DECODER_H_
#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_INCLUDE_AUDIO_DECODER_H_
#include <memory>
#include <vector>
#ifndef WEBRTC_MODULES_AUDIO_CODING_CODECS_AUDIO_DECODER_H_
#define WEBRTC_MODULES_AUDIO_CODING_CODECS_AUDIO_DECODER_H_
#include <memory>
#include <vector>
@ -66,7 +63,7 @@ class AudioDecoder {
struct ParseResult {
ParseResult();
ParseResult(uint32_t timestamp,
bool primary,
int priority,
std::unique_ptr<EncodedAudioFrame> frame);
ParseResult(ParseResult&& b);
~ParseResult();
@ -75,7 +72,10 @@ class AudioDecoder {
// The timestamp of the frame is in samples per channel.
uint32_t timestamp;
bool primary;
// The relative priority of the frame compared to other frames of the same
// payload and the same timeframe. A higher value means a lower priority.
// The highest priority is zero - negative values are not allowed.
int priority;
std::unique_ptr<EncodedAudioFrame> frame;
};
@ -86,8 +86,7 @@ class AudioDecoder {
// buffer. |timestamp| is the input timestamp, in samples, corresponding to
// the start of the payload.
virtual std::vector<ParseResult> ParsePayload(rtc::Buffer&& payload,
uint32_t timestamp,
bool is_primary);
uint32_t timestamp);
// Decodes |encode_len| bytes from |encoded| and writes the result in
// |decoded|. The maximum bytes allowed to be written into |decoded| is
@ -177,4 +176,4 @@ class AudioDecoder {
};
} // namespace webrtc
#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_INCLUDE_AUDIO_DECODER_H_
#endif // WEBRTC_MODULES_AUDIO_CODING_CODECS_AUDIO_DECODER_H_

10
webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc
View File

@ -19,10 +19,9 @@ void AudioDecoderPcmU::Reset() {}
std::vector<AudioDecoder::ParseResult> AudioDecoderPcmU::ParsePayload(
rtc::Buffer&& payload,
uint32_t timestamp,
bool is_primary) {
uint32_t timestamp) {
return LegacyEncodedAudioFrame::SplitBySamples(
this, std::move(payload), timestamp, is_primary, 8 * num_channels_, 8);
this, std::move(payload), timestamp, 8 * num_channels_, 8);
}
int AudioDecoderPcmU::SampleRateHz() const {
@ -55,10 +54,9 @@ void AudioDecoderPcmA::Reset() {}
std::vector<AudioDecoder::ParseResult> AudioDecoderPcmA::ParsePayload(
rtc::Buffer&& payload,
uint32_t timestamp,
bool is_primary) {
uint32_t timestamp) {
return LegacyEncodedAudioFrame::SplitBySamples(
this, std::move(payload), timestamp, is_primary, 8 * num_channels_, 8);
this, std::move(payload), timestamp, 8 * num_channels_, 8);
}
int AudioDecoderPcmA::SampleRateHz() const {

6
webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h
View File

@ -24,8 +24,7 @@ class AudioDecoderPcmU final : public AudioDecoder {
}
void Reset() override;
std::vector<ParseResult> ParsePayload(rtc::Buffer&& payload,
uint32_t timestamp,
bool is_primary) override;
uint32_t timestamp) override;
int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override;
int SampleRateHz() const override;
size_t Channels() const override;
@ -49,8 +48,7 @@ class AudioDecoderPcmA final : public AudioDecoder {
}
void Reset() override;
std::vector<ParseResult> ParsePayload(rtc::Buffer&& payload,
uint32_t timestamp,
bool is_primary) override;
uint32_t timestamp) override;
int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override;
int SampleRateHz() const override;
size_t Channels() const override;

12
webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc
View File

@ -50,10 +50,9 @@ void AudioDecoderG722::Reset() {
std::vector<AudioDecoder::ParseResult> AudioDecoderG722::ParsePayload(
rtc::Buffer&& payload,
uint32_t timestamp,
bool is_primary) {
uint32_t timestamp) {
return LegacyEncodedAudioFrame::SplitBySamples(this, std::move(payload),
timestamp, is_primary, 8, 16);
timestamp, 8, 16);
}
int AudioDecoderG722::PacketDuration(const uint8_t* encoded,
@ -128,10 +127,9 @@ void AudioDecoderG722Stereo::Reset() {
std::vector<AudioDecoder::ParseResult> AudioDecoderG722Stereo::ParsePayload(
rtc::Buffer&& payload,
uint32_t timestamp,
bool is_primary) {
return LegacyEncodedAudioFrame::SplitBySamples(
this, std::move(payload), timestamp, is_primary, 2 * 8, 16);
uint32_t timestamp) {
return LegacyEncodedAudioFrame::SplitBySamples(this, std::move(payload),
timestamp, 2 * 8, 16);
}
// Split the stereo packet and place left and right channel after each other

6
webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h
View File

@ -25,8 +25,7 @@ class AudioDecoderG722 final : public AudioDecoder {
bool HasDecodePlc() const override;
void Reset() override;
std::vector<ParseResult> ParsePayload(rtc::Buffer&& payload,
uint32_t timestamp,
bool is_primary) override;
uint32_t timestamp) override;
int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override;
int SampleRateHz() const override;
size_t Channels() const override;
@ -49,8 +48,7 @@ class AudioDecoderG722Stereo final : public AudioDecoder {
~AudioDecoderG722Stereo() override;
void Reset() override;
std::vector<ParseResult> ParsePayload(rtc::Buffer&& payload,
uint32_t timestamp,
bool is_primary) override;
uint32_t timestamp) override;
int SampleRateHz() const override;
size_t Channels() const override;

15
webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc
View File

@ -10,6 +10,8 @@
#include "webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h"
#include <utility>
#include "webrtc/base/checks.h"
#include "webrtc/base/logging.h"
#include "webrtc/modules/audio_coding/codecs/ilbc/ilbc.h"
@ -53,8 +55,7 @@ void AudioDecoderIlbc::Reset() {
std::vector<AudioDecoder::ParseResult> AudioDecoderIlbc::ParsePayload(
rtc::Buffer&& payload,
uint32_t timestamp,
bool is_primary) {
uint32_t timestamp) {
std::vector<ParseResult> results;
size_t bytes_per_frame;
int timestamps_per_frame;
@ -78,8 +79,8 @@ std::vector<AudioDecoder::ParseResult> AudioDecoderIlbc::ParsePayload(
RTC_DCHECK_EQ(0u, payload.size() % bytes_per_frame);
if (payload.size() == bytes_per_frame) {
std::unique_ptr<EncodedAudioFrame> frame(
new LegacyEncodedAudioFrame(this, std::move(payload), is_primary));
results.emplace_back(timestamp, is_primary, std::move(frame));
new LegacyEncodedAudioFrame(this, std::move(payload)));
results.emplace_back(timestamp, 0, std::move(frame));
} else {
size_t byte_offset;
uint32_t timestamp_offset;
@ -87,11 +88,9 @@ std::vector<AudioDecoder::ParseResult> AudioDecoderIlbc::ParsePayload(
byte_offset < payload.size();
byte_offset += bytes_per_frame,
timestamp_offset += timestamps_per_frame) {
rtc::Buffer new_payload(payload.data() + byte_offset, bytes_per_frame);
std::unique_ptr<EncodedAudioFrame> frame(new LegacyEncodedAudioFrame(
this, std::move(new_payload), is_primary));
results.emplace_back(timestamp + timestamp_offset, is_primary,
std::move(frame));
this, rtc::Buffer(payload.data() + byte_offset, bytes_per_frame)));
results.emplace_back(timestamp + timestamp_offset, 0, std::move(frame));
}
}

3
webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h
View File

@ -26,8 +26,7 @@ class AudioDecoderIlbc final : public AudioDecoder {
size_t DecodePlc(size_t num_frames, int16_t* decoded) override;
void Reset() override;
std::vector<ParseResult> ParsePayload(rtc::Buffer&& payload,
uint32_t timestamp,
bool is_primary) override;
uint32_t timestamp) override;
int SampleRateHz() const override;
size_t Channels() const override;

6
webrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc
View File

@ -81,7 +81,7 @@ TEST_P(SplitIlbcTest, NumFrames) {
};
const auto results = decoder.ParsePayload(
generate_payload(frame_length_bytes_ * num_frames_), 0, true);
generate_payload(frame_length_bytes_ * num_frames_), 0);
EXPECT_EQ(num_frames_, results.size());
size_t frame_num = 0;
@ -123,7 +123,7 @@ TEST(IlbcTest, SplitTooLargePayload) {
AudioDecoderIlbc decoder;
constexpr size_t kPayloadLengthBytes = 950;
const auto results =
decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0, true);
decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0);
EXPECT_TRUE(results.empty());
}
@ -132,7 +132,7 @@ TEST(IlbcTest, SplitUnevenPayload) {
AudioDecoderIlbc decoder;
constexpr size_t kPayloadLengthBytes = 39; // Not an even number of frames.
const auto results =
decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0, true);
decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0);
EXPECT_TRUE(results.empty());
}

38
webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc
View File

@ -17,37 +17,22 @@
namespace webrtc {
LegacyEncodedAudioFrame::LegacyEncodedAudioFrame(AudioDecoder* decoder,
rtc::Buffer&& payload,
bool is_primary_payload)
: decoder_(decoder),
payload_(std::move(payload)),
is_primary_payload_(is_primary_payload) {}
rtc::Buffer&& payload)
: decoder_(decoder), payload_(std::move(payload)) {}
LegacyEncodedAudioFrame::~LegacyEncodedAudioFrame() = default;
size_t LegacyEncodedAudioFrame::Duration() const {
int ret;
if (is_primary_payload_) {
ret = decoder_->PacketDuration(payload_.data(), payload_.size());
} else {
ret = decoder_->PacketDurationRedundant(payload_.data(), payload_.size());
}
const int ret = decoder_->PacketDuration(payload_.data(), payload_.size());
return (ret < 0) ? 0 : static_cast<size_t>(ret);
}
rtc::Optional<AudioDecoder::EncodedAudioFrame::DecodeResult>
LegacyEncodedAudioFrame::Decode(rtc::ArrayView<int16_t> decoded) const {
AudioDecoder::SpeechType speech_type = AudioDecoder::kSpeech;
int ret;
if (is_primary_payload_) {
ret = decoder_->Decode(
payload_.data(), payload_.size(), decoder_->SampleRateHz(),
decoded.size() * sizeof(int16_t), decoded.data(), &speech_type);
} else {
ret = decoder_->DecodeRedundant(
payload_.data(), payload_.size(), decoder_->SampleRateHz(),
decoded.size() * sizeof(int16_t), decoded.data(), &speech_type);
}
const int ret = decoder_->Decode(
payload_.data(), payload_.size(), decoder_->SampleRateHz(),
decoded.size() * sizeof(int16_t), decoded.data(), &speech_type);
if (ret < 0)
return rtc::Optional<DecodeResult>();
@ -59,7 +44,6 @@ std::vector<AudioDecoder::ParseResult> LegacyEncodedAudioFrame::SplitBySamples(
AudioDecoder* decoder,
rtc::Buffer&& payload,
uint32_t timestamp,
bool is_primary,
size_t bytes_per_ms,
uint32_t timestamps_per_ms) {
RTC_DCHECK(payload.data());
@ -70,8 +54,8 @@ std::vector<AudioDecoder::ParseResult> LegacyEncodedAudioFrame::SplitBySamples(
const size_t min_chunk_size = bytes_per_ms * 20;
if (min_chunk_size >= payload.size()) {
std::unique_ptr<LegacyEncodedAudioFrame> frame(
new LegacyEncodedAudioFrame(decoder, std::move(payload), is_primary));
results.emplace_back(timestamp, is_primary, std::move(frame));
new LegacyEncodedAudioFrame(decoder, std::move(payload)));
results.emplace_back(timestamp, 0, std::move(frame));
} else {
// Reduce the split size by half as long as |split_size_bytes| is at least
// twice the minimum chunk size (so that the resulting size is at least as
@ -92,10 +76,8 @@ std::vector<AudioDecoder::ParseResult> LegacyEncodedAudioFrame::SplitBySamples(
std::min(split_size_bytes, payload.size() - byte_offset);
rtc::Buffer new_payload(payload.data() + byte_offset, split_size_bytes);
std::unique_ptr<LegacyEncodedAudioFrame> frame(
new LegacyEncodedAudioFrame(decoder, std::move(new_payload),
is_primary));
results.emplace_back(timestamp + timestamp_offset, is_primary,
std::move(frame));
new LegacyEncodedAudioFrame(decoder, std::move(new_payload)));
results.emplace_back(timestamp + timestamp_offset, 0, std::move(frame));
}
}

6
webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h
View File

@ -20,16 +20,13 @@ namespace webrtc {
class LegacyEncodedAudioFrame final : public AudioDecoder::EncodedAudioFrame {
public:
LegacyEncodedAudioFrame(AudioDecoder* decoder,
rtc::Buffer&& payload,
bool is_primary_payload);
LegacyEncodedAudioFrame(AudioDecoder* decoder, rtc::Buffer&& payload);
~LegacyEncodedAudioFrame() override;
static std::vector<AudioDecoder::ParseResult> SplitBySamples(
AudioDecoder* decoder,
rtc::Buffer&& payload,
uint32_t timestamp,
bool is_primary,
size_t bytes_per_ms,
uint32_t timestamps_per_ms);
@ -44,7 +41,6 @@ class LegacyEncodedAudioFrame final : public AudioDecoder::EncodedAudioFrame {
private:
AudioDecoder* const decoder_;
const rtc::Buffer payload_;
const bool is_primary_payload_;
};
} // namespace webrtc

2
webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc
View File

@ -123,7 +123,7 @@ TEST_P(SplitBySamplesTest, PayloadSizes) {
const auto results = LegacyEncodedAudioFrame::SplitBySamples(
nullptr,
generate_payload(expected_split.payload_size_ms * bytes_per_ms_),
kBaseTimestamp, true, bytes_per_ms_, samples_per_ms_);
kBaseTimestamp, bytes_per_ms_, samples_per_ms_);
EXPECT_EQ(expected_split.num_frames, results.size());
uint32_t expected_timestamp = kBaseTimestamp;

70
webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc
View File

@ -10,10 +10,60 @@
#include "webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h"
#include <utility>
#include "webrtc/base/checks.h"
namespace webrtc {
namespace {
class OpusFrame : public AudioDecoder::EncodedAudioFrame {
public:
OpusFrame(AudioDecoderOpus* decoder,
rtc::Buffer&& payload,
bool is_primary_payload)
: decoder_(decoder),
payload_(std::move(payload)),
is_primary_payload_(is_primary_payload) {}
size_t Duration() const override {
int ret;
if (is_primary_payload_) {
ret = decoder_->PacketDuration(payload_.data(), payload_.size());
} else {
ret = decoder_->PacketDurationRedundant(payload_.data(), payload_.size());
}
return (ret < 0) ? 0 : static_cast<size_t>(ret);
}
rtc::Optional<DecodeResult> Decode(
rtc::ArrayView<int16_t> decoded) const override {
AudioDecoder::SpeechType speech_type = AudioDecoder::kSpeech;
int ret;
if (is_primary_payload_) {
ret = decoder_->Decode(
payload_.data(), payload_.size(), decoder_->SampleRateHz(),
decoded.size() * sizeof(int16_t), decoded.data(), &speech_type);
} else {
ret = decoder_->DecodeRedundant(
payload_.data(), payload_.size(), decoder_->SampleRateHz(),
decoded.size() * sizeof(int16_t), decoded.data(), &speech_type);
}
if (ret < 0)
return rtc::Optional<DecodeResult>();
return rtc::Optional<DecodeResult>({static_cast<size_t>(ret), speech_type});
}
private:
AudioDecoderOpus* const decoder_;
const rtc::Buffer payload_;
const bool is_primary_payload_;
};
} // namespace
AudioDecoderOpus::AudioDecoderOpus(size_t num_channels)
: channels_(num_channels) {
RTC_DCHECK(num_channels == 1 || num_channels == 2);
@ -25,6 +75,26 @@ AudioDecoderOpus::~AudioDecoderOpus() {
WebRtcOpus_DecoderFree(dec_state_);
}
std::vector<AudioDecoder::ParseResult> AudioDecoderOpus::ParsePayload(
rtc::Buffer&& payload,
uint32_t timestamp) {
std::vector<ParseResult> results;
if (PacketHasFec(payload.data(), payload.size())) {
const int duration =
PacketDurationRedundant(payload.data(), payload.size());
RTC_DCHECK_GE(duration, 0);
rtc::Buffer payload_copy(payload.data(), payload.size());
std::unique_ptr<EncodedAudioFrame> fec_frame(
new OpusFrame(this, std::move(payload_copy), false));
results.emplace_back(timestamp - duration, 1, std::move(fec_frame));
}
std::unique_ptr<EncodedAudioFrame> frame(
new OpusFrame(this, std::move(payload), true));
results.emplace_back(timestamp, 0, std::move(frame));
return results;
}
int AudioDecoderOpus::DecodeInternal(const uint8_t* encoded,
size_t encoded_len,
int sample_rate_hz,

2
webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h
View File

@ -22,6 +22,8 @@ class AudioDecoderOpus final : public AudioDecoder {
explicit AudioDecoderOpus(size_t num_channels);
~AudioDecoderOpus() override;
std::vector<ParseResult> ParsePayload(rtc::Buffer&& payload,
uint32_t timestamp) override;
void Reset() override;
int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override;
int PacketDurationRedundant(const uint8_t* encoded,

7
webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc
View File

@ -47,12 +47,11 @@ int AudioDecoderPcm16B::DecodeInternal(const uint8_t* encoded,
std::vector<AudioDecoder::ParseResult> AudioDecoderPcm16B::ParsePayload(
rtc::Buffer&& payload,
uint32_t timestamp,
bool is_primary) {
uint32_t timestamp) {
const int samples_per_ms = rtc::CheckedDivExact(sample_rate_hz_, 1000);
return LegacyEncodedAudioFrame::SplitBySamples(
this, std::move(payload), timestamp, is_primary,
samples_per_ms * 2 * num_channels_, samples_per_ms);
this, std::move(payload), timestamp, samples_per_ms * 2 * num_channels_,
samples_per_ms);
}
int AudioDecoderPcm16B::PacketDuration(const uint8_t* encoded,

3
webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h
View File

@ -21,8 +21,7 @@ class AudioDecoderPcm16B final : public AudioDecoder {
AudioDecoderPcm16B(int sample_rate_hz, size_t num_channels);
void Reset() override;
std::vector<ParseResult> ParsePayload(rtc::Buffer&& payload,
uint32_t timestamp,
bool is_primary) override;
uint32_t timestamp) override;
int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override;
int SampleRateHz() const override;
size_t Channels() const override;

1
webrtc/modules/audio_coding/neteq/include/neteq.h
View File

@ -126,7 +126,6 @@ class NetEq {
kStereoNotSupported,
kSampleUnderrun,
kDecodedTooMuch,
kFrameSplitError,
kRedundancySplitError,
kPacketBufferCorruption
};

31
webrtc/modules/audio_coding/neteq/mock/mock_payload_splitter.h
View File

@ -1,31 +0,0 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_PAYLOAD_SPLITTER_H_
#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_PAYLOAD_SPLITTER_H_
#include "webrtc/modules/audio_coding/neteq/payload_splitter.h"
#include "testing/gmock/include/gmock/gmock.h"
namespace webrtc {
class MockPayloadSplitter : public PayloadSplitter {
public:
MOCK_METHOD1(SplitRed,
int(PacketList* packet_list));
MOCK_METHOD2(SplitFec,
int(PacketList* packet_list, DecoderDatabase* decoder_database));
MOCK_METHOD2(CheckRedPayloads,
int(PacketList* packet_list, const DecoderDatabase& decoder_database));
};
} // namespace webrtc
#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_PAYLOAD_SPLITTER_H_

29
webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h Normal file
View File

@ -0,0 +1,29 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_RED_PAYLOAD_SPLITTER_H_
#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_RED_PAYLOAD_SPLITTER_H_
#include "webrtc/modules/audio_coding/neteq/red_payload_splitter.h"
#include "testing/gmock/include/gmock/gmock.h"
namespace webrtc {
class MockRedPayloadSplitter : public RedPayloadSplitter {
public:
MOCK_METHOD1(SplitRed, bool(PacketList* packet_list));
MOCK_METHOD2(CheckRedPayloads,
int(PacketList* packet_list,
const DecoderDatabase& decoder_database));
};
} // namespace webrtc
#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_RED_PAYLOAD_SPLITTER_H_

4
webrtc/modules/audio_coding/neteq/neteq.gypi
View File

@ -113,8 +113,8 @@
'packet.h',
'packet_buffer.cc',
'packet_buffer.h',
'payload_splitter.cc',
'payload_splitter.h',
'red_payload_splitter.cc',
'red_payload_splitter.h',
'post_decode_vad.cc',
'post_decode_vad.h',
'preemptive_expand.cc',

39
webrtc/modules/audio_coding/neteq/neteq_impl.cc
View File

@ -41,7 +41,7 @@
#include "webrtc/modules/audio_coding/neteq/normal.h"
#include "webrtc/modules/audio_coding/neteq/packet_buffer.h"
#include "webrtc/modules/audio_coding/neteq/packet.h"
#include "webrtc/modules/audio_coding/neteq/payload_splitter.h"
#include "webrtc/modules/audio_coding/neteq/red_payload_splitter.h"
#include "webrtc/modules/audio_coding/neteq/post_decode_vad.h"
#include "webrtc/modules/audio_coding/neteq/preemptive_expand.h"
#include "webrtc/modules/audio_coding/neteq/sync_buffer.h"
@ -65,7 +65,7 @@ NetEqImpl::Dependencies::Dependencies(
dtmf_tone_generator(new DtmfToneGenerator),
packet_buffer(
new PacketBuffer(config.max_packets_in_buffer, tick_timer.get())),
payload_splitter(new PayloadSplitter),
red_payload_splitter(new RedPayloadSplitter),
timestamp_scaler(new TimestampScaler(*decoder_database)),
accelerate_factory(new AccelerateFactory),
expand_factory(new ExpandFactory),
@ -84,7 +84,7 @@ NetEqImpl::NetEqImpl(const NetEq::Config& config,
dtmf_buffer_(std::move(deps.dtmf_buffer)),
dtmf_tone_generator_(std::move(deps.dtmf_tone_generator)),
packet_buffer_(std::move(deps.packet_buffer)),
payload_splitter_(std::move(deps.payload_splitter)),
red_payload_splitter_(std::move(deps.red_payload_splitter)),
timestamp_scaler_(std::move(deps.timestamp_scaler)),
vad_(new PostDecodeVad()),
expand_factory_(std::move(deps.expand_factory)),
@ -567,7 +567,6 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header,
packet->header.ssrc = rtp_header.header.ssrc;
packet->header.numCSRCs = 0;
packet->payload.SetData(payload.data(), payload.size());
packet->primary = true;
// Waiting time will be set upon inserting the packet in the buffer.
RTC_DCHECK(!packet->waiting_time);
// Insert packet in a packet list.
@ -609,13 +608,13 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header,
// Check for RED payload type, and separate payloads into several packets.
if (decoder_database_->IsRed(main_header.payloadType)) {
if (payload_splitter_->SplitRed(&packet_list) != PayloadSplitter::kOK) {
if (!red_payload_splitter_->SplitRed(&packet_list)) {
PacketBuffer::DeleteAllPackets(&packet_list);
return kRedundancySplitError;
}
// Only accept a few RED payloads of the same type as the main data,
// DTMF events and CNG.
payload_splitter_->CheckRedPayloads(&packet_list, *decoder_database_);
red_payload_splitter_->CheckRedPayloads(&packet_list, *decoder_database_);
// Update the stored main payload header since the main payload has now
// changed.
memcpy(&main_header, &packet_list.front()->header, sizeof(main_header));
@ -658,18 +657,6 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header,
}
}
// Check for FEC in packets, and separate payloads into several packets.
int ret = payload_splitter_->SplitFec(&packet_list, decoder_database_.get());
if (ret != PayloadSplitter::kOK) {
PacketBuffer::DeleteAllPackets(&packet_list);
switch (ret) {
case PayloadSplitter::kUnknownPayloadType:
return kUnknownRtpPayloadType;
default:
return kOtherError;
}
}
// Update bandwidth estimate, if the packet is not comfort noise.
if (!packet_list.empty() &&
!decoder_database_->IsComfortNoise(main_header.payloadType)) {
@ -702,9 +689,9 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header,
} else {
std::vector<AudioDecoder::ParseResult> results =
info->GetDecoder()->ParsePayload(std::move(packet->payload),
packet->header.timestamp,
packet->primary);
packet->header.timestamp);
const RTPHeader& original_header = packet->header;
const Packet::Priority original_priority = packet->priority;
for (auto& result : results) {
RTC_DCHECK(result.frame);
// Reuse the packet if possible.
@ -713,8 +700,9 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header,
packet->header = original_header;
}
packet->header.timestamp = result.timestamp;
// TODO(ossu): Move from primary to some sort of priority level.
packet->primary = result.primary;
RTC_DCHECK_GE(result.priority, 0);
packet->priority.codec_level = result.priority;
packet->priority.red_level = original_priority.red_level;
packet->frame = std::move(result.frame);
parsed_packet_list.push_back(packet.release());
}
@ -734,7 +722,7 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header,
// Insert packets in buffer.
const size_t buffer_length_before_insert =
packet_buffer_->NumPacketsInBuffer();
ret = packet_buffer_->InsertPacketList(
const int ret = packet_buffer_->InsertPacketList(
&parsed_packet_list, *decoder_database_, &current_rtp_payload_type_,
&current_cng_rtp_payload_type_);
if (ret == PacketBuffer::kFlushed) {
@ -1979,9 +1967,8 @@ int NetEqImpl::ExtractPackets(size_t required_samples,
size_t packet_duration = 0;
if (packet->frame) {
packet_duration = packet->frame->Duration();
// TODO(ossu): Is this the correct way to track samples decoded from a
// redundant packet?
if (packet_duration > 0 && !packet->primary) {
// TODO(ossu): Is this the correct way to track Opus FEC packets?
if (packet->priority.codec_level > 0) {
stats_.SecondaryDecodedSamples(rtc::checked_cast<int>(packet_duration));
}
} else if (!decoder_database_->IsComfortNoise(packet->header.payloadType)) {

6
webrtc/modules/audio_coding/neteq/neteq_impl.h
View File

@ -46,7 +46,7 @@ class Merge;
class NackTracker;
class Normal;
class PacketBuffer;
class PayloadSplitter;
class RedPayloadSplitter;
class PostDecodeVad;
class PreemptiveExpand;
class RandomVector;
@ -86,7 +86,7 @@ class NetEqImpl : public webrtc::NetEq {
std::unique_ptr<DtmfBuffer> dtmf_buffer;
std::unique_ptr<DtmfToneGenerator> dtmf_tone_generator;
std::unique_ptr<PacketBuffer> packet_buffer;
std::unique_ptr<PayloadSplitter> payload_splitter;
std::unique_ptr<RedPayloadSplitter> red_payload_splitter;
std::unique_ptr<TimestampScaler> timestamp_scaler;
std::unique_ptr<AccelerateFactory> accelerate_factory;
std::unique_ptr<ExpandFactory> expand_factory;
@ -353,7 +353,7 @@ class NetEqImpl : public webrtc::NetEq {
const std::unique_ptr<DtmfToneGenerator> dtmf_tone_generator_
GUARDED_BY(crit_sect_);
const std::unique_ptr<PacketBuffer> packet_buffer_ GUARDED_BY(crit_sect_);
const std::unique_ptr<PayloadSplitter> payload_splitter_
const std::unique_ptr<RedPayloadSplitter> red_payload_splitter_
GUARDED_BY(crit_sect_);
const std::unique_ptr<TimestampScaler> timestamp_scaler_
GUARDED_BY(crit_sect_);

17
webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc
View File

@ -28,7 +28,7 @@
#include "webrtc/modules/audio_coding/neteq/mock/mock_dtmf_buffer.h"
#include "webrtc/modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h"
#include "webrtc/modules/audio_coding/neteq/mock/mock_packet_buffer.h"
#include "webrtc/modules/audio_coding/neteq/mock/mock_payload_splitter.h"
#include "webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h"
#include "webrtc/modules/audio_coding/neteq/preemptive_expand.h"
#include "webrtc/modules/audio_coding/neteq/sync_buffer.h"
#include "webrtc/modules/audio_coding/neteq/timestamp_scaler.h"
@ -124,11 +124,11 @@ class NetEqImplTest : public ::testing::Test {
packet_buffer_ = deps.packet_buffer.get();
if (use_mock_payload_splitter_) {
std::unique_ptr<MockPayloadSplitter> mock(new MockPayloadSplitter);
std::unique_ptr<MockRedPayloadSplitter> mock(new MockRedPayloadSplitter);
mock_payload_splitter_ = mock.get();
deps.payload_splitter = std::move(mock);
deps.red_payload_splitter = std::move(mock);
}
payload_splitter_ = deps.payload_splitter.get();
red_payload_splitter_ = deps.red_payload_splitter.get();
deps.timestamp_scaler = std::unique_ptr<TimestampScaler>(
new TimestampScaler(*deps.decoder_database.get()));
@ -197,8 +197,8 @@ class NetEqImplTest : public ::testing::Test {
MockPacketBuffer* mock_packet_buffer_ = nullptr;
PacketBuffer* packet_buffer_ = nullptr;
bool use_mock_packet_buffer_ = true;
MockPayloadSplitter* mock_payload_splitter_ = nullptr;
PayloadSplitter* payload_splitter_ = nullptr;
MockRedPayloadSplitter* mock_payload_splitter_ = nullptr;
RedPayloadSplitter* red_payload_splitter_ = nullptr;
bool use_mock_payload_splitter_ = true;
};
@ -332,11 +332,6 @@ TEST_F(NetEqImplTest, InsertPacket) {
.WillOnce(Return(0));
}
// Expectations for payload splitter.
EXPECT_CALL(*mock_payload_splitter_, SplitFec(_, _))
.Times(2)
.WillRepeatedly(Return(PayloadSplitter::kOK));
// Insert first packet.
neteq_->InsertPacket(rtp_header, payload, kFirstReceiveTime);

67
webrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc
View File

@ -39,18 +39,57 @@ class MockAudioDecoder final : public AudioDecoder {
MOCK_METHOD0(Reset, void());
int PacketDuration(const uint8_t* encoded,
size_t encoded_len) const /* override */ {
return kPacketDuration;
class MockFrame : public AudioDecoder::EncodedAudioFrame {
public:
MockFrame(size_t num_channels) : num_channels_(num_channels) {}
size_t Duration() const override { return kPacketDuration; }
rtc::Optional<DecodeResult> Decode(
rtc::ArrayView<int16_t> decoded) const override {
const size_t output_size =
sizeof(int16_t) * kPacketDuration * num_channels_;
if (decoded.size() >= output_size) {
memset(decoded.data(), 0,
sizeof(int16_t) * kPacketDuration * num_channels_);
return rtc::Optional<DecodeResult>(
{kPacketDuration * num_channels_, kSpeech});
} else {
ADD_FAILURE() << "Expected decoded.size() to be >= output_size ("
<< decoded.size() << " vs. " << output_size << ")";
return rtc::Optional<DecodeResult>();
}
}
private:
const size_t num_channels_;
};
std::vector<ParseResult> ParsePayload(rtc::Buffer&& payload,
uint32_t timestamp) /* override */ {
std::vector<ParseResult> results;
if (fec_enabled_) {
std::unique_ptr<MockFrame> fec_frame(new MockFrame(num_channels_));
results.emplace_back(timestamp - kPacketDuration, 1,
std::move(fec_frame));
}
std::unique_ptr<MockFrame> frame(new MockFrame(num_channels_));
results.emplace_back(timestamp, 0, std::move(frame));
return results;
}
int PacketDurationRedundant(const uint8_t* encoded,
size_t encoded_len) const /* override */ {
int PacketDuration(const uint8_t* encoded, size_t encoded_len) const
/* override */ {
ADD_FAILURE() << "Since going through ParsePayload, PacketDuration should "
"never get called.";
return kPacketDuration;
}
bool PacketHasFec(
const uint8_t* encoded, size_t encoded_len) const /* override */ {
ADD_FAILURE() << "Since going through ParsePayload, PacketHasFec should "
"never get called.";
return fec_enabled_;
}
@ -63,24 +102,14 @@ class MockAudioDecoder final : public AudioDecoder {
bool fec_enabled() const { return fec_enabled_; }
protected:
// Override the following methods such that no actual payload is needed.
int DecodeInternal(const uint8_t* encoded,
size_t encoded_len,
int /*sample_rate_hz*/,
int sample_rate_hz,
int16_t* decoded,
SpeechType* speech_type) /* override */ {
*speech_type = kSpeech;
memset(decoded, 0, sizeof(int16_t) * kPacketDuration * Channels());
return kPacketDuration * Channels();
}
int DecodeRedundantInternal(const uint8_t* encoded,
size_t encoded_len,
int sample_rate_hz,
int16_t* decoded,
SpeechType* speech_type) /* override */ {
return DecodeInternal(encoded, encoded_len, sample_rate_hz, decoded,
speech_type);
ADD_FAILURE() << "Since going through ParsePayload, DecodeInternal should "
"never get called.";
return -1;
}
private:

55
webrtc/modules/audio_coding/neteq/packet.h
View File

@ -24,10 +24,52 @@ namespace webrtc {
// Struct for holding RTP packets.
struct Packet {
struct Priority {
Priority() : codec_level(0), red_level(0) {}
Priority(int codec_level, int red_level)
: codec_level(codec_level), red_level(red_level) {
CheckInvariant();
}
int codec_level;
int red_level;
// Priorities are sorted low-to-high, first on the level the codec
// prioritizes it, then on the level of RED packet it is; i.e. if it is a
// primary or secondary payload of a RED packet. For example: with Opus, an
// Fec packet (which the decoder prioritizes lower than a regular packet)
// will not be used if there is _any_ RED payload for the same
// timeframe. The highest priority packet will have levels {0, 0}. Negative
// priorities are not allowed.
bool operator<(const Priority& b) const {
CheckInvariant();
b.CheckInvariant();
if (codec_level == b.codec_level)
return red_level < b.red_level;
return codec_level < b.codec_level;
}
bool operator==(const Priority& b) const {
CheckInvariant();
b.CheckInvariant();
return codec_level == b.codec_level && red_level == b.red_level;
}
bool operator!=(const Priority& b) const { return !(*this == b); }
bool operator>(const Priority& b) const { return b < *this; }
bool operator<=(const Priority& b) const { return !(b > *this); }
bool operator>=(const Priority& b) const { return !(b < *this); }
private:
void CheckInvariant() const {
RTC_DCHECK_GE(codec_level, 0);
RTC_DCHECK_GE(red_level, 0);
}
};
RTPHeader header;
// Datagram excluding RTP header and header extension.
rtc::Buffer payload;
bool primary = true; // Primary, i.e., not redundant payload.
Priority priority;
std::unique_ptr<TickTimer::Stopwatch> waiting_time;
std::unique_ptr<AudioDecoder::EncodedAudioFrame> frame;
@ -41,17 +83,16 @@ struct Packet {
// primary payload is considered "smaller" than a secondary.
bool operator==(const Packet& rhs) const {
return (this->header.timestamp == rhs.header.timestamp &&
this->header.sequenceNumber == rhs.header.sequenceNumber &&
this->primary == rhs.primary);
this->header.sequenceNumber == rhs.header.sequenceNumber &&
this->priority == rhs.priority);
}
bool operator!=(const Packet& rhs) const { return !operator==(rhs); }
bool operator<(const Packet& rhs) const {
if (this->header.timestamp == rhs.header.timestamp) {
if (this->header.sequenceNumber == rhs.header.sequenceNumber) {
// Timestamp and sequence numbers are identical - deem the left
// hand side to be "smaller" (i.e., "earlier") if it is primary, and
// right hand side is not.
return (this->primary && !rhs.primary);
// Timestamp and sequence numbers are identical - deem the left hand
// side to be "smaller" (i.e., "earlier") if it has higher priority.
return this->priority < rhs.priority;
}
return (static_cast<uint16_t>(rhs.header.sequenceNumber
- this->header.sequenceNumber) < 0xFFFF / 2);

9
webrtc/modules/audio_coding/neteq/packet_buffer.cc
View File

@ -76,6 +76,9 @@ int PacketBuffer::InsertPacket(Packet* packet) {
return kInvalidPacket;
}
RTC_DCHECK_GE(packet->priority.codec_level, 0);
RTC_DCHECK_GE(packet->priority.red_level, 0);
int return_val = kOK;
packet->waiting_time = tick_timer_->GetNewStopwatch();
@ -262,7 +265,7 @@ int PacketBuffer::DiscardAllOldPackets(uint32_t timestamp_limit) {
void PacketBuffer::DiscardPacketsWithPayloadType(uint8_t payload_type) {
for (auto it = buffer_.begin(); it != buffer_.end(); /* */) {
Packet *packet = *it;
Packet* packet = *it;
if (packet->header.payloadType == payload_type) {
delete packet;
it = buffer_.erase(it);
@ -281,7 +284,9 @@ size_t PacketBuffer::NumSamplesInBuffer(size_t last_decoded_length) const {
size_t last_duration = last_decoded_length;
for (Packet* packet : buffer_) {
if (packet->frame) {
if (!packet->primary) {
// TODO(hlundin): Verify that it's fine to count all packets and remove
// this check.
if (packet->priority != Packet::Priority(0, 0)) {
continue;
}
size_t duration = packet->frame->Duration();

63
webrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc
View File

@ -60,7 +60,6 @@ Packet* PacketGenerator::NextPacket(int payload_size_bytes) {
packet->header.ssrc = 0x12345678;
packet->header.numCSRCs = 0;
packet->header.paddingLength = 0;
packet->primary = true;
packet->payload.SetSize(payload_size_bytes);
++seq_no_;
ts_ += frame_size_;
@ -284,7 +283,7 @@ TEST(PacketBuffer, ExtractOrderRedundancy) {
packet_facts[i].payload_type,
kFrameSize);
Packet* packet = gen.NextPacket(kPayloadLength);
packet->primary = packet_facts[i].primary;
packet->priority.red_level = packet_facts[i].primary ? 0 : 1;
EXPECT_EQ(PacketBuffer::kOK, buffer.InsertPacket(packet));
if (packet_facts[i].extract_order >= 0) {
expect_order[packet_facts[i].extract_order] = packet;
@ -512,8 +511,8 @@ TEST(PacketBuffer, Failures) {
// The function should return true if the first packet "goes before" the second.
TEST(PacketBuffer, ComparePackets) {
PacketGenerator gen(0, 0, 0, 10);
Packet* a = gen.NextPacket(10); // SN = 0, TS = 0.
Packet* b = gen.NextPacket(10); // SN = 1, TS = 10.
std::unique_ptr<Packet> a(gen.NextPacket(10)); // SN = 0, TS = 0.
std::unique_ptr<Packet> b(gen.NextPacket(10)); // SN = 1, TS = 10.
EXPECT_FALSE(*a == *b);
EXPECT_TRUE(*a != *b);
EXPECT_TRUE(*a < *b);
@ -556,10 +555,11 @@ TEST(PacketBuffer, ComparePackets) {
EXPECT_TRUE(*a <= *b);
EXPECT_FALSE(*a >= *b);
// Test equal timestamps and sequence numbers, but only 'b' is primary.
// Test equal timestamps and sequence numbers, but differing priorities.
a->header.sequenceNumber = b->header.sequenceNumber;
a->primary = false;
b->primary = true;
a->priority = {1, 0};
b->priority = {0, 0};
// a after b
EXPECT_FALSE(*a == *b);
EXPECT_TRUE(*a != *b);
EXPECT_FALSE(*a < *b);
@ -567,8 +567,53 @@ TEST(PacketBuffer, ComparePackets) {
EXPECT_FALSE(*a <= *b);
EXPECT_TRUE(*a >= *b);
delete a;
delete b;
std::unique_ptr<Packet> c(gen.NextPacket(0)); // SN = 2, TS = 20.
std::unique_ptr<Packet> d(gen.NextPacket(0)); // SN = 3, TS = 20.
c->header.timestamp = b->header.timestamp;
d->header.timestamp = b->header.timestamp;
c->header.sequenceNumber = b->header.sequenceNumber;
d->header.sequenceNumber = b->header.sequenceNumber;
c->priority = {1, 1};
d->priority = {0, 1};
// c after d
EXPECT_FALSE(*c == *d);
EXPECT_TRUE(*c != *d);
EXPECT_FALSE(*c < *d);
EXPECT_TRUE(*c > *d);
EXPECT_FALSE(*c <= *d);
EXPECT_TRUE(*c >= *d);
// c after a
EXPECT_FALSE(*c == *a);
EXPECT_TRUE(*c != *a);
EXPECT_FALSE(*c < *a);
EXPECT_TRUE(*c > *a);
EXPECT_FALSE(*c <= *a);
EXPECT_TRUE(*c >= *a);
// c after b
EXPECT_FALSE(*c == *b);
EXPECT_TRUE(*c != *b);
EXPECT_FALSE(*c < *b);
EXPECT_TRUE(*c > *b);
EXPECT_FALSE(*c <= *b);
EXPECT_TRUE(*c >= *b);
// a after d
EXPECT_FALSE(*a == *d);
EXPECT_TRUE(*a != *d);
EXPECT_FALSE(*a < *d);
EXPECT_TRUE(*a > *d);
EXPECT_FALSE(*a <= *d);
EXPECT_TRUE(*a >= *d);
// d after b
EXPECT_FALSE(*d == *b);
EXPECT_TRUE(*d != *b);
EXPECT_FALSE(*d < *b);
EXPECT_TRUE(*d > *b);
EXPECT_FALSE(*d <= *b);
EXPECT_TRUE(*d >= *b);
}
// Test the DeleteFirstPacket DeleteAllPackets methods.

139
webrtc/modules/audio_coding/neteq/payload_splitter.cc → webrtc/modules/audio_coding/neteq/red_payload_splitter.cc
View File

@ -8,12 +8,14 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include "webrtc/modules/audio_coding/neteq/payload_splitter.h"
#include "webrtc/modules/audio_coding/neteq/red_payload_splitter.h"
#include <assert.h>
#include <vector>
#include "webrtc/base/checks.h"
#include "webrtc/base/logging.h"
#include "webrtc/base/safe_conversions.h"
#include "webrtc/modules/audio_coding/neteq/decoder_database.h"
namespace webrtc {
@ -25,8 +27,11 @@ namespace webrtc {
// is replaced by the new ones in |new_packets|, so that |packet_list| becomes:
// {A1, A2, ..., B, C, ...}. The method then continues with B, and C, until all
// the original packets have been replaced by their split payloads.
int PayloadSplitter::SplitRed(PacketList* packet_list) {
int ret = kOK;
bool RedPayloadSplitter::SplitRed(PacketList* packet_list) {
// Too many RED blocks indicates that something is wrong. Clamp it at some
// reasonable value.
const size_t kMaxRedBlocks = 32;
bool ret = true;
PacketList::iterator it = packet_list->begin();
while (it != packet_list->end()) {
const Packet* red_packet = (*it);
@ -50,7 +55,6 @@ int PayloadSplitter::SplitRed(PacketList* packet_list) {
uint8_t payload_type;
uint32_t timestamp;
size_t payload_length;
bool primary;
};
std::vector<RedHeader> new_headers;
@ -67,17 +71,15 @@ int PayloadSplitter::SplitRed(PacketList* packet_list) {
++sum_length; // Account for RED header size of 1 byte.
new_header.timestamp = red_packet->header.timestamp;
new_header.payload_length = red_packet->payload.size() - sum_length;
new_header.primary = true; // Last block is always primary.
payload_ptr += 1; // Advance to first payload byte.
} else {
// Bits 8 through 21 are timestamp offset.
int timestamp_offset = (payload_ptr[1] << 6) +
((payload_ptr[2] & 0xFC) >> 2);
int timestamp_offset =
(payload_ptr[1] << 6) + ((payload_ptr[2] & 0xFC) >> 2);
new_header.timestamp = red_packet->header.timestamp - timestamp_offset;
// Bits 22 through 31 are payload length.
new_header.payload_length =
((payload_ptr[2] & 0x03) << 8) + payload_ptr[3];
new_header.primary = false;
payload_ptr += 4; // Advance to next RED header.
}
sum_length += new_header.payload_length;
@ -86,33 +88,41 @@ int PayloadSplitter::SplitRed(PacketList* packet_list) {
new_headers.push_back(new_header);
}
// Populate the new packets with payload data.
// |payload_ptr| now points at the first payload byte.
PacketList new_packets; // An empty list to store the split packets in.
for (const auto& new_header : new_headers) {
size_t payload_length = new_header.payload_length;
if (payload_ptr + payload_length >
red_packet->payload.data() + red_packet->payload.size()) {
// The block lengths in the RED headers do not match the overall packet
// length. Something is corrupt. Discard this and the remaining
// payloads from this packet.
LOG(LS_WARNING) << "SplitRed length mismatch";
ret = kRedLengthMismatch;
break;
if (new_headers.size() <= kMaxRedBlocks) {
// Populate the new packets with payload data.
// |payload_ptr| now points at the first payload byte.
PacketList new_packets; // An empty list to store the split packets in.
for (size_t i = 0; i != new_headers.size(); ++i) {
const auto& new_header = new_headers[i];
size_t payload_length = new_header.payload_length;
if (payload_ptr + payload_length >
red_packet->payload.data() + red_packet->payload.size()) {
// The block lengths in the RED headers do not match the overall
// packet length. Something is corrupt. Discard this and the remaining
// payloads from this packet.
LOG(LS_WARNING) << "SplitRed length mismatch";
ret = false;
break;
}
Packet* new_packet = new Packet;
new_packet->header = red_packet->header;
new_packet->header.timestamp = new_header.timestamp;
new_packet->header.payloadType = new_header.payload_type;
new_packet->priority.red_level =
rtc::checked_cast<int>((new_headers.size() - 1) - i);
new_packet->payload.SetData(payload_ptr, payload_length);
new_packets.push_front(new_packet);
payload_ptr += payload_length;
}
Packet* new_packet = new Packet;
new_packet->header = red_packet->header;
new_packet->header.timestamp = new_header.timestamp;
new_packet->header.payloadType = new_header.payload_type;
new_packet->primary = new_header.primary;
new_packet->payload.SetData(payload_ptr, payload_length);
new_packets.push_front(new_packet);
payload_ptr += payload_length;
// Insert new packets into original list, before the element pointed to by
// iterator |it|.
packet_list->splice(it, new_packets, new_packets.begin(),
new_packets.end());
} else {
LOG(LS_WARNING) << "SplitRed too many blocks: " << new_headers.size();
ret = false;
}
// Insert new packets into original list, before the element pointed to by
// iterator |it|.
packet_list->splice(it, new_packets, new_packets.begin(),
new_packets.end());
// Delete old packet payload.
delete (*it);
// Remove |it| from the packet list. This operation effectively moves the
@ -123,66 +133,9 @@ int PayloadSplitter::SplitRed(PacketList* packet_list) {
return ret;
}
int PayloadSplitter::SplitFec(PacketList* packet_list,
DecoderDatabase* decoder_database) {
PacketList::iterator it = packet_list->begin();
// Iterate through all packets in |packet_list|.
while (it != packet_list->end()) {
Packet* packet = (*it); // Just to make the notation more intuitive.
// Get codec type for this payload.
uint8_t payload_type = packet->header.payloadType;
const DecoderDatabase::DecoderInfo* info =
decoder_database->GetDecoderInfo(payload_type);
if (!info) {
LOG(LS_WARNING) << "SplitFec unknown payload type";
return kUnknownPayloadType;
}
// Not an FEC packet.
AudioDecoder* decoder = decoder_database->GetDecoder(payload_type);
// decoder should not return NULL, except for comfort noise payloads which
// are handled separately.
assert(decoder != NULL || decoder_database->IsComfortNoise(payload_type));
if (!decoder ||
!decoder->PacketHasFec(packet->payload.data(),
packet->payload.size())) {
++it;
continue;
}
switch (info->codec_type) {
case NetEqDecoder::kDecoderOpus:
case NetEqDecoder::kDecoderOpus_2ch: {
// The main payload of this packet should be decoded as a primary
// payload, even if it comes as a secondary payload in a RED packet.
packet->primary = true;
Packet* new_packet = new Packet;
new_packet->header = packet->header;
int duration = decoder->PacketDurationRedundant(packet->payload.data(),
packet->payload.size());
new_packet->header.timestamp -= duration;
new_packet->payload.SetData(packet->payload);
new_packet->primary = false;
// Waiting time should not be set here.
RTC_DCHECK(!packet->waiting_time);
packet_list->insert(it, new_packet);
break;
}
default: {
LOG(LS_WARNING) << "SplitFec wrong payload type";
return kFecSplitError;
}
}
++it;
}
return kOK;
}
int PayloadSplitter::CheckRedPayloads(PacketList* packet_list,
const DecoderDatabase& decoder_database) {
int RedPayloadSplitter::CheckRedPayloads(
PacketList* packet_list,
const DecoderDatabase& decoder_database) {
PacketList::iterator it = packet_list->begin();
int main_payload_type = -1;
int num_deleted_packets = 0;

41
webrtc/modules/audio_coding/neteq/payload_splitter.h → webrtc/modules/audio_coding/neteq/red_payload_splitter.h
View File

@ -8,8 +8,8 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_PAYLOAD_SPLITTER_H_
#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_PAYLOAD_SPLITTER_H_
#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_RED_PAYLOAD_SPLITTER_H_
#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_RED_PAYLOAD_SPLITTER_H_
#include "webrtc/base/constructormagic.h"
#include "webrtc/modules/audio_coding/neteq/packet.h"
@ -19,24 +19,14 @@ namespace webrtc {
// Forward declarations.
class DecoderDatabase;
// This class handles splitting of payloads into smaller parts.
// For RED and FEC the splitting is done internally. Other codecs' packets are
// split by calling AudioDecoder::SplitPacket.
class PayloadSplitter {
// This class handles splitting of RED payloads into smaller parts.
// Codec-specific packet splitting can be performed by
// AudioDecoder::ParsePayload.
class RedPayloadSplitter {
public:
enum SplitterReturnCodes {
kOK = 0,
kNoSplit = 1,
kFrameSplitError = -2,
kUnknownPayloadType = -3,
kRedLengthMismatch = -4,
kFecSplitError = -5,
};
RedPayloadSplitter() {}
PayloadSplitter() {}
virtual ~PayloadSplitter() {}
virtual ~RedPayloadSplitter() {}
// Splits each packet in |packet_list| into its separate RED payloads. Each
// RED payload is packetized into a Packet. The original elements in
@ -44,23 +34,18 @@ class PayloadSplitter {
// Note that all packets in |packet_list| must be RED payloads, i.e., have
// RED headers according to RFC 2198 at the very beginning of the payload.
// Returns kOK or an error.
virtual int SplitRed(PacketList* packet_list);
// Iterates through |packet_list| and, duplicate each audio payload that has
// FEC as new packet for redundant decoding. The decoder database is needed to
// get information about which payload type each packet contains.
virtual int SplitFec(PacketList* packet_list,
DecoderDatabase* decoder_database);
virtual bool SplitRed(PacketList* packet_list);
// Checks all packets in |packet_list|. Packets that are DTMF events or
// comfort noise payloads are kept. Except that, only one single payload type
// is accepted. Any packet with another payload type is discarded.
// is accepted. Any packet with another payload type is discarded. Returns
// the number of discarded packets.
virtual int CheckRedPayloads(PacketList* packet_list,
const DecoderDatabase& decoder_database);
private:
RTC_DISALLOW_COPY_AND_ASSIGN(PayloadSplitter);
RTC_DISALLOW_COPY_AND_ASSIGN(RedPayloadSplitter);
};
} // namespace webrtc
#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_PAYLOAD_SPLITTER_H_
#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_RED_PAYLOAD_SPLITTER_H_

176
webrtc/modules/audio_coding/neteq/payload_splitter_unittest.cc → webrtc/modules/audio_coding/neteq/red_payload_splitter_unittest.cc
View File

@ -8,9 +8,9 @@
* be found in the AUTHORS file in the root of the source tree.
*/
// Unit tests for PayloadSplitter class.
// Unit tests for RedPayloadSplitter class.
#include "webrtc/modules/audio_coding/neteq/payload_splitter.h"
#include "webrtc/modules/audio_coding/neteq/red_payload_splitter.h"
#include <assert.h>
@ -46,7 +46,8 @@ static const uint32_t kBaseTimestamp = 0x12345678;
// : |
// | |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
void CreateOpusFecPayload(uint8_t* payload, size_t payload_length,
void CreateOpusFecPayload(uint8_t* payload,
size_t payload_length,
uint8_t payload_value) {
if (payload_length < 2) {
return;
@ -121,8 +122,10 @@ Packet* CreateRedPayload(size_t num_payloads,
}
// Create a packet with all payload bytes set to |payload_value|.
Packet* CreatePacket(uint8_t payload_type, size_t payload_length,
uint8_t payload_value, bool opus_fec = false) {
Packet* CreatePacket(uint8_t payload_type,
size_t payload_length,
uint8_t payload_value,
bool opus_fec = false) {
Packet* packet = new Packet;
packet->header.payloadType = payload_type;
packet->header.timestamp = kBaseTimestamp;
@ -144,22 +147,34 @@ void VerifyPacket(const Packet* packet,
uint16_t sequence_number,
uint32_t timestamp,
uint8_t payload_value,
bool primary = true) {
Packet::Priority priority) {
EXPECT_EQ(payload_length, packet->payload.size());
EXPECT_EQ(payload_type, packet->header.payloadType);
EXPECT_EQ(sequence_number, packet->header.sequenceNumber);
EXPECT_EQ(timestamp, packet->header.timestamp);
EXPECT_EQ(primary, packet->primary);
EXPECT_EQ(priority, packet->priority);
ASSERT_FALSE(packet->payload.empty());
for (size_t i = 0; i < packet->payload.size(); ++i) {
ASSERT_EQ(payload_value, packet->payload.data()[i]);
}
}
void VerifyPacket(const Packet* packet,
size_t payload_length,
uint8_t payload_type,
uint16_t sequence_number,
uint32_t timestamp,
uint8_t payload_value,
bool primary) {
return VerifyPacket(packet, payload_length, payload_type, sequence_number,
timestamp, payload_value,
Packet::Priority{0, primary ? 0 : 1});
}
// Start of test definitions.
TEST(PayloadSplitter, CreateAndDestroy) {
PayloadSplitter* splitter = new PayloadSplitter;
TEST(RedPayloadSplitter, CreateAndDestroy) {
RedPayloadSplitter* splitter = new RedPayloadSplitter;
delete splitter;
}
@ -170,8 +185,8 @@ TEST(RedPayloadSplitter, OnePacketTwoPayloads) {
Packet* packet = CreateRedPayload(2, payload_types, kTimestampOffset);
PacketList packet_list;
packet_list.push_back(packet);
PayloadSplitter splitter;
EXPECT_EQ(PayloadSplitter::kOK, splitter.SplitRed(&packet_list));
RedPayloadSplitter splitter;
EXPECT_TRUE(splitter.SplitRed(&packet_list));
ASSERT_EQ(2u, packet_list.size());
// Check first packet. The first in list should always be the primary payload.
packet = packet_list.front();
@ -201,8 +216,8 @@ TEST(RedPayloadSplitter, TwoPacketsOnePayload) {
packet->header.timestamp += kTimestampOffset;
packet->header.sequenceNumber++;
packet_list.push_back(packet);
PayloadSplitter splitter;
EXPECT_EQ(PayloadSplitter::kOK, splitter.SplitRed(&packet_list));
RedPayloadSplitter splitter;
EXPECT_TRUE(splitter.SplitRed(&packet_list));
ASSERT_EQ(2u, packet_list.size());
// Check first packet.
packet = packet_list.front();
@ -239,43 +254,43 @@ TEST(RedPayloadSplitter, TwoPacketsThreePayloads) {
packet->header.timestamp += kTimestampOffset;
packet->header.sequenceNumber++;
packet_list.push_back(packet);
PayloadSplitter splitter;
EXPECT_EQ(PayloadSplitter::kOK, splitter.SplitRed(&packet_list));
RedPayloadSplitter splitter;
EXPECT_TRUE(splitter.SplitRed(&packet_list));
ASSERT_EQ(6u, packet_list.size());
// Check first packet, A1.
packet = packet_list.front();
VerifyPacket(packet, kPayloadLength, payload_types[2], kSequenceNumber,
kBaseTimestamp, 2, true);
kBaseTimestamp, 2, {0, 0});
delete packet;
packet_list.pop_front();
// Check second packet, A2.
packet = packet_list.front();
VerifyPacket(packet, kPayloadLength, payload_types[1], kSequenceNumber,
kBaseTimestamp - kTimestampOffset, 1, false);
kBaseTimestamp - kTimestampOffset, 1, {0, 1});
delete packet;
packet_list.pop_front();
// Check third packet, A3.
packet = packet_list.front();
VerifyPacket(packet, kPayloadLength, payload_types[0], kSequenceNumber,
kBaseTimestamp - 2 * kTimestampOffset, 0, false);
kBaseTimestamp - 2 * kTimestampOffset, 0, {0, 2});
delete packet;
packet_list.pop_front();
// Check fourth packet, B1.
packet = packet_list.front();
VerifyPacket(packet, kPayloadLength, payload_types[2], kSequenceNumber + 1,
kBaseTimestamp + kTimestampOffset, 2, true);
kBaseTimestamp + kTimestampOffset, 2, {0, 0});
delete packet;
packet_list.pop_front();
// Check fifth packet, B2.
packet = packet_list.front();
VerifyPacket(packet, kPayloadLength, payload_types[1], kSequenceNumber + 1,
kBaseTimestamp, 1, false);
kBaseTimestamp, 1, {0, 1});
delete packet;
packet_list.pop_front();
// Check sixth packet, B3.
packet = packet_list.front();
VerifyPacket(packet, kPayloadLength, payload_types[0], kSequenceNumber + 1,
kBaseTimestamp - kTimestampOffset, 0, false);
kBaseTimestamp - kTimestampOffset, 0, {0, 2});
delete packet;
}
@ -305,7 +320,7 @@ TEST(RedPayloadSplitter, CheckRedPayloads) {
decoder_database.RegisterPayload(2, NetEqDecoder::kDecoderAVT, "avt");
decoder_database.RegisterPayload(3, NetEqDecoder::kDecoderILBC, "ilbc");
PayloadSplitter splitter;
RedPayloadSplitter splitter;
splitter.CheckRedPayloads(&packet_list, decoder_database);
ASSERT_EQ(3u, packet_list.size()); // Should have dropped the last packet.
@ -332,124 +347,13 @@ TEST(RedPayloadSplitter, WrongPayloadLength) {
packet->payload.SetSize(packet->payload.size() - (kPayloadLength + 1));
PacketList packet_list;
packet_list.push_back(packet);
PayloadSplitter splitter;
EXPECT_EQ(PayloadSplitter::kRedLengthMismatch,
splitter.SplitRed(&packet_list));
RedPayloadSplitter splitter;
EXPECT_FALSE(splitter.SplitRed(&packet_list));
ASSERT_EQ(1u, packet_list.size());
// Check first packet.
packet = packet_list.front();
VerifyPacket(packet, kPayloadLength, payload_types[0], kSequenceNumber,
kBaseTimestamp - 2 * kTimestampOffset, 0, false);
delete packet;
packet_list.pop_front();
}
TEST(FecPayloadSplitter, MixedPayload) {
PacketList packet_list;
DecoderDatabase decoder_database(CreateBuiltinAudioDecoderFactory());
decoder_database.RegisterPayload(0, NetEqDecoder::kDecoderOpus, "opus");
decoder_database.RegisterPayload(1, NetEqDecoder::kDecoderPCMu, "pcmu");
Packet* packet = CreatePacket(0, 10, 0xFF, true);
packet_list.push_back(packet);
packet = CreatePacket(0, 10, 0); // Non-FEC Opus payload.
packet_list.push_back(packet);
packet = CreatePacket(1, 10, 0); // Non-Opus payload.
packet_list.push_back(packet);
PayloadSplitter splitter;
EXPECT_EQ(PayloadSplitter::kOK,
splitter.SplitFec(&packet_list, &decoder_database));
EXPECT_EQ(4u, packet_list.size());
// Check first packet.
packet = packet_list.front();
EXPECT_EQ(0, packet->header.payloadType);
EXPECT_EQ(kBaseTimestamp - 20 * 48, packet->header.timestamp);
EXPECT_EQ(10U, packet->payload.size());
EXPECT_FALSE(packet->primary);
delete packet;
packet_list.pop_front();
// Check second packet.
packet = packet_list.front();
EXPECT_EQ(0, packet->header.payloadType);
EXPECT_EQ(kBaseTimestamp, packet->header.timestamp);
EXPECT_EQ(10U, packet->payload.size());
EXPECT_TRUE(packet->primary);
delete packet;
packet_list.pop_front();
// Check third packet.
packet = packet_list.front();
VerifyPacket(packet, 10, 0, kSequenceNumber, kBaseTimestamp, 0, true);
delete packet;
packet_list.pop_front();
// Check fourth packet.
packet = packet_list.front();
VerifyPacket(packet, 10, 1, kSequenceNumber, kBaseTimestamp, 0, true);
delete packet;
}
TEST(FecPayloadSplitter, EmbedFecInRed) {
PacketList packet_list;
DecoderDatabase decoder_database(CreateBuiltinAudioDecoderFactory());
const int kTimestampOffset = 20 * 48; // 20 ms * 48 kHz.
uint8_t payload_types[] = {0, 0};
decoder_database.RegisterPayload(0, NetEqDecoder::kDecoderOpus, "opus");
Packet* packet = CreateRedPayload(2, payload_types, kTimestampOffset, true);
packet_list.push_back(packet);
PayloadSplitter splitter;
EXPECT_EQ(PayloadSplitter::kOK,
splitter.SplitRed(&packet_list));
EXPECT_EQ(PayloadSplitter::kOK,
splitter.SplitFec(&packet_list, &decoder_database));
EXPECT_EQ(4u, packet_list.size());
// Check first packet. FEC packet copied from primary payload in RED.
packet = packet_list.front();
EXPECT_EQ(0, packet->header.payloadType);
EXPECT_EQ(kBaseTimestamp - kTimestampOffset, packet->header.timestamp);
EXPECT_EQ(kPayloadLength, packet->payload.size());
EXPECT_FALSE(packet->primary);
EXPECT_EQ(packet->payload[3], 1);
delete packet;
packet_list.pop_front();
// Check second packet. Normal packet copied from primary payload in RED.
packet = packet_list.front();
EXPECT_EQ(0, packet->header.payloadType);
EXPECT_EQ(kBaseTimestamp, packet->header.timestamp);
EXPECT_EQ(kPayloadLength, packet->payload.size());
EXPECT_TRUE(packet->primary);
EXPECT_EQ(packet->payload[3], 1);
delete packet;
packet_list.pop_front();
// Check third packet. FEC packet copied from secondary payload in RED.
packet = packet_list.front();
EXPECT_EQ(0, packet->header.payloadType);
EXPECT_EQ(kBaseTimestamp - 2 * kTimestampOffset, packet->header.timestamp);
EXPECT_EQ(kPayloadLength, packet->payload.size());
EXPECT_FALSE(packet->primary);
EXPECT_EQ(packet->payload[3], 0);
delete packet;
packet_list.pop_front();
// Check fourth packet. Normal packet copied from primary payload in RED.
packet = packet_list.front();
EXPECT_EQ(0, packet->header.payloadType);
EXPECT_EQ(kBaseTimestamp - kTimestampOffset, packet->header.timestamp);
EXPECT_EQ(kPayloadLength, packet->payload.size());
EXPECT_TRUE(packet->primary);
EXPECT_EQ(packet->payload[3], 0);
kBaseTimestamp - 2 * kTimestampOffset, 0, {0, 2});
delete packet;
packet_list.pop_front();
}