diff --git a/webrtc/modules/audio_coding/codecs/opus/interface/opus_interface.h b/webrtc/modules/audio_coding/codecs/opus/interface/opus_interface.h index 8226f45439..7998fdbdeb 100644 --- a/webrtc/modules/audio_coding/codecs/opus/interface/opus_interface.h +++ b/webrtc/modules/audio_coding/codecs/opus/interface/opus_interface.h @@ -60,6 +60,49 @@ int16_t WebRtcOpus_Encode(OpusEncInst* inst, int16_t* audio_in, int16_t samples, int16_t WebRtcOpus_SetBitRate(OpusEncInst* inst, int32_t rate); /**************************************************************************** + * WebRtcOpus_SetPacketLossRate(...) + * + * This function configures the encoder's expected packet loss percentage. + * + * Input: + * - inst : Encoder context + * - loss_rate : loss percentage in the range 0-100, inclusive. + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_SetPacketLossRate(OpusEncInst* inst, int32_t loss_rate); + +/* TODO(minyue): Check whether an API to check the FEC and the packet loss rate + * is needed. It might not be very useful since there are not many use cases and + * the caller can always maintain the states. */ + +/**************************************************************************** + * WebRtcOpus_EnableFec() + * + * This function enables FEC for encoding. + * + * Input: + * - inst : Encoder context + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_EnableFec(OpusEncInst* inst); + +/**************************************************************************** + * WebRtcOpus_DisableFec() + * + * This function disables FEC for encoding. + * + * Input: + * - inst : Encoder context + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_DisableFec(OpusEncInst* inst); + +/* * WebRtcOpus_SetComplexity(...) * * This function adjusts the computational complexity. The effect is the same as @@ -128,6 +171,7 @@ int16_t WebRtcOpus_Decode(OpusDecInst* inst, const int16_t* encoded, int16_t WebRtcOpus_DecodeSlave(OpusDecInst* inst, const int16_t* encoded, int16_t encoded_bytes, int16_t* decoded, int16_t* audio_type); + /**************************************************************************** * WebRtcOpus_DecodePlc(...) * TODO(tlegrand): Remove master and slave functions when NetEq4 is in place. @@ -152,6 +196,28 @@ int16_t WebRtcOpus_DecodePlcMaster(OpusDecInst* inst, int16_t* decoded, int16_t WebRtcOpus_DecodePlcSlave(OpusDecInst* inst, int16_t* decoded, int16_t number_of_lost_frames); +/**************************************************************************** + * WebRtcOpus_DecodeFec(...) + * + * This function decodes the FEC data from an Opus packet into one or more audio + * frames at the ACM interface's sampling rate (32 kHz). + * + * Input: + * - inst : Decoder context + * - encoded : Encoded data + * - encoded_bytes : Bytes in encoded vector + * + * Output: + * - decoded : The decoded vector (previous frame) + * + * Return value : >0 - Samples per channel in decoded vector + * 0 - No FEC data in the packet + * -1 - Error + */ +int16_t WebRtcOpus_DecodeFec(OpusDecInst* inst, const uint8_t* encoded, + int16_t encoded_bytes, int16_t* decoded, + int16_t* audio_type); + /**************************************************************************** * WebRtcOpus_DurationEst(...) * @@ -167,6 +233,40 @@ int WebRtcOpus_DurationEst(OpusDecInst* inst, const uint8_t* payload, int payload_length_bytes); +/* TODO(minyue): Check whether it is needed to add a decoder context to the + * arguments, like WebRtcOpus_DurationEst(...). In fact, the packet itself tells + * the duration. The decoder context in WebRtcOpus_DurationEst(...) is not used. + * So it may be advisable to remove it from WebRtcOpus_DurationEst(...). */ + +/**************************************************************************** + * WebRtcOpus_FecDurationEst(...) + * + * This function calculates the duration of the FEC data within an opus packet. + * Input: + * - payload : Encoded data pointer + * - payload_length_bytes : Bytes of encoded data + * + * Return value : >0 - The duration of the FEC data in the + * packet in samples. + * 0 - No FEC data in the packet. + */ +int WebRtcOpus_FecDurationEst(const uint8_t* payload, + int payload_length_bytes); + +/**************************************************************************** + * WebRtcOpus_PacketHasFec(...) + * + * This function detects if an opus packet has FEC. + * Input: + * - payload : Encoded data pointer + * - payload_length_bytes : Bytes of encoded data + * + * Return value : 0 - the packet does NOT contain FEC. + * 1 - the packet contains FEC. + */ +int WebRtcOpus_PacketHasFec(const uint8_t* payload, + int payload_length_bytes); + #ifdef __cplusplus } // extern "C" #endif diff --git a/webrtc/modules/audio_coding/codecs/opus/opus.gypi b/webrtc/modules/audio_coding/codecs/opus/opus.gypi index 406870232b..b1dedd7d4a 100644 --- a/webrtc/modules/audio_coding/codecs/opus/opus.gypi +++ b/webrtc/modules/audio_coding/codecs/opus/opus.gypi @@ -32,4 +32,26 @@ ], }, ], + 'conditions': [ + ['include_tests==1', { + 'targets': [ + { + 'target_name': 'webrtc_opus_fec_test', + 'type': 'executable', + 'dependencies': [ + 'webrtc_opus', + '<(webrtc_root)/common_audio/common_audio.gyp:common_audio', + '<(webrtc_root)/test/test.gyp:test_support_main', + '<(DEPTH)/testing/gtest.gyp:gtest', + ], + 'include_dirs': [ + '<(webrtc_root)', + ], + 'sources': [ + 'opus_fec_test.cc', + ], + }, + ], + }], + ], } diff --git a/webrtc/modules/audio_coding/codecs/opus/opus_fec_test.cc b/webrtc/modules/audio_coding/codecs/opus/opus_fec_test.cc new file mode 100644 index 0000000000..fb4cb04f36 --- /dev/null +++ b/webrtc/modules/audio_coding/codecs/opus/opus_fec_test.cc @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "testing/gtest/include/gtest/gtest.h" +#include "webrtc/modules/audio_coding/codecs/opus/interface/opus_interface.h" +#include "webrtc/test/testsupport/fileutils.h" +#include "webrtc/system_wrappers/interface/scoped_ptr.h" + +using ::std::string; +using ::std::tr1::tuple; +using ::std::tr1::make_tuple; +using ::std::tr1::get; +using ::testing::TestWithParam; +using ::testing::ValuesIn; + +namespace webrtc { + +// Define coding parameter as . +typedef tuple coding_param; +typedef struct mode mode; + +struct mode { + bool fec; + uint8_t target_packet_loss_rate; +}; + +const int kOpusBlockDurationMs = 20; +const int kOpusInputSamplingKhz = 48; +const int kOpusOutputSamplingKhz = 32; + +class OpusFecTest : public TestWithParam { + protected: + OpusFecTest(); + + virtual void SetUp(); + virtual void TearDown(); + + virtual void EncodeABlock(); + + virtual void DecodeABlock(bool lost_previous, bool lost_current); + + int block_duration_ms_; + int input_sampling_khz_; + int output_sampling_khz_; + + // Number of samples-per-channel in a frame. + int input_length_sample_; + + // Expected output number of samples-per-channel in a frame. + int output_length_sample_; + + int channels_; + int bit_rate_; + + size_t data_pointer_; + size_t loop_length_samples_; + int max_bytes_; + int encoded_bytes_; + + WebRtcOpusEncInst* opus_encoder_; + WebRtcOpusDecInst* opus_decoder_; + + string in_filename_; + + scoped_ptr in_data_; + scoped_ptr out_data_; + scoped_ptr bit_stream_; +}; + +void OpusFecTest::SetUp() { + channels_ = get<0>(GetParam()); + bit_rate_ = get<1>(GetParam()); + printf("Coding %d channel signal at %d bps.\n", channels_, bit_rate_); + + in_filename_ = test::ResourcePath(get<2>(GetParam()), get<3>(GetParam())); + + FILE* fp = fopen(in_filename_.c_str(), "rb"); + ASSERT_FALSE(fp == NULL); + + // Obtain file size. + fseek(fp, 0, SEEK_END); + loop_length_samples_ = ftell(fp) / sizeof(int16_t); + rewind(fp); + + // Allocate memory to contain the whole file. + in_data_.reset(new int16_t[loop_length_samples_ + + input_length_sample_ * channels_]); + + // Copy the file into the buffer. + ASSERT_EQ(fread(&in_data_[0], sizeof(int16_t), loop_length_samples_, fp), + loop_length_samples_); + fclose(fp); + + // The audio will be used in a looped manner. To ease the acquisition of an + // audio frame that crosses the end of the excerpt, we add an extra block + // length of samples to the end of the array, starting over again from the + // beginning of the array. Audio frames cross the end of the excerpt always + // appear as a continuum of memory. + memcpy(&in_data_[loop_length_samples_], &in_data_[0], + input_length_sample_ * channels_ * sizeof(int16_t)); + + // Maximum number of bytes in output bitstream. + max_bytes_ = input_length_sample_ * channels_ * sizeof(int16_t); + + out_data_.reset(new int16_t[2 * output_length_sample_ * channels_]); + bit_stream_.reset(new uint8_t[max_bytes_]); + + // Create encoder memory. + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(&opus_encoder_, channels_)); + EXPECT_EQ(0, WebRtcOpus_DecoderCreate(&opus_decoder_, channels_)); + // Set bitrate. + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_encoder_, bit_rate_)); +} + +void OpusFecTest::TearDown() { + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +OpusFecTest::OpusFecTest() + : block_duration_ms_(kOpusBlockDurationMs), + input_sampling_khz_(kOpusInputSamplingKhz), + output_sampling_khz_(kOpusOutputSamplingKhz), + input_length_sample_(block_duration_ms_ * input_sampling_khz_), + output_length_sample_(block_duration_ms_ * output_sampling_khz_), + data_pointer_(0), + max_bytes_(0), + encoded_bytes_(0), + opus_encoder_(NULL), + opus_decoder_(NULL) { +} + +void OpusFecTest::EncodeABlock() { + int16_t value = WebRtcOpus_Encode(opus_encoder_, + &in_data_[data_pointer_], + input_length_sample_, + max_bytes_, &bit_stream_[0]); + EXPECT_GT(value, 0); + + encoded_bytes_ = value; +} + +void OpusFecTest::DecodeABlock(bool lost_previous, bool lost_current) { + int16_t audio_type; + int16_t value_1 = 0, value_2 = 0; + + if (lost_previous) { + // Decode previous frame. + if (!lost_current && + WebRtcOpus_PacketHasFec(&bit_stream_[0], encoded_bytes_) == 1) { + value_1 = WebRtcOpus_DecodeFec(opus_decoder_, &bit_stream_[0], + encoded_bytes_, &out_data_[0], + &audio_type); + } else { + value_1 = WebRtcOpus_DecodePlc(opus_decoder_, &out_data_[0], 1); + } + EXPECT_EQ(output_length_sample_, value_1); + } + + if (!lost_current) { + // Decode current frame. + value_2 = WebRtcOpus_DecodeNew(opus_decoder_, &bit_stream_[0], + encoded_bytes_, + &out_data_[value_1 * channels_], + &audio_type); + EXPECT_EQ(output_length_sample_, value_2); + } +} + +TEST_P(OpusFecTest, RandomPacketLossTest) { + const int kDurationMs = 200000; + int time_now_ms, fec_frames; + int actual_packet_loss_rate; + bool lost_current, lost_previous; + mode mode_set[3] = {{true, 0}, + {false, 0}, + {true, 50}}; + + lost_current = false; + for (int i = 0; i < 3; i++) { + if (mode_set[i].fec) { + EXPECT_EQ(0, WebRtcOpus_EnableFec(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_SetPacketLossRate(opus_encoder_, + mode_set[i].target_packet_loss_rate)); + printf("FEC is ON, target at packet loss rate %d percent.\n", + mode_set[i].target_packet_loss_rate); + } else { + EXPECT_EQ(0, WebRtcOpus_DisableFec(opus_encoder_)); + printf("FEC is OFF.\n"); + } + // In this test, we let the target packet loss rate match the actual rate. + actual_packet_loss_rate = mode_set[i].target_packet_loss_rate; + // Run every mode a certain time. + time_now_ms = 0; + fec_frames = 0; + while (time_now_ms < kDurationMs) { + // Encode & decode. + EncodeABlock(); + + // Check if payload has FEC. + int16_t fec = WebRtcOpus_PacketHasFec(&bit_stream_[0], encoded_bytes_); + + // If FEC is disabled or the target packet loss rate is set to 0, there + // should be no FEC in the bit stream. + if (!mode_set[i].fec || mode_set[i].target_packet_loss_rate == 0) { + EXPECT_EQ(fec, 0); + } else if (fec == 1) { + fec_frames++; + } + + lost_previous = lost_current; + lost_current = rand() < actual_packet_loss_rate * (RAND_MAX / 100); + DecodeABlock(lost_previous, lost_current); + + time_now_ms += block_duration_ms_; + + // |data_pointer_| is incremented and wrapped across + // |loop_length_samples_|. + data_pointer_ = (data_pointer_ + input_length_sample_ * channels_) % + loop_length_samples_; + } + if (mode_set[i].fec) { + printf("%.2f percent frames has FEC.\n", + static_cast(fec_frames) * block_duration_ms_ / 2000); + } + } +} + +const coding_param param_set[] = + {make_tuple(1, 64000, string("audio_coding/testfile32kHz"), + string("pcm")), + make_tuple(1, 32000, string("audio_coding/testfile32kHz"), + string("pcm")), + make_tuple(2, 64000, string("audio_coding/teststereo32kHz"), + string("pcm"))}; + +// 64 kbps, stereo +INSTANTIATE_TEST_CASE_P(AllTest, OpusFecTest, + ValuesIn(param_set)); + +} // namespace webrtc diff --git a/webrtc/modules/audio_coding/codecs/opus/opus_interface.c b/webrtc/modules/audio_coding/codecs/opus/opus_interface.c index c4380a3f09..24fc4fc405 100644 --- a/webrtc/modules/audio_coding/codecs/opus/opus_interface.c +++ b/webrtc/modules/audio_coding/codecs/opus/opus_interface.c @@ -109,6 +109,31 @@ int16_t WebRtcOpus_SetBitRate(OpusEncInst* inst, int32_t rate) { } } +int16_t WebRtcOpus_SetPacketLossRate(OpusEncInst* inst, int32_t loss_rate) { + if (inst) { + return opus_encoder_ctl(inst->encoder, + OPUS_SET_PACKET_LOSS_PERC(loss_rate)); + } else { + return -1; + } +} + +int16_t WebRtcOpus_EnableFec(OpusEncInst* inst) { + if (inst) { + return opus_encoder_ctl(inst->encoder, OPUS_SET_INBAND_FEC(1)); + } else { + return -1; + } +} + +int16_t WebRtcOpus_DisableFec(OpusEncInst* inst) { + if (inst) { + return opus_encoder_ctl(inst->encoder, OPUS_SET_INBAND_FEC(0)); + } else { + return -1; + } +} + int16_t WebRtcOpus_SetComplexity(OpusEncInst* inst, int32_t complexity) { if (inst) { return opus_encoder_ctl(inst->encoder, OPUS_SET_COMPLEXITY(complexity)); @@ -225,6 +250,23 @@ static int DecodeNative(OpusDecoder* inst, const int16_t* encoded, return -1; } +static int DecodeFec(OpusDecoder* inst, const int16_t* encoded, + int16_t encoded_bytes, int frame_size, + int16_t* decoded, int16_t* audio_type) { + unsigned char* coded = (unsigned char*) encoded; + opus_int16* audio = (opus_int16*) decoded; + + int res = opus_decode(inst, coded, encoded_bytes, audio, frame_size, 1); + + /* TODO(tlegrand): set to DTX for zero-length packets? */ + *audio_type = 0; + + if (res > 0) { + return res; + } + return -1; +} + /* Resample from 48 to 32 kHz. Length of state is assumed to be * kWebRtcOpusStateSize (7). */ @@ -550,6 +592,52 @@ int16_t WebRtcOpus_DecodePlcSlave(OpusDecInst* inst, int16_t* decoded, return resampled_samples; } +int16_t WebRtcOpus_DecodeFec(OpusDecInst* inst, const uint8_t* encoded, + int16_t encoded_bytes, int16_t* decoded, + int16_t* audio_type) { + /* |buffer| is big enough for 120 ms (the largest Opus packet size) of stereo + * audio at 48 kHz. */ + int16_t buffer[kWebRtcOpusMaxFrameSize]; + int16_t* coded = (int16_t*)encoded; + int decoded_samples; + int resampled_samples; + int fec_samples; + + if (WebRtcOpus_PacketHasFec(encoded, encoded_bytes) != 1) { + return 0; + } + + fec_samples = opus_packet_get_samples_per_frame(encoded, 48000); + + /* Decode to a temporary buffer. */ + decoded_samples = DecodeFec(inst->decoder_left, coded, encoded_bytes, + fec_samples, buffer, audio_type); + if (decoded_samples < 0) { + return -1; + } + + /* If mono case, just do a regular call to the decoder. + * If stereo, we need to de-interleave the stereo output into blocks with + * left and right channel. Each block is resampled to 32 kHz, and then + * interleaved again. */ + if (inst->channels == 2) { + /* De-interleave and resample. */ + resampled_samples = WebRtcOpus_DeInterleaveResample(inst, + buffer, + decoded_samples, + decoded); + } else { + /* Resample from 48 kHz to 32 kHz. Filter state memory for left channel is + * used for mono signals. */ + resampled_samples = WebRtcOpus_Resample48to32(buffer, + decoded_samples, + inst->state_48_32_left, + decoded); + } + + return resampled_samples; +} + int WebRtcOpus_DurationEst(OpusDecInst* inst, const uint8_t* payload, int payload_length_bytes) { @@ -570,3 +658,79 @@ int WebRtcOpus_DurationEst(OpusDecInst* inst, samples = samples * 2 / 3; return samples; } + +int WebRtcOpus_FecDurationEst(const uint8_t* payload, + int payload_length_bytes) { + int samples; + if (WebRtcOpus_PacketHasFec(payload, payload_length_bytes) != 1) { + return 0; + } + + samples = opus_packet_get_samples_per_frame(payload, 48000); + if (samples < 480 || samples > 5760) { + /* Invalid payload duration. */ + return 0; + } + /* Compensate for the down-sampling from 48 kHz to 32 kHz. + * This should be removed when the resampling in WebRtcOpus_Decode is + * removed. */ + samples = samples * 2 / 3; + return samples; +} + +int WebRtcOpus_PacketHasFec(const uint8_t* payload, + int payload_length_bytes) { + int frames, channels, payload_length_ms; + int n; + opus_int16 frame_sizes[48]; + const unsigned char *frame_data[48]; + + if (payload == NULL || payload_length_bytes <= 0) + return 0; + + /* In CELT_ONLY mode, packets should not have FEC. */ + if (payload[0] & 0x80) + return 0; + + payload_length_ms = opus_packet_get_samples_per_frame(payload, 48000) / 48; + if (10 > payload_length_ms) + payload_length_ms = 10; + + channels = opus_packet_get_nb_channels(payload); + + switch (payload_length_ms) { + case 10: + case 20: { + frames = 1; + break; + } + case 40: { + frames = 2; + break; + } + case 60: { + frames = 3; + break; + } + default: { + return 0; // It is actually even an invalid packet. + } + } + + /* The following is to parse the LBRR flags. */ + if (opus_packet_parse(payload, payload_length_bytes, NULL, frame_data, + frame_sizes, NULL) < 0) { + return 0; + } + + if (frame_sizes[0] <= 1) { + return 0; + } + + for (n = 0; n < channels; n++) { + if (frame_data[0][0] & (0x80 >> ((n + 1) * (frames + 1) - 1))) + return 1; + } + + return 0; +} diff --git a/webrtc/modules/audio_coding/codecs/opus/opus_unittest.cc b/webrtc/modules/audio_coding/codecs/opus/opus_unittest.cc index 5b29c2336c..ed876cd105 100644 --- a/webrtc/modules/audio_coding/codecs/opus/opus_unittest.cc +++ b/webrtc/modules/audio_coding/codecs/opus/opus_unittest.cc @@ -286,6 +286,47 @@ TEST_F(OpusTest, OpusDecodeInit) { EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_stereo_decoder_new_)); } +TEST_F(OpusTest, OpusEnableDisableFec) { + // Test without creating encoder memory. + EXPECT_EQ(-1, WebRtcOpus_EnableFec(opus_mono_encoder_)); + EXPECT_EQ(-1, WebRtcOpus_DisableFec(opus_stereo_encoder_)); + + // Create encoder memory, try with different bitrates. + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(&opus_mono_encoder_, 1)); + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(&opus_stereo_encoder_, 2)); + + EXPECT_EQ(0, WebRtcOpus_EnableFec(opus_mono_encoder_)); + EXPECT_EQ(0, WebRtcOpus_EnableFec(opus_stereo_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DisableFec(opus_mono_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DisableFec(opus_stereo_encoder_)); + + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_mono_encoder_)); + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_stereo_encoder_)); +} + +TEST_F(OpusTest, OpusSetPacketLossRate) { + // Test without creating encoder memory. + EXPECT_EQ(-1, WebRtcOpus_SetPacketLossRate(opus_mono_encoder_, 50)); + EXPECT_EQ(-1, WebRtcOpus_SetPacketLossRate(opus_stereo_encoder_, 50)); + + // Create encoder memory, try with different bitrates. + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(&opus_mono_encoder_, 1)); + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(&opus_stereo_encoder_, 2)); + + EXPECT_EQ(0, WebRtcOpus_SetPacketLossRate(opus_mono_encoder_, 50)); + EXPECT_EQ(0, WebRtcOpus_SetPacketLossRate(opus_stereo_encoder_, 50)); + EXPECT_EQ(-1, WebRtcOpus_SetPacketLossRate(opus_mono_encoder_, -1)); + EXPECT_EQ(-1, WebRtcOpus_SetPacketLossRate(opus_stereo_encoder_, -1)); + EXPECT_EQ(-1, WebRtcOpus_SetPacketLossRate(opus_mono_encoder_, 101)); + EXPECT_EQ(-1, WebRtcOpus_SetPacketLossRate(opus_stereo_encoder_, 101)); + + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_mono_encoder_)); + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_stereo_encoder_)); +} + + // PLC in mono mode. TEST_F(OpusTest, OpusDecodePlcMono) { // Create encoder memory.