diff --git a/src/common_types.h b/src/common_types.h index e59b985944..5ce4542033 100644 --- a/src/common_types.h +++ b/src/common_types.h @@ -434,6 +434,7 @@ enum RawVideoType enum { kConfigParameterSize = 128}; enum { kPayloadNameSize = 32}; enum { kMaxSimulcastStreams = 4}; +enum { kMaxTemporalStreams = 4}; // H.263 specific struct VideoCodecH263 diff --git a/src/modules/video_coding/codecs/interface/video_codec_interface.h b/src/modules/video_coding/codecs/interface/video_codec_interface.h index 9a96a23ce7..87d34bfd4b 100644 --- a/src/modules/video_coding/codecs/interface/video_codec_interface.h +++ b/src/modules/video_coding/codecs/interface/video_codec_interface.h @@ -33,6 +33,7 @@ struct CodecSpecificInfoVP8 bool nonReference; WebRtc_UWord8 simulcastIdx; WebRtc_UWord8 temporalIdx; + int tl0PicIdx; // Negative value to skip tl0PicIdx WebRtc_Word8 keyIdx; // negative value to skip keyIdx }; diff --git a/src/modules/video_coding/codecs/vp8/main/interface/vp8.h b/src/modules/video_coding/codecs/vp8/main/interface/vp8.h index 24b1a26412..944fa2fcea 100644 --- a/src/modules/video_coding/codecs/vp8/main/interface/vp8.h +++ b/src/modules/video_coding/codecs/vp8/main/interface/vp8.h @@ -29,6 +29,7 @@ struct vpx_codec_cx_pkt; namespace webrtc { +class TemporalLayers; class ReferencePictureSelection; @@ -139,7 +140,7 @@ public: WebRtc_Word32 length); private: -// Call encoder initialize function and set control settings. + // Call encoder initialize function and set control settings. WebRtc_Word32 InitAndSetControlSettings(); void PopulateCodecSpecific(CodecSpecificInfo* codec_specific, @@ -174,6 +175,7 @@ private: WebRtc_UWord32 _rcMaxIntraTarget; int _tokenPartitions; ReferencePictureSelection* _rps; + TemporalLayers* _temporalLayers; vpx_codec_ctx_t* _encoder; vpx_codec_enc_cfg_t* _cfg; @@ -270,9 +272,7 @@ private: vpx_ref_frame_t* _refFrame; int _propagationCnt; bool _latestKeyFrameComplete; - };// end of VP8Decoder class - } // namespace webrtc #endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_H_ diff --git a/src/modules/video_coding/codecs/vp8/main/source/temporal_layers.cc b/src/modules/video_coding/codecs/vp8/main/source/temporal_layers.cc new file mode 100644 index 0000000000..b5d59f8001 --- /dev/null +++ b/src/modules/video_coding/codecs/vp8/main/source/temporal_layers.cc @@ -0,0 +1,202 @@ +/* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. +* +* Use of this source code is governed by a BSD-style license +* that can be found in the LICENSE file in the root of the source +* tree. An additional intellectual property rights grant can be found +* in the file PATENTS. All contributing project authors may +* be found in the AUTHORS file in the root of the source tree. +*/ + +#include "temporal_layers.h" + +#include +#include +#include + +#include "module_common_types.h" +#include "video_codec_interface.h" + +#include "vpx/vpx_encoder.h" +#include "vpx/vp8cx.h" + +namespace webrtc { + +TemporalLayers::TemporalLayers(int numberOfTemporalLayers) + : number_of_temporal_layers_(numberOfTemporalLayers), + temporal_ids_length_(0), + temporal_pattern_length_(0), + tl0_pic_idx_(rand()), + pattern_idx_(255) { + assert(kMaxTemporalStreams >= numberOfTemporalLayers); + memset(temporal_ids_, 0, sizeof(temporal_ids_)); + memset(temporal_pattern_, 0, sizeof(temporal_pattern_)); +} + +bool TemporalLayers::ConfigureBitrates(int bitrateKbit, + vpx_codec_enc_cfg_t* cfg) { + switch (number_of_temporal_layers_) { + case 0: + case 1: + // Do nothing. + break; + case 2: + temporal_ids_length_ = 2; + temporal_ids_[0] = 0; + temporal_ids_[1] = 1; + cfg->ts_number_layers = number_of_temporal_layers_; + cfg->ts_periodicity = temporal_ids_length_; + // Split stream 60% 40%. + // Bitrate API for VP8 is the agregated bitrate for all lower layers. + cfg->ts_target_bitrate[0] = bitrateKbit * 3 / 5; + cfg->ts_target_bitrate[1] = bitrateKbit; + cfg->ts_rate_decimator[0] = 2; + cfg->ts_rate_decimator[1] = 1; + memcpy(cfg->ts_layer_id, + temporal_ids_, + sizeof(unsigned int) * temporal_ids_length_); + temporal_pattern_length_ = 8; + temporal_pattern_[0] = kTemporalUpdateLast; + temporal_pattern_[1] = kTemporalUpdateGoldenWithoutDependency; + temporal_pattern_[2] = kTemporalUpdateLast; + temporal_pattern_[3] = kTemporalUpdateGolden; + temporal_pattern_[4] = kTemporalUpdateLast; + temporal_pattern_[5] = kTemporalUpdateGolden; + temporal_pattern_[6] = kTemporalUpdateLast; + temporal_pattern_[7] = kTemporalUpdateNone; + break; + case 3: + temporal_ids_length_ = 4; + temporal_ids_[0] = 0; + temporal_ids_[1] = 2; + temporal_ids_[2] = 1; + temporal_ids_[3] = 2; + cfg->ts_number_layers = number_of_temporal_layers_; + cfg->ts_periodicity = temporal_ids_length_; + // Split stream 40% 20% 40%. + // Bitrate API for VP8 is the agregated bitrate for all lower layers. + cfg->ts_target_bitrate[0] = bitrateKbit * 2 / 5; + cfg->ts_target_bitrate[1] = bitrateKbit * 3 / 5; + cfg->ts_target_bitrate[2] = bitrateKbit; + cfg->ts_rate_decimator[0] = 4; + cfg->ts_rate_decimator[1] = 2; + cfg->ts_rate_decimator[2] = 1; + memcpy(cfg->ts_layer_id, + temporal_ids_, + sizeof(unsigned int) * temporal_ids_length_); + temporal_pattern_length_ = 8; + temporal_pattern_[0] = kTemporalUpdateLast; + temporal_pattern_[1] = kTemporalUpdateAltrefWithoutDependency; + temporal_pattern_[2] = kTemporalUpdateGoldenWithoutDependency; + temporal_pattern_[3] = kTemporalUpdateAltref; + temporal_pattern_[4] = kTemporalUpdateLast; + temporal_pattern_[5] = kTemporalUpdateAltref; + temporal_pattern_[6] = kTemporalUpdateGolden; + temporal_pattern_[7] = kTemporalUpdateNone; + break; + case 4: + temporal_ids_length_ = 8; + temporal_ids_[0] = 0; + temporal_ids_[1] = 3; + temporal_ids_[2] = 2; + temporal_ids_[3] = 3; + temporal_ids_[4] = 1; + temporal_ids_[5] = 3; + temporal_ids_[6] = 2; + temporal_ids_[7] = 3; + // Split stream 25% 15% 20% 40%. + // Bitrate API for VP8 is the agregated bitrate for all lower layers. + cfg->ts_number_layers = 4; + cfg->ts_periodicity = temporal_ids_length_; + cfg->ts_target_bitrate[0] = bitrateKbit / 4; + cfg->ts_target_bitrate[1] = bitrateKbit * 2 / 5; + cfg->ts_target_bitrate[2] = bitrateKbit * 3 / 5; + cfg->ts_target_bitrate[3] = bitrateKbit; + cfg->ts_rate_decimator[0] = 8; + cfg->ts_rate_decimator[1] = 4; + cfg->ts_rate_decimator[2] = 2; + cfg->ts_rate_decimator[3] = 1; + memcpy(cfg->ts_layer_id, + temporal_ids_, + sizeof(unsigned int) * temporal_ids_length_); + temporal_pattern_length_ = 16; + temporal_pattern_[0] = kTemporalUpdateLast; + temporal_pattern_[1] = kTemporalUpdateNone; + temporal_pattern_[2] = kTemporalUpdateAltrefWithoutDependency; + temporal_pattern_[3] = kTemporalUpdateNone; + temporal_pattern_[4] = kTemporalUpdateGoldenWithoutDependency; + temporal_pattern_[5] = kTemporalUpdateNone; + temporal_pattern_[6] = kTemporalUpdateAltref; + temporal_pattern_[7] = kTemporalUpdateNone; + temporal_pattern_[8] = kTemporalUpdateLast; + temporal_pattern_[9] = kTemporalUpdateNone; + temporal_pattern_[10] = kTemporalUpdateAltref; + temporal_pattern_[11] = kTemporalUpdateNone; + temporal_pattern_[12] = kTemporalUpdateGolden; + temporal_pattern_[13] = kTemporalUpdateNone; + temporal_pattern_[14] = kTemporalUpdateAltref; + temporal_pattern_[15] = kTemporalUpdateNone; + break; + default: + assert(false); + return false; + } + return true; +} + +int TemporalLayers::EncodeFlags() { + assert(number_of_temporal_layers_ > 1); + assert(kMaxTemporalPattern >= temporal_pattern_length_); + assert(0 < temporal_pattern_length_); + + int flags = 0; + int patternIdx = ++pattern_idx_ % temporal_pattern_length_; + assert(kMaxTemporalPattern >= patternIdx); + switch (temporal_pattern_[patternIdx]) { + case kTemporalUpdateLast: + flags |= VP8_EFLAG_NO_UPD_GF; + flags |= VP8_EFLAG_NO_UPD_ARF; + flags |= VP8_EFLAG_NO_REF_GF; + flags |= VP8_EFLAG_NO_REF_ARF; + break; + case kTemporalUpdateGoldenWithoutDependency: + flags |= VP8_EFLAG_NO_REF_GF; + // Deliberetely no break here. + case kTemporalUpdateGolden: + flags |= VP8_EFLAG_NO_REF_ARF; + flags |= VP8_EFLAG_NO_UPD_ARF; + flags |= VP8_EFLAG_NO_UPD_LAST; + break; + case kTemporalUpdateAltrefWithoutDependency: + flags |= VP8_EFLAG_NO_REF_ARF; + // Deliberetely no break here. + case kTemporalUpdateAltref: + flags |= VP8_EFLAG_NO_UPD_GF; + flags |= VP8_EFLAG_NO_UPD_LAST; + break; + case kTemporalUpdateNone: + flags |= VP8_EFLAG_NO_UPD_GF; + flags |= VP8_EFLAG_NO_UPD_ARF; + flags |= VP8_EFLAG_NO_UPD_LAST; + flags |= VP8_EFLAG_NO_UPD_ENTROPY; + break; + } + return flags; +} + +void TemporalLayers::PopulateCodecSpecific(bool key_frame, + CodecSpecificInfoVP8 *vp8_info) { + assert(number_of_temporal_layers_ > 1); + assert(0 < temporal_ids_length_); + + if (key_frame) { + // Keyframe is always temporal layer 0 + vp8_info->temporalIdx = 0; + } else { + vp8_info->temporalIdx = temporal_ids_[pattern_idx_ % temporal_ids_length_]; + } + if (vp8_info->temporalIdx == 0) { + tl0_pic_idx_++; + } + vp8_info->tl0PicIdx = tl0_pic_idx_; +} +} // namespace webrtc diff --git a/src/modules/video_coding/codecs/vp8/main/source/temporal_layers.h b/src/modules/video_coding/codecs/vp8/main/source/temporal_layers.h new file mode 100644 index 0000000000..6be0582a08 --- /dev/null +++ b/src/modules/video_coding/codecs/vp8/main/source/temporal_layers.h @@ -0,0 +1,65 @@ +/* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. +* +* Use of this source code is governed by a BSD-style license +* that can be found in the LICENSE file in the root of the source +* tree. An additional intellectual property rights grant can be found +* in the file PATENTS. All contributing project authors may +* be found in the AUTHORS file in the root of the source tree. +*/ +/* +* This file defines classes for doing temporal layers with VP8. +*/ +#ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_ +#define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_ + +#include + + // VPX forward declaration +typedef struct vpx_codec_enc_cfg vpx_codec_enc_cfg_t; + +namespace webrtc { + +struct CodecSpecificInfoVP8; + +class TemporalLayers { + public: + TemporalLayers(int number_of_temporal_layers); + + // Returns the recommended VP8 encode flags needed. May refresh the decoder + // and/or update the reference buffers. + int EncodeFlags(); + + bool ConfigureBitrates(int bitrate_kbit, vpx_codec_enc_cfg_t* cfg); + + void PopulateCodecSpecific(bool key_frame, CodecSpecificInfoVP8 *vp8_info); + + private: + enum TemporalReferences { + // Highest enhancement layer. + kTemporalUpdateNone = 5, + // Second enhancement layer. + kTemporalUpdateAltref = 4, + // Second enhancement layer without dependency on previous frames in + // the second enhancement layer. + kTemporalUpdateAltrefWithoutDependency = 3, + // First enhancement layer. + kTemporalUpdateGolden = 2, + // First enhancement layer without dependency on previous frames in + // the first enhancement layer. + kTemporalUpdateGoldenWithoutDependency = 1, + // Base layer. + kTemporalUpdateLast = 0, + }; + enum { kMaxTemporalPattern = 16 }; + + int number_of_temporal_layers_; + int temporal_ids_length_; + int temporal_ids_[kMaxTemporalPattern]; + int temporal_pattern_length_; + TemporalReferences temporal_pattern_[kMaxTemporalPattern]; + uint8_t tl0_pic_idx_; + uint8_t pattern_idx_; +}; +} // namespace webrtc +#endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_ + diff --git a/src/modules/video_coding/codecs/vp8/main/source/temporal_layers_unittest.cc b/src/modules/video_coding/codecs/vp8/main/source/temporal_layers_unittest.cc new file mode 100644 index 0000000000..45d9ede488 --- /dev/null +++ b/src/modules/video_coding/codecs/vp8/main/source/temporal_layers_unittest.cc @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + + +#include "gtest/gtest.h" +#include "temporal_layers.h" +#include "video_codec_interface.h" + +#include "vpx/vpx_encoder.h" +#include "vpx/vp8cx.h" + +namespace webrtc { + +enum { + kTemporalUpdateLast = VP8_EFLAG_NO_UPD_GF | + VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_REF_GF | + VP8_EFLAG_NO_REF_ARF, + kTemporalUpdateGoldenWithoutDependency = VP8_EFLAG_NO_REF_GF | + VP8_EFLAG_NO_REF_ARF | + VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_UPD_LAST, + kTemporalUpdateGolden = VP8_EFLAG_NO_REF_ARF | + VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_UPD_LAST, + kTemporalUpdateAltrefWithoutDependency = VP8_EFLAG_NO_REF_ARF | + VP8_EFLAG_NO_UPD_GF | + VP8_EFLAG_NO_UPD_LAST, + kTemporalUpdateAltref = VP8_EFLAG_NO_UPD_GF | + VP8_EFLAG_NO_UPD_LAST, + kTemporalUpdateNone = VP8_EFLAG_NO_UPD_GF | + VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_UPD_LAST | + VP8_EFLAG_NO_UPD_ENTROPY, +}; + +TEST(TemporalLayersTest, 2Layers) { + TemporalLayers tl(2); + vpx_codec_enc_cfg_t cfg; + CodecSpecificInfoVP8 vp8_info; + tl.ConfigureBitrates(500, &cfg); + + int expected_flags[16] = { kTemporalUpdateLast, + kTemporalUpdateGoldenWithoutDependency, + kTemporalUpdateLast, + kTemporalUpdateGolden, + kTemporalUpdateLast, + kTemporalUpdateGolden, + kTemporalUpdateLast, + kTemporalUpdateNone, + kTemporalUpdateLast, + kTemporalUpdateGoldenWithoutDependency, + kTemporalUpdateLast, + kTemporalUpdateGolden, + kTemporalUpdateLast, + kTemporalUpdateGolden, + kTemporalUpdateLast, + kTemporalUpdateNone + }; + int expected_temporal_idx[16] = + { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 }; + + for (int i = 0; i < 16; ++i) { + EXPECT_EQ(expected_flags[i], tl.EncodeFlags()); + tl.PopulateCodecSpecific(false, &vp8_info); + EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); + } +} + +TEST(TemporalLayersTest, 3Layers) { + TemporalLayers tl(3); + vpx_codec_enc_cfg_t cfg; + CodecSpecificInfoVP8 vp8_info; + tl.ConfigureBitrates(500, &cfg); + + int expected_flags[16] = { kTemporalUpdateLast, + kTemporalUpdateAltrefWithoutDependency, + kTemporalUpdateGoldenWithoutDependency, + kTemporalUpdateAltref, + kTemporalUpdateLast, + kTemporalUpdateAltref, + kTemporalUpdateGolden, + kTemporalUpdateNone, + kTemporalUpdateLast, + kTemporalUpdateAltrefWithoutDependency, + kTemporalUpdateGoldenWithoutDependency, + kTemporalUpdateAltref, + kTemporalUpdateLast, + kTemporalUpdateAltref, + kTemporalUpdateGolden, + kTemporalUpdateNone, + }; + int expected_temporal_idx[16] = + { 0, 2, 1, 2, 0, 2, 1, 2, 0, 2, 1, 2, 0, 2, 1, 2 }; + + for (int i = 0; i < 16; ++i) { + EXPECT_EQ(expected_flags[i], tl.EncodeFlags()); + tl.PopulateCodecSpecific(false, &vp8_info); + EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); + } +} + +TEST(TemporalLayersTest, 4Layers) { + TemporalLayers tl(4); + vpx_codec_enc_cfg_t cfg; + CodecSpecificInfoVP8 vp8_info; + tl.ConfigureBitrates(500, &cfg); + int expected_flags[16] = { + kTemporalUpdateLast, + kTemporalUpdateNone, + kTemporalUpdateAltrefWithoutDependency, + kTemporalUpdateNone, + kTemporalUpdateGoldenWithoutDependency, + kTemporalUpdateNone, + kTemporalUpdateAltref, + kTemporalUpdateNone, + kTemporalUpdateLast, + kTemporalUpdateNone, + kTemporalUpdateAltref, + kTemporalUpdateNone, + kTemporalUpdateGolden, + kTemporalUpdateNone, + kTemporalUpdateAltref, + kTemporalUpdateNone, + }; + int expected_temporal_idx[16] = + { 0, 3, 2, 3, 1, 3, 2, 3, 0, 3, 2, 3, 1, 3, 2, 3 }; + + for (int i = 0; i < 16; ++i) { + EXPECT_EQ(expected_flags[i], tl.EncodeFlags()); + tl.PopulateCodecSpecific(false, &vp8_info); + EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); + } +} + +TEST(TemporalLayersTest, KeyFrame) { + TemporalLayers tl(3); + vpx_codec_enc_cfg_t cfg; + CodecSpecificInfoVP8 vp8_info; + tl.ConfigureBitrates(500, &cfg); + + int expected_flags[8] = { + kTemporalUpdateLast, + kTemporalUpdateAltrefWithoutDependency, + kTemporalUpdateGoldenWithoutDependency, + kTemporalUpdateAltref, + kTemporalUpdateLast, + kTemporalUpdateAltref, + kTemporalUpdateGolden, + kTemporalUpdateNone, + }; + int expected_temporal_idx[8] = + { 0, 0, 0, 0, 0, 0, 0, 2}; + + for (int i = 0; i < 7; ++i) { + EXPECT_EQ(expected_flags[i], tl.EncodeFlags()); + tl.PopulateCodecSpecific(true, &vp8_info); + EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); + } + EXPECT_EQ(expected_flags[7], tl.EncodeFlags()); + tl.PopulateCodecSpecific(false, &vp8_info); + EXPECT_EQ(expected_temporal_idx[7], vp8_info.temporalIdx); +} +} // namespace webrtc diff --git a/src/modules/video_coding/codecs/vp8/main/source/vp8.cc b/src/modules/video_coding/codecs/vp8/main/source/vp8.cc index 50c3e08609..ffa64c9b3b 100644 --- a/src/modules/video_coding/codecs/vp8/main/source/vp8.cc +++ b/src/modules/video_coding/codecs/vp8/main/source/vp8.cc @@ -22,6 +22,7 @@ #include "module_common_types.h" #include "reference_picture_selection.h" +#include "temporal_layers.h" #include "tick_util.h" #include "vpx/vpx_encoder.h" #include "vpx/vpx_decoder.h" @@ -48,6 +49,7 @@ VP8Encoder::VP8Encoder(): _rcMaxIntraTarget(0), _tokenPartitions(VP8_ONE_TOKENPARTITION), _rps(new ReferencePictureSelection), + _temporalLayers(NULL), _encoder(NULL), _cfg(NULL), _raw(NULL) @@ -111,6 +113,11 @@ VP8Encoder::Release() delete _raw; _raw = NULL; } + if (_temporalLayers != NULL) + { + delete _temporalLayers; + _temporalLayers = NULL; + } _inited = false; return WEBRTC_VIDEO_CODEC_OK; @@ -163,21 +170,12 @@ VP8Encoder::SetRates(WebRtc_UWord32 newBitRateKbit, WebRtc_UWord32 newFrameRate) { newBitRateKbit = _maxBitRateKbit; } - _cfg->rc_target_bitrate = newBitRateKbit; // in kbit/s -/* TODO(pwestin) use number of temoral layers config - int ids[3] = {0,1,2}; - _cfg->ts_number_layers = 3; - _cfg->ts_periodicity = 3; - _cfg->ts_target_bitrate[0] = (newBitRateKbit*2/5); - _cfg->ts_target_bitrate[1] = (newBitRateKbit*3/5); - _cfg->ts_target_bitrate[2] = (newBitRateKbit); - _cfg->ts_rate_decimator[0] = 4; - _cfg->ts_rate_decimator[1] = 2; - _cfg->ts_rate_decimator[2] = 1; - memcpy(_cfg->ts_layer_id, ids, sizeof(ids)); -*/ + if (_temporalLayers) + { + _temporalLayers->ConfigureBitrates(newBitRateKbit, _cfg); + } _maxFrameRate = newFrameRate; // update encoder context @@ -240,8 +238,15 @@ VP8Encoder::InitEncode(const VideoCodec* inst, _width = inst->width; _height = inst->height; - // random start 16 bits is enough - _pictureID = ((WebRtc_UWord16)rand()) % 0x7FFF; + if (inst->codecSpecific.VP8.numberOfTemporalLayers > 1) + { + assert(_temporalLayers == NULL); + _temporalLayers = + new TemporalLayers(inst->codecSpecific.VP8.numberOfTemporalLayers); + } + + // random start 16 bits is enough. + _pictureID = ((WebRtc_UWord16)rand()) & 0x7FFF; // allocate memory for encoded image if (_encodedImage._buffer != NULL) @@ -258,7 +263,6 @@ VP8Encoder::InitEncode(const VideoCodec* inst, { return WEBRTC_VIDEO_CODEC_ERROR; } - _cfg->g_w = inst->width; _cfg->g_h = inst->height; if (_maxBitRateKbit > 0 && @@ -270,19 +274,10 @@ VP8Encoder::InitEncode(const VideoCodec* inst, { _cfg->rc_target_bitrate = inst->startBitrate; // in kbit/s } -/* TODO(pwestin) use number of temoral layers config - int ids[3] = {0,1,2}; - _cfg->ts_number_layers = 3; - _cfg->ts_periodicity = 3; - _cfg->ts_target_bitrate[0] = (inst->startBitrate*2/5); - _cfg->ts_target_bitrate[1] = (inst->startBitrate*3/5); - _cfg->ts_target_bitrate[2] = (inst->startBitrate); - _cfg->ts_rate_decimator[0] = 4; - _cfg->ts_rate_decimator[1] = 2; - _cfg->ts_rate_decimator[2] = 1; - memcpy(_cfg->ts_layer_id, ids, sizeof(ids)); -*/ - + if (_temporalLayers) + { + _temporalLayers->ConfigureBitrates(inst->startBitrate, _cfg); + } // setting the time base of the codec _cfg->g_timebase.num = 1; _cfg->g_timebase.den = 90000; @@ -328,7 +323,6 @@ VP8Encoder::InitEncode(const VideoCodec* inst, _cfg->kf_mode = VPX_KF_AUTO; _cfg->kf_max_dist = 3000; } - switch (inst->codecSpecific.VP8.complexity) { case kComplexityHigh: @@ -352,14 +346,12 @@ VP8Encoder::InitEncode(const VideoCodec* inst, break; } } - _rps->Init(); return InitAndSetControlSettings(); - - } + WebRtc_Word32 VP8Encoder::InitAndSetControlSettings() { @@ -369,7 +361,7 @@ VP8Encoder::InitAndSetControlSettings() // TODO(holmer): We should make a smarter decision on the number of // partitions. Eight is probably not the optimal number for low resolution // video. - _tokenPartitions = VP8_EIGHT_TOKENPARTITION; + #if WEBRTC_LIBVPX_VERSION >= 971 flags |= VPX_CODEC_USE_OUTPUT_PARTITION; #endif @@ -444,10 +436,15 @@ VP8Encoder::Encode(const RawImage& inputImage, _raw->planes[PLANE_V] = &inputImage._buffer[_height * _width * 5 >> 2]; int flags = 0; - if (frameTypes && *frameTypes == kKeyFrame) { + if (_temporalLayers) { + flags |= _temporalLayers->EncodeFlags(); + } + bool sendKeyFrame = frameTypes && (*frameTypes == kKeyFrame); + if (sendKeyFrame) + { // Key frame request from caller. // Will update both golden and alt-ref. - flags |= VPX_EFLAG_FORCE_KF; + flags = VPX_EFLAG_FORCE_KF; } else if (_feedbackModeOn && codecSpecificInfo) { // Handle RPSI and SLI messages and set up the appropriate encode flags. bool sendRefresh = false; @@ -490,10 +487,16 @@ void VP8Encoder::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, codec_specific->codecType = kVideoCodecVP8; CodecSpecificInfoVP8 *vp8Info = &(codec_specific->codecSpecific.VP8); vp8Info->pictureId = _pictureID; - vp8Info->simulcastIdx = _simulcastIdx;; - vp8Info->temporalIdx = kNoTemporalIdx; // TODO(pwestin) need to populate this + vp8Info->simulcastIdx = _simulcastIdx; vp8Info->keyIdx = kNoKeyIdx; // TODO(hlundin) populate this vp8Info->nonReference = (pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE); + if (_temporalLayers) { + _temporalLayers->PopulateCodecSpecific( + (pkt.data.frame.flags & VPX_FRAME_IS_KEY) ? true : false, vp8Info); + } else { + vp8Info->temporalIdx = kNoTemporalIdx; + vp8Info->tl0PicIdx = kNoTl0PicIdx; + } _pictureID = (_pictureID + 1) % 0x7FFF; // prepare next } @@ -568,6 +571,7 @@ VP8Encoder::GetEncodedPartitions(const RawImage& input_image) { RTPFragmentationHeader frag_info; frag_info.VerifyAndAllocateFragmentationHeader((1 << _tokenPartitions) + 1); CodecSpecificInfo codecSpecific; + const vpx_codec_cx_pkt_t *pkt = NULL; while ((pkt = vpx_codec_get_cx_data(_encoder, &iter)) != NULL) { switch(pkt->kind) { diff --git a/src/modules/video_coding/codecs/vp8/main/source/vp8.gypi b/src/modules/video_coding/codecs/vp8/main/source/vp8.gypi index 481a454528..4ffefcddf4 100644 --- a/src/modules/video_coding/codecs/vp8/main/source/vp8.gypi +++ b/src/modules/video_coding/codecs/vp8/main/source/vp8.gypi @@ -50,6 +50,8 @@ 'reference_picture_selection.cc', '../interface/vp8.h', '../interface/vp8_simulcast.h', + 'temporal_layers.h', + 'temporal_layers.cc', 'vp8.cc', 'vp8_simulcast.cc', ], @@ -103,6 +105,7 @@ ], 'sources': [ 'reference_picture_selection_unittest.cc', + 'temporal_layers_unittest.cc', ], }, ], # targets diff --git a/src/modules/video_coding/main/source/generic_encoder.cc b/src/modules/video_coding/main/source/generic_encoder.cc index 8c29e9727e..2838a86f79 100644 --- a/src/modules/video_coding/main/source/generic_encoder.cc +++ b/src/modules/video_coding/main/source/generic_encoder.cc @@ -267,6 +267,8 @@ void VCMEncodedFrameCallback::CopyCodecSpecific(const CodecSpecificInfo& info, info.codecSpecific.VP8.nonReference; (*rtp)->codecHeader.VP8.temporalIdx = info.codecSpecific.VP8.temporalIdx; + (*rtp)->codecHeader.VP8.tl0PicIdx = + info.codecSpecific.VP8.tl0PicIdx; (*rtp)->codecHeader.VP8.keyIdx = info.codecSpecific.VP8.keyIdx; (*rtp)->simulcastIdx = info.codecSpecific.VP8.simulcastIdx; diff --git a/src/video_engine/test/auto_test/interface/tb_external_transport.h b/src/video_engine/test/auto_test/interface/tb_external_transport.h index ce4b4074bc..1a71b43790 100644 --- a/src/video_engine/test/auto_test/interface/tb_external_transport.h +++ b/src/video_engine/test/auto_test/interface/tb_external_transport.h @@ -44,6 +44,7 @@ public: WebRtc_Word32& numDroppedPackets, WebRtc_Word32& numRtcpPackets); + void SetTemporalToggle(unsigned char layers); void EnableSSRCCheck(); unsigned int ReceivedSSRC(); @@ -87,6 +88,14 @@ private: webrtc::ListWrapper _rtpPackets; webrtc::ListWrapper _rtcpPackets; + unsigned char _temporalLayers; + unsigned short _seqNum; + unsigned short _sendPID; + unsigned char _receivedPID; + bool _switchLayer; + unsigned char _currentRelayLayer; + unsigned int _lastTimeMs; + bool _checkSSRC; WebRtc_UWord32 _lastSSRC; bool _filterSSRC; diff --git a/src/video_engine/test/auto_test/source/tb_external_transport.cc b/src/video_engine/test/auto_test/source/tb_external_transport.cc index 5ac18f7e69..f50f2eb16a 100644 --- a/src/video_engine/test/auto_test/source/tb_external_transport.cc +++ b/src/video_engine/test/auto_test/source/tb_external_transport.cc @@ -14,9 +14,11 @@ #include "tb_external_transport.h" +#include // printf #include // rand +#include + #if defined(WEBRTC_LINUX) || defined(__linux__) -#include #include #endif #if defined(WEBRTC_MAC) @@ -48,6 +50,13 @@ TbExternalTransport::TbExternalTransport(webrtc::ViENetwork& vieNetwork) : _dropCount(0), _rtpPackets(), _rtcpPackets(), + _temporalLayers(0), + _seqNum(0), + _sendPID(0), + _receivedPID(0), + _switchLayer(false), + _currentRelayLayer(0), + _lastTimeMs(webrtc::TickTime::MillisecondTimestamp()), _checkSSRC(false), _lastSSRC(0), _filterSSRC(false), @@ -84,7 +93,68 @@ int TbExternalTransport::SendPacket(int channel, const void *data, int len) ssrc += ptr[11]; if (ssrc != _SSRC) { - return len; // return len to avoif error in trace file + return len; // return len to avoid error in trace file + } + } + if (_temporalLayers) { + // parse out vp8 temporal layers + // 12 bytes RTP + WebRtc_UWord8* ptr = (WebRtc_UWord8*)data; + + if (ptr[12] & 0x80 && // X-bit + ptr[13] & 0x20) // T-bit + { + int offset = 1; + if (ptr[13] & 0x80) // PID-bit + { + offset++; + if (ptr[14] & 0x80) // 2 byte PID + { + offset++; + } + } + if (ptr[13] & 0x40) + { + offset++; + } + unsigned char TID = (ptr[13 + offset] >> 5); + unsigned int timeMs = NowMs(); + + // Every 5 second switch layer + if (_lastTimeMs + 5000 < timeMs) + { + _lastTimeMs = timeMs; + _switchLayer = true; + } + // Switch at the non ref frame + if (_switchLayer && (ptr[12] & 0x20)) + { // N-bit + _currentRelayLayer++; + if (_currentRelayLayer >= _temporalLayers) + _currentRelayLayer = 0; + + _switchLayer = false; + printf("\t Switching to layer:%d\n", _currentRelayLayer); + } + if (_currentRelayLayer < TID) + { + return len; // return len to avoid error in trace file + } + if (ptr[14] & 0x80) // 2 byte PID + { + if(_receivedPID != ptr[15]) + { + _sendPID++; + _receivedPID = ptr[15]; + } + } else + { + if(_receivedPID != ptr[14]) + { + _sendPID++; + _receivedPID = ptr[14]; + } + } } } _statCrit.Enter(); @@ -103,6 +173,24 @@ int TbExternalTransport::SendPacket(int channel, const void *data, int len) VideoPacket* newPacket = new VideoPacket(); memcpy(newPacket->packetBuffer, data, len); + + if (_temporalLayers) + { + // rewrite seqNum + newPacket->packetBuffer[2] = _seqNum >> 8; + newPacket->packetBuffer[3] = _seqNum; + _seqNum++; + + // rewrite PID + if (newPacket->packetBuffer[14] & 0x80) // 2 byte PID + { + newPacket->packetBuffer[14] = (_sendPID >> 8) | 0x80; + newPacket->packetBuffer[15] = _sendPID; + } else + { + newPacket->packetBuffer[14] = (_sendPID & 0x7f); + } + } newPacket->length = len; newPacket->channel = channel; @@ -114,6 +202,12 @@ int TbExternalTransport::SendPacket(int channel, const void *data, int len) return len; } +// Set to 0 to disable. +void TbExternalTransport::SetTemporalToggle(unsigned char layers) +{ + _temporalLayers = layers; +} + int TbExternalTransport::SendRTCPPacket(int channel, const void *data, int len) { _statCrit.Enter(); diff --git a/src/video_engine/test/auto_test/source/vie_autotest_loopback.cc b/src/video_engine/test/auto_test/source/vie_autotest_loopback.cc index 4b7e0554bf..ed4883c536 100644 --- a/src/video_engine/test/auto_test/source/vie_autotest_loopback.cc +++ b/src/video_engine/test/auto_test/source/vie_autotest_loopback.cc @@ -212,7 +212,6 @@ int VideoEngineSampleCode(void* window1, void* window2) printf("ERROR in ViERTP_RTCP::SetKeyFrameRequestMethod\n"); return -1; } - error = ptrViERtpRtcp->SetTMMBRStatus(videoChannel, true); if (error == -1) { @@ -343,29 +342,36 @@ int VideoEngineSampleCode(void* window1, void* window2) switch (resolnOption) { case 1: - videoCodec.width = 176; - videoCodec.height = 144; - break; - + videoCodec.width = 176; + videoCodec.height = 144; + break; case 2: - videoCodec.width = 352; - videoCodec.height = 288; - break; - + videoCodec.width = 352; + videoCodec.height = 288; + break; case 3: - videoCodec.width = 640; - videoCodec.height = 480; - break; - + videoCodec.width = 640; + videoCodec.height = 480; + break; case 4: - videoCodec.width = 704; - videoCodec.height = 576; - break; - + videoCodec.width = 704; + videoCodec.height = 576; + break; case 5: - videoCodec.width = 1280; - videoCodec.height = 720; - break; + videoCodec.width = 1280; + videoCodec.height = 720; + break; + } + + // Set number of temporal layers. + std::cout << std::endl; + std::cout << "Choose number of temporal layers (1 to 4)."; + std::cout << "Press enter for default: \n"; + std::getline(std::cin, str); + int numTemporalLayers = atoi(str.c_str()); + if(numTemporalLayers != 0) + { + videoCodec.codecSpecific.VP8.numberOfTemporalLayers = numTemporalLayers; } // Set start bit rate @@ -439,6 +445,12 @@ int VideoEngineSampleCode(void* window1, void* window2) // Setting External transport TbExternalTransport extTransport(*(ptrViENetwork)); + if (numTemporalLayers > 1) { + extTransport.SetTemporalToggle(numTemporalLayers); + } else { + // Disabled + extTransport.SetTemporalToggle(0); + } int testMode = 0; std::cout << std::endl; @@ -449,8 +461,11 @@ int VideoEngineSampleCode(void* window1, void* window2) testMode = atoi(test_str.c_str()); if (testMode == 1) { + // Avoid changing SSRC due to collision. + error = ptrViERtpRtcp->SetLocalSSRC(videoChannel, 1); + error = ptrViENetwork->RegisterSendTransport(videoChannel, - extTransport); + extTransport); if (error == -1) { printf("ERROR in ViECodec::RegisterSendTransport \n");