diff --git a/webrtc/modules/interface/module_common_types.h b/webrtc/modules/interface/module_common_types.h index b500962cec..c8205ee60b 100644 --- a/webrtc/modules/interface/module_common_types.h +++ b/webrtc/modules/interface/module_common_types.h @@ -160,6 +160,7 @@ struct RTPVideoHeaderVP9 { inter_layer_predicted = false; gof_idx = kNoGofIdx; num_ref_pics = 0; + num_spatial_layers = 1; } bool inter_pic_predicted; // This layer frame is dependent on previously @@ -191,7 +192,7 @@ struct RTPVideoHeaderVP9 { int16_t ref_picture_id[kMaxVp9RefPics]; // PictureID of reference pictures. // SS data. - size_t num_spatial_layers; + size_t num_spatial_layers; // Always populated. bool spatial_layer_resolution_present; uint16_t width[kMaxVp9NumberOfSpatialLayers]; uint16_t height[kMaxVp9NumberOfSpatialLayers]; diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format_vp9.cc b/webrtc/modules/rtp_rtcp/source/rtp_format_vp9.cc index 64bd5aa25c..2f5e2e9b1e 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_format_vp9.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_format_vp9.cc @@ -578,7 +578,9 @@ bool RtpPacketizerVp9::NextPacket(uint8_t* buffer, if (!WriteHeaderAndPayload(packet_info, buffer, bytes_to_send)) { return false; } - *last_packet = packets_.empty(); + *last_packet = + packets_.empty() && (hdr_.spatial_idx == kNoSpatialIdx || + hdr_.spatial_idx == hdr_.num_spatial_layers - 1); return true; } diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format_vp9_unittest.cc b/webrtc/modules/rtp_rtcp/source/rtp_format_vp9_unittest.cc index 1f57c92dc0..fad0d1b435 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_format_vp9_unittest.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_format_vp9_unittest.cc @@ -258,6 +258,7 @@ TEST_F(RtpPacketizerVp9Test, TestLayerInfoWithNonFlexibleMode) { const size_t kPacketSize = 25; expected_.gof_idx = 3; + expected_.num_spatial_layers = 3; expected_.spatial_idx = 2; expected_.inter_layer_predicted = true; // D expected_.tl0_pic_idx = 117; @@ -281,6 +282,7 @@ TEST_F(RtpPacketizerVp9Test, TestLayerInfoWithFlexibleMode) { expected_.flexible_mode = true; expected_.temporal_idx = 3; expected_.temporal_up_switch = true; // U + expected_.num_spatial_layers = 3; expected_.spatial_idx = 2; expected_.inter_layer_predicted = false; // D Init(kFrameSize, kPacketSize); diff --git a/webrtc/modules/video_coding/codecs/interface/video_codec_interface.h b/webrtc/modules/video_coding/codecs/interface/video_codec_interface.h index 411fbfdc79..6363ab7332 100644 --- a/webrtc/modules/video_coding/codecs/interface/video_codec_interface.h +++ b/webrtc/modules/video_coding/codecs/interface/video_codec_interface.h @@ -63,7 +63,7 @@ struct CodecSpecificInfoVP9 { uint8_t gof_idx; // SS data. - size_t num_spatial_layers; + size_t num_spatial_layers; // Always populated. bool spatial_layer_resolution_present; uint16_t width[kMaxVp9NumberOfSpatialLayers]; uint16_t height[kMaxVp9NumberOfSpatialLayers]; diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc index 31484b6879..2e9f5ae30f 100644 --- a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -212,8 +212,8 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst, if (inst->codecSpecific.VP9.numberOfTemporalLayers > 3) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } - // For now, only support one spatial layer. - if (inst->codecSpecific.VP9.numberOfSpatialLayers != 1) { + // libvpx currently supports only one or two spatial layers. + if (inst->codecSpecific.VP9.numberOfSpatialLayers > 2) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } int retVal = Release(); @@ -351,13 +351,8 @@ int VP9EncoderImpl::NumberOfThreads(int width, } int VP9EncoderImpl::InitAndSetControlSettings(const VideoCodec* inst) { - config_->ss_number_layers = num_spatial_layers_; - if (num_spatial_layers_ > 1) { - config_->rc_min_quantizer = 0; - config_->rc_max_quantizer = 63; - } int scaling_factor_num = 256; for (int i = num_spatial_layers_ - 1; i >= 0; --i) { svc_internal_.svc_params.max_quantizers[i] = config_->rc_max_quantizer; @@ -549,8 +544,10 @@ void VP9EncoderImpl::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, vp9_info->tl0_pic_idx = tl0_pic_idx_; } + // Always populate this, so that the packetizer can properly set the marker + // bit. + vp9_info->num_spatial_layers = num_spatial_layers_; if (vp9_info->ss_data_available) { - vp9_info->num_spatial_layers = num_spatial_layers_; vp9_info->spatial_layer_resolution_present = true; for (size_t i = 0; i < vp9_info->num_spatial_layers; ++i) { vp9_info->width[i] = codec_.width * diff --git a/webrtc/modules/video_coding/main/source/generic_encoder.cc b/webrtc/modules/video_coding/main/source/generic_encoder.cc index c0925b95b2..e4408d1f7e 100644 --- a/webrtc/modules/video_coding/main/source/generic_encoder.cc +++ b/webrtc/modules/video_coding/main/source/generic_encoder.cc @@ -55,9 +55,11 @@ void CopyCodecSpecific(const CodecSpecificInfo* info, RTPVideoHeader* rtp) { info->codecSpecific.VP9.inter_layer_predicted; rtp->codecHeader.VP9.gof_idx = info->codecSpecific.VP9.gof_idx; + // Packetizer needs to know the number of spatial layers to correctly set + // the marker bit, even when the number won't be written in the packet. + rtp->codecHeader.VP9.num_spatial_layers = + info->codecSpecific.VP9.num_spatial_layers; if (info->codecSpecific.VP9.ss_data_available) { - rtp->codecHeader.VP9.num_spatial_layers = - info->codecSpecific.VP9.num_spatial_layers; rtp->codecHeader.VP9.spatial_layer_resolution_present = info->codecSpecific.VP9.spatial_layer_resolution_present; if (info->codecSpecific.VP9.spatial_layer_resolution_present) { diff --git a/webrtc/test/layer_filtering_transport.cc b/webrtc/test/layer_filtering_transport.cc new file mode 100644 index 0000000000..102f63eb3f --- /dev/null +++ b/webrtc/test/layer_filtering_transport.cc @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/base/checks.h" +#include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h" +#include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp_defines.h" +#include "webrtc/modules/rtp_rtcp/source/byte_io.h" +#include "webrtc/modules/rtp_rtcp/source/rtp_format.h" +#include "webrtc/modules/rtp_rtcp/source/rtp_utility.h" +#include "webrtc/test/layer_filtering_transport.h" + +namespace webrtc { +namespace test { + +LayerFilteringTransport::LayerFilteringTransport( + const FakeNetworkPipe::Config& config, + uint8_t vp8_video_payload_type, + uint8_t vp9_video_payload_type, + uint8_t tl_discard_threshold, + uint8_t sl_discard_threshold) + : test::DirectTransport(config), + vp8_video_payload_type_(vp8_video_payload_type), + vp9_video_payload_type_(vp9_video_payload_type), + tl_discard_threshold_(tl_discard_threshold), + sl_discard_threshold_(sl_discard_threshold), + current_seq_num_(10000) { +} // TODO(ivica): random seq num? + +bool LayerFilteringTransport::SendRtp(const uint8_t* packet, size_t length) { + if (tl_discard_threshold_ == 0 && sl_discard_threshold_ == 0) { + // Nothing to change, forward the packet immediately. + return test::DirectTransport::SendRtp(packet, length); + } + + bool set_marker_bit = false; + rtc::scoped_ptr parser(RtpHeaderParser::Create()); + RTPHeader header; + parser->Parse(packet, length, &header); + + if (header.payloadType == vp8_video_payload_type_ || + header.payloadType == vp9_video_payload_type_) { + const uint8_t* payload = packet + header.headerLength; + DCHECK_GT(length, header.headerLength); + const size_t payload_length = length - header.headerLength; + DCHECK_GT(payload_length, header.paddingLength); + const size_t payload_data_length = payload_length - header.paddingLength; + + const bool is_vp8 = header.payloadType == vp8_video_payload_type_; + rtc::scoped_ptr depacketizer( + RtpDepacketizer::Create(is_vp8 ? kRtpVideoVp8 : kRtpVideoVp9)); + RtpDepacketizer::ParsedPayload parsed_payload; + if (depacketizer->Parse(&parsed_payload, payload, payload_data_length)) { + const uint8_t temporalIdx = + is_vp8 ? parsed_payload.type.Video.codecHeader.VP8.temporalIdx + : parsed_payload.type.Video.codecHeader.VP9.temporal_idx; + const uint8_t spatialIdx = + is_vp8 ? kNoSpatialIdx + : parsed_payload.type.Video.codecHeader.VP9.spatial_idx; + if (sl_discard_threshold_ > 0 && + spatialIdx == sl_discard_threshold_ - 1 && + parsed_payload.type.Video.codecHeader.VP9.end_of_frame) { + // This layer is now the last in the superframe. + set_marker_bit = true; + } + if ((tl_discard_threshold_ > 0 && temporalIdx != kNoTemporalIdx && + temporalIdx >= tl_discard_threshold_) || + (sl_discard_threshold_ > 0 && spatialIdx != kNoSpatialIdx && + spatialIdx >= sl_discard_threshold_)) { + return true; // Discard the packet. + } + } else { + RTC_NOTREACHED() << "Parse error"; + } + } + + uint8_t temp_buffer[IP_PACKET_SIZE]; + memcpy(temp_buffer, packet, length); + + // We are discarding some of the packets (specifically, whole layers), so + // make sure the marker bit is set properly, and that sequence numbers are + // continuous. + if (set_marker_bit) { + temp_buffer[1] |= kRtpMarkerBitMask; + } + ByteWriter::WriteBigEndian(&temp_buffer[2], current_seq_num_); + + ++current_seq_num_; // Increase only if packet not discarded. + + return test::DirectTransport::SendRtp(temp_buffer, length); +} + +} // namespace test +} // namespace webrtc diff --git a/webrtc/test/layer_filtering_transport.h b/webrtc/test/layer_filtering_transport.h new file mode 100644 index 0000000000..96a2cba8a2 --- /dev/null +++ b/webrtc/test/layer_filtering_transport.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef WEBRTC_TEST_LAYER_FILTERING_TRANSPORT_H_ +#define WEBRTC_TEST_LAYER_FILTERING_TRANSPORT_H_ + +#include "webrtc/test/direct_transport.h" +#include "webrtc/test/fake_network_pipe.h" + +namespace webrtc { + +namespace test { + +class LayerFilteringTransport : public test::DirectTransport { + public: + LayerFilteringTransport(const FakeNetworkPipe::Config& config, + uint8_t vp8_video_payload_type, + uint8_t vp9_video_payload_type, + uint8_t tl_discard_threshold, + uint8_t sl_discard_threshold); + bool SendRtp(const uint8_t* data, size_t length) override; + + private: + // Used to distinguish between VP8 and VP9. + const uint8_t vp8_video_payload_type_; + const uint8_t vp9_video_payload_type_; + // Discard all temporal/spatial layers with id greater or equal the + // threshold. 0 to disable. + const uint8_t tl_discard_threshold_; + const uint8_t sl_discard_threshold_; + uint16_t current_seq_num_; +}; + +} // namespace test +} // namespace webrtc + +#endif // WEBRTC_TEST_LAYER_FILTERING_TRANSPORT_H_ diff --git a/webrtc/test/webrtc_test_common.gyp b/webrtc/test/webrtc_test_common.gyp index ea6e4052db..17f4551113 100644 --- a/webrtc/test/webrtc_test_common.gyp +++ b/webrtc/test/webrtc_test_common.gyp @@ -32,6 +32,8 @@ 'fake_network_pipe.h', 'frame_generator_capturer.cc', 'frame_generator_capturer.h', + 'layer_filtering_transport.cc', + 'layer_filtering_transport.h', 'mock_transport.h', 'null_transport.cc', 'null_transport.h', diff --git a/webrtc/video/loopback.cc b/webrtc/video/loopback.cc index 7a753fb481..9083500c1e 100644 --- a/webrtc/video/loopback.cc +++ b/webrtc/video/loopback.cc @@ -19,11 +19,14 @@ #include "webrtc/base/checks.h" #include "webrtc/base/scoped_ptr.h" #include "webrtc/call.h" +#include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h" +#include "webrtc/modules/rtp_rtcp/source/rtp_format.h" +#include "webrtc/modules/rtp_rtcp/source/rtp_utility.h" #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" #include "webrtc/system_wrappers/interface/clock.h" -#include "webrtc/test/direct_transport.h" #include "webrtc/test/encoder_settings.h" #include "webrtc/test/fake_encoder.h" +#include "webrtc/test/layer_filtering_transport.h" #include "webrtc/test/run_loop.h" #include "webrtc/test/testsupport/trace_to_stderr.h" #include "webrtc/test/video_capturer.h" @@ -40,7 +43,8 @@ static const uint32_t kSendRtxSsrc = 0x654322; static const uint32_t kReceiverLocalSsrc = 0x123456; static const uint8_t kRtxVideoPayloadType = 96; -static const uint8_t kVideoPayloadType = 124; +static const uint8_t kVideoPayloadTypeVP8 = 124; +static const uint8_t kVideoPayloadTypeVP9 = 125; Loopback::Loopback(const Config& config) : config_(config), clock_(Clock::GetRealTimeClock()) { @@ -76,7 +80,11 @@ void Loopback::Run() { pipe_config.queue_length_packets = config_.queue_size; pipe_config.queue_delay_ms = config_.avg_propagation_delay_ms; pipe_config.delay_standard_deviation_ms = config_.std_propagation_delay_ms; - test::DirectTransport send_transport(pipe_config); + LayerFilteringTransport send_transport( + pipe_config, kVideoPayloadTypeVP8, kVideoPayloadTypeVP9, + static_cast(config_.tl_discard_threshold), + static_cast(config_.sl_discard_threshold)); + // Loopback, call sends to itself. send_transport.SetReceiver(call->Receiver()); @@ -99,9 +107,11 @@ void Loopback::Run() { RTC_NOTREACHED() << "Codec not supported!"; return; } + const int payload_type = + config_.codec == "VP8" ? kVideoPayloadTypeVP8 : kVideoPayloadTypeVP9; send_config.encoder_settings.encoder = encoder.get(); send_config.encoder_settings.payload_name = config_.codec; - send_config.encoder_settings.payload_type = kVideoPayloadType; + send_config.encoder_settings.payload_type = payload_type; VideoEncoderConfig encoder_config(CreateEncoderConfig()); @@ -115,8 +125,8 @@ void Loopback::Run() { receive_config.rtp.local_ssrc = kReceiverLocalSsrc; receive_config.rtp.nack.rtp_history_ms = 1000; receive_config.rtp.remb = true; - receive_config.rtp.rtx[kVideoPayloadType].ssrc = kSendRtxSsrc; - receive_config.rtp.rtx[kVideoPayloadType].payload_type = kRtxVideoPayloadType; + receive_config.rtp.rtx[payload_type].ssrc = kSendRtxSsrc; + receive_config.rtp.rtx[payload_type].payload_type = kRtxVideoPayloadType; receive_config.rtp.extensions.push_back( RtpExtension(RtpExtension::kAbsSendTime, kAbsSendTimeExtensionId)); receive_config.renderer = loopback_video.get(); diff --git a/webrtc/video/loopback.h b/webrtc/video/loopback.h index 0a8bb9793b..c671de4e98 100644 --- a/webrtc/video/loopback.h +++ b/webrtc/video/loopback.h @@ -33,6 +33,11 @@ class Loopback { int32_t min_transmit_bitrate_kbps; std::string codec; size_t num_temporal_layers; + size_t num_spatial_layers; + // Discard all temporal/spatial layers with id greater or equal the + // threshold. 0 to disable. + size_t tl_discard_threshold; + size_t sl_discard_threshold; int32_t loss_percent; int32_t link_capacity_kbps; int32_t queue_size; diff --git a/webrtc/video/screenshare_loopback.cc b/webrtc/video/screenshare_loopback.cc index 4a13629734..a221e9c52f 100644 --- a/webrtc/video/screenshare_loopback.cc +++ b/webrtc/video/screenshare_loopback.cc @@ -78,6 +78,27 @@ int NumTemporalLayers() { return static_cast(FLAGS_num_temporal_layers); } +DEFINE_int32(num_spatial_layers, 1, "Number of spatial layers to use."); +int NumSpatialLayers() { + return static_cast(FLAGS_num_spatial_layers); +} + +DEFINE_int32( + tl_discard_threshold, + 0, + "Discard TLs with id greater or equal the threshold. 0 to disable."); +int TLDiscardThreshold() { + return static_cast(FLAGS_tl_discard_threshold); +} + +DEFINE_int32( + sl_discard_threshold, + 0, + "Discard SLs with id greater or equal the threshold. 0 to disable."); +int SLDiscardThreshold() { + return static_cast(FLAGS_sl_discard_threshold); +} + DEFINE_int32(min_transmit_bitrate, 400, "Min transmit bitrate incl. padding."); int MinTransmitBitrate() { return FLAGS_min_transmit_bitrate; @@ -135,6 +156,12 @@ class ScreenshareLoopback : public test::Loopback { explicit ScreenshareLoopback(const Config& config) : Loopback(config) { CHECK_GE(config.num_temporal_layers, 1u); CHECK_LE(config.num_temporal_layers, 2u); + CHECK_GE(config.num_spatial_layers, 1u); + CHECK_LE(config.num_spatial_layers, 5u); + CHECK(config.num_spatial_layers == 1 || config.codec == "VP9"); + CHECK(config.num_spatial_layers == 1 || config.num_temporal_layers == 1); + CHECK_LT(config.tl_discard_threshold, config.num_temporal_layers); + CHECK_LT(config.sl_discard_threshold, config.num_spatial_layers); vp8_settings_ = VideoEncoder::GetDefaultVp8Settings(); vp8_settings_.denoisingOn = false; @@ -147,6 +174,8 @@ class ScreenshareLoopback : public test::Loopback { vp9_settings_.frameDroppingOn = false; vp9_settings_.numberOfTemporalLayers = static_cast(config.num_temporal_layers); + vp9_settings_.numberOfSpatialLayers = + static_cast(config.num_spatial_layers); } virtual ~ScreenshareLoopback() {} @@ -219,6 +248,9 @@ void Loopback() { flags::MinTransmitBitrate(), flags::Codec(), flags::NumTemporalLayers(), + flags::NumSpatialLayers(), + flags::TLDiscardThreshold(), + flags::SLDiscardThreshold(), flags::LossPercent(), flags::LinkCapacity(), flags::QueueSize(),