From 7f6a6fc0b23795cd4f0aacbf707618c1f3d0a878 Mon Sep 17 00:00:00 2001 From: ivica Date: Tue, 8 Sep 2015 02:40:29 -0700 Subject: [PATCH] Enabling spatial layers in VP9Impl. Filter layers in the loopback test. Handling the case when encoder drops only the higher layer. Added options to screenshare loopback test to discard high temporal or spatial layers (to view the lower layers). Review URL: https://codereview.webrtc.org/1287643002 Cr-Commit-Position: refs/heads/master@{#9883} --- .../modules/interface/module_common_types.h | 3 +- .../modules/rtp_rtcp/source/rtp_format_vp9.cc | 4 +- .../source/rtp_format_vp9_unittest.cc | 2 + .../codecs/interface/video_codec_interface.h | 2 +- .../video_coding/codecs/vp9/vp9_impl.cc | 13 +-- .../main/source/generic_encoder.cc | 6 +- webrtc/test/layer_filtering_transport.cc | 100 ++++++++++++++++++ webrtc/test/layer_filtering_transport.h | 43 ++++++++ webrtc/test/webrtc_test_common.gyp | 2 + webrtc/video/loopback.cc | 22 ++-- webrtc/video/loopback.h | 5 + webrtc/video/screenshare_loopback.cc | 32 ++++++ 12 files changed, 215 insertions(+), 19 deletions(-) create mode 100644 webrtc/test/layer_filtering_transport.cc create mode 100644 webrtc/test/layer_filtering_transport.h diff --git a/webrtc/modules/interface/module_common_types.h b/webrtc/modules/interface/module_common_types.h index b500962cec..c8205ee60b 100644 --- a/webrtc/modules/interface/module_common_types.h +++ b/webrtc/modules/interface/module_common_types.h @@ -160,6 +160,7 @@ struct RTPVideoHeaderVP9 { inter_layer_predicted = false; gof_idx = kNoGofIdx; num_ref_pics = 0; + num_spatial_layers = 1; } bool inter_pic_predicted; // This layer frame is dependent on previously @@ -191,7 +192,7 @@ struct RTPVideoHeaderVP9 { int16_t ref_picture_id[kMaxVp9RefPics]; // PictureID of reference pictures. // SS data. - size_t num_spatial_layers; + size_t num_spatial_layers; // Always populated. bool spatial_layer_resolution_present; uint16_t width[kMaxVp9NumberOfSpatialLayers]; uint16_t height[kMaxVp9NumberOfSpatialLayers]; diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format_vp9.cc b/webrtc/modules/rtp_rtcp/source/rtp_format_vp9.cc index 64bd5aa25c..2f5e2e9b1e 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_format_vp9.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_format_vp9.cc @@ -578,7 +578,9 @@ bool RtpPacketizerVp9::NextPacket(uint8_t* buffer, if (!WriteHeaderAndPayload(packet_info, buffer, bytes_to_send)) { return false; } - *last_packet = packets_.empty(); + *last_packet = + packets_.empty() && (hdr_.spatial_idx == kNoSpatialIdx || + hdr_.spatial_idx == hdr_.num_spatial_layers - 1); return true; } diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format_vp9_unittest.cc b/webrtc/modules/rtp_rtcp/source/rtp_format_vp9_unittest.cc index 1f57c92dc0..fad0d1b435 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_format_vp9_unittest.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_format_vp9_unittest.cc @@ -258,6 +258,7 @@ TEST_F(RtpPacketizerVp9Test, TestLayerInfoWithNonFlexibleMode) { const size_t kPacketSize = 25; expected_.gof_idx = 3; + expected_.num_spatial_layers = 3; expected_.spatial_idx = 2; expected_.inter_layer_predicted = true; // D expected_.tl0_pic_idx = 117; @@ -281,6 +282,7 @@ TEST_F(RtpPacketizerVp9Test, TestLayerInfoWithFlexibleMode) { expected_.flexible_mode = true; expected_.temporal_idx = 3; expected_.temporal_up_switch = true; // U + expected_.num_spatial_layers = 3; expected_.spatial_idx = 2; expected_.inter_layer_predicted = false; // D Init(kFrameSize, kPacketSize); diff --git a/webrtc/modules/video_coding/codecs/interface/video_codec_interface.h b/webrtc/modules/video_coding/codecs/interface/video_codec_interface.h index 411fbfdc79..6363ab7332 100644 --- a/webrtc/modules/video_coding/codecs/interface/video_codec_interface.h +++ b/webrtc/modules/video_coding/codecs/interface/video_codec_interface.h @@ -63,7 +63,7 @@ struct CodecSpecificInfoVP9 { uint8_t gof_idx; // SS data. - size_t num_spatial_layers; + size_t num_spatial_layers; // Always populated. bool spatial_layer_resolution_present; uint16_t width[kMaxVp9NumberOfSpatialLayers]; uint16_t height[kMaxVp9NumberOfSpatialLayers]; diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc index 31484b6879..2e9f5ae30f 100644 --- a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -212,8 +212,8 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst, if (inst->codecSpecific.VP9.numberOfTemporalLayers > 3) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } - // For now, only support one spatial layer. - if (inst->codecSpecific.VP9.numberOfSpatialLayers != 1) { + // libvpx currently supports only one or two spatial layers. + if (inst->codecSpecific.VP9.numberOfSpatialLayers > 2) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } int retVal = Release(); @@ -351,13 +351,8 @@ int VP9EncoderImpl::NumberOfThreads(int width, } int VP9EncoderImpl::InitAndSetControlSettings(const VideoCodec* inst) { - config_->ss_number_layers = num_spatial_layers_; - if (num_spatial_layers_ > 1) { - config_->rc_min_quantizer = 0; - config_->rc_max_quantizer = 63; - } int scaling_factor_num = 256; for (int i = num_spatial_layers_ - 1; i >= 0; --i) { svc_internal_.svc_params.max_quantizers[i] = config_->rc_max_quantizer; @@ -549,8 +544,10 @@ void VP9EncoderImpl::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, vp9_info->tl0_pic_idx = tl0_pic_idx_; } + // Always populate this, so that the packetizer can properly set the marker + // bit. + vp9_info->num_spatial_layers = num_spatial_layers_; if (vp9_info->ss_data_available) { - vp9_info->num_spatial_layers = num_spatial_layers_; vp9_info->spatial_layer_resolution_present = true; for (size_t i = 0; i < vp9_info->num_spatial_layers; ++i) { vp9_info->width[i] = codec_.width * diff --git a/webrtc/modules/video_coding/main/source/generic_encoder.cc b/webrtc/modules/video_coding/main/source/generic_encoder.cc index c0925b95b2..e4408d1f7e 100644 --- a/webrtc/modules/video_coding/main/source/generic_encoder.cc +++ b/webrtc/modules/video_coding/main/source/generic_encoder.cc @@ -55,9 +55,11 @@ void CopyCodecSpecific(const CodecSpecificInfo* info, RTPVideoHeader* rtp) { info->codecSpecific.VP9.inter_layer_predicted; rtp->codecHeader.VP9.gof_idx = info->codecSpecific.VP9.gof_idx; + // Packetizer needs to know the number of spatial layers to correctly set + // the marker bit, even when the number won't be written in the packet. + rtp->codecHeader.VP9.num_spatial_layers = + info->codecSpecific.VP9.num_spatial_layers; if (info->codecSpecific.VP9.ss_data_available) { - rtp->codecHeader.VP9.num_spatial_layers = - info->codecSpecific.VP9.num_spatial_layers; rtp->codecHeader.VP9.spatial_layer_resolution_present = info->codecSpecific.VP9.spatial_layer_resolution_present; if (info->codecSpecific.VP9.spatial_layer_resolution_present) { diff --git a/webrtc/test/layer_filtering_transport.cc b/webrtc/test/layer_filtering_transport.cc new file mode 100644 index 0000000000..102f63eb3f --- /dev/null +++ b/webrtc/test/layer_filtering_transport.cc @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/base/checks.h" +#include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h" +#include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp_defines.h" +#include "webrtc/modules/rtp_rtcp/source/byte_io.h" +#include "webrtc/modules/rtp_rtcp/source/rtp_format.h" +#include "webrtc/modules/rtp_rtcp/source/rtp_utility.h" +#include "webrtc/test/layer_filtering_transport.h" + +namespace webrtc { +namespace test { + +LayerFilteringTransport::LayerFilteringTransport( + const FakeNetworkPipe::Config& config, + uint8_t vp8_video_payload_type, + uint8_t vp9_video_payload_type, + uint8_t tl_discard_threshold, + uint8_t sl_discard_threshold) + : test::DirectTransport(config), + vp8_video_payload_type_(vp8_video_payload_type), + vp9_video_payload_type_(vp9_video_payload_type), + tl_discard_threshold_(tl_discard_threshold), + sl_discard_threshold_(sl_discard_threshold), + current_seq_num_(10000) { +} // TODO(ivica): random seq num? + +bool LayerFilteringTransport::SendRtp(const uint8_t* packet, size_t length) { + if (tl_discard_threshold_ == 0 && sl_discard_threshold_ == 0) { + // Nothing to change, forward the packet immediately. + return test::DirectTransport::SendRtp(packet, length); + } + + bool set_marker_bit = false; + rtc::scoped_ptr parser(RtpHeaderParser::Create()); + RTPHeader header; + parser->Parse(packet, length, &header); + + if (header.payloadType == vp8_video_payload_type_ || + header.payloadType == vp9_video_payload_type_) { + const uint8_t* payload = packet + header.headerLength; + DCHECK_GT(length, header.headerLength); + const size_t payload_length = length - header.headerLength; + DCHECK_GT(payload_length, header.paddingLength); + const size_t payload_data_length = payload_length - header.paddingLength; + + const bool is_vp8 = header.payloadType == vp8_video_payload_type_; + rtc::scoped_ptr depacketizer( + RtpDepacketizer::Create(is_vp8 ? kRtpVideoVp8 : kRtpVideoVp9)); + RtpDepacketizer::ParsedPayload parsed_payload; + if (depacketizer->Parse(&parsed_payload, payload, payload_data_length)) { + const uint8_t temporalIdx = + is_vp8 ? parsed_payload.type.Video.codecHeader.VP8.temporalIdx + : parsed_payload.type.Video.codecHeader.VP9.temporal_idx; + const uint8_t spatialIdx = + is_vp8 ? kNoSpatialIdx + : parsed_payload.type.Video.codecHeader.VP9.spatial_idx; + if (sl_discard_threshold_ > 0 && + spatialIdx == sl_discard_threshold_ - 1 && + parsed_payload.type.Video.codecHeader.VP9.end_of_frame) { + // This layer is now the last in the superframe. + set_marker_bit = true; + } + if ((tl_discard_threshold_ > 0 && temporalIdx != kNoTemporalIdx && + temporalIdx >= tl_discard_threshold_) || + (sl_discard_threshold_ > 0 && spatialIdx != kNoSpatialIdx && + spatialIdx >= sl_discard_threshold_)) { + return true; // Discard the packet. + } + } else { + RTC_NOTREACHED() << "Parse error"; + } + } + + uint8_t temp_buffer[IP_PACKET_SIZE]; + memcpy(temp_buffer, packet, length); + + // We are discarding some of the packets (specifically, whole layers), so + // make sure the marker bit is set properly, and that sequence numbers are + // continuous. + if (set_marker_bit) { + temp_buffer[1] |= kRtpMarkerBitMask; + } + ByteWriter::WriteBigEndian(&temp_buffer[2], current_seq_num_); + + ++current_seq_num_; // Increase only if packet not discarded. + + return test::DirectTransport::SendRtp(temp_buffer, length); +} + +} // namespace test +} // namespace webrtc diff --git a/webrtc/test/layer_filtering_transport.h b/webrtc/test/layer_filtering_transport.h new file mode 100644 index 0000000000..96a2cba8a2 --- /dev/null +++ b/webrtc/test/layer_filtering_transport.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef WEBRTC_TEST_LAYER_FILTERING_TRANSPORT_H_ +#define WEBRTC_TEST_LAYER_FILTERING_TRANSPORT_H_ + +#include "webrtc/test/direct_transport.h" +#include "webrtc/test/fake_network_pipe.h" + +namespace webrtc { + +namespace test { + +class LayerFilteringTransport : public test::DirectTransport { + public: + LayerFilteringTransport(const FakeNetworkPipe::Config& config, + uint8_t vp8_video_payload_type, + uint8_t vp9_video_payload_type, + uint8_t tl_discard_threshold, + uint8_t sl_discard_threshold); + bool SendRtp(const uint8_t* data, size_t length) override; + + private: + // Used to distinguish between VP8 and VP9. + const uint8_t vp8_video_payload_type_; + const uint8_t vp9_video_payload_type_; + // Discard all temporal/spatial layers with id greater or equal the + // threshold. 0 to disable. + const uint8_t tl_discard_threshold_; + const uint8_t sl_discard_threshold_; + uint16_t current_seq_num_; +}; + +} // namespace test +} // namespace webrtc + +#endif // WEBRTC_TEST_LAYER_FILTERING_TRANSPORT_H_ diff --git a/webrtc/test/webrtc_test_common.gyp b/webrtc/test/webrtc_test_common.gyp index ea6e4052db..17f4551113 100644 --- a/webrtc/test/webrtc_test_common.gyp +++ b/webrtc/test/webrtc_test_common.gyp @@ -32,6 +32,8 @@ 'fake_network_pipe.h', 'frame_generator_capturer.cc', 'frame_generator_capturer.h', + 'layer_filtering_transport.cc', + 'layer_filtering_transport.h', 'mock_transport.h', 'null_transport.cc', 'null_transport.h', diff --git a/webrtc/video/loopback.cc b/webrtc/video/loopback.cc index 7a753fb481..9083500c1e 100644 --- a/webrtc/video/loopback.cc +++ b/webrtc/video/loopback.cc @@ -19,11 +19,14 @@ #include "webrtc/base/checks.h" #include "webrtc/base/scoped_ptr.h" #include "webrtc/call.h" +#include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h" +#include "webrtc/modules/rtp_rtcp/source/rtp_format.h" +#include "webrtc/modules/rtp_rtcp/source/rtp_utility.h" #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" #include "webrtc/system_wrappers/interface/clock.h" -#include "webrtc/test/direct_transport.h" #include "webrtc/test/encoder_settings.h" #include "webrtc/test/fake_encoder.h" +#include "webrtc/test/layer_filtering_transport.h" #include "webrtc/test/run_loop.h" #include "webrtc/test/testsupport/trace_to_stderr.h" #include "webrtc/test/video_capturer.h" @@ -40,7 +43,8 @@ static const uint32_t kSendRtxSsrc = 0x654322; static const uint32_t kReceiverLocalSsrc = 0x123456; static const uint8_t kRtxVideoPayloadType = 96; -static const uint8_t kVideoPayloadType = 124; +static const uint8_t kVideoPayloadTypeVP8 = 124; +static const uint8_t kVideoPayloadTypeVP9 = 125; Loopback::Loopback(const Config& config) : config_(config), clock_(Clock::GetRealTimeClock()) { @@ -76,7 +80,11 @@ void Loopback::Run() { pipe_config.queue_length_packets = config_.queue_size; pipe_config.queue_delay_ms = config_.avg_propagation_delay_ms; pipe_config.delay_standard_deviation_ms = config_.std_propagation_delay_ms; - test::DirectTransport send_transport(pipe_config); + LayerFilteringTransport send_transport( + pipe_config, kVideoPayloadTypeVP8, kVideoPayloadTypeVP9, + static_cast(config_.tl_discard_threshold), + static_cast(config_.sl_discard_threshold)); + // Loopback, call sends to itself. send_transport.SetReceiver(call->Receiver()); @@ -99,9 +107,11 @@ void Loopback::Run() { RTC_NOTREACHED() << "Codec not supported!"; return; } + const int payload_type = + config_.codec == "VP8" ? kVideoPayloadTypeVP8 : kVideoPayloadTypeVP9; send_config.encoder_settings.encoder = encoder.get(); send_config.encoder_settings.payload_name = config_.codec; - send_config.encoder_settings.payload_type = kVideoPayloadType; + send_config.encoder_settings.payload_type = payload_type; VideoEncoderConfig encoder_config(CreateEncoderConfig()); @@ -115,8 +125,8 @@ void Loopback::Run() { receive_config.rtp.local_ssrc = kReceiverLocalSsrc; receive_config.rtp.nack.rtp_history_ms = 1000; receive_config.rtp.remb = true; - receive_config.rtp.rtx[kVideoPayloadType].ssrc = kSendRtxSsrc; - receive_config.rtp.rtx[kVideoPayloadType].payload_type = kRtxVideoPayloadType; + receive_config.rtp.rtx[payload_type].ssrc = kSendRtxSsrc; + receive_config.rtp.rtx[payload_type].payload_type = kRtxVideoPayloadType; receive_config.rtp.extensions.push_back( RtpExtension(RtpExtension::kAbsSendTime, kAbsSendTimeExtensionId)); receive_config.renderer = loopback_video.get(); diff --git a/webrtc/video/loopback.h b/webrtc/video/loopback.h index 0a8bb9793b..c671de4e98 100644 --- a/webrtc/video/loopback.h +++ b/webrtc/video/loopback.h @@ -33,6 +33,11 @@ class Loopback { int32_t min_transmit_bitrate_kbps; std::string codec; size_t num_temporal_layers; + size_t num_spatial_layers; + // Discard all temporal/spatial layers with id greater or equal the + // threshold. 0 to disable. + size_t tl_discard_threshold; + size_t sl_discard_threshold; int32_t loss_percent; int32_t link_capacity_kbps; int32_t queue_size; diff --git a/webrtc/video/screenshare_loopback.cc b/webrtc/video/screenshare_loopback.cc index 4a13629734..a221e9c52f 100644 --- a/webrtc/video/screenshare_loopback.cc +++ b/webrtc/video/screenshare_loopback.cc @@ -78,6 +78,27 @@ int NumTemporalLayers() { return static_cast(FLAGS_num_temporal_layers); } +DEFINE_int32(num_spatial_layers, 1, "Number of spatial layers to use."); +int NumSpatialLayers() { + return static_cast(FLAGS_num_spatial_layers); +} + +DEFINE_int32( + tl_discard_threshold, + 0, + "Discard TLs with id greater or equal the threshold. 0 to disable."); +int TLDiscardThreshold() { + return static_cast(FLAGS_tl_discard_threshold); +} + +DEFINE_int32( + sl_discard_threshold, + 0, + "Discard SLs with id greater or equal the threshold. 0 to disable."); +int SLDiscardThreshold() { + return static_cast(FLAGS_sl_discard_threshold); +} + DEFINE_int32(min_transmit_bitrate, 400, "Min transmit bitrate incl. padding."); int MinTransmitBitrate() { return FLAGS_min_transmit_bitrate; @@ -135,6 +156,12 @@ class ScreenshareLoopback : public test::Loopback { explicit ScreenshareLoopback(const Config& config) : Loopback(config) { CHECK_GE(config.num_temporal_layers, 1u); CHECK_LE(config.num_temporal_layers, 2u); + CHECK_GE(config.num_spatial_layers, 1u); + CHECK_LE(config.num_spatial_layers, 5u); + CHECK(config.num_spatial_layers == 1 || config.codec == "VP9"); + CHECK(config.num_spatial_layers == 1 || config.num_temporal_layers == 1); + CHECK_LT(config.tl_discard_threshold, config.num_temporal_layers); + CHECK_LT(config.sl_discard_threshold, config.num_spatial_layers); vp8_settings_ = VideoEncoder::GetDefaultVp8Settings(); vp8_settings_.denoisingOn = false; @@ -147,6 +174,8 @@ class ScreenshareLoopback : public test::Loopback { vp9_settings_.frameDroppingOn = false; vp9_settings_.numberOfTemporalLayers = static_cast(config.num_temporal_layers); + vp9_settings_.numberOfSpatialLayers = + static_cast(config.num_spatial_layers); } virtual ~ScreenshareLoopback() {} @@ -219,6 +248,9 @@ void Loopback() { flags::MinTransmitBitrate(), flags::Codec(), flags::NumTemporalLayers(), + flags::NumSpatialLayers(), + flags::TLDiscardThreshold(), + flags::SLDiscardThreshold(), flags::LossPercent(), flags::LinkCapacity(), flags::QueueSize(),