diff --git a/webrtc/common_types.h b/webrtc/common_types.h index 07faf6aefc..048485fa2f 100644 --- a/webrtc/common_types.h +++ b/webrtc/common_types.h @@ -547,6 +547,7 @@ enum RawVideoType enum { kConfigParameterSize = 128}; enum { kPayloadNameSize = 32}; enum { kMaxSimulcastStreams = 4}; +enum { kMaxSpatialLayers = 5 }; enum { kMaxTemporalStreams = 4}; enum VideoCodecComplexity @@ -676,6 +677,13 @@ struct SimulcastStream { } }; +struct SpatialLayer { + int scaling_factor_num; + int scaling_factor_den; + int target_bitrate_bps; + // TODO(ivica): Add max_quantizer and min_quantizer? +}; + enum VideoCodecMode { kRealtimeVideo, kScreensharing @@ -702,6 +710,7 @@ struct VideoCodec { unsigned int qpMax; unsigned char numberOfSimulcastStreams; SimulcastStream simulcastStream[kMaxSimulcastStreams]; + SpatialLayer spatialLayers[kMaxSpatialLayers]; VideoCodecMode mode; diff --git a/webrtc/config.h b/webrtc/config.h index 52711636ac..4b863c8d23 100644 --- a/webrtc/config.h +++ b/webrtc/config.h @@ -104,6 +104,7 @@ struct VideoEncoderConfig { std::string ToString() const; std::vector streams; + std::vector spatial_layers; ContentType content_type; void* encoder_specific_settings; diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc index 0ca7eeabe9..97f08462bf 100644 --- a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -112,42 +112,72 @@ int VP9EncoderImpl::Release() { return WEBRTC_VIDEO_CODEC_OK; } +bool VP9EncoderImpl::ExplicitlyConfiguredSpatialLayers() const { + // We check target_bitrate_bps of the 0th layer to see if the spatial layers + // (i.e. bitrates) were explicitly configured. + return num_spatial_layers_ > 1 && + codec_.spatialLayers[0].target_bitrate_bps > 0; +} + bool VP9EncoderImpl::SetSvcRates() { - float rate_ratio[VPX_MAX_LAYERS] = {0}; - float total = 0; uint8_t i = 0; - for (i = 0; i < num_spatial_layers_; ++i) { - if (svc_internal_.svc_params.scaling_factor_num[i] <= 0 || - svc_internal_.svc_params.scaling_factor_den[i] <= 0) { + if (ExplicitlyConfiguredSpatialLayers()) { + if (num_temporal_layers_ > 1) { + LOG(LS_ERROR) << "Multiple temporal layers when manually specifying " + "spatial layers not implemented yet!"; return false; } - rate_ratio[i] = static_cast( - svc_internal_.svc_params.scaling_factor_num[i]) / - svc_internal_.svc_params.scaling_factor_den[i]; - total += rate_ratio[i]; - } + int total_bitrate_bps = 0; + for (i = 0; i < num_spatial_layers_; ++i) + total_bitrate_bps += codec_.spatialLayers[i].target_bitrate_bps; + // If total bitrate differs now from what has been specified at the + // beginning, update the bitrates in the same ratio as before. + for (i = 0; i < num_spatial_layers_; ++i) { + config_->ss_target_bitrate[i] = config_->layer_target_bitrate[i] = + static_cast(static_cast(config_->rc_target_bitrate) * + codec_.spatialLayers[i].target_bitrate_bps / + total_bitrate_bps); + } + } else { + float rate_ratio[VPX_MAX_LAYERS] = {0}; + float total = 0; - for (i = 0; i < num_spatial_layers_; ++i) { - config_->ss_target_bitrate[i] = static_cast( - config_->rc_target_bitrate * rate_ratio[i] / total); - if (num_temporal_layers_ == 1) { - config_->layer_target_bitrate[i] = config_->ss_target_bitrate[i]; - } else if (num_temporal_layers_ == 2) { - config_->layer_target_bitrate[i * num_temporal_layers_] = - config_->ss_target_bitrate[i] * 2 / 3; - config_->layer_target_bitrate[i * num_temporal_layers_ + 1] = - config_->ss_target_bitrate[i]; - } else if (num_temporal_layers_ == 3) { - config_->layer_target_bitrate[i * num_temporal_layers_] = - config_->ss_target_bitrate[i] / 2; - config_->layer_target_bitrate[i * num_temporal_layers_ + 1] = - config_->layer_target_bitrate[i * num_temporal_layers_] + - (config_->ss_target_bitrate[i] / 4); - config_->layer_target_bitrate[i * num_temporal_layers_ + 2] = - config_->ss_target_bitrate[i]; - } else { - return false; + for (i = 0; i < num_spatial_layers_; ++i) { + if (svc_internal_.svc_params.scaling_factor_num[i] <= 0 || + svc_internal_.svc_params.scaling_factor_den[i] <= 0) { + LOG(LS_ERROR) << "Scaling factors not specified!"; + return false; + } + rate_ratio[i] = + static_cast(svc_internal_.svc_params.scaling_factor_num[i]) / + svc_internal_.svc_params.scaling_factor_den[i]; + total += rate_ratio[i]; + } + + for (i = 0; i < num_spatial_layers_; ++i) { + config_->ss_target_bitrate[i] = static_cast( + config_->rc_target_bitrate * rate_ratio[i] / total); + if (num_temporal_layers_ == 1) { + config_->layer_target_bitrate[i] = config_->ss_target_bitrate[i]; + } else if (num_temporal_layers_ == 2) { + config_->layer_target_bitrate[i * num_temporal_layers_] = + config_->ss_target_bitrate[i] * 2 / 3; + config_->layer_target_bitrate[i * num_temporal_layers_ + 1] = + config_->ss_target_bitrate[i]; + } else if (num_temporal_layers_ == 3) { + config_->layer_target_bitrate[i * num_temporal_layers_] = + config_->ss_target_bitrate[i] / 2; + config_->layer_target_bitrate[i * num_temporal_layers_ + 1] = + config_->layer_target_bitrate[i * num_temporal_layers_] + + (config_->ss_target_bitrate[i] / 4); + config_->layer_target_bitrate[i * num_temporal_layers_ + 2] = + config_->ss_target_bitrate[i]; + } else { + LOG(LS_ERROR) << "Unsupported number of temporal layers: " + << num_temporal_layers_; + return false; + } } } @@ -349,14 +379,24 @@ int VP9EncoderImpl::NumberOfThreads(int width, int VP9EncoderImpl::InitAndSetControlSettings(const VideoCodec* inst) { config_->ss_number_layers = num_spatial_layers_; - int scaling_factor_num = 256; - for (int i = num_spatial_layers_ - 1; i >= 0; --i) { - svc_internal_.svc_params.max_quantizers[i] = config_->rc_max_quantizer; - svc_internal_.svc_params.min_quantizers[i] = config_->rc_min_quantizer; - // 1:2 scaling in each dimension. - svc_internal_.svc_params.scaling_factor_num[i] = scaling_factor_num; - svc_internal_.svc_params.scaling_factor_den[i] = 256; - scaling_factor_num /= 2; + if (ExplicitlyConfiguredSpatialLayers()) { + for (int i = 0; i < num_spatial_layers_; ++i) { + const auto& layer = codec_.spatialLayers[i]; + svc_internal_.svc_params.max_quantizers[i] = config_->rc_max_quantizer; + svc_internal_.svc_params.min_quantizers[i] = config_->rc_min_quantizer; + svc_internal_.svc_params.scaling_factor_num[i] = layer.scaling_factor_num; + svc_internal_.svc_params.scaling_factor_den[i] = layer.scaling_factor_den; + } + } else { + int scaling_factor_num = 256; + for (int i = num_spatial_layers_ - 1; i >= 0; --i) { + svc_internal_.svc_params.max_quantizers[i] = config_->rc_max_quantizer; + svc_internal_.svc_params.min_quantizers[i] = config_->rc_min_quantizer; + // 1:2 scaling in each dimension. + svc_internal_.svc_params.scaling_factor_num[i] = scaling_factor_num; + svc_internal_.svc_params.scaling_factor_den[i] = 256; + scaling_factor_num /= 2; + } } if (!SetSvcRates()) { diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h index f9c123079e..ecc04651e5 100644 --- a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h +++ b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h @@ -56,6 +56,7 @@ class VP9EncoderImpl : public VP9Encoder { const vpx_codec_cx_pkt& pkt, uint32_t timestamp); + bool ExplicitlyConfiguredSpatialLayers() const; bool SetSvcRates(); virtual int GetEncodedLayerFrame(const vpx_codec_cx_pkt* pkt); diff --git a/webrtc/test/layer_filtering_transport.cc b/webrtc/test/layer_filtering_transport.cc index a4ebf47f93..5533a4cd46 100644 --- a/webrtc/test/layer_filtering_transport.cc +++ b/webrtc/test/layer_filtering_transport.cc @@ -9,9 +9,7 @@ */ #include "webrtc/base/checks.h" -#include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h" #include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp_defines.h" -#include "webrtc/modules/rtp_rtcp/source/byte_io.h" #include "webrtc/modules/rtp_rtcp/source/rtp_format.h" #include "webrtc/modules/rtp_rtcp/source/rtp_utility.h" #include "webrtc/test/layer_filtering_transport.h" @@ -24,33 +22,35 @@ LayerFilteringTransport::LayerFilteringTransport( Call* send_call, uint8_t vp8_video_payload_type, uint8_t vp9_video_payload_type, - uint8_t tl_discard_threshold, - uint8_t sl_discard_threshold) + int selected_tl, + int selected_sl) : test::DirectTransport(config, send_call), vp8_video_payload_type_(vp8_video_payload_type), vp9_video_payload_type_(vp9_video_payload_type), - tl_discard_threshold_(tl_discard_threshold), - sl_discard_threshold_(sl_discard_threshold) {} + selected_tl_(selected_tl), + selected_sl_(selected_sl), + discarded_last_packet_(false) {} -uint16_t LayerFilteringTransport::NextSequenceNumber(uint32_t ssrc) { - auto it = current_seq_nums_.find(ssrc); - if (it == current_seq_nums_.end()) - return current_seq_nums_[ssrc] = 10000; - return ++it->second; +bool LayerFilteringTransport::DiscardedLastPacket() const { + return discarded_last_packet_; } bool LayerFilteringTransport::SendRtp(const uint8_t* packet, size_t length, const PacketOptions& options) { - if (tl_discard_threshold_ == 0 && sl_discard_threshold_ == 0) { + if (selected_tl_ == -1 && selected_sl_ == -1) { // Nothing to change, forward the packet immediately. return test::DirectTransport::SendRtp(packet, length, options); } bool set_marker_bit = false; - rtc::scoped_ptr parser(RtpHeaderParser::Create()); + RtpUtility::RtpHeaderParser parser(packet, length); RTPHeader header; - parser->Parse(packet, length, &header); + parser.Parse(header); + + RTC_DCHECK_LE(length, static_cast(IP_PACKET_SIZE)); + uint8_t temp_buffer[IP_PACKET_SIZE]; + memcpy(temp_buffer, packet, length); if (header.payloadType == vp8_video_payload_type_ || header.payloadType == vp9_video_payload_type_) { @@ -65,40 +65,38 @@ bool LayerFilteringTransport::SendRtp(const uint8_t* packet, RtpDepacketizer::Create(is_vp8 ? kRtpVideoVp8 : kRtpVideoVp9)); RtpDepacketizer::ParsedPayload parsed_payload; if (depacketizer->Parse(&parsed_payload, payload, payload_data_length)) { - const uint8_t temporalIdx = + const int temporal_idx = static_cast( is_vp8 ? parsed_payload.type.Video.codecHeader.VP8.temporalIdx - : parsed_payload.type.Video.codecHeader.VP9.temporal_idx; - const uint8_t spatialIdx = + : parsed_payload.type.Video.codecHeader.VP9.temporal_idx); + const int spatial_idx = static_cast( is_vp8 ? kNoSpatialIdx - : parsed_payload.type.Video.codecHeader.VP9.spatial_idx; - if (sl_discard_threshold_ > 0 && - spatialIdx == sl_discard_threshold_ - 1 && + : parsed_payload.type.Video.codecHeader.VP9.spatial_idx); + if (selected_sl_ >= 0 && spatial_idx == selected_sl_ && parsed_payload.type.Video.codecHeader.VP9.end_of_frame) { // This layer is now the last in the superframe. set_marker_bit = true; - } - if ((tl_discard_threshold_ > 0 && temporalIdx != kNoTemporalIdx && - temporalIdx >= tl_discard_threshold_) || - (sl_discard_threshold_ > 0 && spatialIdx != kNoSpatialIdx && - spatialIdx >= sl_discard_threshold_)) { - return true; // Discard the packet. + } else if ((selected_tl_ >= 0 && temporal_idx != kNoTemporalIdx && + temporal_idx > selected_tl_) || + (selected_sl_ >= 0 && spatial_idx != kNoSpatialIdx && + spatial_idx > selected_sl_)) { + // Truncate packet to a padding packet. + length = header.headerLength + 1; + temp_buffer[0] |= (1 << 5); // P = 1. + temp_buffer[1] &= 0x7F; // M = 0. + discarded_last_packet_ = true; + temp_buffer[header.headerLength] = 1; // One byte of padding. } } else { RTC_NOTREACHED() << "Parse error"; } } - uint8_t temp_buffer[IP_PACKET_SIZE]; - memcpy(temp_buffer, packet, length); - // We are discarding some of the packets (specifically, whole layers), so // make sure the marker bit is set properly, and that sequence numbers are // continuous. if (set_marker_bit) temp_buffer[1] |= kRtpMarkerBitMask; - uint16_t seq_num = NextSequenceNumber(header.ssrc); - ByteWriter::WriteBigEndian(&temp_buffer[2], seq_num); return test::DirectTransport::SendRtp(temp_buffer, length, options); } diff --git a/webrtc/test/layer_filtering_transport.h b/webrtc/test/layer_filtering_transport.h index 3f2389a51b..d453556235 100644 --- a/webrtc/test/layer_filtering_transport.h +++ b/webrtc/test/layer_filtering_transport.h @@ -26,23 +26,22 @@ class LayerFilteringTransport : public test::DirectTransport { Call* send_call, uint8_t vp8_video_payload_type, uint8_t vp9_video_payload_type, - uint8_t tl_discard_threshold, - uint8_t sl_discard_threshold); + int selected_tl, + int selected_sl); + bool DiscardedLastPacket() const; bool SendRtp(const uint8_t* data, size_t length, const PacketOptions& options) override; private: - uint16_t NextSequenceNumber(uint32_t ssrc); // Used to distinguish between VP8 and VP9. const uint8_t vp8_video_payload_type_; const uint8_t vp9_video_payload_type_; - // Discard all temporal/spatial layers with id greater or equal the - // threshold. 0 to disable. - const uint8_t tl_discard_threshold_; - const uint8_t sl_discard_threshold_; - // Current sequence number for each SSRC separately. - std::map current_seq_nums_; + // Discard or invalidate all temporal/spatial layers with id greater than the + // selected one. -1 to disable filtering. + const int selected_tl_; + const int selected_sl_; + bool discarded_last_packet_; }; } // namespace test diff --git a/webrtc/video/full_stack.cc b/webrtc/video/full_stack.cc index 8511b8281e..2810cd610e 100644 --- a/webrtc/video/full_stack.cc +++ b/webrtc/video/full_stack.cc @@ -23,6 +23,15 @@ class FullStackTest : public VideoQualityTest { } }; +// VideoQualityTest::Params params = { +// { ... }, // Common. +// { ... }, // Video-specific settings. +// { ... }, // Screenshare-specific settings. +// { ... }, // Analyzer settings. +// pipe, // FakeNetworkPipe::Config +// { ... }, // Spatial scalability. +// logs // bool +// }; TEST_F(FullStackTest, ParisQcifWithoutPacketLoss) { VideoQualityTest::Params paris_qcif = { @@ -120,16 +129,16 @@ TEST_F(FullStackTest, ForemanCif1000kbps100msLimitedQueue) { TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL) { VideoQualityTest::Params screenshare = { - {1850, 1110, 5, 50000, 200000, 2000000, "VP8", 2, 400000}, - {}, // Video-specific. - {true, 10}, // Screenshare-specific. + {1850, 1110, 5, 50000, 200000, 2000000, "VP8", 2, 1, 400000}, + {}, + {true, 10}, {"screenshare_slides", 0.0, 0.0, kFullStackTestDurationSecs}}; RunTest(screenshare); } TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_Scroll) { VideoQualityTest::Params config = { - {1850, 1110 / 2, 5, 50000, 200000, 2000000, "VP8", 2, 400000}, + {1850, 1110 / 2, 5, 50000, 200000, 2000000, "VP8", 2, 1, 400000}, {}, {true, 10, 2}, {"screenshare_slides_scrolling", 0.0, 0.0, kFullStackTestDurationSecs}}; @@ -138,7 +147,7 @@ TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_Scroll) { TEST_F(FullStackTest, ScreenshareSlidesVP9_2TL) { VideoQualityTest::Params screenshare = { - {1850, 1110, 5, 50000, 200000, 2000000, "VP9", 2, 400000}, + {1850, 1110, 5, 50000, 200000, 2000000, "VP9", 2, 1, 400000}, {}, {true, 10}, {"screenshare_slides_vp9_2tl", 0.0, 0.0, kFullStackTestDurationSecs}}; diff --git a/webrtc/video/screenshare_loopback.cc b/webrtc/video/screenshare_loopback.cc index 9897783eb9..6479aa4ebb 100644 --- a/webrtc/video/screenshare_loopback.cc +++ b/webrtc/video/screenshare_loopback.cc @@ -20,6 +20,7 @@ namespace webrtc { namespace flags { +// Flags common with video loopback, with different default values. DEFINE_int32(width, 1850, "Video width (crops source)."); size_t Width() { return static_cast(FLAGS_width); @@ -35,21 +36,6 @@ int Fps() { return static_cast(FLAGS_fps); } -DEFINE_int32(slide_change_interval, - 10, - "Interval (in seconds) between simulated slide changes."); -int SlideChangeInterval() { - return static_cast(FLAGS_slide_change_interval); -} - -DEFINE_int32( - scroll_duration, - 0, - "Duration (in seconds) during which a slide will be scrolled into place."); -int ScrollDuration() { - return static_cast(FLAGS_scroll_duration); -} - DEFINE_int32(min_bitrate, 50, "Call and stream min bitrate in kbps."); int MinBitrateKbps() { return static_cast(FLAGS_min_bitrate); @@ -71,28 +57,43 @@ int MaxBitrateKbps() { } DEFINE_int32(num_temporal_layers, 2, "Number of temporal layers to use."); -size_t NumTemporalLayers() { - return static_cast(FLAGS_num_temporal_layers); -} - -DEFINE_int32( - tl_discard_threshold, - 0, - "Discard TLs with id greater or equal the threshold. 0 to disable."); -size_t TLDiscardThreshold() { - return static_cast(FLAGS_tl_discard_threshold); -} - -DEFINE_int32(min_transmit_bitrate, 400, "Min transmit bitrate incl. padding."); -int MinTransmitBitrateKbps() { - return FLAGS_min_transmit_bitrate; +int NumTemporalLayers() { + return static_cast(FLAGS_num_temporal_layers); } +// Flags common with video loopback, with equal default values. DEFINE_string(codec, "VP8", "Video codec to use."); std::string Codec() { return static_cast(FLAGS_codec); } +DEFINE_int32(selected_tl, + -1, + "Temporal layer to show or analyze. -1 to disable filtering."); +int SelectedTL() { + return static_cast(FLAGS_selected_tl); +} + +DEFINE_int32( + duration, + 0, + "Duration of the test in seconds. If 0, rendered will be shown instead."); +int DurationSecs() { + return static_cast(FLAGS_duration); +} + +DEFINE_string(output_filename, "", "Target graph data filename."); +std::string OutputFilename() { + return static_cast(FLAGS_output_filename); +} + +DEFINE_string(graph_title, + "", + "If empty, title will be generated automatically."); +std::string GraphTitle() { + return static_cast(FLAGS_graph_title); +} + DEFINE_int32(loss_percent, 0, "Percentage of packets randomly lost."); int LossPercent() { return static_cast(FLAGS_loss_percent); @@ -124,21 +125,53 @@ int StdPropagationDelayMs() { return static_cast(FLAGS_std_propagation_delay_ms); } +DEFINE_int32(selected_stream, 0, "ID of the stream to show or analyze."); +int SelectedStream() { + return static_cast(FLAGS_selected_stream); +} + +DEFINE_int32(num_spatial_layers, 1, "Number of spatial layers to use."); +int NumSpatialLayers() { + return static_cast(FLAGS_num_spatial_layers); +} + +DEFINE_int32(selected_sl, + -1, + "Spatial layer to show or analyze. -1 to disable filtering."); +int SelectedSL() { + return static_cast(FLAGS_selected_sl); +} + +DEFINE_string(stream0, + "", + "Comma separated values describing VideoStream for stream #0."); +std::string Stream0() { + return static_cast(FLAGS_stream0); +} + +DEFINE_string(stream1, + "", + "Comma separated values describing VideoStream for stream #1."); +std::string Stream1() { + return static_cast(FLAGS_stream1); +} + +DEFINE_string(sl0, + "", + "Comma separated values describing SpatialLayer for layer #0."); +std::string SL0() { + return static_cast(FLAGS_sl0); +} + +DEFINE_string(sl1, + "", + "Comma separated values describing SpatialLayer for layer #1."); +std::string SL1() { + return static_cast(FLAGS_sl1); +} + DEFINE_bool(logs, false, "print logs to stderr"); -DEFINE_string( - output_filename, - "", - "Name of a target graph data file. If set, no preview will be shown."); -std::string OutputFilename() { - return static_cast(FLAGS_output_filename); -} - -DEFINE_int32(duration, 60, "Duration of the test in seconds."); -int DurationSecs() { - return static_cast(FLAGS_duration); -} - DEFINE_bool(send_side_bwe, true, "Use send-side bandwidth estimation"); DEFINE_string( @@ -148,6 +181,28 @@ DEFINE_string( "E.g. running with --force_fieldtrials=WebRTC-FooFeature/Enable/" " will assign the group Enable to field trial WebRTC-FooFeature. Multiple " "trials are separated by \"/\""); + +// Screenshare-specific flags. +DEFINE_int32(min_transmit_bitrate, 400, "Min transmit bitrate incl. padding."); +int MinTransmitBitrateKbps() { + return FLAGS_min_transmit_bitrate; +} + +DEFINE_int32(slide_change_interval, + 10, + "Interval (in seconds) between simulated slide changes."); +int SlideChangeInterval() { + return static_cast(FLAGS_slide_change_interval); +} + +DEFINE_int32( + scroll_duration, + 0, + "Duration (in seconds) during which a slide will be scrolled into place."); +int ScrollDuration() { + return static_cast(FLAGS_scroll_duration); +} + } // namespace flags void Loopback() { @@ -167,20 +222,32 @@ void Loopback() { {flags::Width(), flags::Height(), flags::Fps(), flags::MinBitrateKbps() * 1000, flags::TargetBitrateKbps() * 1000, flags::MaxBitrateKbps() * 1000, flags::Codec(), - flags::NumTemporalLayers(), flags::MinTransmitBitrateKbps() * 1000, - call_bitrate_config, flags::TLDiscardThreshold(), + flags::NumTemporalLayers(), flags::SelectedTL(), + flags::MinTransmitBitrateKbps() * 1000, call_bitrate_config, flags::FLAGS_send_side_bwe}, {}, // Video specific. {true, flags::SlideChangeInterval(), flags::ScrollDuration()}, - {"screenshare", 0.0, 0.0, flags::DurationSecs(), flags::OutputFilename()}, + {"screenshare", 0.0, 0.0, flags::DurationSecs(), flags::OutputFilename(), + flags::GraphTitle()}, pipe_config, flags::FLAGS_logs}; + std::vector stream_descriptors; + stream_descriptors.push_back(flags::Stream0()); + stream_descriptors.push_back(flags::Stream1()); + std::vector SL_descriptors; + SL_descriptors.push_back(flags::SL0()); + SL_descriptors.push_back(flags::SL1()); + VideoQualityTest::FillScalabilitySettings( + ¶ms, stream_descriptors, flags::SelectedStream(), + flags::NumSpatialLayers(), flags::SelectedSL(), SL_descriptors); + VideoQualityTest test; - if (flags::OutputFilename().empty()) - test.RunWithVideoRenderer(params); - else + if (flags::DurationSecs()) { test.RunWithAnalyzer(params); + } else { + test.RunWithVideoRenderer(params); + } } } // namespace webrtc diff --git a/webrtc/video/video_loopback.cc b/webrtc/video/video_loopback.cc index 0c06f85fcc..2338a84a43 100644 --- a/webrtc/video/video_loopback.cc +++ b/webrtc/video/video_loopback.cc @@ -20,6 +20,7 @@ namespace webrtc { namespace flags { +// Flags common with screenshare loopback, with different default values. DEFINE_int32(width, 640, "Video width."); size_t Width() { return static_cast(FLAGS_width); @@ -55,11 +56,46 @@ int MaxBitrateKbps() { return static_cast(FLAGS_max_bitrate); } +DEFINE_int32(num_temporal_layers, + 1, + "Number of temporal layers. Set to 1-4 to override."); +int NumTemporalLayers() { + return static_cast(FLAGS_num_temporal_layers); +} + +// Flags common with screenshare loopback, with equal default values. DEFINE_string(codec, "VP8", "Video codec to use."); std::string Codec() { return static_cast(FLAGS_codec); } +DEFINE_int32(selected_tl, + -1, + "Temporal layer to show or analyze. -1 to disable filtering."); +int SelectedTL() { + return static_cast(FLAGS_selected_tl); +} + +DEFINE_int32( + duration, + 0, + "Duration of the test in seconds. If 0, rendered will be shown instead."); +int DurationSecs() { + return static_cast(FLAGS_duration); +} + +DEFINE_string(output_filename, "", "Target graph data filename."); +std::string OutputFilename() { + return static_cast(FLAGS_output_filename); +} + +DEFINE_string(graph_title, + "", + "If empty, title will be generated automatically."); +std::string GraphTitle() { + return static_cast(FLAGS_graph_title); +} + DEFINE_int32(loss_percent, 0, "Percentage of packets randomly lost."); int LossPercent() { return static_cast(FLAGS_loss_percent); @@ -91,8 +127,55 @@ int StdPropagationDelayMs() { return static_cast(FLAGS_std_propagation_delay_ms); } +DEFINE_int32(selected_stream, 0, "ID of the stream to show or analyze."); +int SelectedStream() { + return static_cast(FLAGS_selected_stream); +} + +DEFINE_int32(num_spatial_layers, 1, "Number of spatial layers to use."); +int NumSpatialLayers() { + return static_cast(FLAGS_num_spatial_layers); +} + +DEFINE_int32(selected_sl, + -1, + "Spatial layer to show or analyze. -1 to disable filtering."); +int SelectedSL() { + return static_cast(FLAGS_selected_sl); +} + +DEFINE_string(stream0, + "", + "Comma separated values describing VideoStream for stream #0."); +std::string Stream0() { + return static_cast(FLAGS_stream0); +} + +DEFINE_string(stream1, + "", + "Comma separated values describing VideoStream for stream #1."); +std::string Stream1() { + return static_cast(FLAGS_stream1); +} + +DEFINE_string(sl0, + "", + "Comma separated values describing SpatialLayer for layer #0."); +std::string SL0() { + return static_cast(FLAGS_sl0); +} + +DEFINE_string(sl1, + "", + "Comma separated values describing SpatialLayer for layer #1."); +std::string SL1() { + return static_cast(FLAGS_sl1); +} + DEFINE_bool(logs, false, "print logs to stderr"); +DEFINE_bool(send_side_bwe, true, "Use send-side bandwidth estimation"); + DEFINE_string( force_fieldtrials, "", @@ -101,21 +184,7 @@ DEFINE_string( " will assign the group Enable to field trial WebRTC-FooFeature. Multiple " "trials are separated by \"/\""); -DEFINE_int32(num_temporal_layers, - 1, - "Number of temporal layers. Set to 1-4 to override."); -size_t NumTemporalLayers() { - return static_cast(FLAGS_num_temporal_layers); -} - -DEFINE_int32( - tl_discard_threshold, - 0, - "Discard TLs with id greater or equal the threshold. 0 to disable."); -size_t TLDiscardThreshold() { - return static_cast(FLAGS_tl_discard_threshold); -} - +// Video-specific flags. DEFINE_string(clip, "", "Name of the clip to show. If empty, using chroma generator."); @@ -123,21 +192,6 @@ std::string Clip() { return static_cast(FLAGS_clip); } -DEFINE_string( - output_filename, - "", - "Name of a target graph data file. If set, no preview will be shown."); -std::string OutputFilename() { - return static_cast(FLAGS_output_filename); -} - -DEFINE_int32(duration, 60, "Duration of the test in seconds."); -int DurationSecs() { - return static_cast(FLAGS_duration); -} - -DEFINE_bool(send_side_bwe, true, "Use send-side bandwidth estimation"); - } // namespace flags void Loopback() { @@ -153,27 +207,36 @@ void Loopback() { call_bitrate_config.start_bitrate_bps = flags::StartBitrateKbps() * 1000; call_bitrate_config.max_bitrate_bps = flags::MaxBitrateKbps() * 1000; - std::string clip = flags::Clip(); - std::string graph_title = clip.empty() ? "" : "video " + clip; VideoQualityTest::Params params{ {flags::Width(), flags::Height(), flags::Fps(), flags::MinBitrateKbps() * 1000, flags::TargetBitrateKbps() * 1000, flags::MaxBitrateKbps() * 1000, flags::Codec(), - flags::NumTemporalLayers(), + flags::NumTemporalLayers(), flags::SelectedTL(), 0, // No min transmit bitrate. - call_bitrate_config, flags::TLDiscardThreshold(), - flags::FLAGS_send_side_bwe}, - {clip}, + call_bitrate_config, flags::FLAGS_send_side_bwe}, + {flags::Clip()}, {}, // Screenshare specific. - {graph_title, 0.0, 0.0, flags::DurationSecs(), flags::OutputFilename()}, + {"video", 0.0, 0.0, flags::DurationSecs(), flags::OutputFilename(), + flags::GraphTitle()}, pipe_config, flags::FLAGS_logs}; + std::vector stream_descriptors; + stream_descriptors.push_back(flags::Stream0()); + stream_descriptors.push_back(flags::Stream1()); + std::vector SL_descriptors; + SL_descriptors.push_back(flags::SL0()); + SL_descriptors.push_back(flags::SL1()); + VideoQualityTest::FillScalabilitySettings( + ¶ms, stream_descriptors, flags::SelectedStream(), + flags::NumSpatialLayers(), flags::SelectedSL(), SL_descriptors); + VideoQualityTest test; - if (flags::OutputFilename().empty()) - test.RunWithVideoRenderer(params); - else + if (flags::DurationSecs()) { test.RunWithAnalyzer(params); + } else { + test.RunWithVideoRenderer(params); + } } } // namespace webrtc diff --git a/webrtc/video/video_quality_test.cc b/webrtc/video/video_quality_test.cc index 0f45fa6632..333f00dc43 100644 --- a/webrtc/video/video_quality_test.cc +++ b/webrtc/video/video_quality_test.cc @@ -12,6 +12,7 @@ #include #include #include +#include #include #include "testing/gtest/include/gtest/gtest.h" @@ -21,7 +22,7 @@ #include "webrtc/base/scoped_ptr.h" #include "webrtc/call.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" -#include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h" +#include "webrtc/modules/rtp_rtcp/source/rtp_utility.h" #include "webrtc/system_wrappers/include/cpu_info.h" #include "webrtc/test/layer_filtering_transport.h" #include "webrtc/test/run_loop.h" @@ -43,18 +44,22 @@ class VideoAnalyzer : public PacketReceiver, public EncodedFrameObserver, public EncodingTimeObserver { public: - VideoAnalyzer(Transport* transport, + VideoAnalyzer(test::LayerFilteringTransport* transport, const std::string& test_label, double avg_psnr_threshold, double avg_ssim_threshold, int duration_frames, - FILE* graph_data_output_file) + FILE* graph_data_output_file, + const std::string& graph_title, + uint32_t ssrc_to_analyze) : input_(nullptr), transport_(transport), receiver_(nullptr), send_stream_(nullptr), test_label_(test_label), graph_data_output_file_(graph_data_output_file), + graph_title_(graph_title), + ssrc_to_analyze_(ssrc_to_analyze), frames_to_process_(duration_frames), frames_recorded_(0), frames_processed_(0), @@ -93,7 +98,6 @@ class VideoAnalyzer : public PacketReceiver, stats_polling_thread_ = ThreadWrapper::CreateThread(&PollStatsThread, this, "StatsPoller"); - EXPECT_TRUE(stats_polling_thread_->Start()); } ~VideoAnalyzer() { @@ -109,9 +113,9 @@ class VideoAnalyzer : public PacketReceiver, const uint8_t* packet, size_t length, const PacketTime& packet_time) override { - rtc::scoped_ptr parser(RtpHeaderParser::Create()); + RtpUtility::RtpHeaderParser parser(packet, length); RTPHeader header; - parser->Parse(packet, length, &header); + parser.Parse(header); { rtc::CritScope lock(&crit_); recv_times_[header.timestamp - rtp_timestamp_delta_] = @@ -145,10 +149,13 @@ class VideoAnalyzer : public PacketReceiver, bool SendRtp(const uint8_t* packet, size_t length, const PacketOptions& options) override { - rtc::scoped_ptr parser(RtpHeaderParser::Create()); + RtpUtility::RtpHeaderParser parser(packet, length); RTPHeader header; - parser->Parse(packet, length, &header); + parser.Parse(header); + int64_t current_time = + Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); + bool result = transport_->SendRtp(packet, length, options); { rtc::CritScope lock(&crit_); if (rtp_timestamp_delta_ == 0) { @@ -156,13 +163,14 @@ class VideoAnalyzer : public PacketReceiver, first_send_frame_.Reset(); } uint32_t timestamp = header.timestamp - rtp_timestamp_delta_; - send_times_[timestamp] = - Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); - encoded_frame_sizes_[timestamp] += - length - (header.headerLength + header.paddingLength); + send_times_[timestamp] = current_time; + if (!transport_->DiscardedLastPacket() && + header.ssrc == ssrc_to_analyze_) { + encoded_frame_sizes_[timestamp] += + length - (header.headerLength + header.paddingLength); + } } - - return transport_->SendRtp(packet, length, options); + return result; } bool SendRtcp(const uint8_t* packet, size_t length) override { @@ -192,6 +200,11 @@ class VideoAnalyzer : public PacketReceiver, VideoFrame reference_frame = frames_.front(); frames_.pop_front(); assert(!reference_frame.IsZeroSize()); + if (send_timestamp == reference_frame.timestamp() - 1) { + // TODO(ivica): Make this work for > 2 streams. + // Look at rtp_sender.c:RTPSender::BuildRTPHeader. + ++send_timestamp; + } EXPECT_EQ(reference_frame.timestamp(), send_timestamp); assert(reference_frame.timestamp() == send_timestamp); @@ -207,6 +220,8 @@ class VideoAnalyzer : public PacketReceiver, // at time-out check if frames_processed is going up. If so, give it more // time, otherwise fail. Hopefully this will reduce test flakiness. + EXPECT_TRUE(stats_polling_thread_->Start()); + int last_frames_processed = -1; EventTypeWrapper eventType; int iteration = 0; @@ -245,7 +260,7 @@ class VideoAnalyzer : public PacketReceiver, } VideoCaptureInput* input_; - Transport* const transport_; + test::LayerFilteringTransport* const transport_; PacketReceiver* receiver_; VideoSendStream* send_stream_; @@ -320,8 +335,13 @@ class VideoAnalyzer : public PacketReceiver, int64_t recv_time_ms = recv_times_[reference.timestamp()]; recv_times_.erase(reference.timestamp()); - size_t encoded_size = encoded_frame_sizes_[reference.timestamp()]; - encoded_frame_sizes_.erase(reference.timestamp()); + // TODO(ivica): Make this work for > 2 streams. + auto it = encoded_frame_sizes_.find(reference.timestamp()); + if (it == encoded_frame_sizes_.end()) + it = encoded_frame_sizes_.find(reference.timestamp() - 1); + size_t encoded_size = it == encoded_frame_sizes_.end() ? 0 : it->second; + if (it != encoded_frame_sizes_.end()) + encoded_frame_sizes_.erase(it); VideoFrame reference_copy; VideoFrame render_copy; @@ -509,7 +529,7 @@ class VideoAnalyzer : public PacketReceiver, return A.input_time_ms < B.input_time_ms; }); - fprintf(out, "%s\n", test_label_.c_str()); + fprintf(out, "%s\n", graph_title_.c_str()); fprintf(out, "%" PRIuS "\n", samples_.size()); fprintf(out, "dropped " @@ -547,6 +567,8 @@ class VideoAnalyzer : public PacketReceiver, const std::string test_label_; FILE* const graph_data_output_file_; + const std::string graph_title_; + const uint32_t ssrc_to_analyze_; std::vector samples_ GUARDED_BY(comparison_lock_); std::map samples_encode_time_ms_ GUARDED_BY(comparison_lock_); test::Statistics sender_time_ GUARDED_BY(comparison_lock_); @@ -588,28 +610,188 @@ class VideoAnalyzer : public PacketReceiver, VideoQualityTest::VideoQualityTest() : clock_(Clock::GetRealTimeClock()) {} -void VideoQualityTest::ValidateParams(const Params& params) { - RTC_CHECK_GE(params.common.max_bitrate_bps, params.common.target_bitrate_bps); - RTC_CHECK_GE(params.common.target_bitrate_bps, params.common.min_bitrate_bps); - RTC_CHECK_LT(params.common.tl_discard_threshold, - params.common.num_temporal_layers); -} - void VideoQualityTest::TestBody() {} -void VideoQualityTest::SetupFullStack(const Params& params, - Transport* send_transport, - Transport* recv_transport) { - if (params.logs) +std::string VideoQualityTest::GenerateGraphTitle() const { + std::stringstream ss; + ss << params_.common.codec; + ss << " (" << params_.common.target_bitrate_bps / 1000 << "kbps"; + ss << ", " << params_.common.fps << " FPS"; + if (params_.screenshare.scroll_duration) + ss << ", " << params_.screenshare.scroll_duration << "s scroll"; + if (params_.ss.streams.size() > 1) + ss << ", Stream #" << params_.ss.selected_stream; + if (params_.ss.num_spatial_layers > 1) + ss << ", Layer #" << params_.ss.selected_sl; + ss << ")"; + return ss.str(); +} + +void VideoQualityTest::CheckParams() { + // Add a default stream in none specified. + if (params_.ss.streams.empty()) + params_.ss.streams.push_back(VideoQualityTest::DefaultVideoStream(params_)); + if (params_.ss.num_spatial_layers == 0) + params_.ss.num_spatial_layers = 1; + + if (params_.pipe.loss_percent != 0 || + params_.pipe.queue_length_packets != 0) { + // Since LayerFilteringTransport changes the sequence numbers, we can't + // use that feature with pack loss, since the NACK request would end up + // retransmitting the wrong packets. + RTC_CHECK(params_.ss.selected_sl == -1 || + params_.ss.num_spatial_layers == 1); + RTC_CHECK(params_.common.selected_tl == -1 || + params_.common.num_temporal_layers == 1); + } + + // TODO(ivica): Should max_bitrate_bps == -1 represent inf max bitrate, as it + // does in some parts of the code? + RTC_CHECK_GE(params_.common.max_bitrate_bps, + params_.common.target_bitrate_bps); + RTC_CHECK_GE(params_.common.target_bitrate_bps, + params_.common.min_bitrate_bps); + RTC_CHECK_LT(params_.common.selected_tl, params_.common.num_temporal_layers); + RTC_CHECK_LT(params_.ss.selected_stream, params_.ss.streams.size()); + for (const VideoStream& stream : params_.ss.streams) { + RTC_CHECK_GE(stream.min_bitrate_bps, 0); + RTC_CHECK_GE(stream.target_bitrate_bps, stream.min_bitrate_bps); + RTC_CHECK_GE(stream.max_bitrate_bps, stream.target_bitrate_bps); + RTC_CHECK_EQ(static_cast(stream.temporal_layer_thresholds_bps.size()), + params_.common.num_temporal_layers - 1); + } + // TODO(ivica): Should we check if the sum of all streams/layers is equal to + // the total bitrate? We anyway have to update them in the case bitrate + // estimator changes the total bitrates. + RTC_CHECK_GE(params_.ss.num_spatial_layers, 1); + RTC_CHECK_LE(params_.ss.selected_sl, params_.ss.num_spatial_layers); + RTC_CHECK(params_.ss.spatial_layers.empty() || + params_.ss.spatial_layers.size() == + static_cast(params_.ss.num_spatial_layers)); + if (params_.common.codec == "VP8") { + RTC_CHECK_EQ(params_.ss.num_spatial_layers, 1); + } else if (params_.common.codec == "VP9") { + RTC_CHECK_EQ(params_.ss.streams.size(), 1u); + } +} + +// Static. +std::vector VideoQualityTest::ParseCSV(const std::string& str) { + // Parse comma separated nonnegative integers, where some elements may be + // empty. The empty values are replaced with -1. + // E.g. "10,-20,,30,40" --> {10, 20, -1, 30,40} + // E.g. ",,10,,20," --> {-1, -1, 10, -1, 20, -1} + std::vector result; + if (str.empty()) + return result; + + const char* p = str.c_str(); + int value = -1; + int pos; + while (*p) { + if (*p == ',') { + result.push_back(value); + value = -1; + ++p; + continue; + } + RTC_CHECK_EQ(sscanf(p, "%d%n", &value, &pos), 1) + << "Unexpected non-number value."; + p += pos; + } + result.push_back(value); + return result; +} + +// Static. +VideoStream VideoQualityTest::DefaultVideoStream(const Params& params) { + VideoStream stream; + stream.width = params.common.width; + stream.height = params.common.height; + stream.max_framerate = params.common.fps; + stream.min_bitrate_bps = params.common.min_bitrate_bps; + stream.target_bitrate_bps = params.common.target_bitrate_bps; + stream.max_bitrate_bps = params.common.max_bitrate_bps; + stream.max_qp = 52; + if (params.common.num_temporal_layers == 2) + stream.temporal_layer_thresholds_bps.push_back(stream.target_bitrate_bps); + return stream; +} + +// Static. +void VideoQualityTest::FillScalabilitySettings( + Params* params, + const std::vector& stream_descriptors, + size_t selected_stream, + int num_spatial_layers, + int selected_sl, + const std::vector& sl_descriptors) { + // Read VideoStream and SpatialLayer elements from a list of comma separated + // lists. To use a default value for an element, use -1 or leave empty. + // Validity checks performed in CheckParams. + + RTC_CHECK(params->ss.streams.empty()); + for (auto descriptor : stream_descriptors) { + if (descriptor.empty()) + continue; + VideoStream stream = VideoQualityTest::DefaultVideoStream(*params); + std::vector v = VideoQualityTest::ParseCSV(descriptor); + if (v[0] != -1) + stream.width = static_cast(v[0]); + if (v[1] != -1) + stream.height = static_cast(v[1]); + if (v[2] != -1) + stream.max_framerate = v[2]; + if (v[3] != -1) + stream.min_bitrate_bps = v[3]; + if (v[4] != -1) + stream.target_bitrate_bps = v[4]; + if (v[5] != -1) + stream.max_bitrate_bps = v[5]; + if (v.size() > 6 && v[6] != -1) + stream.max_qp = v[6]; + if (v.size() > 7) { + stream.temporal_layer_thresholds_bps.clear(); + stream.temporal_layer_thresholds_bps.insert( + stream.temporal_layer_thresholds_bps.end(), v.begin() + 7, v.end()); + } else { + // Automatic TL thresholds for more than two layers not supported. + RTC_CHECK_LE(params->common.num_temporal_layers, 2); + } + params->ss.streams.push_back(stream); + } + params->ss.selected_stream = selected_stream; + + params->ss.num_spatial_layers = num_spatial_layers ? num_spatial_layers : 1; + params->ss.selected_sl = selected_sl; + RTC_CHECK(params->ss.spatial_layers.empty()); + for (auto descriptor : sl_descriptors) { + if (descriptor.empty()) + continue; + std::vector v = VideoQualityTest::ParseCSV(descriptor); + RTC_CHECK_GT(v[2], 0); + + SpatialLayer layer; + layer.scaling_factor_num = v[0] == -1 ? 1 : v[0]; + layer.scaling_factor_den = v[1] == -1 ? 1 : v[1]; + layer.target_bitrate_bps = v[2]; + params->ss.spatial_layers.push_back(layer); + } +} + +void VideoQualityTest::SetupCommon(Transport* send_transport, + Transport* recv_transport) { + if (params_.logs) trace_to_stderr_.reset(new test::TraceToStderr); - CreateSendConfig(1, send_transport); + size_t num_streams = params_.ss.streams.size(); + CreateSendConfig(num_streams, send_transport); int payload_type; - if (params.common.codec == "VP8") { + if (params_.common.codec == "VP8") { encoder_.reset(VideoEncoder::Create(VideoEncoder::kVp8)); payload_type = kPayloadTypeVP8; - } else if (params.common.codec == "VP9") { + } else if (params_.common.codec == "VP9") { encoder_.reset(VideoEncoder::Create(VideoEncoder::kVp9)); payload_type = kPayloadTypeVP9; } else { @@ -617,15 +799,15 @@ void VideoQualityTest::SetupFullStack(const Params& params, return; } send_config_.encoder_settings.encoder = encoder_.get(); - send_config_.encoder_settings.payload_name = params.common.codec; + send_config_.encoder_settings.payload_name = params_.common.codec; send_config_.encoder_settings.payload_type = payload_type; - send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs; - send_config_.rtp.rtx.ssrcs.push_back(kSendRtxSsrcs[0]); send_config_.rtp.rtx.payload_type = kSendRtxPayloadType; + for (size_t i = 0; i < num_streams; ++i) + send_config_.rtp.rtx.ssrcs.push_back(kSendRtxSsrcs[i]); send_config_.rtp.extensions.clear(); - if (params.common.send_side_bwe) { + if (params_.common.send_side_bwe) { send_config_.rtp.extensions.push_back( RtpExtension(RtpExtension::kTransportSequenceNumber, test::kTransportSequenceNumberExtensionId)); @@ -634,49 +816,41 @@ void VideoQualityTest::SetupFullStack(const Params& params, RtpExtension::kAbsSendTime, test::kAbsSendTimeExtensionId)); } - // Automatically fill out streams[0] with params. - VideoStream* stream = &encoder_config_.streams[0]; - stream->width = params.common.width; - stream->height = params.common.height; - stream->min_bitrate_bps = params.common.min_bitrate_bps; - stream->target_bitrate_bps = params.common.target_bitrate_bps; - stream->max_bitrate_bps = params.common.max_bitrate_bps; - stream->max_framerate = static_cast(params.common.fps); - - stream->temporal_layer_thresholds_bps.clear(); - if (params.common.num_temporal_layers > 1) { - stream->temporal_layer_thresholds_bps.push_back(stream->target_bitrate_bps); - } + encoder_config_.min_transmit_bitrate_bps = params_.common.min_transmit_bps; + encoder_config_.streams = params_.ss.streams; + encoder_config_.spatial_layers = params_.ss.spatial_layers; CreateMatchingReceiveConfigs(recv_transport); - receive_configs_[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs; - receive_configs_[0].rtp.rtx[kSendRtxPayloadType].ssrc = kSendRtxSsrcs[0]; - receive_configs_[0].rtp.rtx[kSendRtxPayloadType].payload_type = - kSendRtxPayloadType; - - encoder_config_.min_transmit_bitrate_bps = params.common.min_transmit_bps; + for (size_t i = 0; i < num_streams; ++i) { + receive_configs_[i].rtp.nack.rtp_history_ms = kNackRtpHistoryMs; + receive_configs_[i].rtp.rtx[kSendRtxPayloadType].ssrc = kSendRtxSsrcs[i]; + receive_configs_[i].rtp.rtx[kSendRtxPayloadType].payload_type = + kSendRtxPayloadType; + } } -void VideoQualityTest::SetupScreenshare(const Params& params) { - RTC_CHECK(params.screenshare.enabled); +void VideoQualityTest::SetupScreenshare() { + RTC_CHECK(params_.screenshare.enabled); // Fill out codec settings. encoder_config_.content_type = VideoEncoderConfig::ContentType::kScreen; - if (params.common.codec == "VP8") { + if (params_.common.codec == "VP8") { codec_settings_.VP8 = VideoEncoder::GetDefaultVp8Settings(); codec_settings_.VP8.denoisingOn = false; codec_settings_.VP8.frameDroppingOn = false; codec_settings_.VP8.numberOfTemporalLayers = - static_cast(params.common.num_temporal_layers); + static_cast(params_.common.num_temporal_layers); encoder_config_.encoder_specific_settings = &codec_settings_.VP8; - } else if (params.common.codec == "VP9") { + } else if (params_.common.codec == "VP9") { codec_settings_.VP9 = VideoEncoder::GetDefaultVp9Settings(); codec_settings_.VP9.denoisingOn = false; codec_settings_.VP9.frameDroppingOn = false; codec_settings_.VP9.numberOfTemporalLayers = - static_cast(params.common.num_temporal_layers); + static_cast(params_.common.num_temporal_layers); encoder_config_.encoder_specific_settings = &codec_settings_.VP9; + codec_settings_.VP9.numberOfSpatialLayers = + static_cast(params_.ss.num_spatial_layers); } // Setup frame generator. @@ -688,71 +862,67 @@ void VideoQualityTest::SetupScreenshare(const Params& params) { slides.push_back(test::ResourcePath("photo_1850_1110", "yuv")); slides.push_back(test::ResourcePath("difficult_photo_1850_1110", "yuv")); - if (params.screenshare.scroll_duration == 0) { + if (params_.screenshare.scroll_duration == 0) { // Cycle image every slide_change_interval seconds. frame_generator_.reset(test::FrameGenerator::CreateFromYuvFile( slides, kWidth, kHeight, - params.screenshare.slide_change_interval * params.common.fps)); + params_.screenshare.slide_change_interval * params_.common.fps)); } else { - RTC_CHECK_LE(params.common.width, kWidth); - RTC_CHECK_LE(params.common.height, kHeight); - RTC_CHECK_GT(params.screenshare.slide_change_interval, 0); - const int kPauseDurationMs = (params.screenshare.slide_change_interval - - params.screenshare.scroll_duration) * 1000; - RTC_CHECK_LE(params.screenshare.scroll_duration, - params.screenshare.slide_change_interval); + RTC_CHECK_LE(params_.common.width, kWidth); + RTC_CHECK_LE(params_.common.height, kHeight); + RTC_CHECK_GT(params_.screenshare.slide_change_interval, 0); + const int kPauseDurationMs = (params_.screenshare.slide_change_interval - + params_.screenshare.scroll_duration) * + 1000; + RTC_CHECK_LE(params_.screenshare.scroll_duration, + params_.screenshare.slide_change_interval); - if (params.screenshare.scroll_duration) { - frame_generator_.reset( - test::FrameGenerator::CreateScrollingInputFromYuvFiles( - clock_, slides, kWidth, kHeight, params.common.width, - params.common.height, params.screenshare.scroll_duration * 1000, - kPauseDurationMs)); - } else { - frame_generator_.reset(test::FrameGenerator::CreateFromYuvFile( - slides, kWidth, kHeight, - params.screenshare.slide_change_interval * params.common.fps)); - } + frame_generator_.reset( + test::FrameGenerator::CreateScrollingInputFromYuvFiles( + clock_, slides, kWidth, kHeight, params_.common.width, + params_.common.height, params_.screenshare.scroll_duration * 1000, + kPauseDurationMs)); } } -void VideoQualityTest::CreateCapturer(const Params& params, - VideoCaptureInput* input) { - if (params.screenshare.enabled) { - test::FrameGeneratorCapturer *frame_generator_capturer = +void VideoQualityTest::CreateCapturer(VideoCaptureInput* input) { + if (params_.screenshare.enabled) { + test::FrameGeneratorCapturer* frame_generator_capturer = new test::FrameGeneratorCapturer( - clock_, input, frame_generator_.release(), params.common.fps); + clock_, input, frame_generator_.release(), params_.common.fps); EXPECT_TRUE(frame_generator_capturer->Init()); capturer_.reset(frame_generator_capturer); } else { - if (params.video.clip_name.empty()) { - capturer_.reset(test::VideoCapturer::Create( - input, params.common.width, params.common.height, params.common.fps, - clock_)); + if (params_.video.clip_name.empty()) { + capturer_.reset(test::VideoCapturer::Create(input, params_.common.width, + params_.common.height, + params_.common.fps, clock_)); } else { capturer_.reset(test::FrameGeneratorCapturer::CreateFromYuvFile( - input, test::ResourcePath(params.video.clip_name, "yuv"), - params.common.width, params.common.height, params.common.fps, + input, test::ResourcePath(params_.video.clip_name, "yuv"), + params_.common.width, params_.common.height, params_.common.fps, clock_)); ASSERT_TRUE(capturer_.get() != nullptr) - << "Could not create capturer for " << params.video.clip_name + << "Could not create capturer for " << params_.video.clip_name << ".yuv. Is this resource file present?"; } } } void VideoQualityTest::RunWithAnalyzer(const Params& params) { + params_ = params; + // TODO(ivica): Merge with RunWithRenderer and use a flag / argument to // differentiate between the analyzer and the renderer case. - ValidateParams(params); + CheckParams(); FILE* graph_data_output_file = nullptr; - if (!params.analyzer.graph_data_output_filename.empty()) { + if (!params_.analyzer.graph_data_output_filename.empty()) { graph_data_output_file = - fopen(params.analyzer.graph_data_output_filename.c_str(), "w"); + fopen(params_.analyzer.graph_data_output_filename.c_str(), "w"); RTC_CHECK(graph_data_output_file != nullptr) - << "Can't open the file " - << params.analyzer.graph_data_output_filename << "!"; + << "Can't open the file " << params_.analyzer.graph_data_output_filename + << "!"; } Call::Config call_config; @@ -761,33 +931,60 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { test::LayerFilteringTransport send_transport( params.pipe, sender_call_.get(), kPayloadTypeVP8, kPayloadTypeVP9, - static_cast(params.common.tl_discard_threshold), 0); + params.common.selected_tl, params_.ss.selected_sl); test::DirectTransport recv_transport(params.pipe, receiver_call_.get()); + std::string graph_title = params_.analyzer.graph_title; + if (graph_title.empty()) + graph_title = VideoQualityTest::GenerateGraphTitle(); + + // In the case of different resolutions, the functions calculating PSNR and + // SSIM return -1.0, instead of a positive value as usual. VideoAnalyzer + // aborts if the average psnr/ssim are below the given threshold, which is + // 0.0 by default. Setting the thresholds to -1.1 prevents the unnecessary + // abort. + VideoStream& selected_stream = params_.ss.streams[params_.ss.selected_stream]; + int selected_sl = params_.ss.selected_sl != -1 + ? params_.ss.selected_sl + : params_.ss.num_spatial_layers - 1; + bool disable_quality_check = + selected_stream.width != params_.common.width || + selected_stream.height != params_.common.height || + (!params_.ss.spatial_layers.empty() && + params_.ss.spatial_layers[selected_sl].scaling_factor_num != + params_.ss.spatial_layers[selected_sl].scaling_factor_den); + if (disable_quality_check) { + fprintf(stderr, + "Warning: Calculating PSNR and SSIM for downsized resolution " + "not implemented yet! Skipping PSNR and SSIM calculations!"); + } + VideoAnalyzer analyzer( - &send_transport, params.analyzer.test_label, - params.analyzer.avg_psnr_threshold, params.analyzer.avg_ssim_threshold, - params.analyzer.test_durations_secs * params.common.fps, - graph_data_output_file); + &send_transport, params_.analyzer.test_label, + disable_quality_check ? -1.1 : params_.analyzer.avg_psnr_threshold, + disable_quality_check ? -1.1 : params_.analyzer.avg_ssim_threshold, + params_.analyzer.test_durations_secs * params_.common.fps, + graph_data_output_file, graph_title, + kSendSsrcs[params_.ss.selected_stream]); analyzer.SetReceiver(receiver_call_->Receiver()); send_transport.SetReceiver(&analyzer); recv_transport.SetReceiver(sender_call_->Receiver()); - SetupFullStack(params, &analyzer, &recv_transport); + SetupCommon(&analyzer, &recv_transport); send_config_.encoding_time_observer = &analyzer; - receive_configs_[0].renderer = &analyzer; + receive_configs_[params_.ss.selected_stream].renderer = &analyzer; for (auto& config : receive_configs_) config.pre_decode_callback = &analyzer; - if (params.screenshare.enabled) - SetupScreenshare(params); + if (params_.screenshare.enabled) + SetupScreenshare(); CreateStreams(); analyzer.input_ = send_stream_->Input(); analyzer.send_stream_ = send_stream_; - CreateCapturer(params, &analyzer); + CreateCapturer(&analyzer); send_stream_->Start(); for (size_t i = 0; i < receive_streams_.size(); ++i) @@ -811,40 +1008,49 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { } void VideoQualityTest::RunWithVideoRenderer(const Params& params) { - ValidateParams(params); + params_ = params; + CheckParams(); rtc::scoped_ptr local_preview( - test::VideoRenderer::Create("Local Preview", params.common.width, - params.common.height)); + test::VideoRenderer::Create("Local Preview", params_.common.width, + params_.common.height)); + size_t stream_id = params_.ss.selected_stream; + char title[32]; + if (params_.ss.streams.size() == 1) { + sprintf(title, "Loopback Video"); + } else { + sprintf(title, "Loopback Video - Stream #%" PRIuS, stream_id); + } rtc::scoped_ptr loopback_video( - test::VideoRenderer::Create("Loopback Video", params.common.width, - params.common.height)); + test::VideoRenderer::Create(title, params_.ss.streams[stream_id].width, + params_.ss.streams[stream_id].height)); // TODO(ivica): Remove bitrate_config and use the default Call::Config(), to // match the full stack tests. Call::Config call_config; - call_config.bitrate_config = params.common.call_bitrate_config; + call_config.bitrate_config = params_.common.call_bitrate_config; rtc::scoped_ptr call(Call::Create(call_config)); test::LayerFilteringTransport transport( params.pipe, call.get(), kPayloadTypeVP8, kPayloadTypeVP9, - static_cast(params.common.tl_discard_threshold), 0); + params.common.selected_tl, params_.ss.selected_sl); // TODO(ivica): Use two calls to be able to merge with RunWithAnalyzer or at // least share as much code as possible. That way this test would also match // the full stack tests better. transport.SetReceiver(call->Receiver()); - SetupFullStack(params, &transport, &transport); - send_config_.local_renderer = local_preview.get(); - receive_configs_[0].renderer = loopback_video.get(); + SetupCommon(&transport, &transport); - if (params.screenshare.enabled) - SetupScreenshare(params); + send_config_.local_renderer = local_preview.get(); + receive_configs_[stream_id].renderer = loopback_video.get(); + + if (params_.screenshare.enabled) + SetupScreenshare(); send_stream_ = call->CreateVideoSendStream(send_config_, encoder_config_); VideoReceiveStream* receive_stream = - call->CreateVideoReceiveStream(receive_configs_[0]); - CreateCapturer(params, send_stream_->Input()); + call->CreateVideoReceiveStream(receive_configs_[stream_id]); + CreateCapturer(send_stream_->Input()); receive_stream->Start(); send_stream_->Start(); diff --git a/webrtc/video/video_quality_test.h b/webrtc/video/video_quality_test.h index 7b62fb3dce..b88c513853 100644 --- a/webrtc/video/video_quality_test.h +++ b/webrtc/video/video_quality_test.h @@ -33,11 +33,11 @@ class VideoQualityTest : public test::CallTest { int target_bitrate_bps; int max_bitrate_bps; std::string codec; - size_t num_temporal_layers; + int num_temporal_layers; + int selected_tl; int min_transmit_bps; Call::Config::BitrateConfig call_bitrate_config; - size_t tl_discard_threshold; bool send_side_bwe; } common; struct { // Video-specific settings. @@ -50,30 +50,56 @@ class VideoQualityTest : public test::CallTest { } screenshare; struct { // Analyzer settings. std::string test_label; - double avg_psnr_threshold; - double avg_ssim_threshold; + double avg_psnr_threshold; // (*) + double avg_ssim_threshold; // (*) int test_durations_secs; std::string graph_data_output_filename; + std::string graph_title; } analyzer; FakeNetworkPipe::Config pipe; bool logs; + struct { // Spatial scalability. + std::vector streams; // If empty, one stream is assumed. + size_t selected_stream; + int num_spatial_layers; + int selected_sl; + // If empty, bitrates are generated in VP9Impl automatically. + std::vector spatial_layers; + } ss; }; + // (*) Set to -1.1 if generating graph data for simulcast or SVC and the + // selected stream/layer doesn't have the same resolution as the largest + // stream/layer (to ignore the PSNR and SSIM calculation errors). VideoQualityTest(); void RunWithAnalyzer(const Params& params); void RunWithVideoRenderer(const Params& params); + static void FillScalabilitySettings( + Params* params, + const std::vector& stream_descriptors, + size_t selected_stream, + int num_spatial_layers, + int selected_sl, + const std::vector& sl_descriptors); + protected: // No-op implementation to be able to instantiate this class from non-TEST_F // locations. void TestBody() override; - void CreateCapturer(const Params& params, VideoCaptureInput* input); - void ValidateParams(const Params& params); - void SetupFullStack(const Params& params, - Transport* send_transport, - Transport* recv_transport); - void SetupScreenshare(const Params& params); + // Helper methods accessing only params_. + std::string GenerateGraphTitle() const; + void CheckParams(); + + // Helper static methods. + static VideoStream DefaultVideoStream(const Params& params); + static std::vector ParseCSV(const std::string& str); + + // Helper methods for setting up the call. + void CreateCapturer(VideoCaptureInput* input); + void SetupCommon(Transport* send_transport, Transport* recv_transport); + void SetupScreenshare(); // We need a more general capturer than the FrameGeneratorCapturer. rtc::scoped_ptr capturer_; @@ -82,6 +108,8 @@ class VideoQualityTest : public test::CallTest { rtc::scoped_ptr encoder_; VideoCodecUnion codec_settings_; Clock* const clock_; + + Params params_; }; } // namespace webrtc diff --git a/webrtc/video/video_send_stream.cc b/webrtc/video/video_send_stream.cc index 4ec923f788..67e8a9c595 100644 --- a/webrtc/video/video_send_stream.cc +++ b/webrtc/video/video_send_stream.cc @@ -370,6 +370,16 @@ bool VideoSendStream::ReconfigureVideoEncoder( static_cast(streams.size()); video_codec.minBitrate = streams[0].min_bitrate_bps / 1000; RTC_DCHECK_LE(streams.size(), static_cast(kMaxSimulcastStreams)); + if (video_codec.codecType == kVideoCodecVP9) { + // If the vector is empty, bitrates will be configured automatically. + RTC_DCHECK(config.spatial_layers.empty() || + config.spatial_layers.size() == + video_codec.codecSpecific.VP9.numberOfSpatialLayers); + RTC_DCHECK_LE(video_codec.codecSpecific.VP9.numberOfSpatialLayers, + kMaxSimulcastStreams); + for (size_t i = 0; i < config.spatial_layers.size(); ++i) + video_codec.spatialLayers[i] = config.spatial_layers[i]; + } for (size_t i = 0; i < streams.size(); ++i) { SimulcastStream* sim_stream = &video_codec.simulcastStream[i]; RTC_DCHECK_GT(streams[i].width, 0u);