From cb8c1467bdf2b90423781b811201b4cb1ea96dec Mon Sep 17 00:00:00 2001 From: ilnik Date: Thu, 9 Mar 2017 09:23:30 -0800 Subject: [PATCH] Add FullStack test for simulcast screenshare mode. BUG=webrtc:4172 Review-Url: https://codereview.webrtc.org/2745523002 Cr-Commit-Position: refs/heads/master@{#17150} --- webrtc/common_video/include/frame_callback.h | 12 ++-- .../codecs/vp8/simulcast_rate_allocator.cc | 1 + webrtc/video/full_stack_tests.cc | 27 ++++++++ webrtc/video/video_quality_test.cc | 68 +++++++++++++------ webrtc/video/video_quality_test.h | 3 + webrtc/video/video_receive_stream.cc | 11 +-- webrtc/video/video_send_stream.cc | 11 +-- webrtc/video/vie_encoder.cc | 7 +- 8 files changed, 101 insertions(+), 39 deletions(-) diff --git a/webrtc/common_video/include/frame_callback.h b/webrtc/common_video/include/frame_callback.h index 45624f1554..0a2a905008 100644 --- a/webrtc/common_video/include/frame_callback.h +++ b/webrtc/common_video/include/frame_callback.h @@ -26,27 +26,23 @@ struct EncodedFrame { : data_(nullptr), length_(0), frame_type_(kEmptyFrame), - encoded_width_(0), - encoded_height_(0), + stream_id_(0), timestamp_(0) {} EncodedFrame(const uint8_t* data, size_t length, FrameType frame_type, - uint32_t encoded_width, - uint32_t encoded_height, + size_t stream_id, uint32_t timestamp) : data_(data), length_(length), frame_type_(frame_type), - encoded_width_(encoded_width), - encoded_height_(encoded_height), + stream_id_(stream_id), timestamp_(timestamp) {} const uint8_t* data_; const size_t length_; const FrameType frame_type_; - const uint32_t encoded_width_; - const uint32_t encoded_height_; + const size_t stream_id_; const uint32_t timestamp_; }; diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_rate_allocator.cc b/webrtc/modules/video_coding/codecs/vp8/simulcast_rate_allocator.cc index af073cd370..39c43c9b6d 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_rate_allocator.cc +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_rate_allocator.cc @@ -30,6 +30,7 @@ SimulcastRateAllocator::SimulcastRateAllocator( void SimulcastRateAllocator::OnTemporalLayersCreated(int simulcast_id, TemporalLayers* layers) { RTC_DCHECK(temporal_layers_.find(simulcast_id) == temporal_layers_.end()); + RTC_DCHECK(layers); temporal_layers_[simulcast_id] = layers; } diff --git a/webrtc/video/full_stack_tests.cc b/webrtc/video/full_stack_tests.cc index e94acdae32..e59280a055 100644 --- a/webrtc/video/full_stack_tests.cc +++ b/webrtc/video/full_stack_tests.cc @@ -9,6 +9,7 @@ */ #include +#include "webrtc/test/field_trial.h" #include "webrtc/test/gtest.h" #include "webrtc/video/video_quality_test.h" @@ -308,6 +309,32 @@ TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL) { RunTest(screenshare); } +TEST_F(FullStackTest, ScreenshareSlidesVP8_3TL_Simulcast) { + test::ScopedFieldTrials field_trial("WebRTC-SimulcastScreenshare/Enabled/"); + VideoQualityTest::Params screenshare; + screenshare.call.send_side_bwe = true; + screenshare.screenshare = {true, 10}; + screenshare.video = {true, 1850, 1110, 5, 800000, 2500000, + 2500000, false, "VP8", 3, 2, 400000, + false, false, "", ""}; + screenshare.analyzer = {"screenshare_slides_simulcast", 0.0, 0.0, + kFullStackTestDurationSecs}; + VideoQualityTest::Params screenshare_params_high; + screenshare_params_high.video = {true, 1850, 1110, 5, 800000, 2500000, + 2500000, false, "VP8", 3, 0, 400000, + false, false, "", ""}; + VideoQualityTest::Params screenshare_params_low; + screenshare_params_low.video = {true, 1850, 1110, 5, 50000, 200000, + 2000000, false, "VP8", 2, 0, 400000, + false, false, "", ""}; + + std::vector streams = { + DefaultVideoStream(screenshare_params_low), + DefaultVideoStream(screenshare_params_high)}; + screenshare.ss = {streams, 1, 1, 0}; + RunTest(screenshare); +} + TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_Scroll) { VideoQualityTest::Params config; config.call.send_side_bwe = true; diff --git a/webrtc/video/video_quality_test.cc b/webrtc/video/video_quality_test.cc index a938bb8a52..cfee0a3793 100644 --- a/webrtc/video/video_quality_test.cc +++ b/webrtc/video/video_quality_test.cc @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,7 @@ #include "webrtc/modules/rtp_rtcp/source/rtp_utility.h" #include "webrtc/modules/video_coding/codecs/h264/include/h264.h" #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" +#include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h" #include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h" #include "webrtc/system_wrappers/include/cpu_info.h" #include "webrtc/system_wrappers/include/field_trial.h" @@ -141,8 +143,7 @@ class VideoAnalyzer : public PacketReceiver, const std::string& graph_title, uint32_t ssrc_to_analyze, uint32_t rtx_ssrc_to_analyze, - uint32_t selected_stream_width, - uint32_t selected_stream_height, + size_t selected_stream, int selected_sl, int selected_tl, bool is_quick_test_enabled) @@ -156,8 +157,7 @@ class VideoAnalyzer : public PacketReceiver, graph_title_(graph_title), ssrc_to_analyze_(ssrc_to_analyze), rtx_ssrc_to_analyze_(rtx_ssrc_to_analyze), - selected_stream_width_(selected_stream_width), - selected_stream_height_(selected_stream_height), + selected_stream_(selected_stream), selected_sl_(selected_sl), selected_tl_(selected_tl), pre_encode_proxy_(this), @@ -286,8 +286,7 @@ class VideoAnalyzer : public PacketReceiver, void PostEncodeFrameCallback(const EncodedFrame& encoded_frame) { rtc::CritScope lock(&crit_); if (!first_sent_timestamp_ && - encoded_frame.encoded_width_ == selected_stream_width_ && - encoded_frame.encoded_height_ == selected_stream_height_) { + encoded_frame.stream_id_ == selected_stream_) { first_sent_timestamp_ = rtc::Optional(encoded_frame.timestamp_); } } @@ -959,8 +958,7 @@ class VideoAnalyzer : public PacketReceiver, const std::string graph_title_; const uint32_t ssrc_to_analyze_; const uint32_t rtx_ssrc_to_analyze_; - const uint32_t selected_stream_width_; - const uint32_t selected_stream_height_; + const size_t selected_stream_; const int selected_sl_; const int selected_tl_; PreEncodeProxy pre_encode_proxy_; @@ -1020,6 +1018,27 @@ class VideoAnalyzer : public PacketReceiver, rtc::Event done_; }; +class Vp8EncoderFactory : public VideoEncoderFactory { + public: + Vp8EncoderFactory() = default; + ~Vp8EncoderFactory() override { RTC_CHECK(live_encoders_.empty()); } + + VideoEncoder* Create() override { + VideoEncoder* encoder = VP8Encoder::Create(); + live_encoders_.insert(encoder); + return encoder; + } + + void Destroy(VideoEncoder* encoder) override { + auto it = live_encoders_.find(encoder); + RTC_CHECK(it != live_encoders_.end()); + live_encoders_.erase(it); + delete encoder; + } + + std::set live_encoders_; +}; + VideoQualityTest::VideoQualityTest() : clock_(Clock::GetRealTimeClock()), receive_logs_(0), send_logs_(0) {} @@ -1085,7 +1104,7 @@ void VideoQualityTest::CheckParams() { RTC_CHECK_GE(stream.min_bitrate_bps, 0); RTC_CHECK_GE(stream.target_bitrate_bps, stream.min_bitrate_bps); RTC_CHECK_GE(stream.max_bitrate_bps, stream.target_bitrate_bps); - RTC_CHECK_EQ(stream.temporal_layer_thresholds_bps.size(), + RTC_CHECK_LE(stream.temporal_layer_thresholds_bps.size(), params_.video.num_temporal_layers - 1); } // TODO(ivica): Should we check if the sum of all streams/layers is equal to @@ -1148,11 +1167,15 @@ VideoStream VideoQualityTest::DefaultVideoStream(const Params& params) { stream.target_bitrate_bps = params.video.target_bitrate_bps; stream.max_bitrate_bps = params.video.max_bitrate_bps; stream.max_qp = 52; - if (params.video.num_temporal_layers == 2) { - stream.temporal_layer_thresholds_bps.push_back(stream.target_bitrate_bps); - } else if (params.video.num_temporal_layers == 3) { - stream.temporal_layer_thresholds_bps.push_back(stream.max_bitrate_bps / 4); - stream.temporal_layer_thresholds_bps.push_back(stream.target_bitrate_bps); + if (params.video.num_temporal_layers > 1) { + RTC_CHECK_LE(params.video.num_temporal_layers, kMaxTemporalStreams); + if (params.video.codec == "VP8") { + for (int i = 0; i < params.video.num_temporal_layers - 1; ++i) { + stream.temporal_layer_thresholds_bps.push_back(static_cast( + stream.target_bitrate_bps * + kVp8LayerRateAlloction[params.video.num_temporal_layers][i])); + } + } } return stream; } @@ -1245,7 +1268,15 @@ void VideoQualityTest::SetupVideo(Transport* send_transport, video_encoder_.reset(H264Encoder::Create(cricket::VideoCodec("H264"))); payload_type = kPayloadTypeH264; } else if (params_.video.codec == "VP8") { - video_encoder_.reset(VP8Encoder::Create()); + if (params_.screenshare.enabled && params_.ss.streams.size() > 1) { + // Simulcast screenshare needs a simulcast encoder adapter to work, since + // encoders usually can't natively do simulcast with different frame rates + // for the different layers. + video_encoder_.reset( + new SimulcastEncoderAdapter(new Vp8EncoderFactory())); + } else { + video_encoder_.reset(VP8Encoder::Create()); + } payload_type = kPayloadTypeVP8; } else if (params_.video.codec == "VP9") { video_encoder_.reset(VP9Encoder::Create()); @@ -1580,8 +1611,6 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { if (graph_title.empty()) graph_title = VideoQualityTest::GenerateGraphTitle(); - VideoStream& selected_stream = params_.ss.streams[params_.ss.selected_stream]; - bool is_quick_test_enabled = field_trial::IsEnabled("WebRTC-QuickPerfTest"); VideoAnalyzer analyzer( &send_transport, params_.analyzer.test_label, @@ -1592,8 +1621,7 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { graph_data_output_file, graph_title, kVideoSendSsrcs[params_.ss.selected_stream], kSendRtxSsrcs[params_.ss.selected_stream], - static_cast(selected_stream.width), - static_cast(selected_stream.height), params.ss.selected_sl, + static_cast(params_.ss.selected_stream), params.ss.selected_sl, params_.video.selected_tl, is_quick_test_enabled); analyzer.SetReceiver(receiver_call_->Receiver()); send_transport.SetReceiver(&analyzer); @@ -1863,6 +1891,7 @@ void VideoQualityTest::StartEncodedFrameLogs(VideoSendStream* stream) { 10000000); } } + void VideoQualityTest::StartEncodedFrameLogs(VideoReceiveStream* stream) { if (!params_.video.encoded_frame_base_path.empty()) { std::ostringstream str; @@ -1873,5 +1902,4 @@ void VideoQualityTest::StartEncodedFrameLogs(VideoReceiveStream* stream) { 10000000); } } - } // namespace webrtc diff --git a/webrtc/video/video_quality_test.h b/webrtc/video/video_quality_test.h index 564d49936b..2631d35da0 100644 --- a/webrtc/video/video_quality_test.h +++ b/webrtc/video/video_quality_test.h @@ -14,6 +14,7 @@ #include #include +#include "webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h" #include "webrtc/test/call_test.h" #include "webrtc/test/frame_generator.h" #include "webrtc/test/testsupport/trace_to_stderr.h" @@ -129,6 +130,8 @@ class VideoQualityTest : public test::CallTest { std::unique_ptr trace_to_stderr_; std::unique_ptr frame_generator_; std::unique_ptr video_encoder_; + std::unique_ptr vp8_encoder_factory_; + std::vector> thumbnail_encoders_; std::vector thumbnail_send_configs_; std::vector thumbnail_encoder_configs_; diff --git a/webrtc/video/video_receive_stream.cc b/webrtc/video/video_receive_stream.cc index edee9094a8..eb2404c174 100644 --- a/webrtc/video/video_receive_stream.cc +++ b/webrtc/video/video_receive_stream.cc @@ -392,11 +392,14 @@ EncodedImageCallback::Result VideoReceiveStream::OnEncodedImage( const CodecSpecificInfo* codec_specific_info, const RTPFragmentationHeader* fragmentation) { stats_proxy_.OnPreDecode(encoded_image, codec_specific_info); + size_t simulcast_idx = 0; + if (codec_specific_info->codecType == kVideoCodecVP8) { + simulcast_idx = codec_specific_info->codecSpecific.VP8.simulcastIdx; + } if (config_.pre_decode_callback) { - config_.pre_decode_callback->EncodedFrameCallback( - EncodedFrame(encoded_image._buffer, encoded_image._length, - encoded_image._frameType, encoded_image._encodedWidth, - encoded_image._encodedHeight, encoded_image._timeStamp)); + config_.pre_decode_callback->EncodedFrameCallback(EncodedFrame( + encoded_image._buffer, encoded_image._length, encoded_image._frameType, + simulcast_idx, encoded_image._timeStamp)); } { rtc::CritScope lock(&ivf_writer_lock_); diff --git a/webrtc/video/video_send_stream.cc b/webrtc/video/video_send_stream.cc index 9e4dff6238..38a2e5df39 100644 --- a/webrtc/video/video_send_stream.cc +++ b/webrtc/video/video_send_stream.cc @@ -1032,11 +1032,14 @@ EncodedImageCallback::Result VideoSendStreamImpl::OnEncodedImage( // Encoded is called on whatever thread the real encoder implementation run // on. In the case of hardware encoders, there might be several encoders // running in parallel on different threads. + size_t simulcast_idx = 0; + if (codec_specific_info->codecType == kVideoCodecVP8) { + simulcast_idx = codec_specific_info->codecSpecific.VP8.simulcastIdx; + } if (config_->post_encode_callback) { - config_->post_encode_callback->EncodedFrameCallback( - EncodedFrame(encoded_image._buffer, encoded_image._length, - encoded_image._frameType, encoded_image._encodedWidth, - encoded_image._encodedHeight, encoded_image._timeStamp)); + config_->post_encode_callback->EncodedFrameCallback(EncodedFrame( + encoded_image._buffer, encoded_image._length, encoded_image._frameType, + simulcast_idx, encoded_image._timeStamp)); } { rtc::CritScope lock(&encoder_activity_crit_sect_); diff --git a/webrtc/video/vie_encoder.cc b/webrtc/video/vie_encoder.cc index c30752b6df..4591d89599 100644 --- a/webrtc/video/vie_encoder.cc +++ b/webrtc/video/vie_encoder.cc @@ -438,12 +438,13 @@ void ViEEncoder::ReconfigureEncoder() { static_cast(max_data_payload_length_)) == VCM_OK; if (!success) { LOG(LS_ERROR) << "Failed to configure encoder."; + rate_allocator_.reset(); RTC_DCHECK(success); + } else { + video_sender_.UpdateChannelParemeters(rate_allocator_.get(), + bitrate_observer_); } - video_sender_.UpdateChannelParemeters(rate_allocator_.get(), - bitrate_observer_); - int framerate = stats_proxy_->GetSendFrameRate(); if (framerate == 0) framerate = codec.maxFramerate;