diff --git a/webrtc/media/engine/fakewebrtcvideoengine.h b/webrtc/media/engine/fakewebrtcvideoengine.h index d93431e263..898277ff2f 100644 --- a/webrtc/media/engine/fakewebrtcvideoengine.h +++ b/webrtc/media/engine/fakewebrtcvideoengine.h @@ -39,13 +39,12 @@ static const int kMaxVideoBitrate = 1000; // renderer for a channel or it is adding a renderer for a capturer. static const int kViEChannelIdBase = 0; static const int kViEChannelIdMax = 1000; +static const int kEventTimeoutMs = 10000; // Fake class for mocking out webrtc::VideoDecoder class FakeWebRtcVideoDecoder : public webrtc::VideoDecoder { public: - FakeWebRtcVideoDecoder() - : num_frames_received_(0) { - } + FakeWebRtcVideoDecoder() : num_frames_received_(0) {} virtual int32_t InitDecode(const webrtc::VideoCodec*, int32_t) { return WEBRTC_VIDEO_CODEC_OK; @@ -121,16 +120,20 @@ class FakeWebRtcVideoDecoderFactory : public WebRtcVideoDecoderFactory { // Fake class for mocking out webrtc::VideoEnoder class FakeWebRtcVideoEncoder : public webrtc::VideoEncoder { public: - FakeWebRtcVideoEncoder() : num_frames_encoded_(0) {} + FakeWebRtcVideoEncoder() + : init_encode_event_(false, false), num_frames_encoded_(0) {} virtual int32_t InitEncode(const webrtc::VideoCodec* codecSettings, int32_t numberOfCores, size_t maxPayloadSize) { rtc::CritScope lock(&crit_); codec_settings_ = *codecSettings; + init_encode_event_.Set(); return WEBRTC_VIDEO_CODEC_OK; } + bool WaitForInitEncode() { return init_encode_event_.Wait(kEventTimeoutMs); } + webrtc::VideoCodec GetCodecSettings() { rtc::CritScope lock(&crit_); return codec_settings_; @@ -141,6 +144,7 @@ class FakeWebRtcVideoEncoder : public webrtc::VideoEncoder { const std::vector* frame_types) { rtc::CritScope lock(&crit_); ++num_frames_encoded_; + init_encode_event_.Set(); return WEBRTC_VIDEO_CODEC_OK; } @@ -166,6 +170,7 @@ class FakeWebRtcVideoEncoder : public webrtc::VideoEncoder { private: rtc::CriticalSection crit_; + rtc::Event init_encode_event_; int num_frames_encoded_ GUARDED_BY(crit_); webrtc::VideoCodec codec_settings_ GUARDED_BY(crit_); }; @@ -174,20 +179,33 @@ class FakeWebRtcVideoEncoder : public webrtc::VideoEncoder { class FakeWebRtcVideoEncoderFactory : public WebRtcVideoEncoderFactory { public: FakeWebRtcVideoEncoderFactory() - : num_created_encoders_(0), encoders_have_internal_sources_(false) {} + : created_video_encoder_event_(false, false), + num_created_encoders_(0), + encoders_have_internal_sources_(false) {} virtual webrtc::VideoEncoder* CreateVideoEncoder( webrtc::VideoCodecType type) { + rtc::CritScope lock(&crit_); if (supported_codec_types_.count(type) == 0) { return NULL; } FakeWebRtcVideoEncoder* encoder = new FakeWebRtcVideoEncoder(); encoders_.push_back(encoder); num_created_encoders_++; + created_video_encoder_event_.Set(); return encoder; } + bool WaitForCreatedVideoEncoders(int num_encoders) { + while (created_video_encoder_event_.Wait(kEventTimeoutMs)) { + if (GetNumCreatedEncoders() >= num_encoders) + return true; + } + return false; + } + virtual void DestroyVideoEncoder(webrtc::VideoEncoder* encoder) { + rtc::CritScope lock(&crit_); encoders_.erase( std::remove(encoders_.begin(), encoders_.end(), encoder), encoders_.end()); @@ -216,18 +234,22 @@ class FakeWebRtcVideoEncoderFactory : public WebRtcVideoEncoderFactory { } int GetNumCreatedEncoders() { + rtc::CritScope lock(&crit_); return num_created_encoders_; } - const std::vector& encoders() { + const std::vector encoders() { + rtc::CritScope lock(&crit_); return encoders_; } private: + rtc::CriticalSection crit_; + rtc::Event created_video_encoder_event_; std::set supported_codec_types_; std::vector codecs_; - std::vector encoders_; - int num_created_encoders_; + std::vector encoders_ GUARDED_BY(crit_); + int num_created_encoders_ GUARDED_BY(crit_); bool encoders_have_internal_sources_; }; diff --git a/webrtc/media/engine/webrtcvideoengine2_unittest.cc b/webrtc/media/engine/webrtcvideoengine2_unittest.cc index 8e445d1066..f3f1ee855e 100644 --- a/webrtc/media/engine/webrtcvideoengine2_unittest.cc +++ b/webrtc/media/engine/webrtcvideoengine2_unittest.cc @@ -340,6 +340,7 @@ TEST_F(WebRtcVideoEngine2Test, UseExternalFactoryForVp8WhenSupported) { EXPECT_TRUE( channel->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrc))); + ASSERT_TRUE(encoder_factory.WaitForCreatedVideoEncoders(1)); ASSERT_EQ(1u, encoder_factory.encoders().size()); EXPECT_TRUE(channel->SetSend(true)); @@ -348,11 +349,13 @@ TEST_F(WebRtcVideoEngine2Test, UseExternalFactoryForVp8WhenSupported) { EXPECT_EQ(cricket::CS_RUNNING, capturer.Start(capturer.GetSupportedFormats()->front())); EXPECT_TRUE(capturer.CaptureFrame()); + // Sending one frame will have reallocated the encoder since input size + // changes from a small default to the actual frame width/height. Wait for + // that to happen then for the frame to be sent. + ASSERT_TRUE(encoder_factory.WaitForCreatedVideoEncoders(2)); EXPECT_TRUE_WAIT(encoder_factory.encoders()[0]->GetNumEncodedFrames() > 0, kTimeout); - // Sending one frame will have reallocated the encoder since input size - // changes from a small default to the actual frame width/height. int num_created_encoders = encoder_factory.GetNumCreatedEncoders(); EXPECT_EQ(num_created_encoders, 2); @@ -583,12 +586,13 @@ TEST_F(WebRtcVideoEngine2Test, UsesSimulcastAdapterForVp8Factories) { capturer.Start(capturer.GetSupportedFormats()->front())); EXPECT_TRUE(capturer.CaptureFrame()); - EXPECT_GT(encoder_factory.encoders().size(), 1u); + ASSERT_TRUE(encoder_factory.WaitForCreatedVideoEncoders(2)); // Verify that encoders are configured for simulcast through adapter // (increasing resolution and only configured to send one stream each). int prev_width = -1; for (size_t i = 0; i < encoder_factory.encoders().size(); ++i) { + ASSERT_TRUE(encoder_factory.encoders()[i]->WaitForInitEncode()); webrtc::VideoCodec codec_settings = encoder_factory.encoders()[i]->GetCodecSettings(); EXPECT_EQ(0, codec_settings.numberOfSimulcastStreams); @@ -664,7 +668,8 @@ TEST_F(WebRtcVideoEngine2Test, capturer.Start(capturer.GetSupportedFormats()->front())); EXPECT_TRUE(capturer.CaptureFrame()); - ASSERT_GT(encoder_factory.encoders().size(), 1u); + ASSERT_TRUE(encoder_factory.WaitForCreatedVideoEncoders(2)); + ASSERT_TRUE(encoder_factory.encoders()[0]->WaitForInitEncode()); EXPECT_EQ(webrtc::kVideoCodecVP8, encoder_factory.encoders()[0]->GetCodecSettings().codecType); @@ -688,6 +693,7 @@ TEST_F(WebRtcVideoEngine2Test, EXPECT_TRUE( channel->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrc))); ASSERT_EQ(1u, encoder_factory.encoders().size()); + ASSERT_TRUE(encoder_factory.encoders()[0]->WaitForInitEncode()); EXPECT_EQ(webrtc::kVideoCodecH264, encoder_factory.encoders()[0]->GetCodecSettings().codecType); @@ -719,6 +725,7 @@ TEST_F(WebRtcVideoEngine2Test, SimulcastDisabledForH264) { ASSERT_EQ(1u, encoder_factory.encoders().size()); FakeWebRtcVideoEncoder* encoder = encoder_factory.encoders()[0]; + ASSERT_TRUE(encoder_factory.encoders()[0]->WaitForInitEncode()); EXPECT_EQ(webrtc::kVideoCodecH264, encoder->GetCodecSettings().codecType); EXPECT_EQ(1u, encoder->GetCodecSettings().numberOfSimulcastStreams); EXPECT_TRUE(channel->SetCapturer(ssrcs[0], nullptr)); diff --git a/webrtc/video/video_quality_test.cc b/webrtc/video/video_quality_test.cc index c8829925e4..fb07e88a72 100644 --- a/webrtc/video/video_quality_test.cc +++ b/webrtc/video/video_quality_test.cc @@ -43,6 +43,7 @@ static const int kPayloadTypeVP9 = 124; class VideoAnalyzer : public PacketReceiver, public Transport, + public I420FrameCallback, public VideoRenderer, public VideoCaptureInput, public EncodedFrameObserver { @@ -68,6 +69,8 @@ class VideoAnalyzer : public PacketReceiver, frames_recorded_(0), frames_processed_(0), dropped_frames_(0), + dropped_frames_before_first_encode_(0), + dropped_frames_before_rendering_(0), last_render_time_(0), rtp_timestamp_delta_(0), avg_psnr_threshold_(avg_psnr_threshold), @@ -137,18 +140,26 @@ class VideoAnalyzer : public PacketReceiver, void IncomingCapturedFrame(const VideoFrame& video_frame) override { VideoFrame copy = video_frame; copy.set_timestamp(copy.ntp_time_ms() * 90); - { rtc::CritScope lock(&crit_); - if (first_send_frame_.IsZeroSize() && rtp_timestamp_delta_ == 0) - first_send_frame_ = copy; - frames_.push_back(copy); } input_->IncomingCapturedFrame(video_frame); } + void FrameCallback(VideoFrame* video_frame) { + rtc::CritScope lock(&crit_); + if (first_send_frame_.IsZeroSize() && rtp_timestamp_delta_ == 0) { + while (frames_.front().timestamp() != video_frame->timestamp()) { + ++dropped_frames_before_first_encode_; + frames_.pop_front(); + RTC_CHECK(!frames_.empty()); + } + first_send_frame_ = *video_frame; + } + } + bool SendRtp(const uint8_t* packet, size_t length, const PacketOptions& options) override { @@ -162,7 +173,7 @@ class VideoAnalyzer : public PacketReceiver, { rtc::CritScope lock(&crit_); - if (rtp_timestamp_delta_ == 0) { + if (!first_send_frame_.IsZeroSize()) { rtp_timestamp_delta_ = header.timestamp - first_send_frame_.timestamp(); first_send_frame_.Reset(); } @@ -198,9 +209,18 @@ class VideoAnalyzer : public PacketReceiver, wrap_handler_.Unwrap(video_frame.timestamp() - rtp_timestamp_delta_); while (wrap_handler_.Unwrap(frames_.front().timestamp()) < send_timestamp) { + if (last_rendered_frame_.IsZeroSize()) { + // No previous frame rendered, this one was dropped after sending but + // before rendering. + ++dropped_frames_before_rendering_; + frames_.pop_front(); + RTC_CHECK(!frames_.empty()); + continue; + } AddFrameComparison(frames_.front(), last_rendered_frame_, true, render_time_ms); frames_.pop_front(); + RTC_DCHECK(!frames_.empty()); } VideoFrame reference_frame = frames_.front(); @@ -352,6 +372,7 @@ class VideoAnalyzer : public PacketReceiver, bool dropped, int64_t render_time_ms) EXCLUSIVE_LOCKS_REQUIRED(crit_) { + RTC_DCHECK(!render.IsZeroSize()); int64_t reference_timestamp = wrap_handler_.Unwrap(reference.timestamp()); int64_t send_time_ms = send_times_[reference_timestamp]; send_times_.erase(reference_timestamp); @@ -484,8 +505,6 @@ class VideoAnalyzer : public PacketReceiver, PrintResult("psnr", psnr_, " dB"); PrintResult("ssim", ssim_, " score"); PrintResult("sender_time", sender_time_, " ms"); - printf("RESULT dropped_frames: %s = %d frames\n", test_label_.c_str(), - dropped_frames_); PrintResult("receiver_time", receiver_time_, " ms"); PrintResult("total_delay_incl_network", end_to_end_, " ms"); PrintResult("time_between_rendered_frames", rendered_delta_, " ms"); @@ -495,6 +514,13 @@ class VideoAnalyzer : public PacketReceiver, PrintResult("encode_usage_percent", encode_usage_percent, " percent"); PrintResult("media_bitrate", media_bitrate_bps, " bps"); + printf("RESULT dropped_frames: %s = %d frames\n", test_label_.c_str(), + dropped_frames_); + printf("RESULT dropped_frames_before_first_encode: %s = %d frames\n", + test_label_.c_str(), dropped_frames_before_first_encode_); + printf("RESULT dropped_frames_before_rendering: %s = %d frames\n", + test_label_.c_str(), dropped_frames_before_rendering_); + EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_); EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_); } @@ -609,6 +635,8 @@ class VideoAnalyzer : public PacketReceiver, int frames_recorded_; int frames_processed_; int dropped_frames_; + int dropped_frames_before_first_encode_; + int dropped_frames_before_rendering_; int64_t last_render_time_; uint32_t rtp_timestamp_delta_; @@ -1002,6 +1030,7 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { SetupCommon(&analyzer, &recv_transport); video_receive_configs_[params_.ss.selected_stream].renderer = &analyzer; + video_send_config_.pre_encode_callback = &analyzer; for (auto& config : video_receive_configs_) config.pre_decode_callback = &analyzer; RTC_DCHECK(!video_send_config_.post_encode_callback); diff --git a/webrtc/video/video_send_stream.cc b/webrtc/video/video_send_stream.cc index 658cd495ba..21a22dc914 100644 --- a/webrtc/video/video_send_stream.cc +++ b/webrtc/video/video_send_stream.cc @@ -148,6 +148,149 @@ CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) { } return options; } + +VideoCodec VideoEncoderConfigToVideoCodec(const VideoEncoderConfig& config, + const std::string& payload_name, + int payload_type) { + const std::vector& streams = config.streams; + static const int kEncoderMinBitrateKbps = 30; + RTC_DCHECK(!streams.empty()); + RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0); + + VideoCodec video_codec; + memset(&video_codec, 0, sizeof(video_codec)); + video_codec.codecType = PayloadNameToCodecType(payload_name); + + switch (config.content_type) { + case VideoEncoderConfig::ContentType::kRealtimeVideo: + video_codec.mode = kRealtimeVideo; + break; + case VideoEncoderConfig::ContentType::kScreen: + video_codec.mode = kScreensharing; + if (config.streams.size() == 1 && + config.streams[0].temporal_layer_thresholds_bps.size() == 1) { + video_codec.targetBitrate = + config.streams[0].temporal_layer_thresholds_bps[0] / 1000; + } + break; + } + + switch (video_codec.codecType) { + case kVideoCodecVP8: { + if (config.encoder_specific_settings) { + video_codec.codecSpecific.VP8 = *reinterpret_cast( + config.encoder_specific_settings); + } else { + video_codec.codecSpecific.VP8 = VideoEncoder::GetDefaultVp8Settings(); + } + video_codec.codecSpecific.VP8.numberOfTemporalLayers = + static_cast( + streams.back().temporal_layer_thresholds_bps.size() + 1); + break; + } + case kVideoCodecVP9: { + if (config.encoder_specific_settings) { + video_codec.codecSpecific.VP9 = *reinterpret_cast( + config.encoder_specific_settings); + if (video_codec.mode == kScreensharing) { + video_codec.codecSpecific.VP9.flexibleMode = true; + // For now VP9 screensharing use 1 temporal and 2 spatial layers. + RTC_DCHECK_EQ(video_codec.codecSpecific.VP9.numberOfTemporalLayers, + 1); + RTC_DCHECK_EQ(video_codec.codecSpecific.VP9.numberOfSpatialLayers, 2); + } + } else { + video_codec.codecSpecific.VP9 = VideoEncoder::GetDefaultVp9Settings(); + } + video_codec.codecSpecific.VP9.numberOfTemporalLayers = + static_cast( + streams.back().temporal_layer_thresholds_bps.size() + 1); + break; + } + case kVideoCodecH264: { + if (config.encoder_specific_settings) { + video_codec.codecSpecific.H264 = + *reinterpret_cast( + config.encoder_specific_settings); + } else { + video_codec.codecSpecific.H264 = VideoEncoder::GetDefaultH264Settings(); + } + break; + } + default: + // TODO(pbos): Support encoder_settings codec-agnostically. + RTC_DCHECK(config.encoder_specific_settings == nullptr) + << "Encoder-specific settings for codec type not wired up."; + break; + } + + strncpy(video_codec.plName, payload_name.c_str(), kPayloadNameSize - 1); + video_codec.plName[kPayloadNameSize - 1] = '\0'; + video_codec.plType = payload_type; + video_codec.numberOfSimulcastStreams = + static_cast(streams.size()); + video_codec.minBitrate = streams[0].min_bitrate_bps / 1000; + if (video_codec.minBitrate < kEncoderMinBitrateKbps) + video_codec.minBitrate = kEncoderMinBitrateKbps; + RTC_DCHECK_LE(streams.size(), static_cast(kMaxSimulcastStreams)); + if (video_codec.codecType == kVideoCodecVP9) { + // If the vector is empty, bitrates will be configured automatically. + RTC_DCHECK(config.spatial_layers.empty() || + config.spatial_layers.size() == + video_codec.codecSpecific.VP9.numberOfSpatialLayers); + RTC_DCHECK_LE(video_codec.codecSpecific.VP9.numberOfSpatialLayers, + kMaxSimulcastStreams); + for (size_t i = 0; i < config.spatial_layers.size(); ++i) + video_codec.spatialLayers[i] = config.spatial_layers[i]; + } + for (size_t i = 0; i < streams.size(); ++i) { + SimulcastStream* sim_stream = &video_codec.simulcastStream[i]; + RTC_DCHECK_GT(streams[i].width, 0u); + RTC_DCHECK_GT(streams[i].height, 0u); + RTC_DCHECK_GT(streams[i].max_framerate, 0); + // Different framerates not supported per stream at the moment. + RTC_DCHECK_EQ(streams[i].max_framerate, streams[0].max_framerate); + RTC_DCHECK_GE(streams[i].min_bitrate_bps, 0); + RTC_DCHECK_GE(streams[i].target_bitrate_bps, streams[i].min_bitrate_bps); + RTC_DCHECK_GE(streams[i].max_bitrate_bps, streams[i].target_bitrate_bps); + RTC_DCHECK_GE(streams[i].max_qp, 0); + + sim_stream->width = static_cast(streams[i].width); + sim_stream->height = static_cast(streams[i].height); + sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000; + sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000; + sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000; + sim_stream->qpMax = streams[i].max_qp; + sim_stream->numberOfTemporalLayers = static_cast( + streams[i].temporal_layer_thresholds_bps.size() + 1); + + video_codec.width = std::max(video_codec.width, + static_cast(streams[i].width)); + video_codec.height = std::max( + video_codec.height, static_cast(streams[i].height)); + video_codec.minBitrate = + std::min(static_cast(video_codec.minBitrate), + static_cast(streams[i].min_bitrate_bps / 1000)); + video_codec.maxBitrate += streams[i].max_bitrate_bps / 1000; + video_codec.qpMax = std::max(video_codec.qpMax, + static_cast(streams[i].max_qp)); + } + + if (video_codec.maxBitrate == 0) { + // Unset max bitrate -> cap to one bit per pixel. + video_codec.maxBitrate = + (video_codec.width * video_codec.height * video_codec.maxFramerate) / + 1000; + } + if (video_codec.maxBitrate < kEncoderMinBitrateKbps) + video_codec.maxBitrate = kEncoderMinBitrateKbps; + + RTC_DCHECK_GT(streams[0].max_framerate, 0); + video_codec.maxFramerate = streams[0].max_framerate; + + return video_codec; +} + } // namespace namespace internal { @@ -286,11 +429,6 @@ VideoSendStream::VideoSendStream( RTC_DCHECK(config.encoder_settings.encoder != nullptr); RTC_DCHECK_GE(config.encoder_settings.payload_type, 0); RTC_DCHECK_LE(config.encoder_settings.payload_type, 127); - RTC_CHECK_EQ(0, vie_encoder_.RegisterExternalEncoder( - config.encoder_settings.encoder, - config.encoder_settings.payload_type, - config.encoder_settings.internal_source)); - ReconfigureVideoEncoder(encoder_config); vie_channel_.RegisterSendSideDelayObserver(&stats_proxy_); @@ -298,11 +436,6 @@ VideoSendStream::VideoSendStream( if (config_.post_encode_callback) vie_encoder_.RegisterPostEncodeImageCallback(&encoded_frame_proxy_); - if (config_.suspend_below_min_bitrate) { - vcm_->SuspendBelowMinBitrate(); - bitrate_allocator_->EnforceMinBitrate(false); - } - vie_channel_.RegisterRtcpPacketTypeCounterObserver(&stats_proxy_); vie_channel_.RegisterSendBitrateObserver(&stats_proxy_); vie_channel_.RegisterSendFrameCountObserver(&stats_proxy_); @@ -329,8 +462,6 @@ VideoSendStream::~VideoSendStream() { vie_channel_.RegisterSendBitrateObserver(nullptr); vie_channel_.RegisterRtcpPacketTypeCounterObserver(nullptr); - vie_encoder_.DeRegisterExternalEncoder(config_.encoder_settings.payload_type); - call_stats_->DeregisterStatsObserver(vie_channel_.GetStatsObserver()); rtp_rtcp_modules_[0]->SetREMBStatus(false); remb_->RemoveRembSender(rtp_rtcp_modules_[0]); @@ -373,158 +504,63 @@ bool VideoSendStream::EncoderThreadFunction(void* obj) { } void VideoSendStream::EncoderProcess() { + RTC_CHECK_EQ(0, vie_encoder_.RegisterExternalEncoder( + config_.encoder_settings.encoder, + config_.encoder_settings.payload_type, + config_.encoder_settings.internal_source)); + while (true) { encoder_wakeup_event_.Wait(rtc::Event::kForever); if (rtc::AtomicOps::AcquireLoad(&stop_encoder_thread_)) - return; + break; + rtc::Optional encoder_settings; + { + rtc::CritScope lock(&encoder_settings_crit_); + if (pending_encoder_settings_) { + encoder_settings = pending_encoder_settings_; + pending_encoder_settings_ = rtc::Optional(); + } + } + if (encoder_settings) { + encoder_settings->video_codec.startBitrate = + bitrate_allocator_->AddObserver( + this, encoder_settings->video_codec.minBitrate * 1000, + encoder_settings->video_codec.maxBitrate * 1000) / + 1000; + vie_encoder_.SetEncoder(encoder_settings->video_codec, + encoder_settings->min_transmit_bitrate_bps); + if (config_.suspend_below_min_bitrate) { + vcm_->SuspendBelowMinBitrate(); + bitrate_allocator_->EnforceMinBitrate(false); + } + // We might've gotten new settings while configuring the encoder settings, + // restart from the top to see if that's the case before trying to encode + // a frame (which might correspond to the last frame size). + encoder_wakeup_event_.Set(); + continue; + } VideoFrame frame; if (input_.GetVideoFrame(&frame)) vie_encoder_.EncodeVideoFrame(frame); } + vie_encoder_.DeRegisterExternalEncoder(config_.encoder_settings.payload_type); } void VideoSendStream::ReconfigureVideoEncoder( const VideoEncoderConfig& config) { TRACE_EVENT0("webrtc", "VideoSendStream::(Re)configureVideoEncoder"); LOG(LS_INFO) << "(Re)configureVideoEncoder: " << config.ToString(); - const std::vector& streams = config.streams; - static const int kEncoderMinBitrateKbps = 30; - RTC_DCHECK(!streams.empty()); - RTC_DCHECK_GE(config_.rtp.ssrcs.size(), streams.size()); - RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0); - - VideoCodec video_codec; - memset(&video_codec, 0, sizeof(video_codec)); - video_codec.codecType = - PayloadNameToCodecType(config_.encoder_settings.payload_name); - - switch (config.content_type) { - case VideoEncoderConfig::ContentType::kRealtimeVideo: - video_codec.mode = kRealtimeVideo; - break; - case VideoEncoderConfig::ContentType::kScreen: - video_codec.mode = kScreensharing; - if (config.streams.size() == 1 && - config.streams[0].temporal_layer_thresholds_bps.size() == 1) { - video_codec.targetBitrate = - config.streams[0].temporal_layer_thresholds_bps[0] / 1000; - } - break; + RTC_DCHECK_GE(config_.rtp.ssrcs.size(), config.streams.size()); + VideoCodec video_codec = VideoEncoderConfigToVideoCodec( + config, config_.encoder_settings.payload_name, + config_.encoder_settings.payload_type); + { + rtc::CritScope lock(&encoder_settings_crit_); + pending_encoder_settings_ = rtc::Optional( + {video_codec, config.min_transmit_bitrate_bps}); } - - if (video_codec.codecType == kVideoCodecVP8) { - video_codec.codecSpecific.VP8 = VideoEncoder::GetDefaultVp8Settings(); - } else if (video_codec.codecType == kVideoCodecVP9) { - video_codec.codecSpecific.VP9 = VideoEncoder::GetDefaultVp9Settings(); - } else if (video_codec.codecType == kVideoCodecH264) { - video_codec.codecSpecific.H264 = VideoEncoder::GetDefaultH264Settings(); - } - - if (video_codec.codecType == kVideoCodecVP8) { - if (config.encoder_specific_settings != nullptr) { - video_codec.codecSpecific.VP8 = *reinterpret_cast( - config.encoder_specific_settings); - } - video_codec.codecSpecific.VP8.numberOfTemporalLayers = - static_cast( - streams.back().temporal_layer_thresholds_bps.size() + 1); - } else if (video_codec.codecType == kVideoCodecVP9) { - if (config.encoder_specific_settings != nullptr) { - video_codec.codecSpecific.VP9 = *reinterpret_cast( - config.encoder_specific_settings); - if (video_codec.mode == kScreensharing) { - video_codec.codecSpecific.VP9.flexibleMode = true; - // For now VP9 screensharing use 1 temporal and 2 spatial layers. - RTC_DCHECK_EQ(video_codec.codecSpecific.VP9.numberOfTemporalLayers, 1); - RTC_DCHECK_EQ(video_codec.codecSpecific.VP9.numberOfSpatialLayers, 2); - } - } - video_codec.codecSpecific.VP9.numberOfTemporalLayers = - static_cast( - streams.back().temporal_layer_thresholds_bps.size() + 1); - } else if (video_codec.codecType == kVideoCodecH264) { - if (config.encoder_specific_settings != nullptr) { - video_codec.codecSpecific.H264 = *reinterpret_cast( - config.encoder_specific_settings); - } - } else { - // TODO(pbos): Support encoder_settings codec-agnostically. - RTC_DCHECK(config.encoder_specific_settings == nullptr) - << "Encoder-specific settings for codec type not wired up."; - } - - strncpy(video_codec.plName, - config_.encoder_settings.payload_name.c_str(), - kPayloadNameSize - 1); - video_codec.plName[kPayloadNameSize - 1] = '\0'; - video_codec.plType = config_.encoder_settings.payload_type; - video_codec.numberOfSimulcastStreams = - static_cast(streams.size()); - video_codec.minBitrate = streams[0].min_bitrate_bps / 1000; - if (video_codec.minBitrate < kEncoderMinBitrateKbps) - video_codec.minBitrate = kEncoderMinBitrateKbps; - RTC_DCHECK_LE(streams.size(), static_cast(kMaxSimulcastStreams)); - if (video_codec.codecType == kVideoCodecVP9) { - // If the vector is empty, bitrates will be configured automatically. - RTC_DCHECK(config.spatial_layers.empty() || - config.spatial_layers.size() == - video_codec.codecSpecific.VP9.numberOfSpatialLayers); - RTC_DCHECK_LE(video_codec.codecSpecific.VP9.numberOfSpatialLayers, - kMaxSimulcastStreams); - for (size_t i = 0; i < config.spatial_layers.size(); ++i) - video_codec.spatialLayers[i] = config.spatial_layers[i]; - } - for (size_t i = 0; i < streams.size(); ++i) { - SimulcastStream* sim_stream = &video_codec.simulcastStream[i]; - RTC_DCHECK_GT(streams[i].width, 0u); - RTC_DCHECK_GT(streams[i].height, 0u); - RTC_DCHECK_GT(streams[i].max_framerate, 0); - // Different framerates not supported per stream at the moment. - RTC_DCHECK_EQ(streams[i].max_framerate, streams[0].max_framerate); - RTC_DCHECK_GE(streams[i].min_bitrate_bps, 0); - RTC_DCHECK_GE(streams[i].target_bitrate_bps, streams[i].min_bitrate_bps); - RTC_DCHECK_GE(streams[i].max_bitrate_bps, streams[i].target_bitrate_bps); - RTC_DCHECK_GE(streams[i].max_qp, 0); - - sim_stream->width = static_cast(streams[i].width); - sim_stream->height = static_cast(streams[i].height); - sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000; - sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000; - sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000; - sim_stream->qpMax = streams[i].max_qp; - sim_stream->numberOfTemporalLayers = static_cast( - streams[i].temporal_layer_thresholds_bps.size() + 1); - - video_codec.width = std::max(video_codec.width, - static_cast(streams[i].width)); - video_codec.height = std::max( - video_codec.height, static_cast(streams[i].height)); - video_codec.minBitrate = - std::min(static_cast(video_codec.minBitrate), - static_cast(streams[i].min_bitrate_bps / 1000)); - video_codec.maxBitrate += streams[i].max_bitrate_bps / 1000; - video_codec.qpMax = std::max(video_codec.qpMax, - static_cast(streams[i].max_qp)); - } - - if (video_codec.maxBitrate == 0) { - // Unset max bitrate -> cap to one bit per pixel. - video_codec.maxBitrate = - (video_codec.width * video_codec.height * video_codec.maxFramerate) / - 1000; - } - if (video_codec.maxBitrate < kEncoderMinBitrateKbps) - video_codec.maxBitrate = kEncoderMinBitrateKbps; - - RTC_DCHECK_GT(streams[0].max_framerate, 0); - video_codec.maxFramerate = streams[0].max_framerate; - - video_codec.startBitrate = - bitrate_allocator_->AddObserver(this, - video_codec.minBitrate * 1000, - video_codec.maxBitrate * 1000) / 1000; - vie_encoder_.SetEncoder(video_codec, config.min_transmit_bitrate_bps); + encoder_wakeup_event_.Set(); } bool VideoSendStream::DeliverRtcp(const uint8_t* packet, size_t length) { diff --git a/webrtc/video/video_send_stream.h b/webrtc/video/video_send_stream.h index 8b3d064f3e..185f5a5f75 100644 --- a/webrtc/video/video_send_stream.h +++ b/webrtc/video/video_send_stream.h @@ -15,6 +15,7 @@ #include #include "webrtc/call/bitrate_allocator.h" +#include "webrtc/base/criticalsection.h" #include "webrtc/call.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" #include "webrtc/video/encoded_frame_callback_adapter.h" @@ -82,6 +83,10 @@ class VideoSendStream : public webrtc::VideoSendStream, int64_t rtt) override; private: + struct EncoderSettings { + VideoCodec video_codec; + int min_transmit_bitrate_bps; + }; static bool EncoderThreadFunction(void* obj); void EncoderProcess(); @@ -101,6 +106,9 @@ class VideoSendStream : public webrtc::VideoSendStream, rtc::PlatformThread encoder_thread_; rtc::Event encoder_wakeup_event_; volatile int stop_encoder_thread_; + rtc::CriticalSection encoder_settings_crit_; + rtc::Optional pending_encoder_settings_ + GUARDED_BY(encoder_settings_crit_); OveruseFrameDetector overuse_detector_; PayloadRouter payload_router_; diff --git a/webrtc/video/video_send_stream_tests.cc b/webrtc/video/video_send_stream_tests.cc index 4243ee69de..de5f4b0f28 100644 --- a/webrtc/video/video_send_stream_tests.cc +++ b/webrtc/video/video_send_stream_tests.cc @@ -1116,18 +1116,22 @@ TEST_F(VideoSendStreamTest, CanReconfigureToUseStartBitrateAbovePreviousMax) { class StartBitrateObserver : public test::FakeEncoder { public: StartBitrateObserver() - : FakeEncoder(Clock::GetRealTimeClock()), start_bitrate_kbps_(0) {} + : FakeEncoder(Clock::GetRealTimeClock()), + start_bitrate_changed_(false, false), + start_bitrate_kbps_(0) {} int32_t InitEncode(const VideoCodec* config, int32_t number_of_cores, size_t max_payload_size) override { rtc::CritScope lock(&crit_); start_bitrate_kbps_ = config->startBitrate; + start_bitrate_changed_.Set(); return FakeEncoder::InitEncode(config, number_of_cores, max_payload_size); } int32_t SetRates(uint32_t new_target_bitrate, uint32_t framerate) override { rtc::CritScope lock(&crit_); start_bitrate_kbps_ = new_target_bitrate; + start_bitrate_changed_.Set(); return FakeEncoder::SetRates(new_target_bitrate, framerate); } @@ -1136,8 +1140,14 @@ TEST_F(VideoSendStreamTest, CanReconfigureToUseStartBitrateAbovePreviousMax) { return start_bitrate_kbps_; } + bool WaitForStartBitrate() { + return start_bitrate_changed_.Wait( + VideoSendStreamTest::kDefaultTimeoutMs); + } + private: rtc::CriticalSection crit_; + rtc::Event start_bitrate_changed_; int start_bitrate_kbps_ GUARDED_BY(crit_); }; @@ -1156,6 +1166,7 @@ TEST_F(VideoSendStreamTest, CanReconfigureToUseStartBitrateAbovePreviousMax) { CreateVideoStreams(); + EXPECT_TRUE(encoder.WaitForStartBitrate()); EXPECT_EQ(video_encoder_config_.streams[0].max_bitrate_bps / 1000, encoder.GetStartBitrateKbps()); @@ -1166,6 +1177,7 @@ TEST_F(VideoSendStreamTest, CanReconfigureToUseStartBitrateAbovePreviousMax) { // New bitrate should be reconfigured above the previous max. As there's no // network connection this shouldn't be flaky, as no bitrate should've been // reported in between. + EXPECT_TRUE(encoder.WaitForStartBitrate()); EXPECT_EQ(bitrate_config.start_bitrate_bps / 1000, encoder.GetStartBitrateKbps()); @@ -1379,9 +1391,6 @@ TEST_F(VideoSendStreamTest, EncoderIsProperlyInitializedAndDestroyed) { void OnVideoStreamsCreated( VideoSendStream* send_stream, const std::vector& receive_streams) override { - // Encoder initialization should be done in stream construction before - // starting. - EXPECT_TRUE(IsReadyForEncode()); stream_ = send_stream; } @@ -1429,6 +1438,7 @@ TEST_F(VideoSendStreamTest, EncoderSetupPropagatesCommonEncoderConfigValues) { VideoCodecConfigObserver() : SendTest(kDefaultTimeoutMs), FakeEncoder(Clock::GetRealTimeClock()), + init_encode_event_(false, false), num_initializations_(0) {} private: @@ -1457,19 +1467,23 @@ TEST_F(VideoSendStreamTest, EncoderSetupPropagatesCommonEncoderConfigValues) { EXPECT_EQ(kScreensharing, config->mode); } ++num_initializations_; + init_encode_event_.Set(); return FakeEncoder::InitEncode(config, number_of_cores, max_payload_size); } void PerformTest() override { + EXPECT_TRUE(init_encode_event_.Wait(kDefaultTimeoutMs)); EXPECT_EQ(1u, num_initializations_) << "VideoEncoder not initialized."; encoder_config_.content_type = VideoEncoderConfig::ContentType::kScreen; stream_->ReconfigureVideoEncoder(encoder_config_); + EXPECT_TRUE(init_encode_event_.Wait(kDefaultTimeoutMs)); EXPECT_EQ(2u, num_initializations_) << "ReconfigureVideoEncoder did not reinitialize the encoder with " "new encoder settings."; } + rtc::Event init_encode_event_; size_t num_initializations_; VideoSendStream* stream_; VideoEncoderConfig encoder_config_; @@ -1489,6 +1503,7 @@ class VideoCodecConfigObserver : public test::SendTest, FakeEncoder(Clock::GetRealTimeClock()), video_codec_type_(video_codec_type), codec_name_(codec_name), + init_encode_event_(false, false), num_initializations_(0) { memset(&encoder_settings_, 0, sizeof(encoder_settings_)); } @@ -1522,16 +1537,21 @@ class VideoCodecConfigObserver : public test::SendTest, EXPECT_EQ(video_codec_type_, config->codecType); VerifyCodecSpecifics(*config); ++num_initializations_; + init_encode_event_.Set(); return FakeEncoder::InitEncode(config, number_of_cores, max_payload_size); } void VerifyCodecSpecifics(const VideoCodec& config) const; void PerformTest() override { - EXPECT_EQ(1u, num_initializations_) << "VideoEncoder not initialized."; + EXPECT_TRUE( + init_encode_event_.Wait(VideoSendStreamTest::kDefaultTimeoutMs)); + ASSERT_EQ(1u, num_initializations_) << "VideoEncoder not initialized."; encoder_settings_.frameDroppingOn = true; stream_->ReconfigureVideoEncoder(encoder_config_); + ASSERT_TRUE( + init_encode_event_.Wait(VideoSendStreamTest::kDefaultTimeoutMs)); EXPECT_EQ(2u, num_initializations_) << "ReconfigureVideoEncoder did not reinitialize the encoder with " "new encoder settings."; @@ -1547,6 +1567,7 @@ class VideoCodecConfigObserver : public test::SendTest, T encoder_settings_; const VideoCodecType video_codec_type_; const char* const codec_name_; + rtc::Event init_encode_event_; size_t num_initializations_; VideoSendStream* stream_; VideoEncoderConfig encoder_config_; @@ -1735,6 +1756,7 @@ TEST_F(VideoSendStreamTest, EncoderBitrateThresholdObserver() : SendTest(kDefaultTimeoutMs), FakeEncoder(Clock::GetRealTimeClock()), + init_encode_event_(false, false), num_initializations_(0) {} private: @@ -1763,6 +1785,7 @@ TEST_F(VideoSendStreamTest, codecSettings->startBitrate); } ++num_initializations_; + init_encode_event_.Set(); return FakeEncoder::InitEncode(codecSettings, numberOfCores, maxPayloadSize); } @@ -1798,6 +1821,9 @@ TEST_F(VideoSendStreamTest, } void PerformTest() override { + ASSERT_TRUE( + init_encode_event_.Wait(VideoSendStreamTest::kDefaultTimeoutMs)) + << "Timed out while waiting encoder to be configured."; Call::Config::BitrateConfig bitrate_config; bitrate_config.start_bitrate_bps = kIncreasedStartBitrateKbps * 1000; bitrate_config.max_bitrate_bps = kIncreasedMaxBitrateKbps * 1000; @@ -1807,6 +1833,8 @@ TEST_F(VideoSendStreamTest, encoder_config_.streams[0].min_bitrate_bps = 0; encoder_config_.streams[0].max_bitrate_bps = kLowerMaxBitrateKbps * 1000; send_stream_->ReconfigureVideoEncoder(encoder_config_); + ASSERT_TRUE( + init_encode_event_.Wait(VideoSendStreamTest::kDefaultTimeoutMs)); EXPECT_EQ(2, num_initializations_) << "Encoder should have been reconfigured with the new value."; encoder_config_.streams[0].target_bitrate_bps = @@ -1814,10 +1842,13 @@ TEST_F(VideoSendStreamTest, encoder_config_.streams[0].max_bitrate_bps = kIncreasedMaxBitrateKbps * 1000; send_stream_->ReconfigureVideoEncoder(encoder_config_); + ASSERT_TRUE( + init_encode_event_.Wait(VideoSendStreamTest::kDefaultTimeoutMs)); EXPECT_EQ(3, num_initializations_) << "Encoder should have been reconfigured with the new value."; } + rtc::Event init_encode_event_; int num_initializations_; webrtc::Call* call_; webrtc::VideoSendStream* send_stream_;