diff --git a/webrtc/media/base/videoadapter.cc b/webrtc/media/base/videoadapter.cc index 9228e6d84c..9a9666657c 100644 --- a/webrtc/media/base/videoadapter.cc +++ b/webrtc/media/base/videoadapter.cc @@ -495,12 +495,28 @@ void CoordinatedVideoAdapter::OnEncoderResolutionRequest( << " To: " << new_width << "x" << new_height; } +void CoordinatedVideoAdapter::OnCpuResolutionRequest( + rtc::Optional max_pixel_count, + rtc::Optional max_pixel_count_step_up) { + rtc::CritScope cs(&request_critical_section_); + // TODO(perkj): We should support taking larger steps up and down and + // actually look at the values set in max_pixel_count and + // max_pixel_count_step_up. + if (max_pixel_count && *max_pixel_count < GetOutputNumPixels()) { + OnCpuResolutionRequest(DOWNGRADE); + } else if (max_pixel_count_step_up && + *max_pixel_count_step_up >= GetOutputNumPixels()) { + OnCpuResolutionRequest(UPGRADE); + } +} + // A Bandwidth GD request for new resolution void CoordinatedVideoAdapter::OnCpuResolutionRequest(AdaptRequest request) { rtc::CritScope cs(&request_critical_section_); if (!cpu_adaptation_) { return; } + // Update how many times we have downgraded due to the cpu load. switch (request) { case DOWNGRADE: diff --git a/webrtc/media/base/videoadapter.h b/webrtc/media/base/videoadapter.h index 0146182d1d..13a342d86e 100644 --- a/webrtc/media/base/videoadapter.h +++ b/webrtc/media/base/videoadapter.h @@ -13,6 +13,7 @@ #include "webrtc/base/common.h" // For ASSERT #include "webrtc/base/criticalsection.h" +#include "webrtc/base/optional.h" #include "webrtc/base/sigslot.h" #include "webrtc/media/base/videocommon.h" @@ -139,6 +140,9 @@ class CoordinatedVideoAdapter void OnEncoderResolutionRequest(int width, int height, AdaptRequest request); // Handle the resolution request for CPU overuse. void OnCpuResolutionRequest(AdaptRequest request); + void OnCpuResolutionRequest(rtc::Optional max_pixel_count, + rtc::Optional max_pixel_count_step_up); + // Handle the CPU load provided by a CPU monitor. void OnCpuLoadUpdated(int current_cpus, int max_cpus, float process_load, float system_load); diff --git a/webrtc/media/base/videobroadcaster.cc b/webrtc/media/base/videobroadcaster.cc index e369ee07c6..5fa3db733b 100644 --- a/webrtc/media/base/videobroadcaster.cc +++ b/webrtc/media/base/videobroadcaster.cc @@ -10,6 +10,8 @@ #include "webrtc/media/base/videobroadcaster.h" +#include + #include "webrtc/base/checks.h" namespace rtc { @@ -30,12 +32,7 @@ void VideoBroadcaster::AddOrUpdateSink( } else { sink_pair->wants = wants; } - - // Rotation must be applied by the source if one sink wants it. - current_wants_.rotation_applied = false; - for (auto& sink_pair : sinks_) { - current_wants_.rotation_applied |= sink_pair.wants.rotation_applied; - } + UpdateWants(); } void VideoBroadcaster::RemoveSink( @@ -49,6 +46,7 @@ void VideoBroadcaster::RemoveSink( return sink_pair.sink == sink; }), sinks_.end()); + UpdateWants(); } bool VideoBroadcaster::frame_wanted() const { @@ -79,4 +77,36 @@ VideoBroadcaster::SinkPair* VideoBroadcaster::FindSinkPair( return nullptr; } +void VideoBroadcaster::UpdateWants() { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + + VideoSinkWants wants; + wants.rotation_applied = false; + for (auto& sink : sinks_) { + // wants.rotation_applied == ANY(sink.wants.rotation_applied) + if (sink.wants.rotation_applied) { + wants.rotation_applied = true; + } + // wants.max_pixel_count == MIN(sink.wants.max_pixel_count) + if (sink.wants.max_pixel_count && + (!wants.max_pixel_count || + (*sink.wants.max_pixel_count < *wants.max_pixel_count))) { + wants.max_pixel_count = sink.wants.max_pixel_count; + } + // wants.max_pixel_count_step_up == MIN(sink.wants.max_pixel_count_step_up) + if (sink.wants.max_pixel_count_step_up && + (!wants.max_pixel_count_step_up || + (*sink.wants.max_pixel_count_step_up < + *wants.max_pixel_count_step_up))) { + wants.max_pixel_count_step_up = sink.wants.max_pixel_count_step_up; + } + } + + if (wants.max_pixel_count && wants.max_pixel_count_step_up && + *wants.max_pixel_count_step_up >= *wants.max_pixel_count) { + wants.max_pixel_count_step_up = Optional(); + } + current_wants_ = wants; +} + } // namespace rtc diff --git a/webrtc/media/base/videobroadcaster.h b/webrtc/media/base/videobroadcaster.h index 63c22730c7..89c9f3dd49 100644 --- a/webrtc/media/base/videobroadcaster.h +++ b/webrtc/media/base/videobroadcaster.h @@ -47,11 +47,11 @@ class VideoBroadcaster : public VideoSourceInterface, VideoSinkWants wants; }; SinkPair* FindSinkPair(const VideoSinkInterface* sink); + void UpdateWants(); ThreadChecker thread_checker_; VideoSinkWants current_wants_; - std::vector sinks_; }; diff --git a/webrtc/media/base/videobroadcaster_unittest.cc b/webrtc/media/base/videobroadcaster_unittest.cc new file mode 100644 index 0000000000..87d3e932c3 --- /dev/null +++ b/webrtc/media/base/videobroadcaster_unittest.cc @@ -0,0 +1,131 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/base/gunit.h" +#include "webrtc/media/base/videobroadcaster.h" +#include "webrtc/media/engine/webrtcvideoframe.h" + +using rtc::VideoBroadcaster; +using rtc::VideoSinkWants; +using cricket::WebRtcVideoFrame; + +namespace { + +class TestSink : public rtc::VideoSinkInterface { + public: + void OnFrame(const cricket::VideoFrame& frame) override { + ++number_of_rendered_frames_; + } + + int number_of_rendered_frames_ = 0; +}; + +} // namespace + +TEST(VideoBroadcasterTest, frame_wanted) { + VideoBroadcaster broadcaster; + EXPECT_FALSE(broadcaster.frame_wanted()); + + TestSink sink; + broadcaster.AddOrUpdateSink(&sink, rtc::VideoSinkWants()); + EXPECT_TRUE(broadcaster.frame_wanted()); + + broadcaster.RemoveSink(&sink); + EXPECT_FALSE(broadcaster.frame_wanted()); +} + +TEST(VideoBroadcasterTest, OnFrame) { + VideoBroadcaster broadcaster; + + TestSink sink1; + TestSink sink2; + broadcaster.AddOrUpdateSink(&sink1, rtc::VideoSinkWants()); + broadcaster.AddOrUpdateSink(&sink2, rtc::VideoSinkWants()); + + WebRtcVideoFrame frame; + + broadcaster.OnFrame(frame); + EXPECT_EQ(1, sink1.number_of_rendered_frames_); + EXPECT_EQ(1, sink2.number_of_rendered_frames_); + + broadcaster.RemoveSink(&sink1); + broadcaster.OnFrame(frame); + EXPECT_EQ(1, sink1.number_of_rendered_frames_); + EXPECT_EQ(2, sink2.number_of_rendered_frames_); + + broadcaster.AddOrUpdateSink(&sink1, rtc::VideoSinkWants()); + broadcaster.OnFrame(frame); + EXPECT_EQ(2, sink1.number_of_rendered_frames_); + EXPECT_EQ(3, sink2.number_of_rendered_frames_); +} + +TEST(VideoBroadcasterTest, AppliesRotationIfAnySinkWantsRotationApplied) { + VideoBroadcaster broadcaster; + EXPECT_TRUE(broadcaster.wants().rotation_applied); + + TestSink sink1; + VideoSinkWants wants1; + wants1.rotation_applied = false; + + broadcaster.AddOrUpdateSink(&sink1, wants1); + EXPECT_FALSE(broadcaster.wants().rotation_applied); + + TestSink sink2; + VideoSinkWants wants2; + wants2.rotation_applied = true; + + broadcaster.AddOrUpdateSink(&sink2, wants2); + EXPECT_TRUE(broadcaster.wants().rotation_applied); + + broadcaster.RemoveSink(&sink2); + EXPECT_FALSE(broadcaster.wants().rotation_applied); +} + +TEST(VideoBroadcasterTest, AppliesMinOfSinkWantsMaxPixelCount) { + VideoBroadcaster broadcaster; + EXPECT_TRUE(!broadcaster.wants().max_pixel_count); + + TestSink sink1; + VideoSinkWants wants1; + wants1.max_pixel_count = rtc::Optional(1280 * 720); + + broadcaster.AddOrUpdateSink(&sink1, wants1); + EXPECT_EQ(1280 * 720, *broadcaster.wants().max_pixel_count); + + TestSink sink2; + VideoSinkWants wants2; + wants2.max_pixel_count = rtc::Optional(640 * 360); + broadcaster.AddOrUpdateSink(&sink2, wants2); + EXPECT_EQ(640 * 360, *broadcaster.wants().max_pixel_count); + + broadcaster.RemoveSink(&sink2); + EXPECT_EQ(1280 * 720, *broadcaster.wants().max_pixel_count); +} + +TEST(VideoBroadcasterTest, AppliesMinOfSinkWantsMaxPixelCountStepUp) { + VideoBroadcaster broadcaster; + EXPECT_TRUE(!broadcaster.wants().max_pixel_count_step_up); + + TestSink sink1; + VideoSinkWants wants1; + wants1.max_pixel_count_step_up = rtc::Optional(1280 * 720); + + broadcaster.AddOrUpdateSink(&sink1, wants1); + EXPECT_EQ(1280 * 720, *broadcaster.wants().max_pixel_count_step_up); + + TestSink sink2; + VideoSinkWants wants2; + wants2.max_pixel_count_step_up = rtc::Optional(640 * 360); + broadcaster.AddOrUpdateSink(&sink2, wants2); + EXPECT_EQ(640 * 360, *broadcaster.wants().max_pixel_count_step_up); + + broadcaster.RemoveSink(&sink2); + EXPECT_EQ(1280 * 720, *broadcaster.wants().max_pixel_count_step_up); +} diff --git a/webrtc/media/base/videocapturer.cc b/webrtc/media/base/videocapturer.cc index f2933240f9..b9d865ef46 100644 --- a/webrtc/media/base/videocapturer.cc +++ b/webrtc/media/base/videocapturer.cc @@ -320,6 +320,7 @@ void VideoCapturer::GetStats(VariableInfo* adapt_drops_stats, void VideoCapturer::RemoveSink( rtc::VideoSinkInterface* sink) { broadcaster_.RemoveSink(sink); + OnSinkWantsChanged(broadcaster_.wants()); } void VideoCapturer::AddOrUpdateSink( @@ -334,6 +335,11 @@ void VideoCapturer::OnSinkWantsChanged(const rtc::VideoSinkWants& wants) { if (frame_factory_) { frame_factory_->SetApplyRotation(apply_rotation_); } + + if (video_adapter()) { + video_adapter()->OnCpuResolutionRequest(wants.max_pixel_count, + wants.max_pixel_count_step_up); + } } void VideoCapturer::OnFrameCaptured(VideoCapturer*, diff --git a/webrtc/media/base/videocapturer.h b/webrtc/media/base/videocapturer.h index 15089a7dcb..4585d8f6ba 100644 --- a/webrtc/media/base/videocapturer.h +++ b/webrtc/media/base/videocapturer.h @@ -246,11 +246,6 @@ class VideoCapturer : public sigslot::has_slots<>, enable_video_adapter_ = enable_video_adapter; } - CoordinatedVideoAdapter* video_adapter() { return &video_adapter_; } - const CoordinatedVideoAdapter* video_adapter() const { - return &video_adapter_; - } - // Takes ownership. void set_frame_factory(VideoFrameFactory* frame_factory); @@ -286,6 +281,8 @@ class VideoCapturer : public sigslot::has_slots<>, // TODO(perkj): Remove once SignalVideoFrame is removed. void OnFrame(VideoCapturer* capturer, const VideoFrame* frame); + CoordinatedVideoAdapter* video_adapter() { return &video_adapter_; } + void SetCaptureState(CaptureState state); // Marshals SignalStateChange onto thread_. diff --git a/webrtc/media/base/videocapturer_unittest.cc b/webrtc/media/base/videocapturer_unittest.cc index f385d59551..9a33108105 100644 --- a/webrtc/media/base/videocapturer_unittest.cc +++ b/webrtc/media/base/videocapturer_unittest.cc @@ -327,6 +327,71 @@ TEST_F(VideoCapturerTest, TestRotationAppliedBySourceWhenDifferentWants) { EXPECT_EQ(webrtc::kVideoRotation_0, renderer2.rotation()); } +TEST_F(VideoCapturerTest, SinkWantsMaxPixelAndMaxPixelCountStepUp) { + EXPECT_EQ(cricket::CS_RUNNING, + capturer_.Start(cricket::VideoFormat( + 1280, 720, cricket::VideoFormat::FpsToInterval(30), + cricket::FOURCC_I420))); + EXPECT_TRUE(capturer_.IsRunning()); + + EXPECT_EQ(0, renderer_.num_rendered_frames()); + EXPECT_TRUE(capturer_.CaptureFrame()); + EXPECT_EQ(1, renderer_.num_rendered_frames()); + EXPECT_EQ(1280, renderer_.width()); + EXPECT_EQ(720, renderer_.height()); + + // Request a lower resolution. + rtc::VideoSinkWants wants; + wants.max_pixel_count = rtc::Optional(1280 * 720 / 2); + capturer_.AddOrUpdateSink(&renderer_, wants); + EXPECT_TRUE(capturer_.CaptureFrame()); + EXPECT_EQ(2, renderer_.num_rendered_frames()); + EXPECT_EQ(960, renderer_.width()); + EXPECT_EQ(540, renderer_.height()); + + // Request a lower resolution. + wants.max_pixel_count = + rtc::Optional(renderer_.width() * renderer_.height() / 2); + capturer_.AddOrUpdateSink(&renderer_, wants); + EXPECT_TRUE(capturer_.CaptureFrame()); + EXPECT_EQ(3, renderer_.num_rendered_frames()); + EXPECT_EQ(640, renderer_.width()); + EXPECT_EQ(360, renderer_.height()); + + // Adding a new renderer should not affect resolution. + cricket::FakeVideoRenderer renderer2; + capturer_.AddOrUpdateSink(&renderer2, rtc::VideoSinkWants()); + EXPECT_TRUE(capturer_.CaptureFrame()); + EXPECT_EQ(4, renderer_.num_rendered_frames()); + EXPECT_EQ(640, renderer_.width()); + EXPECT_EQ(360, renderer_.height()); + EXPECT_EQ(1, renderer2.num_rendered_frames()); + EXPECT_EQ(640, renderer2.width()); + EXPECT_EQ(360, renderer2.height()); + + // Request higher resolution. + wants.max_pixel_count_step_up = wants.max_pixel_count; + wants.max_pixel_count = rtc::Optional(); + capturer_.AddOrUpdateSink(&renderer_, wants); + EXPECT_TRUE(capturer_.CaptureFrame()); + EXPECT_EQ(5, renderer_.num_rendered_frames()); + EXPECT_EQ(960, renderer_.width()); + EXPECT_EQ(540, renderer_.height()); + EXPECT_EQ(2, renderer2.num_rendered_frames()); + EXPECT_EQ(960, renderer2.width()); + EXPECT_EQ(540, renderer2.height()); + + // Updating with no wants should not affect resolution. + capturer_.AddOrUpdateSink(&renderer2, rtc::VideoSinkWants()); + EXPECT_TRUE(capturer_.CaptureFrame()); + EXPECT_EQ(6, renderer_.num_rendered_frames()); + EXPECT_EQ(960, renderer_.width()); + EXPECT_EQ(540, renderer_.height()); + EXPECT_EQ(3, renderer2.num_rendered_frames()); + EXPECT_EQ(960, renderer2.width()); + EXPECT_EQ(540, renderer2.height()); +} + TEST_F(VideoCapturerTest, ScreencastScaledSuperLarge) { capturer_.SetScreencast(true); diff --git a/webrtc/media/base/videosourceinterface.h b/webrtc/media/base/videosourceinterface.h index 1462458543..8895970df3 100644 --- a/webrtc/media/base/videosourceinterface.h +++ b/webrtc/media/base/videosourceinterface.h @@ -13,20 +13,25 @@ #include "webrtc/media/base/videosinkinterface.h" #include "webrtc/base/callback.h" +#include "webrtc/base/optional.h" namespace rtc { // VideoSinkWants is used for notifying the source of properties a video frame // should have when it is delivered to a certain sink. struct VideoSinkWants { - bool operator==(const VideoSinkWants& rh) const { - return rotation_applied == rh.rotation_applied; - } - bool operator!=(const VideoSinkWants& rh) const { return !operator==(rh); } - // Tells the source whether the sink wants frames with rotation applied. // By default, the rotation is applied by the source. bool rotation_applied = true; + + // Tells the source the maximum number of pixels the sink wants. + rtc::Optional max_pixel_count; + // Like |max_pixel_count| but relative to the given value. The source is + // requested to produce frames with a resolution one "step up" from the given + // value. In practice, this means that the sink can consume this amount of + // pixels but wants more and the source should produce a resolution one + // "step" higher than this but not higher. + rtc::Optional max_pixel_count_step_up; }; template diff --git a/webrtc/media/engine/webrtcvideoengine2.cc b/webrtc/media/engine/webrtcvideoengine2.cc index c6bdafee5a..41a23c1a92 100644 --- a/webrtc/media/engine/webrtcvideoengine2.cc +++ b/webrtc/media/engine/webrtcvideoengine2.cc @@ -314,6 +314,7 @@ static int GetMaxDefaultVideoBitrateKbps(int width, int height) { return 2500; } } + } // namespace // Constants defined in webrtc/media/engine/constants.h @@ -1000,12 +1001,10 @@ bool WebRtcVideoChannel2::AddSendStream(const StreamParams& sp) { send_ssrcs_.insert(used_ssrc); webrtc::VideoSendStream::Config config(this); - config.overuse_callback = this; - - WebRtcVideoSendStream* stream = - new WebRtcVideoSendStream(call_, sp, config, external_encoder_factory_, - bitrate_config_.max_bitrate_bps, send_codec_, - send_rtp_extensions_, send_params_); + WebRtcVideoSendStream* stream = new WebRtcVideoSendStream( + call_, sp, config, external_encoder_factory_, signal_cpu_adaptation_, + bitrate_config_.max_bitrate_bps, send_codec_, send_rtp_extensions_, + send_params_); uint32_t ssrc = sp.first_ssrc(); RTC_DCHECK(ssrc != 0); @@ -1283,10 +1282,6 @@ bool WebRtcVideoChannel2::SetCapturer(uint32_t ssrc, VideoCapturer* capturer) { return false; } } - { - rtc::CritScope lock(&capturer_crit_); - capturers_[ssrc] = capturer; - } return true; } @@ -1412,26 +1407,6 @@ void WebRtcVideoChannel2::SetInterface(NetworkInterface* iface) { kVideoRtpBufferSize); } -void WebRtcVideoChannel2::OnLoadUpdate(Load load) { - // OnLoadUpdate can not take any locks that are held while creating streams - // etc. Doing so establishes lock-order inversions between the webrtc process - // thread on stream creation and locks such as stream_crit_ while calling out. - rtc::CritScope stream_lock(&capturer_crit_); - if (!signal_cpu_adaptation_) - return; - // Do not adapt resolution for screen content as this will likely result in - // blurry and unreadable text. - for (auto& kv : capturers_) { - if (kv.second != nullptr - && !kv.second->IsScreencast() - && kv.second->video_adapter() != nullptr) { - kv.second->video_adapter()->OnCpuResolutionRequest( - load == kOveruse ? CoordinatedVideoAdapter::DOWNGRADE - : CoordinatedVideoAdapter::UPGRADE); - } - } -} - bool WebRtcVideoChannel2::SendRtp(const uint8_t* data, size_t len, const webrtc::PacketOptions& options) { @@ -1495,24 +1470,28 @@ WebRtcVideoChannel2::WebRtcVideoSendStream::WebRtcVideoSendStream( const StreamParams& sp, const webrtc::VideoSendStream::Config& config, WebRtcVideoEncoderFactory* external_encoder_factory, + bool enable_cpu_overuse_detection, int max_bitrate_bps, const rtc::Optional& codec_settings, const std::vector& rtp_extensions, // TODO(deadbeef): Don't duplicate information between send_params, // rtp_extensions, options, etc. const VideoSendParameters& send_params) - : ssrcs_(sp.ssrcs), + : worker_thread_(rtc::Thread::Current()), + ssrcs_(sp.ssrcs), ssrc_groups_(sp.ssrc_groups), call_(call), + cpu_restricted_counter_(0), + number_of_cpu_adapt_changes_(0), + capturer_(nullptr), external_encoder_factory_(external_encoder_factory), - stream_(NULL), + stream_(nullptr), parameters_(config, send_params.options, max_bitrate_bps, codec_settings), pending_encoder_reconfiguration_(false), - allocated_encoder_(NULL, webrtc::kVideoCodecUnknown, false), - capturer_(NULL), + allocated_encoder_(nullptr, webrtc::kVideoCodecUnknown, false), + capturer_is_screencast_(false), sending_(false), muted_(false), - old_adapt_changes_(0), first_frame_timestamp_ms_(0), last_frame_timestamp_ms_(0) { parameters_.config.rtp.max_packet_size = kVideoMtu; @@ -1526,6 +1505,8 @@ WebRtcVideoChannel2::WebRtcVideoSendStream::WebRtcVideoSendStream( parameters_.config.rtp.rtcp_mode = send_params.rtcp.reduced_size ? webrtc::RtcpMode::kReducedSize : webrtc::RtcpMode::kCompound; + parameters_.config.overuse_callback = + enable_cpu_overuse_detection ? this : nullptr; if (codec_settings) { SetCodecAndOptions(*codec_settings, parameters_.options); @@ -1589,7 +1570,7 @@ void WebRtcVideoChannel2::WebRtcVideoSendStream::OnFrame( video_frame.set_render_time_ms(last_frame_timestamp_ms_); // Reconfigure codec if necessary. SetDimensions(video_frame.width(), video_frame.height(), - capturer_->IsScreencast()); + capturer_is_screencast_); last_rotation_ = video_frame.rotation(); stream_->Input()->IncomingCapturedFrame(video_frame); @@ -1598,6 +1579,7 @@ void WebRtcVideoChannel2::WebRtcVideoSendStream::OnFrame( bool WebRtcVideoChannel2::WebRtcVideoSendStream::SetCapturer( VideoCapturer* capturer) { TRACE_EVENT0("webrtc", "WebRtcVideoSendStream::SetCapturer"); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); if (!DisconnectCapturer() && capturer == NULL) { return false; } @@ -1630,10 +1612,10 @@ bool WebRtcVideoChannel2::WebRtcVideoSendStream::SetCapturer( capturer_ = NULL; return true; } - - capturer_ = capturer; - capturer_->AddOrUpdateSink(this, sink_wants_); + capturer_is_screencast_ = capturer->IsScreencast(); } + capturer_ = capturer; + capturer_->AddOrUpdateSink(this, sink_wants_); return true; } @@ -1643,20 +1625,18 @@ void WebRtcVideoChannel2::WebRtcVideoSendStream::MuteStream(bool mute) { } bool WebRtcVideoChannel2::WebRtcVideoSendStream::DisconnectCapturer() { - cricket::VideoCapturer* capturer; - { - rtc::CritScope cs(&lock_); - if (capturer_ == NULL) - return false; - - if (capturer_->video_adapter() != nullptr) - old_adapt_changes_ += capturer_->video_adapter()->adaptation_changes(); - - capturer = capturer_; - capturer_ = NULL; + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + if (capturer_ == NULL) { + return false; } - capturer->RemoveSink(this); + capturer_->RemoveSink(this); + capturer_ = NULL; + // Reset |cpu_restricted_counter_| if the capturer is changed. It is not + // possible to know if the video resolution is restricted by CPU usage after + // the capturer is changed since the next capturer might be screen capture + // with another resolution and frame rate. + cpu_restricted_counter_ = 0; return true; } @@ -1822,8 +1802,7 @@ void WebRtcVideoChannel2::WebRtcVideoSendStream::SetSendParameters( } else { parameters_.options = *params.options; } - } - else if (params.conference_mode && parameters_.codec_settings) { + } else if (params.conference_mode && parameters_.codec_settings) { SetCodecAndOptions(*parameters_.codec_settings, parameters_.options); return; } @@ -1950,10 +1929,66 @@ void WebRtcVideoChannel2::WebRtcVideoSendStream::Stop() { sending_ = false; } +void WebRtcVideoChannel2::WebRtcVideoSendStream::OnLoadUpdate(Load load) { + if (worker_thread_ != rtc::Thread::Current()) { + invoker_.AsyncInvoke( + worker_thread_, + rtc::Bind(&WebRtcVideoChannel2::WebRtcVideoSendStream::OnLoadUpdate, + this, load)); + return; + } + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + LOG(LS_INFO) << "OnLoadUpdate " << load; + if (!capturer_) { + return; + } + { + rtc::CritScope cs(&lock_); + // Do not adapt resolution for screen content as this will likely result in + // blurry and unreadable text. + if (capturer_is_screencast_) + return; + + rtc::Optional max_pixel_count; + rtc::Optional max_pixel_count_step_up; + if (load == kOveruse) { + max_pixel_count = rtc::Optional( + (last_dimensions_.height * last_dimensions_.width) / 2); + // Increase |number_of_cpu_adapt_changes_| if + // sink_wants_.max_pixel_count will be changed since + // last time |capturer_->AddOrUpdateSink| was called. That is, this will + // result in a new request for the capturer to change resolution. + if (!sink_wants_.max_pixel_count || + *sink_wants_.max_pixel_count > *max_pixel_count) { + ++number_of_cpu_adapt_changes_; + ++cpu_restricted_counter_; + } + } else { + RTC_DCHECK(load == kUnderuse); + max_pixel_count_step_up = rtc::Optional(last_dimensions_.height * + last_dimensions_.width); + // Increase |number_of_cpu_adapt_changes_| if + // sink_wants_.max_pixel_count_step_up will be changed since + // last time |capturer_->AddOrUpdateSink| was called. That is, this will + // result in a new request for the capturer to change resolution. + if (sink_wants_.max_pixel_count || + (sink_wants_.max_pixel_count_step_up && + *sink_wants_.max_pixel_count_step_up < *max_pixel_count_step_up)) { + ++number_of_cpu_adapt_changes_; + --cpu_restricted_counter_; + } + } + sink_wants_.max_pixel_count = max_pixel_count; + sink_wants_.max_pixel_count_step_up = max_pixel_count_step_up; + } + capturer_->AddOrUpdateSink(this, sink_wants_); +} + VideoSenderInfo WebRtcVideoChannel2::WebRtcVideoSendStream::GetVideoSenderInfo() { VideoSenderInfo info; webrtc::VideoSendStream::Stats stats; + RTC_DCHECK(thread_checker_.CalledOnValidThread()); { rtc::CritScope cs(&lock_); for (uint32_t ssrc : parameters_.config.rtp.ssrcs) @@ -1975,23 +2010,20 @@ WebRtcVideoChannel2::WebRtcVideoSendStream::GetVideoSenderInfo() { return info; stats = stream_->GetStats(); + } + info.adapt_changes = number_of_cpu_adapt_changes_; + info.adapt_reason = cpu_restricted_counter_ <= 0 + ? CoordinatedVideoAdapter::ADAPTREASON_NONE + : CoordinatedVideoAdapter::ADAPTREASON_CPU; - info.adapt_changes = old_adapt_changes_; - info.adapt_reason = CoordinatedVideoAdapter::ADAPTREASON_NONE; - - if (capturer_ != NULL) { - if (!capturer_->IsMuted()) { - VideoFormat last_captured_frame_format; - capturer_->GetStats(&info.adapt_frame_drops, &info.effects_frame_drops, - &info.capturer_frame_time, - &last_captured_frame_format); - info.input_frame_width = last_captured_frame_format.width; - info.input_frame_height = last_captured_frame_format.height; - } - if (capturer_->video_adapter() != nullptr) { - info.adapt_changes += capturer_->video_adapter()->adaptation_changes(); - info.adapt_reason = capturer_->video_adapter()->adapt_reason(); - } + if (capturer_) { + if (!capturer_->IsMuted()) { + VideoFormat last_captured_frame_format; + capturer_->GetStats(&info.adapt_frame_drops, &info.effects_frame_drops, + &info.capturer_frame_time, + &last_captured_frame_format); + info.input_frame_width = last_captured_frame_format.width; + info.input_frame_height = last_captured_frame_format.height; } } diff --git a/webrtc/media/engine/webrtcvideoengine2.h b/webrtc/media/engine/webrtcvideoengine2.h index 762af58893..3dd05f2400 100644 --- a/webrtc/media/engine/webrtcvideoengine2.h +++ b/webrtc/media/engine/webrtcvideoengine2.h @@ -17,6 +17,7 @@ #include #include +#include "webrtc/base/asyncinvoker.h" #include "webrtc/base/criticalsection.h" #include "webrtc/base/thread_annotations.h" #include "webrtc/base/thread_checker.h" @@ -129,9 +130,7 @@ class WebRtcVideoEngine2 { std::unique_ptr simulcast_encoder_factory_; }; -class WebRtcVideoChannel2 : public VideoMediaChannel, - public webrtc::Transport, - public webrtc::LoadObserver { +class WebRtcVideoChannel2 : public VideoMediaChannel, public webrtc::Transport { public: WebRtcVideoChannel2(webrtc::Call* call, const MediaConfig& config, @@ -168,8 +167,6 @@ class WebRtcVideoChannel2 : public VideoMediaChannel, void OnReadyToSend(bool ready) override; void SetInterface(NetworkInterface* iface) override; - void OnLoadUpdate(Load load) override; - // Implemented for VideoMediaChannelTest. bool sending() const { return sending_; } uint32_t GetDefaultSendChannelSsrc() { return default_send_ssrc_; } @@ -230,13 +227,15 @@ class WebRtcVideoChannel2 : public VideoMediaChannel, // Wrapper for the sender part, this is where the capturer is connected and // frames are then converted from cricket frames to webrtc frames. class WebRtcVideoSendStream - : public rtc::VideoSinkInterface { + : public rtc::VideoSinkInterface, + public webrtc::LoadObserver { public: WebRtcVideoSendStream( webrtc::Call* call, const StreamParams& sp, const webrtc::VideoSendStream::Config& config, WebRtcVideoEncoderFactory* external_encoder_factory, + bool enable_cpu_overuse_detection, int max_bitrate_bps, const rtc::Optional& codec_settings, const std::vector& rtp_extensions, @@ -255,6 +254,9 @@ class WebRtcVideoChannel2 : public VideoMediaChannel, void Start(); void Stop(); + // Implements webrtc::LoadObserver. + void OnLoadUpdate(Load load) override; + const std::vector& GetSsrcs() const; VideoSenderInfo GetVideoSenderInfo(); void FillBandwidthEstimationInfo(BandwidthEstimationInfo* bwe_info); @@ -341,10 +343,20 @@ class WebRtcVideoChannel2 : public VideoMediaChannel, void SetDimensions(int width, int height, bool is_screencast) EXCLUSIVE_LOCKS_REQUIRED(lock_); + rtc::ThreadChecker thread_checker_; + rtc::AsyncInvoker invoker_; + rtc::Thread* worker_thread_; const std::vector ssrcs_; const std::vector ssrc_groups_; webrtc::Call* const call_; rtc::VideoSinkWants sink_wants_; + // Counter used for deciding if the video resolution is currently + // restricted by CPU usage. It is reset if |capturer_| is changed. + int cpu_restricted_counter_; + // Total number of times resolution as been requested to be changed due to + // CPU adaptation. + int number_of_cpu_adapt_changes_; + VideoCapturer* capturer_; WebRtcVideoEncoderFactory* const external_encoder_factory_ GUARDED_BY(lock_); @@ -358,10 +370,9 @@ class WebRtcVideoChannel2 : public VideoMediaChannel, webrtc::VideoRotation last_rotation_ GUARDED_BY(lock_) = webrtc::kVideoRotation_0; - VideoCapturer* capturer_ GUARDED_BY(lock_); + bool capturer_is_screencast_ GUARDED_BY(lock_); bool sending_ GUARDED_BY(lock_); bool muted_ GUARDED_BY(lock_); - int old_adapt_changes_ GUARDED_BY(lock_); // The timestamp of the first frame received // Used to generate the timestamps of subsequent frames @@ -491,12 +502,6 @@ class WebRtcVideoChannel2 : public VideoMediaChannel, const bool signal_cpu_adaptation_; const bool disable_prerenderer_smoothing_; - // Separate list of set capturers used to signal CPU adaptation. These should - // not be locked while calling methods that take other locks to prevent - // lock-order inversions. - rtc::CriticalSection capturer_crit_; - std::map capturers_ GUARDED_BY(capturer_crit_); - rtc::CriticalSection stream_crit_; // Using primary-ssrc (first ssrc) as key. std::map send_streams_ diff --git a/webrtc/media/engine/webrtcvideoengine2_unittest.cc b/webrtc/media/engine/webrtcvideoengine2_unittest.cc index 46fd2049f6..529d77b14b 100644 --- a/webrtc/media/engine/webrtcvideoengine2_unittest.cc +++ b/webrtc/media/engine/webrtcvideoengine2_unittest.cc @@ -1491,10 +1491,13 @@ TEST_F(WebRtcVideoChannel2Test, UsesCorrectSettingsForScreencast) { EXPECT_EQ(0, encoder_config.min_transmit_bitrate_bps) << "Non-screenshare shouldn't use min-transmit bitrate."; - capturer.SetScreencast(true); - EXPECT_TRUE(capturer.CaptureFrame()); - + EXPECT_TRUE(channel_->SetCapturer(last_ssrc_, nullptr)); + // Removing a capturer triggers a black frame to be sent. EXPECT_EQ(2, send_stream->GetNumberOfSwappedFrames()); + capturer.SetScreencast(true); + EXPECT_TRUE(channel_->SetCapturer(last_ssrc_, &capturer)); + EXPECT_TRUE(capturer.CaptureFrame()); + EXPECT_EQ(3, send_stream->GetNumberOfSwappedFrames()); // Verify screencast settings. encoder_config = send_stream->GetEncoderConfig(); @@ -1625,7 +1628,9 @@ TEST_F(WebRtcVideoChannel2Test, VerifyVp8SpecificSettings) { EXPECT_TRUE(vp8_settings.frameDroppingOn); // In screen-share mode, denoising is forced off and simulcast disabled. + EXPECT_TRUE(channel_->SetCapturer(last_ssrc_, NULL)); capturer.SetScreencast(true); + EXPECT_TRUE(channel_->SetCapturer(last_ssrc_, &capturer)); EXPECT_TRUE(capturer.CaptureFrame()); stream = SetDenoisingOption(parameters, false); @@ -1704,7 +1709,10 @@ TEST_F(Vp9SettingsTest, VerifyVp9SpecificSettings) { EXPECT_TRUE(vp9_settings.frameDroppingOn); // In screen-share mode, denoising is forced off. + EXPECT_TRUE(channel_->SetCapturer(last_ssrc_, nullptr)); capturer.SetScreencast(true); + EXPECT_TRUE(channel_->SetCapturer(last_ssrc_, &capturer)); + EXPECT_TRUE(capturer.CaptureFrame()); stream = SetDenoisingOption(parameters, false); @@ -1734,6 +1742,73 @@ TEST_F(WebRtcVideoChannel2Test, DoesNotAdaptOnOveruseWhenScreensharing) { TestCpuAdaptation(true, true); } +TEST_F(WebRtcVideoChannel2Test, AdaptsOnOveruseAndChangeResolution) { + cricket::VideoCodec codec = kVp8Codec720p; + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(codec); + + MediaConfig media_config = MediaConfig(); + channel_.reset( + engine_.CreateChannel(fake_call_.get(), media_config, VideoOptions())); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + + AddSendStream(); + + cricket::FakeVideoCapturer capturer; + capturer.SetScreencast(false); + ASSERT_TRUE(channel_->SetCapturer(last_ssrc_, &capturer)); + ASSERT_EQ(cricket::CS_RUNNING, + capturer.Start(capturer.GetSupportedFormats()->front())); + ASSERT_TRUE(channel_->SetSend(true)); + + ASSERT_EQ(1u, fake_call_->GetVideoSendStreams().size()); + FakeVideoSendStream* send_stream = fake_call_->GetVideoSendStreams().front(); + webrtc::LoadObserver* overuse_callback = + send_stream->GetConfig().overuse_callback; + ASSERT_TRUE(overuse_callback != NULL); + + EXPECT_TRUE(capturer.CaptureCustomFrame(1280, 720, cricket::FOURCC_I420)); + EXPECT_EQ(1, send_stream->GetNumberOfSwappedFrames()); + EXPECT_EQ(1280, send_stream->GetLastWidth()); + EXPECT_EQ(720, send_stream->GetLastHeight()); + + // Trigger overuse. + overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kOveruse); + EXPECT_TRUE(capturer.CaptureCustomFrame(1280, 720, cricket::FOURCC_I420)); + EXPECT_EQ(2, send_stream->GetNumberOfSwappedFrames()); + EXPECT_EQ(1280 * 3 / 4, send_stream->GetLastWidth()); + EXPECT_EQ(720 * 3 / 4, send_stream->GetLastHeight()); + + // Trigger overuse again. + overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kOveruse); + EXPECT_TRUE(capturer.CaptureCustomFrame(1280, 720, cricket::FOURCC_I420)); + EXPECT_EQ(3, send_stream->GetNumberOfSwappedFrames()); + EXPECT_EQ(1280 * 2 / 4, send_stream->GetLastWidth()); + EXPECT_EQ(720 * 2 / 4, send_stream->GetLastHeight()); + + // Change input resolution. + EXPECT_TRUE(capturer.CaptureCustomFrame(1284, 724, cricket::FOURCC_I420)); + EXPECT_EQ(4, send_stream->GetNumberOfSwappedFrames()); + EXPECT_EQ(1284 / 2, send_stream->GetLastWidth()); + EXPECT_EQ(724 / 2, send_stream->GetLastHeight()); + + // Trigger underuse which should go back up in resolution. + overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kUnderuse); + EXPECT_TRUE(capturer.CaptureCustomFrame(1284, 724, cricket::FOURCC_I420)); + EXPECT_EQ(5, send_stream->GetNumberOfSwappedFrames()); + EXPECT_EQ(1284 * 3 / 4, send_stream->GetLastWidth()); + EXPECT_EQ(724 * 3 / 4, send_stream->GetLastHeight()); + + // Trigger underuse which should go back up in resolution. + overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kUnderuse); + EXPECT_TRUE(capturer.CaptureCustomFrame(1284, 724, cricket::FOURCC_I420)); + EXPECT_EQ(6, send_stream->GetNumberOfSwappedFrames()); + EXPECT_EQ(1284, send_stream->GetLastWidth()); + EXPECT_EQ(724, send_stream->GetLastHeight()); + + EXPECT_TRUE(channel_->SetCapturer(last_ssrc_, NULL)); +} + void WebRtcVideoChannel2Test::TestCpuAdaptation(bool enable_overuse, bool is_screenshare) { cricket::VideoCodec codec = kVp8Codec720p; @@ -1764,25 +1839,41 @@ void WebRtcVideoChannel2Test::TestCpuAdaptation(bool enable_overuse, FakeVideoSendStream* send_stream = fake_call_->GetVideoSendStreams().front(); webrtc::LoadObserver* overuse_callback = send_stream->GetConfig().overuse_callback; + + if (!enable_overuse) { + ASSERT_TRUE(overuse_callback == NULL); + + EXPECT_TRUE(capturer.CaptureFrame()); + EXPECT_EQ(1, send_stream->GetNumberOfSwappedFrames()); + + EXPECT_EQ(codec.width, send_stream->GetLastWidth()); + EXPECT_EQ(codec.height, send_stream->GetLastHeight()); + + EXPECT_TRUE(channel_->SetCapturer(last_ssrc_, NULL)); + return; + } + ASSERT_TRUE(overuse_callback != NULL); + EXPECT_TRUE(capturer.CaptureFrame()); + EXPECT_EQ(1, send_stream->GetNumberOfSwappedFrames()); overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kOveruse); EXPECT_TRUE(capturer.CaptureFrame()); - EXPECT_EQ(1, send_stream->GetNumberOfSwappedFrames()); + EXPECT_EQ(2, send_stream->GetNumberOfSwappedFrames()); - if (enable_overuse && !is_screenshare) { - EXPECT_LT(send_stream->GetLastWidth(), codec.width); - EXPECT_LT(send_stream->GetLastHeight(), codec.height); - } else { + if (is_screenshare) { + // Do not adapt screen share. EXPECT_EQ(codec.width, send_stream->GetLastWidth()); EXPECT_EQ(codec.height, send_stream->GetLastHeight()); + } else { + EXPECT_LT(send_stream->GetLastWidth(), codec.width); + EXPECT_LT(send_stream->GetLastHeight(), codec.height); } // Trigger underuse which should go back to normal resolution. overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kUnderuse); EXPECT_TRUE(capturer.CaptureFrame()); - - EXPECT_EQ(2, send_stream->GetNumberOfSwappedFrames()); + EXPECT_EQ(3, send_stream->GetNumberOfSwappedFrames()); EXPECT_EQ(codec.width, send_stream->GetLastWidth()); EXPECT_EQ(codec.height, send_stream->GetLastHeight()); diff --git a/webrtc/media/media_tests.gypi b/webrtc/media/media_tests.gypi index fcac6b4549..276b72dc69 100644 --- a/webrtc/media/media_tests.gypi +++ b/webrtc/media/media_tests.gypi @@ -85,6 +85,7 @@ 'base/streamparams_unittest.cc', 'base/turnutils_unittest.cc', 'base/videoadapter_unittest.cc', + 'base/videobroadcaster_unittest.cc', 'base/videocapturer_unittest.cc', 'base/videocommon_unittest.cc', 'base/videoengine_unittest.h',