diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc index 2480a502e8..869657b972 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -17,6 +17,8 @@ #include "vpx/vpx_encoder.h" #include "vpx/vp8cx.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" +#include "webrtc/system_wrappers/include/clock.h" +#include "webrtc/system_wrappers/include/metrics.h" namespace webrtc { @@ -46,8 +48,10 @@ const int ScreenshareLayers::kTl1SyncFlags = VP8_EFLAG_NO_UPD_LAST; ScreenshareLayers::ScreenshareLayers(int num_temporal_layers, - uint8_t initial_tl0_pic_idx) - : number_of_temporal_layers_(num_temporal_layers), + uint8_t initial_tl0_pic_idx, + Clock* clock) + : clock_(clock), + number_of_temporal_layers_(num_temporal_layers), last_base_layer_sync_(false), tl0_pic_idx_(initial_tl0_pic_idx), active_layer_(-1), @@ -61,6 +65,10 @@ ScreenshareLayers::ScreenshareLayers(int num_temporal_layers, RTC_CHECK_LE(num_temporal_layers, 2); } +ScreenshareLayers::~ScreenshareLayers() { + UpdateHistograms(); +} + int ScreenshareLayers::CurrentLayerId() const { // Codec does not use temporal layers for screenshare. return 0; @@ -72,6 +80,9 @@ int ScreenshareLayers::EncodeFlags(uint32_t timestamp) { return 0; } + if (stats_.first_frame_time_ms_ == -1) + stats_.first_frame_time_ms_ = clock_->TimeInMilliseconds(); + int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp); int flags = 0; @@ -104,6 +115,7 @@ int ScreenshareLayers::EncodeFlags(uint32_t timestamp) { break; case -1: flags = -1; + ++stats_.num_dropped_frames_; break; default: flags = -1; @@ -176,6 +188,7 @@ void ScreenshareLayers::FrameEncoded(unsigned int size, RTC_DCHECK_NE(-1, active_layer_); if (size == 0) { layers_[active_layer_].state = TemporalLayer::State::kDropped; + ++stats_.num_overshoots_; return; } @@ -189,8 +202,14 @@ void ScreenshareLayers::FrameEncoded(unsigned int size, if (active_layer_ == 0) { layers_[0].debt_bytes_ += size; layers_[1].debt_bytes_ += size; + ++stats_.num_tl0_frames_; + stats_.tl0_target_bitrate_sum_ += layers_[0].target_rate_kbps_; + stats_.tl0_qp_sum_ += qp; } else if (active_layer_ == 1) { layers_[1].debt_bytes_ += size; + ++stats_.num_tl1_frames_; + stats_.tl1_target_bitrate_sum_ += layers_[1].target_rate_kbps_; + stats_.tl1_qp_sum_ += qp; } } @@ -283,4 +302,42 @@ void ScreenshareLayers::TemporalLayer::UpdateDebt(int64_t delta_ms) { } } +void ScreenshareLayers::UpdateHistograms() { + if (stats_.first_frame_time_ms_ == -1) + return; + int64_t duration_sec = + (clock_->TimeInMilliseconds() - stats_.first_frame_time_ms_ + 500) / 1000; + if (duration_sec >= metrics::kMinRunTimeInSeconds) { + RTC_HISTOGRAM_COUNTS_10000( + "WebRTC.Video.Screenshare.Layer0.FrameRate", + (stats_.num_tl0_frames_ + (duration_sec / 2)) / duration_sec); + RTC_HISTOGRAM_COUNTS_10000( + "WebRTC.Video.Screenshare.Layer1.FrameRate", + (stats_.num_tl1_frames_ + (duration_sec / 2)) / duration_sec); + int total_frames = stats_.num_tl0_frames_ + stats_.num_tl1_frames_; + RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.Screenshare.FramesPerDrop", + stats_.num_dropped_frames_ == 0 + ? 0 + : total_frames / stats_.num_dropped_frames_); + RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.Screenshare.FramesPerOvershoot", + stats_.num_overshoots_ == 0 + ? 0 + : total_frames / stats_.num_overshoots_); + if (stats_.num_tl0_frames_ > 0) { + RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.Screenshare.Layer0.Qp", + stats_.tl0_qp_sum_ / stats_.num_tl0_frames_); + RTC_HISTOGRAM_COUNTS_10000( + "WebRTC.Video.Screenshare.Layer0.TargetBitrate", + stats_.tl0_target_bitrate_sum_ / stats_.num_tl0_frames_); + } + if (stats_.num_tl1_frames_ > 0) { + RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.Screenshare.Layer1.Qp", + stats_.tl1_qp_sum_ / stats_.num_tl1_frames_); + RTC_HISTOGRAM_COUNTS_10000( + "WebRTC.Video.Screenshare.Layer1.TargetBitrate", + stats_.tl1_target_bitrate_sum_ / stats_.num_tl1_frames_); + } + } +} + } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h index 7628758209..e11f3d27ef 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -21,6 +21,7 @@ namespace webrtc { struct CodecSpecificInfoVP8; +class Clock; class ScreenshareLayers : public TemporalLayers { public: @@ -30,8 +31,10 @@ class ScreenshareLayers : public TemporalLayers { static const int kTl1Flags; static const int kTl1SyncFlags; - ScreenshareLayers(int num_temporal_layers, uint8_t initial_tl0_pic_idx); - virtual ~ScreenshareLayers() {} + ScreenshareLayers(int num_temporal_layers, + uint8_t initial_tl0_pic_idx, + Clock* clock); + virtual ~ScreenshareLayers(); // Returns the recommended VP8 encode flags needed. May refresh the decoder // and/or update the reference buffers. @@ -58,6 +61,8 @@ class ScreenshareLayers : public TemporalLayers { private: bool TimeToSync(int64_t timestamp) const; + Clock* const clock_; + int number_of_temporal_layers_; bool last_base_layer_sync_; uint8_t tl0_pic_idx_; @@ -93,6 +98,20 @@ class ScreenshareLayers : public TemporalLayers { void UpdateDebt(int64_t delta_ms); } layers_[kMaxNumTemporalLayers]; + + void UpdateHistograms(); + // Data for histogram statistics. + struct Stats { + int64_t first_frame_time_ms_ = -1; + int64_t num_tl0_frames_ = 0; + int64_t num_tl1_frames_ = 0; + int64_t num_dropped_frames_ = 0; + int64_t num_overshoots_ = 0; + int64_t tl0_qp_sum_ = 0; + int64_t tl1_qp_sum_ = 0; + int64_t tl0_target_bitrate_sum_ = 0; + int64_t tl1_target_bitrate_sum_ = 0; + } stats_; }; } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc index 2733d7d2be..3117e49788 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc @@ -17,6 +17,9 @@ #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h" #include "webrtc/modules/video_coding/utility/mock/mock_frame_dropper.h" +#include "webrtc/system_wrappers/include/clock.h" +#include "webrtc/system_wrappers/include/metrics.h" +#include "webrtc/test/histogram.h" using ::testing::_; using ::testing::NiceMock; @@ -35,9 +38,12 @@ const int kMaxSyncPeriodSeconds = 10; class ScreenshareLayerTest : public ::testing::Test { protected: - ScreenshareLayerTest() : min_qp_(2), max_qp_(kDefaultQp), frame_size_(-1) {} + ScreenshareLayerTest() + : min_qp_(2), max_qp_(kDefaultQp), frame_size_(-1), clock_(1) {} virtual ~ScreenshareLayerTest() {} + void SetUp() override { layers_.reset(new ScreenshareLayers(2, 0, &clock_)); } + void EncodeFrame(uint32_t timestamp, bool base_sync, CodecSpecificInfoVP8* vp8_info, @@ -105,11 +111,12 @@ class ScreenshareLayerTest : public ::testing::Test { int min_qp_; int max_qp_; int frame_size_; + SimulatedClock clock_; std::unique_ptr layers_; }; TEST_F(ScreenshareLayerTest, 1Layer) { - layers_.reset(new ScreenshareLayers(1, 0)); + layers_.reset(new ScreenshareLayers(1, 0, &clock_)); ConfigureBitrates(); int flags = 0; uint32_t timestamp = 0; @@ -135,7 +142,6 @@ TEST_F(ScreenshareLayerTest, 1Layer) { } TEST_F(ScreenshareLayerTest, 2Layer) { - layers_.reset(new ScreenshareLayers(2, 0)); ConfigureBitrates(); int flags = 0; uint32_t timestamp = 0; @@ -185,7 +191,6 @@ TEST_F(ScreenshareLayerTest, 2Layer) { } TEST_F(ScreenshareLayerTest, 2LayersPeriodicSync) { - layers_.reset(new ScreenshareLayers(2, 0)); ConfigureBitrates(); int flags = 0; uint32_t timestamp = 0; @@ -206,7 +211,6 @@ TEST_F(ScreenshareLayerTest, 2LayersPeriodicSync) { } TEST_F(ScreenshareLayerTest, 2LayersSyncAfterTimeout) { - layers_.reset(new ScreenshareLayers(2, 0)); ConfigureBitrates(); uint32_t timestamp = 0; CodecSpecificInfoVP8 vp8_info; @@ -234,7 +238,6 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterTimeout) { } TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) { - layers_.reset(new ScreenshareLayers(2, 0)); ConfigureBitrates(); uint32_t timestamp = 0; CodecSpecificInfoVP8 vp8_info; @@ -284,7 +287,6 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) { } TEST_F(ScreenshareLayerTest, 2LayersToggling) { - layers_.reset(new ScreenshareLayers(2, 0)); ConfigureBitrates(); int flags = 0; CodecSpecificInfoVP8 vp8_info; @@ -312,7 +314,6 @@ TEST_F(ScreenshareLayerTest, 2LayersToggling) { } TEST_F(ScreenshareLayerTest, AllFitsLayer0) { - layers_.reset(new ScreenshareLayers(2, 0)); ConfigureBitrates(); frame_size_ = ((kDefaultTl0BitrateKbps * 1000) / 8) / kFrameRate; @@ -329,7 +330,6 @@ TEST_F(ScreenshareLayerTest, AllFitsLayer0) { } TEST_F(ScreenshareLayerTest, TooHighBitrate) { - layers_.reset(new ScreenshareLayers(2, 0)); ConfigureBitrates(); frame_size_ = 2 * ((kDefaultTl1BitrateKbps * 1000) / 8) / kFrameRate; int flags = 0; @@ -365,8 +365,6 @@ TEST_F(ScreenshareLayerTest, TooHighBitrate) { } TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL0) { - layers_.reset(new ScreenshareLayers(2, 0)); - vpx_codec_enc_cfg_t cfg; layers_->ConfigureBitrates(100, 1000, 5, &cfg); @@ -376,7 +374,6 @@ TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL0) { } TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL1) { - layers_.reset(new ScreenshareLayers(2, 0)); vpx_codec_enc_cfg_t cfg; layers_->ConfigureBitrates(100, 450, 5, &cfg); @@ -386,7 +383,6 @@ TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL1) { } TEST_F(ScreenshareLayerTest, TargetBitrateBelowTL0) { - layers_.reset(new ScreenshareLayers(2, 0)); vpx_codec_enc_cfg_t cfg; layers_->ConfigureBitrates(100, 100, 5, &cfg); @@ -394,7 +390,6 @@ TEST_F(ScreenshareLayerTest, TargetBitrateBelowTL0) { } TEST_F(ScreenshareLayerTest, EncoderDrop) { - layers_.reset(new ScreenshareLayers(2, 0)); ConfigureBitrates(); CodecSpecificInfoVP8 vp8_info; vpx_codec_enc_cfg_t cfg; @@ -446,4 +441,89 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) { layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); } +TEST_F(ScreenshareLayerTest, UpdatesHistograms) { + ConfigureBitrates(); + vpx_codec_enc_cfg_t cfg; + cfg.rc_max_quantizer = kDefaultQp; + bool trigger_drop = false; + bool dropped_frame = false; + bool overshoot = false; + const int kTl0Qp = 35; + const int kTl1Qp = 30; + for (int64_t timestamp = 0; + timestamp < kTimestampDelta5Fps * 5 * metrics::kMinRunTimeInSeconds; + timestamp += kTimestampDelta5Fps) { + int flags = layers_->EncodeFlags(timestamp); + if (flags != -1) + layers_->UpdateConfiguration(&cfg); + + if (timestamp >= kTimestampDelta5Fps * 5 && !overshoot && flags != -1) { + // Simulate one overshoot. + layers_->FrameEncoded(0, timestamp, 0); + overshoot = true; + flags = layers_->EncodeFlags(timestamp); + } + + if (flags == ScreenshareLayers::kTl0Flags) { + if (timestamp >= kTimestampDelta5Fps * 20 && !trigger_drop) { + // Simulate a too large frame, to cause frame drop. + layers_->FrameEncoded(frame_size_ * 5, timestamp, kTl0Qp); + trigger_drop = true; + } else { + layers_->FrameEncoded(frame_size_, timestamp, kTl0Qp); + } + } else if (flags == ScreenshareLayers::kTl1Flags || + flags == ScreenshareLayers::kTl1SyncFlags) { + layers_->FrameEncoded(frame_size_, timestamp, kTl1Qp); + } else if (flags == -1) { + dropped_frame = true; + } else { + RTC_NOTREACHED() << "Unexpected flags"; + } + clock_.AdvanceTimeMilliseconds(1000 / 5); + } + + EXPECT_TRUE(overshoot); + EXPECT_TRUE(dropped_frame); + + layers_.reset(); // Histograms are reported on destruction. + + EXPECT_EQ(1, test::NumHistogramSamples( + "WebRTC.Video.Screenshare.Layer0.FrameRate")); + EXPECT_EQ(1, test::NumHistogramSamples( + "WebRTC.Video.Screenshare.Layer1.FrameRate")); + EXPECT_EQ( + 1, test::NumHistogramSamples("WebRTC.Video.Screenshare.FramesPerDrop")); + EXPECT_EQ(1, test::NumHistogramSamples( + "WebRTC.Video.Screenshare.FramesPerOvershoot")); + EXPECT_EQ(1, test::NumHistogramSamples("WebRTC.Video.Screenshare.Layer0.Qp")); + EXPECT_EQ(1, test::NumHistogramSamples("WebRTC.Video.Screenshare.Layer1.Qp")); + EXPECT_EQ(1, test::NumHistogramSamples( + "WebRTC.Video.Screenshare.Layer0.TargetBitrate")); + EXPECT_EQ(1, test::NumHistogramSamples( + "WebRTC.Video.Screenshare.Layer1.TargetBitrate")); + + EXPECT_GT( + test::LastHistogramSample("WebRTC.Video.Screenshare.Layer0.FrameRate"), + 1); + EXPECT_GT( + test::LastHistogramSample("WebRTC.Video.Screenshare.Layer1.FrameRate"), + 1); + EXPECT_GT(test::LastHistogramSample("WebRTC.Video.Screenshare.FramesPerDrop"), + 1); + EXPECT_GT( + test::LastHistogramSample("WebRTC.Video.Screenshare.FramesPerOvershoot"), + 1); + EXPECT_EQ(kTl0Qp, + test::LastHistogramSample("WebRTC.Video.Screenshare.Layer0.Qp")); + EXPECT_EQ(kTl1Qp, + test::LastHistogramSample("WebRTC.Video.Screenshare.Layer1.Qp")); + EXPECT_EQ(kDefaultTl0BitrateKbps, + test::LastHistogramSample( + "WebRTC.Video.Screenshare.Layer0.TargetBitrate")); + EXPECT_EQ(kDefaultTl1BitrateKbps, + test::LastHistogramSample( + "WebRTC.Video.Screenshare.Layer1.TargetBitrate")); +} + } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc index 0a17e743f7..d4f4b80a10 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc @@ -17,6 +17,7 @@ #include "webrtc/base/checks.h" #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h" +#include "webrtc/system_wrappers/include/clock.h" namespace { @@ -102,7 +103,8 @@ struct ScreenshareTemporalLayersFactory : webrtc::TemporalLayersFactory { virtual webrtc::TemporalLayers* Create(int num_temporal_layers, uint8_t initial_tl0_pic_idx) const { - return new webrtc::ScreenshareLayers(num_temporal_layers, rand()); + return new webrtc::ScreenshareLayers(num_temporal_layers, rand(), + webrtc::Clock::GetRealTimeClock()); } mutable webrtc::FrameDropper tl0_frame_dropper_; diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc index eac0afd983..31e1d80636 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc @@ -28,6 +28,7 @@ #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h" #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h" #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" +#include "webrtc/system_wrappers/include/clock.h" #include "webrtc/system_wrappers/include/tick_util.h" namespace webrtc { @@ -328,8 +329,8 @@ void VP8EncoderImpl::SetupTemporalLayers(int num_streams, if (num_streams == 1) { if (codec.mode == kScreensharing) { // Special mode when screensharing on a single stream. - temporal_layers_.push_back( - new ScreenshareLayers(num_temporal_layers, rand())); + temporal_layers_.push_back(new ScreenshareLayers( + num_temporal_layers, rand(), webrtc::Clock::GetRealTimeClock())); } else { temporal_layers_.push_back( tl_factory->Create(num_temporal_layers, rand()));