From d19513f3ffbb939fd56b5377b678bb31d3154e14 Mon Sep 17 00:00:00 2001 From: Artem Titov Date: Wed, 25 Mar 2020 11:53:41 +0100 Subject: [PATCH] Move calculation of target_encode_bitrate to DefaultVideoQualityAnalyzer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To migrate on new GetStats API and properly support target encode bitrate for regular, simulcast and svc cases we need to calculate it inside video quality analyzer getting values from SetRates in VideoEncoder. Bug: webrtc:11381 Change-Id: Ia37acac764ed3c30f64cdbfda8906d543fa03ae2 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/171501 Commit-Queue: Artem Titov Reviewed-by: Ilya Nikolaevskiy Reviewed-by: Patrik Höglund Cr-Commit-Position: refs/heads/master@{#30881} --- api/test/video_quality_analyzer_interface.h | 24 ++++++++++++++----- .../video/default_video_quality_analyzer.cc | 12 +++++++--- .../video/default_video_quality_analyzer.h | 9 ++++--- .../default_video_quality_analyzer_test.cc | 21 +++++++++------- .../video/example_video_quality_analyzer.cc | 6 ++--- .../video/example_video_quality_analyzer.h | 6 ++--- .../video/quality_analyzing_video_decoder.cc | 4 +++- .../video/quality_analyzing_video_encoder.cc | 20 +++++++++++++++- .../video/quality_analyzing_video_encoder.h | 1 + .../video/video_quality_metrics_reporter.cc | 9 ------- .../video/video_quality_metrics_reporter.h | 1 - 11 files changed, 74 insertions(+), 39 deletions(-) diff --git a/api/test/video_quality_analyzer_interface.h b/api/test/video_quality_analyzer_interface.h index 990548af9e..0d3f441534 100644 --- a/api/test/video_quality_analyzer_interface.h +++ b/api/test/video_quality_analyzer_interface.h @@ -53,6 +53,20 @@ namespace webrtc_pc_e2e { // The analyzer will be injected in all points from A to F. class VideoQualityAnalyzerInterface : public StatsObserverInterface { public: + // Contains extra statistic provided by video encoder. + struct EncoderStats { + // TODO(hbos) https://crbug.com/webrtc/9547, + // https://crbug.com/webrtc/11443: improve stats API to make available + // there. + uint32_t target_encode_bitrate; + }; + // Contains extra statistic provided by video decoder. + struct DecoderStats { + // Decode time provided by decoder itself. If decoder doesn’t produce such + // information can be omitted. + absl::optional decode_time_ms; + }; + ~VideoQualityAnalyzerInterface() override = default; // Will be called by framework before test. @@ -74,18 +88,16 @@ class VideoQualityAnalyzerInterface : public StatsObserverInterface { // VideoFrame can produce multiple EncodedImages. Each encoded image will // have id from VideoFrame. virtual void OnFrameEncoded(uint16_t frame_id, - const EncodedImage& encoded_image) {} + const EncodedImage& encoded_image, + const EncoderStats& stats) {} // Will be called for each frame dropped by encoder. virtual void OnFrameDropped(EncodedImageCallback::DropReason reason) {} // Will be called before calling the decoder. virtual void OnFramePreDecode(uint16_t frame_id, const EncodedImage& encoded_image) {} - // Will be called after decoding the frame. |decode_time_ms| is a decode - // time provided by decoder itself. If decoder doesn’t produce such - // information can be omitted. + // Will be called after decoding the frame. virtual void OnFrameDecoded(const VideoFrame& frame, - absl::optional decode_time_ms, - absl::optional qp) {} + const DecoderStats& stats) {} // Will be called when frame will be obtained from PeerConnection stack. virtual void OnFrameRendered(const VideoFrame& frame) {} // Will be called if encoder return not WEBRTC_VIDEO_CODEC_OK. diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc index 81a0448c3b..40a885fc0e 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -26,6 +26,7 @@ namespace { constexpr int kMaxActiveComparisons = 10; constexpr int kFreezeThresholdMs = 150; constexpr int kMicrosPerSecond = 1000000; +constexpr int kBitsInByte = 8; void LogFrameCounters(const std::string& name, const FrameCounters& counters) { RTC_LOG(INFO) << "[" << name << "] Captured : " << counters.captured; @@ -180,7 +181,8 @@ void DefaultVideoQualityAnalyzer::OnFramePreEncode( void DefaultVideoQualityAnalyzer::OnFrameEncoded( uint16_t frame_id, - const webrtc::EncodedImage& encoded_image) { + const webrtc::EncodedImage& encoded_image, + const EncoderStats& stats) { rtc::CritScope crit(&lock_); auto it = frame_stats_.find(frame_id); RTC_DCHECK(it != frame_stats_.end()); @@ -193,6 +195,7 @@ void DefaultVideoQualityAnalyzer::OnFrameEncoded( } it->second.encoded_time = Now(); it->second.encoded_image_size = encoded_image.size(); + it->second.target_encode_bitrate = stats.target_encode_bitrate; } void DefaultVideoQualityAnalyzer::OnFrameDropped( @@ -226,8 +229,7 @@ void DefaultVideoQualityAnalyzer::OnFramePreDecode( void DefaultVideoQualityAnalyzer::OnFrameDecoded( const webrtc::VideoFrame& frame, - absl::optional decode_time_ms, - absl::optional qp) { + const DecoderStats& stats) { rtc::CritScope crit(&lock_); auto it = frame_stats_.find(frame.id()); RTC_DCHECK(it != frame_stats_.end()); @@ -517,6 +519,7 @@ void DefaultVideoQualityAnalyzer::ProcessComparison( (frame_stats.encoded_time - frame_stats.pre_encode_time).ms()); stats->encode_frame_rate.AddEvent(frame_stats.encoded_time); stats->total_encoded_images_payload += frame_stats.encoded_image_size; + stats->target_encode_bitrate.AddSample(frame_stats.target_encode_bitrate); } else { if (frame_stats.pre_encode_time.IsFinite()) { stats->dropped_by_encoder++; @@ -670,6 +673,9 @@ void DefaultVideoQualityAnalyzer::ReportResults( /*important=*/false, ImproveDirection::kSmallerIsBetter); ReportResult("max_skipped", test_case_name, stats.skipped_between_rendered, "count", ImproveDirection::kSmallerIsBetter); + ReportResult("target_encode_bitrate", test_case_name, + stats.target_encode_bitrate / kBitsInByte, "bytesPerSecond", + ImproveDirection::kNone); test::PrintResult( "actual_encode_bitrate", "", test_case_name, static_cast(stats.total_encoded_images_payload) / diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h index 31e42675d1..778ccb3092 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h @@ -101,6 +101,7 @@ struct StreamStats { // Mean time between one freeze end and next freeze start. SamplesStatsCounter time_between_freezes_ms; SamplesStatsCounter resolution_of_rendered_frame; + SamplesStatsCounter target_encode_bitrate; int64_t total_encoded_images_payload = 0; int64_t dropped_by_encoder = 0; @@ -138,13 +139,13 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { const VideoFrame& frame) override; void OnFramePreEncode(const VideoFrame& frame) override; void OnFrameEncoded(uint16_t frame_id, - const EncodedImage& encoded_image) override; + const EncodedImage& encoded_image, + const EncoderStats& stats) override; void OnFrameDropped(EncodedImageCallback::DropReason reason) override; void OnFramePreDecode(uint16_t frame_id, const EncodedImage& input_image) override; void OnFrameDecoded(const VideoFrame& frame, - absl::optional decode_time_ms, - absl::optional qp) override; + const DecoderStats& stats) override; void OnFrameRendered(const VideoFrame& frame) override; void OnEncoderError(const VideoFrame& frame, int32_t error_code) override; void OnDecoderError(uint16_t frame_id, int32_t error_code) override; @@ -181,6 +182,8 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { Timestamp rendered_time = Timestamp::MinusInfinity(); Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity(); + uint32_t target_encode_bitrate = 0; + absl::optional rendered_frame_width = absl::nullopt; absl::optional rendered_frame_height = absl::nullopt; diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc index 1a59015e10..1bc29c5f09 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc @@ -83,14 +83,15 @@ TEST(DefaultVideoQualityAnalyzerTest, frames_order.push_back(frame.id()); captured_frames.insert({frame.id(), frame}); analyzer.OnFramePreEncode(frame); - analyzer.OnFrameEncoded(frame.id(), FakeEncode(frame)); + analyzer.OnFrameEncoded(frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); } for (const uint16_t& frame_id : frames_order) { VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); analyzer.OnFramePreDecode(received_frame.id(), FakeEncode(received_frame)); - analyzer.OnFrameDecoded(received_frame, /*decode_time_ms=*/absl::nullopt, - /*qp=*/absl::nullopt); + analyzer.OnFrameDecoded(received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); analyzer.OnFrameRendered(received_frame); } @@ -129,15 +130,16 @@ TEST(DefaultVideoQualityAnalyzerTest, frames_order.push_back(frame.id()); captured_frames.insert({frame.id(), frame}); analyzer.OnFramePreEncode(frame); - analyzer.OnFrameEncoded(frame.id(), FakeEncode(frame)); + analyzer.OnFrameEncoded(frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); } for (size_t i = kMaxFramesInFlightPerStream; i < frames_order.size(); ++i) { uint16_t frame_id = frames_order.at(i); VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); analyzer.OnFramePreDecode(received_frame.id(), FakeEncode(received_frame)); - analyzer.OnFrameDecoded(received_frame, /*decode_time_ms=*/absl::nullopt, - /*qp=*/absl::nullopt); + analyzer.OnFrameDecoded(received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); analyzer.OnFrameRendered(received_frame); } @@ -174,15 +176,16 @@ TEST(DefaultVideoQualityAnalyzerTest, NormalScenario) { frames_order.push_back(frame.id()); captured_frames.insert({frame.id(), frame}); analyzer.OnFramePreEncode(frame); - analyzer.OnFrameEncoded(frame.id(), FakeEncode(frame)); + analyzer.OnFrameEncoded(frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); } for (size_t i = 1; i < frames_order.size(); i += 2) { uint16_t frame_id = frames_order.at(i); VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); analyzer.OnFramePreDecode(received_frame.id(), FakeEncode(received_frame)); - analyzer.OnFrameDecoded(received_frame, /*decode_time_ms=*/absl::nullopt, - /*qp=*/absl::nullopt); + analyzer.OnFrameDecoded(received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); analyzer.OnFrameRendered(received_frame); } diff --git a/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc index caa639a229..d1d1bface7 100644 --- a/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc @@ -52,7 +52,8 @@ void ExampleVideoQualityAnalyzer::OnFramePreEncode( void ExampleVideoQualityAnalyzer::OnFrameEncoded( uint16_t frame_id, - const webrtc::EncodedImage& encoded_image) { + const webrtc::EncodedImage& encoded_image, + const EncoderStats& stats) { rtc::CritScope crit(&lock_); ++frames_encoded_; } @@ -73,8 +74,7 @@ void ExampleVideoQualityAnalyzer::OnFramePreDecode( void ExampleVideoQualityAnalyzer::OnFrameDecoded( const webrtc::VideoFrame& frame, - absl::optional decode_time_ms, - absl::optional qp) { + const DecoderStats& stats) { rtc::CritScope crit(&lock_); ++frames_decoded_; } diff --git a/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h index 8b29e1223b..0d6169f9fa 100644 --- a/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h +++ b/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h @@ -38,13 +38,13 @@ class ExampleVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { const VideoFrame& frame) override; void OnFramePreEncode(const VideoFrame& frame) override; void OnFrameEncoded(uint16_t frame_id, - const EncodedImage& encoded_image) override; + const EncodedImage& encoded_image, + const EncoderStats& stats) override; void OnFrameDropped(EncodedImageCallback::DropReason reason) override; void OnFramePreDecode(uint16_t frame_id, const EncodedImage& encoded_image) override; void OnFrameDecoded(const VideoFrame& frame, - absl::optional decode_time_ms, - absl::optional qp) override; + const DecoderStats& stats) override; void OnFrameRendered(const VideoFrame& frame) override; void OnEncoderError(const VideoFrame& frame, int32_t error_code) override; void OnDecoderError(uint16_t frame_id, int32_t error_code) override; diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc index d4d961153a..228ab8ac02 100644 --- a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc @@ -222,7 +222,9 @@ void QualityAnalyzingVideoDecoder::OnFrameDecoded( // Set frame id to the value, that was extracted from corresponding encoded // image. frame->set_id(frame_id); - analyzer_->OnFrameDecoded(*frame, decode_time_ms, qp); + VideoQualityAnalyzerInterface::DecoderStats stats; + stats.decode_time_ms = decode_time_ms; + analyzer_->OnFrameDecoded(*frame, stats); } QualityAnalyzingVideoDecoderFactory::QualityAnalyzingVideoDecoderFactory( diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc index 96b23b408e..6ab2938f12 100644 --- a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc @@ -161,6 +161,10 @@ void QualityAnalyzingVideoEncoder::SetRates( const VideoEncoder::RateControlParameters& parameters) { RTC_DCHECK_GT(bitrate_multiplier_, 0.0); if (fabs(bitrate_multiplier_ - kNoMultiplier) < kEps) { + { + rtc::CritScope crit(&lock_); + bitrate_allocation_ = parameters.bitrate; + } return delegate_->SetRates(parameters); } @@ -200,6 +204,10 @@ void QualityAnalyzingVideoEncoder::SetRates( RateControlParameters adjusted_params = parameters; adjusted_params.bitrate = multiplied_allocation; + { + rtc::CritScope crit(&lock_); + bitrate_allocation_ = adjusted_params.bitrate; + } return delegate_->SetRates(adjusted_params); } @@ -226,6 +234,7 @@ EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage( const RTPFragmentationHeader* fragmentation) { uint16_t frame_id; bool discard = false; + uint32_t target_encode_bitrate = 0; { rtc::CritScope crit(&lock_); std::pair timestamp_frame_id; @@ -257,11 +266,20 @@ EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage( frame_id = timestamp_frame_id.second; discard = ShouldDiscard(frame_id, encoded_image); + if (!discard) { + std::string stream_label = analyzer_->GetStreamLabel(frame_id); + absl::optional required_spatial_index = + stream_required_spatial_index_[stream_label]; + target_encode_bitrate = bitrate_allocation_.GetSpatialLayerSum( + required_spatial_index.value_or(0)); + } } if (!discard) { // Analyzer should see only encoded images, that weren't discarded. - analyzer_->OnFrameEncoded(frame_id, encoded_image); + VideoQualityAnalyzerInterface::EncoderStats stats; + stats.target_encode_bitrate = target_encode_bitrate; + analyzer_->OnFrameEncoded(frame_id, encoded_image, stats); } // Image data injector injects frame id and discard flag into provided diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h index 247be73212..03231be633 100644 --- a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h @@ -150,6 +150,7 @@ class QualityAnalyzingVideoEncoder : public VideoEncoder, EncodedImageCallback* delegate_callback_ RTC_GUARDED_BY(lock_); std::list> timestamp_to_frame_id_list_ RTC_GUARDED_BY(lock_); + VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(lock_); }; // Produces QualityAnalyzingVideoEncoder, which hold decoders, produced by diff --git a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc index b461c6a424..754a0a468f 100644 --- a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc +++ b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc @@ -42,13 +42,9 @@ void VideoQualityMetricsReporter::OnStatsReports( const webrtc::StatsReport::Value* transmission_bitrate = stats_report->FindValue( StatsReport::StatsValueName::kStatsValueNameTransmitBitrate); - const webrtc::StatsReport::Value* target_encode_bitrate = - stats_report->FindValue( - StatsReport::StatsValueName::kStatsValueNameTargetEncBitrate); RTC_CHECK(available_send_bandwidth); RTC_CHECK(retransmission_bitrate); RTC_CHECK(transmission_bitrate); - RTC_CHECK(target_encode_bitrate); rtc::CritScope crit(&video_bwe_stats_lock_); VideoBweStats& video_bwe_stats = video_bwe_stats_[pc_label]; @@ -58,8 +54,6 @@ void VideoQualityMetricsReporter::OnStatsReports( transmission_bitrate->int_val()); video_bwe_stats.retransmission_bitrate.AddSample( retransmission_bitrate->int_val()); - video_bwe_stats.target_encode_bitrate.AddSample( - target_encode_bitrate->int_val()); } } @@ -87,9 +81,6 @@ void VideoQualityMetricsReporter::ReportVideoBweResults( ReportResult("retransmission_bitrate", test_case_name, video_bwe_stats.retransmission_bitrate / kBitsInByte, "bytesPerSecond"); - ReportResult("target_encode_bitrate", test_case_name, - video_bwe_stats.target_encode_bitrate / kBitsInByte, - "bytesPerSecond"); } void VideoQualityMetricsReporter::ReportResult( diff --git a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h index fe2f1696c4..1688a7b6fc 100644 --- a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h +++ b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h @@ -26,7 +26,6 @@ struct VideoBweStats { SamplesStatsCounter available_send_bandwidth; SamplesStatsCounter transmission_bitrate; SamplesStatsCounter retransmission_bitrate; - SamplesStatsCounter target_encode_bitrate; }; class VideoQualityMetricsReporter