From 23aff9b73788628a0273c22d9c8c79cba031a63e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Bostr=C3=B6m?= Date: Mon, 20 May 2019 15:15:38 +0200 Subject: [PATCH] Implement RTCOutboundRtpStreamStats.totalEncodedBytesTarget. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a standardized metric: https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-totalencodedbytestarget We estimate the target frame size in bytes from the current encoder target bitrate and encoder framerate. We would expect that the average bytes produced by the encoder would over time match the average target, which is calculated by polling getStats() twice and dividing the delta totalEncodedBytesTarget with the delta framesEncoded. This is meant to make googTargetEncBitrate obsolete. Bug: webrtc:10446 Change-Id: Ib10ce236476a2f965582d5c536f419952926d4e6 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/137200 Reviewed-by: Stefan Holmer Commit-Queue: Henrik Boström Cr-Commit-Position: refs/heads/master@{#28022} --- api/stats/rtcstats_objects.h | 1 + call/video_send_stream.h | 2 + media/base/media_channel.h | 2 + media/engine/webrtc_video_engine.cc | 1 + pc/rtc_stats_collector.cc | 2 + pc/rtc_stats_collector_unittest.cc | 2 + pc/rtc_stats_integrationtest.cc | 4 ++ stats/rtcstats_objects.cc | 3 ++ video/BUILD.gn | 1 + video/send_statistics_proxy.cc | 12 ++++++ video/send_statistics_proxy_unittest.cc | 52 +++++++++++++++++++++++++ 11 files changed, 82 insertions(+) diff --git a/api/stats/rtcstats_objects.h b/api/stats/rtcstats_objects.h index 8f8eb203d5..2e6cd3b9bb 100644 --- a/api/stats/rtcstats_objects.h +++ b/api/stats/rtcstats_objects.h @@ -458,6 +458,7 @@ class RTC_EXPORT RTCOutboundRTPStreamStats final : public RTCRTPStreamStats { RTCStatsMember target_bitrate; RTCStatsMember frames_encoded; RTCStatsMember total_encode_time; + RTCStatsMember total_encoded_bytes_target; // TODO(https://crbug.com/webrtc/10635): This is only implemented for video; // implement it for audio as well. RTCStatsMember total_packet_send_delay; diff --git a/call/video_send_stream.h b/call/video_send_stream.h index a06024a177..42760bd300 100644 --- a/call/video_send_stream.h +++ b/call/video_send_stream.h @@ -71,6 +71,8 @@ class VideoSendStream { uint32_t frames_encoded = 0; // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-totalencodetime uint64_t total_encode_time_ms = 0; + // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-totalencodedbytestarget + uint64_t total_encoded_bytes_target = 0; uint32_t frames_dropped_by_capturer = 0; uint32_t frames_dropped_by_encoder_queue = 0; uint32_t frames_dropped_by_rate_limiter = 0; diff --git a/media/base/media_channel.h b/media/base/media_channel.h index 04edd9c467..8bdcd2b55e 100644 --- a/media/base/media_channel.h +++ b/media/base/media_channel.h @@ -551,6 +551,8 @@ struct VideoSenderInfo : public MediaSenderInfo { uint32_t frames_encoded = 0; // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-totalencodetime uint64_t total_encode_time_ms = 0; + // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-totalencodedbytestarget + uint64_t total_encoded_bytes_target = 0; uint64_t total_packet_send_delay_ms = 0; bool has_entered_low_resolution = false; absl::optional qp_sum; diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index d79dc3ff98..eecae16dea 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -2261,6 +2261,7 @@ VideoSenderInfo WebRtcVideoChannel::WebRtcVideoSendStream::GetVideoSenderInfo( info.encode_usage_percent = stats.encode_usage_percent; info.frames_encoded = stats.frames_encoded; info.total_encode_time_ms = stats.total_encode_time_ms; + info.total_encoded_bytes_target = stats.total_encoded_bytes_target; info.qp_sum = stats.qp_sum; info.nominal_bitrate = stats.media_bitrate_bps; diff --git a/pc/rtc_stats_collector.cc b/pc/rtc_stats_collector.cc index 06618486cd..3e11a92061 100644 --- a/pc/rtc_stats_collector.cc +++ b/pc/rtc_stats_collector.cc @@ -347,6 +347,8 @@ void SetOutboundRTPStreamStatsFromVideoSenderInfo( outbound_video->total_encode_time = static_cast(video_sender_info.total_encode_time_ms) / rtc::kNumMillisecsPerSec; + outbound_video->total_encoded_bytes_target = + video_sender_info.total_encoded_bytes_target; outbound_video->total_packet_send_delay = static_cast(video_sender_info.total_packet_send_delay_ms) / rtc::kNumMillisecsPerSec; diff --git a/pc/rtc_stats_collector_unittest.cc b/pc/rtc_stats_collector_unittest.cc index 312e309262..78530dfd63 100644 --- a/pc/rtc_stats_collector_unittest.cc +++ b/pc/rtc_stats_collector_unittest.cc @@ -1849,6 +1849,7 @@ TEST_F(RTCStatsCollectorTest, CollectRTCOutboundRTPStreamStats_Video) { video_media_info.senders[0].codec_payload_type = 42; video_media_info.senders[0].frames_encoded = 8; video_media_info.senders[0].total_encode_time_ms = 9000; + video_media_info.senders[0].total_encoded_bytes_target = 1234; video_media_info.senders[0].total_packet_send_delay_ms = 10000; video_media_info.senders[0].qp_sum = absl::nullopt; video_media_info.senders[0].content_type = VideoContentType::UNSPECIFIED; @@ -1891,6 +1892,7 @@ TEST_F(RTCStatsCollectorTest, CollectRTCOutboundRTPStreamStats_Video) { expected_video.retransmitted_bytes_sent = 60; expected_video.frames_encoded = 8; expected_video.total_encode_time = 9.0; + expected_video.total_encoded_bytes_target = 1234; expected_video.total_packet_send_delay = 10.0; // |expected_video.content_type| should be undefined. // |expected_video.qp_sum| should be undefined. diff --git a/pc/rtc_stats_integrationtest.cc b/pc/rtc_stats_integrationtest.cc index ebb2f38090..438b47b160 100644 --- a/pc/rtc_stats_integrationtest.cc +++ b/pc/rtc_stats_integrationtest.cc @@ -793,6 +793,8 @@ class RTCStatsReportVerifier { verifier.TestMemberIsDefined(outbound_stream.frames_encoded); verifier.TestMemberIsNonNegative( outbound_stream.total_encode_time); + verifier.TestMemberIsNonNegative( + outbound_stream.total_encoded_bytes_target); verifier.TestMemberIsNonNegative( outbound_stream.total_packet_send_delay); // The integration test is not set up to test screen share; don't require @@ -801,6 +803,8 @@ class RTCStatsReportVerifier { } else { verifier.TestMemberIsUndefined(outbound_stream.frames_encoded); verifier.TestMemberIsUndefined(outbound_stream.total_encode_time); + verifier.TestMemberIsUndefined( + outbound_stream.total_encoded_bytes_target); // TODO(https://crbug.com/webrtc/10635): Implement for audio as well. verifier.TestMemberIsUndefined(outbound_stream.total_packet_send_delay); verifier.TestMemberIsUndefined(outbound_stream.content_type); diff --git a/stats/rtcstats_objects.cc b/stats/rtcstats_objects.cc index 40a28747c0..e6c96e0092 100644 --- a/stats/rtcstats_objects.cc +++ b/stats/rtcstats_objects.cc @@ -675,6 +675,7 @@ WEBRTC_RTCSTATS_IMPL( &target_bitrate, &frames_encoded, &total_encode_time, + &total_encoded_bytes_target, &total_packet_send_delay, &content_type) // clang-format on @@ -693,6 +694,7 @@ RTCOutboundRTPStreamStats::RTCOutboundRTPStreamStats(std::string&& id, target_bitrate("targetBitrate"), frames_encoded("framesEncoded"), total_encode_time("totalEncodeTime"), + total_encoded_bytes_target("totalEncodedBytesTarget"), total_packet_send_delay("totalPacketSendDelay"), content_type("contentType") {} @@ -706,6 +708,7 @@ RTCOutboundRTPStreamStats::RTCOutboundRTPStreamStats( target_bitrate(other.target_bitrate), frames_encoded(other.frames_encoded), total_encode_time(other.total_encode_time), + total_encoded_bytes_target(other.total_encoded_bytes_target), total_packet_send_delay(other.total_packet_send_delay), content_type(other.content_type) {} diff --git a/video/BUILD.gn b/video/BUILD.gn index 32e096686d..e15b15eeaf 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -548,6 +548,7 @@ if (rtc_include_tests) { "../api/task_queue:default_task_queue_factory", "../api/test/video:function_video_factory", "../api/units:data_rate", + "../api/units:timestamp", "../api/video:builtin_video_bitrate_allocator_factory", "../api/video:encoded_image", "../api/video:video_bitrate_allocation", diff --git a/video/send_statistics_proxy.cc b/video/send_statistics_proxy.cc index c504e828ee..27e6014854 100644 --- a/video/send_statistics_proxy.cc +++ b/video/send_statistics_proxy.cc @@ -896,6 +896,18 @@ void SendStatisticsProxy::OnSendEncodedImage( rtc::CritScope lock(&crit_); ++stats_.frames_encoded; + // The current encode frame rate is based on previously encoded frames. + double encode_frame_rate = encoded_frame_rate_tracker_.ComputeRate(); + // We assume that less than 1 FPS is not a trustworthy estimate - perhaps we + // just started encoding for the first time or after a pause. Assuming frame + // rate is at least 1 FPS is conservative to avoid too large increments. + if (encode_frame_rate < 1.0) + encode_frame_rate = 1.0; + double target_frame_size_bytes = + stats_.target_media_bitrate_bps / (8.0 * encode_frame_rate); + // |stats_.target_media_bitrate_bps| is set in + // SendStatisticsProxy::OnSetEncoderTargetRate. + stats_.total_encoded_bytes_target += round(target_frame_size_bytes); if (codec_info) { UpdateEncoderFallbackStats( codec_info, encoded_image._encodedWidth * encoded_image._encodedHeight, diff --git a/video/send_statistics_proxy_unittest.cc b/video/send_statistics_proxy_unittest.cc index 08c1230a8b..58514e526c 100644 --- a/video/send_statistics_proxy_unittest.cc +++ b/video/send_statistics_proxy_unittest.cc @@ -17,6 +17,8 @@ #include #include "absl/algorithm/container.h" +#include "api/units/timestamp.h" +#include "rtc_base/fake_clock.h" #include "system_wrappers/include/metrics.h" #include "test/field_trial.h" #include "test/gtest.h" @@ -366,6 +368,56 @@ TEST_F(SendStatisticsProxyTest, OnSendEncodedImageWithoutQpQpSumWontExist) { EXPECT_EQ(absl::nullopt, statistics_proxy_->GetStats().qp_sum); } +TEST_F(SendStatisticsProxyTest, TotalEncodedBytesTargetFirstFrame) { + const uint32_t kTargetBytesPerSecond = 100000; + statistics_proxy_->OnSetEncoderTargetRate(kTargetBytesPerSecond * 8); + EXPECT_EQ(0u, statistics_proxy_->GetStats().total_encoded_bytes_target); + + EncodedImage encoded_image; + statistics_proxy_->OnSendEncodedImage(encoded_image, nullptr); + // On the first frame we don't know the frame rate yet, calculation yields + // zero. Our estimate assumes at least 1 FPS, so we expect the frame size to + // increment by a full |kTargetBytesPerSecond|. + EXPECT_EQ(kTargetBytesPerSecond, + statistics_proxy_->GetStats().total_encoded_bytes_target); +} + +TEST_F(SendStatisticsProxyTest, + TotalEncodedBytesTargetIncrementsBasedOnFrameRate) { + const uint32_t kTargetBytesPerSecond = 100000; + const int kInterframeDelayMs = 100; + + // SendStatisticsProxy uses a RateTracker internally. SendStatisticsProxy uses + // |fake_clock_| for testing, but the RateTracker relies on a global clock. + // This test relies on rtc::ScopedFakeClock to synchronize these two clocks. + // TODO(https://crbug.com/webrtc/10640): When the RateTracker uses a Clock + // this test can stop relying on rtc::ScopedFakeClock. + rtc::ScopedFakeClock fake_global_clock; + fake_global_clock.SetTime(Timestamp::ms(fake_clock_.TimeInMilliseconds())); + + statistics_proxy_->OnSetEncoderTargetRate(kTargetBytesPerSecond * 8); + EncodedImage encoded_image; + + // First frame + statistics_proxy_->OnSendEncodedImage(encoded_image, nullptr); + uint64_t first_total_encoded_bytes_target = + statistics_proxy_->GetStats().total_encoded_bytes_target; + // Second frame + fake_clock_.AdvanceTimeMilliseconds(kInterframeDelayMs); + fake_global_clock.SetTime(Timestamp::ms(fake_clock_.TimeInMilliseconds())); + encoded_image.SetTimestamp(encoded_image.Timestamp() + + 90 * kInterframeDelayMs); + statistics_proxy_->OnSendEncodedImage(encoded_image, nullptr); + + auto stats = statistics_proxy_->GetStats(); + // By the time the second frame arrives, one frame has previously arrived + // during a |kInterframeDelayMs| interval. The estimated encode frame rate at + // the second frame's arrival should be 10 FPS. + uint64_t delta_encoded_bytes_target = + stats.total_encoded_bytes_target - first_total_encoded_bytes_target; + EXPECT_EQ(kTargetBytesPerSecond / 10, delta_encoded_bytes_target); +} + TEST_F(SendStatisticsProxyTest, GetCpuAdaptationStats) { SendStatisticsProxy::AdaptationSteps cpu_counts; SendStatisticsProxy::AdaptationSteps quality_counts;