admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reland "Report total and squared inter frame delays measured in OnRenderedFrame"

Browse Source 
This is a reland of commit d49d49ad89e67d1a3c63fbc638af445af5648875

Fixed seconds to milliseconds conversion in VideoAnalyzer.

Original change's description:
> Report total and squared inter frame delays measured in OnRenderedFrame
>
> After https://webrtc-review.googlesource.com/c/src/+/160042 we ended up with two sets of metrics representing total and total squared inter frame delays: old is measured in OnDecodedFrame and new in OnRenderedFrame. Reporting of old metrics was unshipped in https://webrtc-review.googlesource.com/c/src/+/278100. The metrics are used for calculation of harmonic frame rate and are desired to be measured as close as possible to rendering. This CL removes calculation of inter frame delay metrics from OnDecodedFrame and reports the metrics calculated in OnRenderedFrame to the stats.
>
> Bug: webrtc:11108, b/261512902
> Change-Id: Ia21b321aab3a1ac0b6136dc0df7d95f2f0fd24c6
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/286842
> Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
> Reviewed-by: Erik Språng <sprang@webrtc.org>
> Reviewed-by: Harald Alvestrand <hta@webrtc.org>
> Reviewed-by: Henrik Boström <hbos@webrtc.org>
> Cr-Commit-Position: refs/heads/main@{#38909}

Bug: webrtc:11108, webrtc:14792, b/261512902
Change-Id: Ic5d0bc4622ee0cb46b6c225cdddccc217200e794
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/288641
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Reviewed-by: Johannes Kron <kron@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38929}
This commit is contained in:
Sergey Silkin 2022-12-20 13:58:15 +01:00 committed by WebRTC LUCI CQ
parent 2ba941e6bc
commit ed0dd8e419
7 changed files with 36 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
call/video_receive_stream.h
View File

@ -132,8 +132,6 @@ class VideoReceiveStreamInterface : public MediaReceiveStreamInterface {
uint32_t pause_count = 0;
uint32_t total_freezes_duration_ms = 0;
uint32_t total_pauses_duration_ms = 0;
uint32_t total_frames_duration_ms = 0;
double sum_squared_frame_durations = 0.0;
VideoContentType content_type = VideoContentType::UNSPECIFIED;

2
media/base/media_channel.h
View File

@ -628,8 +628,6 @@ struct VideoReceiverInfo : public MediaReceiverInfo {
uint32_t pause_count = 0;
uint32_t total_freezes_duration_ms = 0;
uint32_t total_pauses_duration_ms = 0;
uint32_t total_frames_duration_ms = 0;
double sum_squared_frame_durations = 0.0;
uint32_t jitter_ms = 0;
webrtc::VideoContentType content_type = webrtc::VideoContentType::UNSPECIFIED;

6
video/g3doc/stats.md
View File

@ -148,8 +148,6 @@ Updated after a frame has been decoded, `VCMDecodedFrameCallback::Decoded`.
* `qp_sum` - sum of quantizer values of decoded frames [[rtcinboundrtpstreamstats-qpsum]].
* `content_type` - content type (UNSPECIFIED/SCREENSHARE).
* `interframe_delay_max_ms` - max inter-frame delay within a time window between decoded frames.
* `total_inter_frame_delay` - sum of inter-frame delay in seconds between decoded frames [[rtcinboundrtpstreamstats-totalinterframedelay]].
* `total_squared_inter_frame_delay` - sum of squared inter-frame delays in seconds between decoded frames [[rtcinboundrtpstreamstats-totalsquaredinterframedelay]].
Updated before a frame is sent to the renderer, `VideoReceiveStream2::OnFrame`.
* `frames_rendered` - total number of rendered frames.
@ -162,8 +160,8 @@ Updated before a frame is sent to the renderer, `VideoReceiveStream2::OnFrame`.
* `pause_count` - total number of detected pauses.
* `total_freezes_duration_ms` - total duration of freezes in ms.
* `total_pauses_duration_ms` - total duration of pauses in ms.
* `total_frames_duration_ms` - time in ms between the last rendered frame and the first rendered frame.
* `sum_squared_frame_durations` - sum of squared inter-frame delays in seconds between rendered frames.
* `total_inter_frame_delay` - sum of inter-frame delay in seconds between rendered frames [[rtcinboundrtpstreamstats-totalinterframedelay]].
* `total_squared_inter_frame_delay` - sum of squared inter-frame delays in seconds between rendered frames [[rtcinboundrtpstreamstats-totalsquaredinterframedelay]].
`ReceiveStatisticsImpl::OnRtpPacket` is updated for received RTP packets. From `ReceiveStatistics`:
* `total_bitrate_bps` - incoming bitrate in bps.

12
video/receive_statistics_proxy2.cc
View File

@ -624,10 +624,12 @@ VideoReceiveStreamInterface::Stats ReceiveStatisticsProxy::GetStats() const {
video_quality_observer_->TotalFreezesDurationMs();
stats_.total_pauses_duration_ms =
video_quality_observer_->TotalPausesDurationMs();
stats_.total_frames_duration_ms =
video_quality_observer_->TotalFramesDurationMs();
stats_.sum_squared_frame_durations =
stats_.total_inter_frame_delay =
static_cast<double>(video_quality_observer_->TotalFramesDurationMs()) /
rtc::kNumMillisecsPerSec;
stats_.total_squared_inter_frame_delay =
video_quality_observer_->SumSquaredFrameDurationsSec();
stats_.content_type = last_content_type_;
stats_.timing_frame_info = timing_frame_info_counter_.Max(now_ms);
stats_.jitter_buffer_delay_seconds =
@ -839,10 +841,6 @@ void ReceiveStatisticsProxy::OnDecodedFrame(
int64_t interframe_delay_ms =
frame_meta.decode_timestamp.ms() - *last_decoded_frame_time_ms_;
RTC_DCHECK_GE(interframe_delay_ms, 0);
double interframe_delay = interframe_delay_ms / 1000.0;
stats_.total_inter_frame_delay += interframe_delay;
stats_.total_squared_inter_frame_delay +=
interframe_delay * interframe_delay;
interframe_delay_max_moving_.Add(interframe_delay_ms,
frame_meta.decode_timestamp.ms());
content_specific_stats->interframe_delay_counter.Add(interframe_delay_ms);

72
video/receive_statistics_proxy2_unittest.cc
View File

@ -328,62 +328,6 @@ TEST_F(ReceiveStatisticsProxy2Test, ReportsContentType) {
videocontenttypehelpers::ToString(FlushAndGetStats().content_type));
}
TEST_F(ReceiveStatisticsProxy2Test, ReportsMaxTotalInterFrameDelay) {
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
const TimeDelta kInterFrameDelay1 = TimeDelta::Millis(100);
const TimeDelta kInterFrameDelay2 = TimeDelta::Millis(200);
const TimeDelta kInterFrameDelay3 = TimeDelta::Millis(300);
double expected_total_inter_frame_delay = 0;
double expected_total_squared_inter_frame_delay = 0;
EXPECT_EQ(expected_total_inter_frame_delay,
statistics_proxy_->GetStats().total_inter_frame_delay);
EXPECT_EQ(expected_total_squared_inter_frame_delay,
statistics_proxy_->GetStats().total_squared_inter_frame_delay);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED);
EXPECT_DOUBLE_EQ(expected_total_inter_frame_delay,
FlushAndGetStats().total_inter_frame_delay);
EXPECT_DOUBLE_EQ(expected_total_squared_inter_frame_delay,
FlushAndGetStats().total_squared_inter_frame_delay);
time_controller_.AdvanceTime(kInterFrameDelay1);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED);
expected_total_inter_frame_delay += kInterFrameDelay1.seconds<double>();
expected_total_squared_inter_frame_delay +=
pow(kInterFrameDelay1.seconds<double>(), 2.0);
EXPECT_DOUBLE_EQ(expected_total_inter_frame_delay,
FlushAndGetStats().total_inter_frame_delay);
EXPECT_DOUBLE_EQ(
expected_total_squared_inter_frame_delay,
statistics_proxy_->GetStats().total_squared_inter_frame_delay);
time_controller_.AdvanceTime(kInterFrameDelay2);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED);
expected_total_inter_frame_delay += kInterFrameDelay2.seconds<double>();
expected_total_squared_inter_frame_delay +=
pow(kInterFrameDelay2.seconds<double>(), 2.0);
EXPECT_DOUBLE_EQ(expected_total_inter_frame_delay,
FlushAndGetStats().total_inter_frame_delay);
EXPECT_DOUBLE_EQ(
expected_total_squared_inter_frame_delay,
statistics_proxy_->GetStats().total_squared_inter_frame_delay);
time_controller_.AdvanceTime(kInterFrameDelay3);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED);
expected_total_inter_frame_delay += kInterFrameDelay3.seconds<double>();
expected_total_squared_inter_frame_delay +=
pow(kInterFrameDelay3.seconds<double>(), 2.0);
EXPECT_DOUBLE_EQ(expected_total_inter_frame_delay,
FlushAndGetStats().total_inter_frame_delay);
EXPECT_DOUBLE_EQ(
expected_total_squared_inter_frame_delay,
statistics_proxy_->GetStats().total_squared_inter_frame_delay);
}
TEST_F(ReceiveStatisticsProxy2Test, ReportsMaxInterframeDelay) {
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
const TimeDelta kInterframeDelay1 = TimeDelta::Millis(100);
@ -503,9 +447,9 @@ TEST_F(ReceiveStatisticsProxy2Test, PauseBeforeFirstAndAfterLastFrameIgnored) {
EXPECT_EQ(0u, stats.total_pauses_duration_ms);
}
TEST_F(ReceiveStatisticsProxy2Test, ReportsFramesDuration) {
TEST_F(ReceiveStatisticsProxy2Test, ReportsTotalInterFrameDelay) {
VideoReceiveStreamInterface::Stats stats = statistics_proxy_->GetStats();
ASSERT_EQ(0u, stats.total_frames_duration_ms);
ASSERT_EQ(0.0, stats.total_inter_frame_delay);
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
@ -519,12 +463,12 @@ TEST_F(ReceiveStatisticsProxy2Test, ReportsFramesDuration) {
}
stats = statistics_proxy_->GetStats();
EXPECT_EQ(10 * 30u, stats.total_frames_duration_ms);
EXPECT_EQ(10 * 30 / 1000.0, stats.total_inter_frame_delay);
}
TEST_F(ReceiveStatisticsProxy2Test, ReportsSumSquaredFrameDurations) {
TEST_F(ReceiveStatisticsProxy2Test, ReportsTotalSquaredInterFrameDelay) {
VideoReceiveStreamInterface::Stats stats = statistics_proxy_->GetStats();
ASSERT_EQ(0u, stats.sum_squared_frame_durations);
ASSERT_EQ(0.0, stats.total_squared_inter_frame_delay);
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
for (int i = 0; i <= 10; ++i) {
@ -533,10 +477,10 @@ TEST_F(ReceiveStatisticsProxy2Test, ReportsSumSquaredFrameDurations) {
}
stats = statistics_proxy_->GetStats();
const double kExpectedSumSquaredFrameDurationsSecs =
const double kExpectedTotalSquaredInterFrameDelaySecs =
10 * (30 / 1000.0 * 30 / 1000.0);
EXPECT_EQ(kExpectedSumSquaredFrameDurationsSecs,
stats.sum_squared_frame_durations);
EXPECT_EQ(kExpectedTotalSquaredInterFrameDelaySecs,
stats.total_squared_inter_frame_delay);
}
TEST_F(ReceiveStatisticsProxy2Test, OnDecodedFrameWithoutQpQpSumWontExist) {

37
video/video_analyzer.cc
View File

@ -99,8 +99,8 @@ VideoAnalyzer::VideoAnalyzer(test::LayerFilteringTransport* transport,
mean_decode_time_ms_(0.0),
freeze_count_(0),
total_freezes_duration_ms_(0),
total_frames_duration_ms_(0),
sum_squared_frame_durations_(0),
total_inter_frame_delay_(0),
total_squared_inter_frame_delay_(0),
decode_frame_rate_(0),
render_frame_rate_(0),
last_fec_bytes_(0),
@ -502,6 +502,14 @@ void VideoAnalyzer::PollStats() {
if (receive_stream_ != nullptr) {
VideoReceiveStreamInterface::Stats receive_stats =
receive_stream_->GetStats();
// Freeze metrics.
freeze_count_ = receive_stats.freeze_count;
total_freezes_duration_ms_ = receive_stats.total_freezes_duration_ms;
total_inter_frame_delay_ = receive_stats.total_inter_frame_delay;
total_squared_inter_frame_delay_ =
receive_stats.total_squared_inter_frame_delay;
// `total_decode_time_ms` gives a good estimate of the mean decode time,
// `decode_ms` is used to keep track of the standard deviation.
if (receive_stats.frames_decoded > 0)
@ -518,20 +526,12 @@ void VideoAnalyzer::PollStats() {
// `frames_decoded` and `frames_rendered` are used because they are more
// accurate than `decode_frame_rate` and `render_frame_rate`.
// The latter two are calculated on a momentary basis.
const double total_frames_duration_sec_double =
static_cast<double>(receive_stats.total_frames_duration_ms) / 1000.0;
if (total_frames_duration_sec_double > 0) {
decode_frame_rate_ = static_cast<double>(receive_stats.frames_decoded) /
total_frames_duration_sec_double;
render_frame_rate_ = static_cast<double>(receive_stats.frames_rendered) /
total_frames_duration_sec_double;
if (total_inter_frame_delay_ > 0) {
decode_frame_rate_ =
receive_stats.frames_decoded / total_inter_frame_delay_;
render_frame_rate_ =
receive_stats.frames_rendered / total_inter_frame_delay_;
}
// Freeze metrics.
freeze_count_ = receive_stats.freeze_count;
total_freezes_duration_ms_ = receive_stats.total_freezes_duration_ms;
total_frames_duration_ms_ = receive_stats.total_frames_duration_ms;
sum_squared_frame_durations_ = receive_stats.sum_squared_frame_durations;
}
if (audio_receive_stream_ != nullptr) {
@ -673,7 +673,7 @@ void VideoAnalyzer::PrintResults() {
const double total_freezes_duration_ms_double =
static_cast<double>(total_freezes_duration_ms_);
const double total_frames_duration_ms_double =
static_cast<double>(total_frames_duration_ms_);
total_inter_frame_delay_ * rtc::kNumMillisecsPerSec;
if (total_frames_duration_ms_double > 0) {
GetGlobalMetricsLogger()->LogSingleValueMetric(
@ -701,10 +701,11 @@ void VideoAnalyzer::PrintResults() {
: 0,
Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter);
if (1000 * sum_squared_frame_durations_ > 0) {
if (total_squared_inter_frame_delay_ > 0) {
GetGlobalMetricsLogger()->LogSingleValueMetric(
"harmonic_frame_rate_fps", test_label_,
total_frames_duration_ms_double / (1000 * sum_squared_frame_durations_),
total_frames_duration_ms_double /
(1000 * total_squared_inter_frame_delay_),
Unit::kHertz, ImprovementDirection::kBiggerIsBetter);
}

4
video/video_analyzer.h
View File

@ -263,8 +263,8 @@ class VideoAnalyzer : public PacketReceiver,
SamplesStatsCounter time_between_freezes_ RTC_GUARDED_BY(comparison_lock_);
uint32_t freeze_count_ RTC_GUARDED_BY(comparison_lock_);
uint32_t total_freezes_duration_ms_ RTC_GUARDED_BY(comparison_lock_);
uint32_t total_frames_duration_ms_ RTC_GUARDED_BY(comparison_lock_);
double sum_squared_frame_durations_ RTC_GUARDED_BY(comparison_lock_);
double total_inter_frame_delay_ RTC_GUARDED_BY(comparison_lock_);
double total_squared_inter_frame_delay_ RTC_GUARDED_BY(comparison_lock_);
double decode_frame_rate_ RTC_GUARDED_BY(comparison_lock_);
double render_frame_rate_ RTC_GUARDED_BY(comparison_lock_);