admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Report total and squared inter frame delays measured in OnRenderedFrame

Browse Source 
After https://webrtc-review.googlesource.com/c/src/+/160042 we ended up with two sets of metrics representing total and total squared inter frame delays: old is measured in OnDecodedFrame and new in OnRenderedFrame. Reporting of old metrics was unshipped in https://webrtc-review.googlesource.com/c/src/+/278100. The metrics are used for calculation of harmonic frame rate and are desired to be measured as close as possible to rendering. This CL removes calculation of inter frame delay metrics from OnDecodedFrame and reports the metrics calculated in OnRenderedFrame to the stats.

Bug: webrtc:11108, b/261512902
Change-Id: Ia21b321aab3a1ac0b6136dc0df7d95f2f0fd24c6
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/286842
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38909}
This commit is contained in:
Sergey Silkin 2022-12-07 15:29:09 +01:00 committed by WebRTC LUCI CQ
parent a6c79d4bcf
commit d49d49ad89
7 changed files with 35 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
call/video_receive_stream.h
View File

@ -132,8 +132,6 @@ class VideoReceiveStreamInterface : public MediaReceiveStreamInterface {
uint32_t pause_count = 0;
uint32_t total_freezes_duration_ms = 0;
uint32_t total_pauses_duration_ms = 0;
uint32_t total_frames_duration_ms = 0;
double sum_squared_frame_durations = 0.0;
VideoContentType content_type = VideoContentType::UNSPECIFIED;

2
media/base/media_channel.h
View File

@ -628,8 +628,6 @@ struct VideoReceiverInfo : public MediaReceiverInfo {
uint32_t pause_count = 0;
uint32_t total_freezes_duration_ms = 0;
uint32_t total_pauses_duration_ms = 0;
uint32_t total_frames_duration_ms = 0;
double sum_squared_frame_durations = 0.0;
uint32_t jitter_ms = 0;
webrtc::VideoContentType content_type = webrtc::VideoContentType::UNSPECIFIED;

6
video/g3doc/stats.md
View File

@ -148,8 +148,6 @@ Updated after a frame has been decoded, `VCMDecodedFrameCallback::Decoded`.
* `qp_sum` - sum of quantizer values of decoded frames [[rtcinboundrtpstreamstats-qpsum]].
* `content_type` - content type (UNSPECIFIED/SCREENSHARE).
* `interframe_delay_max_ms` - max inter-frame delay within a time window between decoded frames.
* `total_inter_frame_delay` - sum of inter-frame delay in seconds between decoded frames [[rtcinboundrtpstreamstats-totalinterframedelay]].
* `total_squared_inter_frame_delay` - sum of squared inter-frame delays in seconds between decoded frames [[rtcinboundrtpstreamstats-totalsquaredinterframedelay]].
Updated before a frame is sent to the renderer, `VideoReceiveStream2::OnFrame`.
* `frames_rendered` - total number of rendered frames.
@ -162,8 +160,8 @@ Updated before a frame is sent to the renderer, `VideoReceiveStream2::OnFrame`.
* `pause_count` - total number of detected pauses.
* `total_freezes_duration_ms` - total duration of freezes in ms.
* `total_pauses_duration_ms` - total duration of pauses in ms.
* `total_frames_duration_ms` - time in ms between the last rendered frame and the first rendered frame.
* `sum_squared_frame_durations` - sum of squared inter-frame delays in seconds between rendered frames.
* `total_inter_frame_delay` - sum of inter-frame delay in seconds between rendered frames [[rtcinboundrtpstreamstats-totalinterframedelay]].
* `total_squared_inter_frame_delay` - sum of squared inter-frame delays in seconds between rendered frames [[rtcinboundrtpstreamstats-totalsquaredinterframedelay]].
`ReceiveStatisticsImpl::OnRtpPacket` is updated for received RTP packets. From `ReceiveStatistics`:
* `total_bitrate_bps` - incoming bitrate in bps.

11
video/receive_statistics_proxy2.cc
View File

@ -624,10 +624,11 @@ VideoReceiveStreamInterface::Stats ReceiveStatisticsProxy::GetStats() const {
video_quality_observer_->TotalFreezesDurationMs();
stats_.total_pauses_duration_ms =
video_quality_observer_->TotalPausesDurationMs();
stats_.total_frames_duration_ms =
video_quality_observer_->TotalFramesDurationMs();
stats_.sum_squared_frame_durations =
stats_.total_inter_frame_delay =
video_quality_observer_->TotalFramesDurationMs() / 1000.0;
stats_.total_squared_inter_frame_delay =
video_quality_observer_->SumSquaredFrameDurationsSec();
stats_.content_type = last_content_type_;
stats_.timing_frame_info = timing_frame_info_counter_.Max(now_ms);
stats_.jitter_buffer_delay_seconds =
@ -839,10 +840,6 @@ void ReceiveStatisticsProxy::OnDecodedFrame(
int64_t interframe_delay_ms =
frame_meta.decode_timestamp.ms() - *last_decoded_frame_time_ms_;
RTC_DCHECK_GE(interframe_delay_ms, 0);
double interframe_delay = interframe_delay_ms / 1000.0;
stats_.total_inter_frame_delay += interframe_delay;
stats_.total_squared_inter_frame_delay +=
interframe_delay * interframe_delay;
interframe_delay_max_moving_.Add(interframe_delay_ms,
frame_meta.decode_timestamp.ms());
content_specific_stats->interframe_delay_counter.Add(interframe_delay_ms);

72
video/receive_statistics_proxy2_unittest.cc
View File

@ -328,62 +328,6 @@ TEST_F(ReceiveStatisticsProxy2Test, ReportsContentType) {
videocontenttypehelpers::ToString(FlushAndGetStats().content_type));
}
TEST_F(ReceiveStatisticsProxy2Test, ReportsMaxTotalInterFrameDelay) {
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
const TimeDelta kInterFrameDelay1 = TimeDelta::Millis(100);
const TimeDelta kInterFrameDelay2 = TimeDelta::Millis(200);
const TimeDelta kInterFrameDelay3 = TimeDelta::Millis(300);
double expected_total_inter_frame_delay = 0;
double expected_total_squared_inter_frame_delay = 0;
EXPECT_EQ(expected_total_inter_frame_delay,
statistics_proxy_->GetStats().total_inter_frame_delay);
EXPECT_EQ(expected_total_squared_inter_frame_delay,
statistics_proxy_->GetStats().total_squared_inter_frame_delay);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED);
EXPECT_DOUBLE_EQ(expected_total_inter_frame_delay,
FlushAndGetStats().total_inter_frame_delay);
EXPECT_DOUBLE_EQ(expected_total_squared_inter_frame_delay,
FlushAndGetStats().total_squared_inter_frame_delay);
time_controller_.AdvanceTime(kInterFrameDelay1);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED);
expected_total_inter_frame_delay += kInterFrameDelay1.seconds<double>();
expected_total_squared_inter_frame_delay +=
pow(kInterFrameDelay1.seconds<double>(), 2.0);
EXPECT_DOUBLE_EQ(expected_total_inter_frame_delay,
FlushAndGetStats().total_inter_frame_delay);
EXPECT_DOUBLE_EQ(
expected_total_squared_inter_frame_delay,
statistics_proxy_->GetStats().total_squared_inter_frame_delay);
time_controller_.AdvanceTime(kInterFrameDelay2);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED);
expected_total_inter_frame_delay += kInterFrameDelay2.seconds<double>();
expected_total_squared_inter_frame_delay +=
pow(kInterFrameDelay2.seconds<double>(), 2.0);
EXPECT_DOUBLE_EQ(expected_total_inter_frame_delay,
FlushAndGetStats().total_inter_frame_delay);
EXPECT_DOUBLE_EQ(
expected_total_squared_inter_frame_delay,
statistics_proxy_->GetStats().total_squared_inter_frame_delay);
time_controller_.AdvanceTime(kInterFrameDelay3);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED);
expected_total_inter_frame_delay += kInterFrameDelay3.seconds<double>();
expected_total_squared_inter_frame_delay +=
pow(kInterFrameDelay3.seconds<double>(), 2.0);
EXPECT_DOUBLE_EQ(expected_total_inter_frame_delay,
FlushAndGetStats().total_inter_frame_delay);
EXPECT_DOUBLE_EQ(
expected_total_squared_inter_frame_delay,
statistics_proxy_->GetStats().total_squared_inter_frame_delay);
}
TEST_F(ReceiveStatisticsProxy2Test, ReportsMaxInterframeDelay) {
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
const TimeDelta kInterframeDelay1 = TimeDelta::Millis(100);
@ -503,9 +447,9 @@ TEST_F(ReceiveStatisticsProxy2Test, PauseBeforeFirstAndAfterLastFrameIgnored) {
EXPECT_EQ(0u, stats.total_pauses_duration_ms);
}
TEST_F(ReceiveStatisticsProxy2Test, ReportsFramesDuration) {
TEST_F(ReceiveStatisticsProxy2Test, ReportsTotalInterFrameDelay) {
VideoReceiveStreamInterface::Stats stats = statistics_proxy_->GetStats();
ASSERT_EQ(0u, stats.total_frames_duration_ms);
ASSERT_EQ(0.0, stats.total_inter_frame_delay);
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
@ -519,12 +463,12 @@ TEST_F(ReceiveStatisticsProxy2Test, ReportsFramesDuration) {
}
stats = statistics_proxy_->GetStats();
EXPECT_EQ(10 * 30u, stats.total_frames_duration_ms);
EXPECT_EQ(10 * 30 / 1000.0, stats.total_inter_frame_delay);
}
TEST_F(ReceiveStatisticsProxy2Test, ReportsSumSquaredFrameDurations) {
TEST_F(ReceiveStatisticsProxy2Test, ReportsTotalSquaredInterFrameDelay) {
VideoReceiveStreamInterface::Stats stats = statistics_proxy_->GetStats();
ASSERT_EQ(0u, stats.sum_squared_frame_durations);
ASSERT_EQ(0.0, stats.total_squared_inter_frame_delay);
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
for (int i = 0; i <= 10; ++i) {
@ -533,10 +477,10 @@ TEST_F(ReceiveStatisticsProxy2Test, ReportsSumSquaredFrameDurations) {
}
stats = statistics_proxy_->GetStats();
const double kExpectedSumSquaredFrameDurationsSecs =
const double kExpectedTotalSquaredInterFrameDelaySecs =
10 * (30 / 1000.0 * 30 / 1000.0);
EXPECT_EQ(kExpectedSumSquaredFrameDurationsSecs,
stats.sum_squared_frame_durations);
EXPECT_EQ(kExpectedTotalSquaredInterFrameDelaySecs,
stats.total_squared_inter_frame_delay);
}
TEST_F(ReceiveStatisticsProxy2Test, OnDecodedFrameWithoutQpQpSumWontExist) {

37
video/video_analyzer.cc
View File

@ -99,8 +99,8 @@ VideoAnalyzer::VideoAnalyzer(test::LayerFilteringTransport* transport,
mean_decode_time_ms_(0.0),
freeze_count_(0),
total_freezes_duration_ms_(0),
total_frames_duration_ms_(0),
sum_squared_frame_durations_(0),
total_inter_frame_delay_(0),
total_squared_inter_frame_delay_(0),
decode_frame_rate_(0),
render_frame_rate_(0),
last_fec_bytes_(0),
@ -502,6 +502,14 @@ void VideoAnalyzer::PollStats() {
if (receive_stream_ != nullptr) {
VideoReceiveStreamInterface::Stats receive_stats =
receive_stream_->GetStats();
// Freeze metrics.
freeze_count_ = receive_stats.freeze_count;
total_freezes_duration_ms_ = receive_stats.total_freezes_duration_ms;
total_inter_frame_delay_ = receive_stats.total_inter_frame_delay;
total_squared_inter_frame_delay_ =
receive_stats.total_squared_inter_frame_delay;
// `total_decode_time_ms` gives a good estimate of the mean decode time,
// `decode_ms` is used to keep track of the standard deviation.
if (receive_stats.frames_decoded > 0)
@ -518,20 +526,12 @@ void VideoAnalyzer::PollStats() {
// `frames_decoded` and `frames_rendered` are used because they are more
// accurate than `decode_frame_rate` and `render_frame_rate`.
// The latter two are calculated on a momentary basis.
const double total_frames_duration_sec_double =
static_cast<double>(receive_stats.total_frames_duration_ms) / 1000.0;
if (total_frames_duration_sec_double > 0) {
decode_frame_rate_ = static_cast<double>(receive_stats.frames_decoded) /
total_frames_duration_sec_double;
render_frame_rate_ = static_cast<double>(receive_stats.frames_rendered) /
total_frames_duration_sec_double;
if (total_inter_frame_delay_ > 0) {
decode_frame_rate_ =
receive_stats.frames_decoded / total_inter_frame_delay_;
render_frame_rate_ =
receive_stats.frames_rendered / total_inter_frame_delay_;
}
// Freeze metrics.
freeze_count_ = receive_stats.freeze_count;
total_freezes_duration_ms_ = receive_stats.total_freezes_duration_ms;
total_frames_duration_ms_ = receive_stats.total_frames_duration_ms;
sum_squared_frame_durations_ = receive_stats.sum_squared_frame_durations;
}
if (audio_receive_stream_ != nullptr) {
@ -673,7 +673,7 @@ void VideoAnalyzer::PrintResults() {
const double total_freezes_duration_ms_double =
static_cast<double>(total_freezes_duration_ms_);
const double total_frames_duration_ms_double =
static_cast<double>(total_frames_duration_ms_);
total_inter_frame_delay_ / rtc::kNumMillisecsPerSec;
if (total_frames_duration_ms_double > 0) {
GetGlobalMetricsLogger()->LogSingleValueMetric(
@ -701,10 +701,11 @@ void VideoAnalyzer::PrintResults() {
: 0,
Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter);
if (1000 * sum_squared_frame_durations_ > 0) {
if (total_squared_inter_frame_delay_ > 0) {
GetGlobalMetricsLogger()->LogSingleValueMetric(
"harmonic_frame_rate_fps", test_label_,
total_frames_duration_ms_double / (1000 * sum_squared_frame_durations_),
total_frames_duration_ms_double /
(1000 * total_squared_inter_frame_delay_),
Unit::kHertz, ImprovementDirection::kBiggerIsBetter);
}

4
video/video_analyzer.h
View File

@ -263,8 +263,8 @@ class VideoAnalyzer : public PacketReceiver,
SamplesStatsCounter time_between_freezes_ RTC_GUARDED_BY(comparison_lock_);
uint32_t freeze_count_ RTC_GUARDED_BY(comparison_lock_);
uint32_t total_freezes_duration_ms_ RTC_GUARDED_BY(comparison_lock_);
uint32_t total_frames_duration_ms_ RTC_GUARDED_BY(comparison_lock_);
double sum_squared_frame_durations_ RTC_GUARDED_BY(comparison_lock_);
double total_inter_frame_delay_ RTC_GUARDED_BY(comparison_lock_);
double total_squared_inter_frame_delay_ RTC_GUARDED_BY(comparison_lock_);
double decode_frame_rate_ RTC_GUARDED_BY(comparison_lock_);
double render_frame_rate_ RTC_GUARDED_BY(comparison_lock_);