admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add some more frame stats to PCLF

Browse Source 
Metrics for calculating sent/recv key frame rate:
num_encoded_frames -- number of encoded frames
num_decoded_frames -- number of decoded frames
num_send_key_frames -- number of sent keyframes
num_recv_key_frames -- number of received keyframes

Metrics for frame size (in bytes) distribution:
recv_key_frame_size_bytes -- key frame size
recv_delta_frame_size_bytes -- delta frame size

The stats are only reported if the user so requests. This is
because the stats may not be useful for regression tracking,
but mainly for adhoc analysis.

Tested:
```
~/s/w/src (frame_stats↑1|✚1) $ autoninja -C out/Debug && out/Debug/test_support_unittests --gtest_filter="PeerConnectionE2EQualityTestSmokeTest.Smoke" 2>/dev/null
Note: Google Test filter = PeerConnectionE2EQualityTestSmokeTest.Smoke
[==========] Running 1 test from 1 test suite.
[----------] Global test environment set-up.
[----------] 1 test from PeerConnectionE2EQualityTestSmokeTest
[ RUN      ] PeerConnectionE2EQualityTestSmokeTest.Smoke
...
RESULT num_encoded_frames: Smoke/alice-video= 31 count_biggerIsBetter
RESULT num_decoded_frames: Smoke/alice-video= 31 count_biggerIsBetter
RESULT num_send_key_frames: Smoke/alice-video= 2 count_biggerIsBetter
RESULT num_recv_key_frames: Smoke/alice-video= 2 count_biggerIsBetter
RESULT recv_key_frame_size_bytes: Smoke/alice-video= {1847,126} count_biggerIsBetter
RESULT recv_delta_frame_size_bytes: Smoke/alice-video= {1150.1034,393.66329} count_biggerIsBetter
...
RESULT num_encoded_frames: Smoke/charlie-video= 31 count_biggerIsBetter
RESULT num_decoded_frames: Smoke/charlie-video= 31 count_biggerIsBetter
RESULT num_send_key_frames: Smoke/charlie-video= 2 count_biggerIsBetter
RESULT num_recv_key_frames: Smoke/charlie-video= 2 count_biggerIsBetter
RESULT recv_key_frame_size_bytes: Smoke/charlie-video= {1847,126} count_biggerIsBetter
RESULT recv_delta_frame_size_bytes: Smoke/charlie-video= {1150.1034,393.66329} count_biggerIsBetter
...
[----------] 1 test from PeerConnectionE2EQualityTestSmokeTest (2196 ms total)

[----------] Global test environment tear-down
[==========] 1 test from 1 test suite ran. (2196 ms total)
[  PASSED  ] 1 test.
```

Bug: webrtc:14019
Change-Id: I3d5ea0cf659a6299c6d94f5004cbf0c763b267e5
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/260924
Reviewed-by: Artem Titov <titovartem@webrtc.org>
Commit-Queue: Rasmus Brandt <brandtr@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#36759}
This commit is contained in:
Rasmus Brandt 2022-05-04 13:54:10 +02:00 committed by WebRTC LUCI CQ
parent 30ec725b6e
commit 6566bd5484
6 changed files with 97 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
test/pc/e2e/BUILD.gn
View File

@ -662,10 +662,12 @@ if (!build_with_chromium) {
"../../../api:array_view",
"../../../api:video_quality_analyzer_api",
"../../../api/numerics",
"../../../api/units:data_size",
"../../../api/units:time_delta",
"../../../api/units:timestamp",
"../../../api/video:encoded_image",
"../../../api/video:video_frame",
"../../../api/video:video_frame_type",
"../../../api/video:video_rtp_headers",
"../../../common_video",
"../../../rtc_base:checks",
@ -706,8 +708,10 @@ if (!build_with_chromium) {
"../../../api:array_view",
"../../../api:scoped_refptr",
"../../../api/numerics:numerics",
"../../../api/units:data_size",
"../../../api/units:timestamp",
"../../../api/video:video_frame",
"../../../api/video:video_frame_type",
"../../../common_video",
"../../../rtc_base:checks",
"../../../rtc_base:platform_thread",

42
test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
View File

@ -317,7 +317,8 @@ void DefaultVideoQualityAnalyzer::OnFrameEncoded(
used_encoder.last_frame_id = frame_id;
used_encoder.switched_on_at = now;
used_encoder.switched_from_at = now;
it->second.OnFrameEncoded(now, encoded_image.size(),
it->second.OnFrameEncoded(now, encoded_image._frameType,
DataSize::Bytes(encoded_image.size()),
stats.target_encode_bitrate, used_encoder);
}
@ -363,7 +364,9 @@ void DefaultVideoQualityAnalyzer::OnFramePreDecode(
->receive_time();
it->second.OnFramePreDecode(peer_index,
/*received_time=*/last_receive_time,
/*decode_start_time=*/Now());
/*decode_start_time=*/Now(),
input_image._frameType,
DataSize::Bytes(input_image.size()));
}
void DefaultVideoQualityAnalyzer::OnFrameDecoded(
@ -867,6 +870,28 @@ void DefaultVideoQualityAnalyzer::ReportResults(
static_cast<double>(stats.total_encoded_images_payload) /
static_cast<double>(test_duration.us()) * kMicrosPerSecond,
"bytesPerSecond", /*important=*/false, ImproveDirection::kNone);
if (options_.report_detailed_frame_stats) {
test::PrintResult("num_encoded_frames", "", test_case_name,
frame_counters.encoded, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
test::PrintResult("num_decoded_frames", "", test_case_name,
frame_counters.decoded, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
test::PrintResult("num_send_key_frames", "", test_case_name,
stats.num_send_key_frames, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
test::PrintResult("num_recv_key_frames", "", test_case_name,
stats.num_recv_key_frames, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
ReportResult("recv_key_frame_size_bytes", test_case_name,
stats.recv_key_frame_size_bytes, "count",
ImproveDirection::kBiggerIsBetter);
ReportResult("recv_delta_frame_size_bytes", test_case_name,
stats.recv_delta_frame_size_bytes, "count",
ImproveDirection::kBiggerIsBetter);
}
}
void DefaultVideoQualityAnalyzer::ReportResult(
@ -1046,10 +1071,12 @@ bool DefaultVideoQualityAnalyzer::FrameInFlight::HaveAllPeersReceived() const {
void DefaultVideoQualityAnalyzer::FrameInFlight::OnFrameEncoded(
webrtc::Timestamp time,
int64_t encoded_image_size,
VideoFrameType frame_type,
DataSize encoded_image_size,
uint32_t target_encode_bitrate,
StreamCodecInfo used_encoder) {
encoded_time_ = time;
frame_type_ = frame_type;
encoded_image_size_ = encoded_image_size;
target_encode_bitrate_ += target_encode_bitrate;
// Update used encoder info. If simulcast/SVC is used, this method can
@ -1070,9 +1097,13 @@ void DefaultVideoQualityAnalyzer::FrameInFlight::OnFrameEncoded(
void DefaultVideoQualityAnalyzer::FrameInFlight::OnFramePreDecode(
size_t peer,
webrtc::Timestamp received_time,
webrtc::Timestamp decode_start_time) {
webrtc::Timestamp decode_start_time,
VideoFrameType frame_type,
DataSize encoded_image_size) {
receiver_stats_[peer].received_time = received_time;
receiver_stats_[peer].decode_start_time = decode_start_time;
receiver_stats_[peer].frame_type = frame_type;
receiver_stats_[peer].encoded_image_size = encoded_image_size;
}
bool DefaultVideoQualityAnalyzer::FrameInFlight::HasReceivedTime(
@ -1134,6 +1165,7 @@ FrameStats DefaultVideoQualityAnalyzer::FrameInFlight::GetStatsForPeer(
stats.pre_encode_time = pre_encode_time_;
stats.encoded_time = encoded_time_;
stats.target_encode_bitrate = target_encode_bitrate_;
stats.encoded_frame_type = frame_type_;
stats.encoded_image_size = encoded_image_size_;
stats.used_encoder = used_encoder_;
@ -1148,6 +1180,8 @@ FrameStats DefaultVideoQualityAnalyzer::FrameInFlight::GetStatsForPeer(
stats.rendered_frame_width = receiver_stats->rendered_frame_width;
stats.rendered_frame_height = receiver_stats->rendered_frame_height;
stats.used_decoder = receiver_stats->used_decoder;
stats.pre_decoded_frame_type = receiver_stats->frame_type;
stats.pre_decoded_image_size = receiver_stats->encoded_image_size;
}
return stats;
}

17
test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
View File

@ -22,9 +22,11 @@
#include "api/array_view.h"
#include "api/numerics/samples_stats_counter.h"
#include "api/test/video_quality_analyzer_interface.h"
#include "api/units/data_size.h"
#include "api/units/timestamp.h"
#include "api/video/encoded_image.h"
#include "api/video/video_frame.h"
#include "api/video/video_frame_type.h"
#include "rtc_base/event.h"
#include "rtc_base/platform_thread.h"
#include "rtc_base/synchronization/mutex.h"
@ -179,6 +181,10 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
Timestamp rendered_time = Timestamp::MinusInfinity();
Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity();
// Type and encoded size of received frame.
VideoFrameType frame_type = VideoFrameType::kEmptyFrame;
DataSize encoded_image_size = DataSize::Bytes(0);
absl::optional<int> rendered_frame_width = absl::nullopt;
absl::optional<int> rendered_frame_height = absl::nullopt;
@ -217,7 +223,8 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
void SetPreEncodeTime(webrtc::Timestamp time) { pre_encode_time_ = time; }
void OnFrameEncoded(webrtc::Timestamp time,
int64_t encoded_image_size,
VideoFrameType frame_type,
DataSize encoded_image_size,
uint32_t target_encode_bitrate,
StreamCodecInfo used_encoder);
@ -225,7 +232,9 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
void OnFramePreDecode(size_t peer,
webrtc::Timestamp received_time,
webrtc::Timestamp decode_start_time);
webrtc::Timestamp decode_start_time,
VideoFrameType frame_type,
DataSize encoded_image_size);
bool HasReceivedTime(size_t peer) const;
@ -267,7 +276,9 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
Timestamp captured_time_;
Timestamp pre_encode_time_ = Timestamp::MinusInfinity();
Timestamp encoded_time_ = Timestamp::MinusInfinity();
int64_t encoded_image_size_ = 0;
// Type and encoded size of sent frame.
VideoFrameType frame_type_ = VideoFrameType::kEmptyFrame;
DataSize encoded_image_size_ = DataSize::Bytes(0);
uint32_t target_encode_bitrate_ = 0;
// Can be not set if frame was dropped by encoder.
absl::optional<StreamCodecInfo> used_encoder_ = absl::nullopt;

23
test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc
View File

@ -18,6 +18,7 @@
#include "api/array_view.h"
#include "api/scoped_refptr.h"
#include "api/video/i420_buffer.h"
#include "api/video/video_frame_type.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "rtc_base/checks.h"
#include "rtc_base/platform_thread.h"
@ -425,9 +426,15 @@ void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison(
(frame_stats.encoded_time - frame_stats.pre_encode_time).ms(),
frame_stats.encoded_time));
stats->encode_frame_rate.AddEvent(frame_stats.encoded_time);
stats->total_encoded_images_payload += frame_stats.encoded_image_size;
stats->total_encoded_images_payload +=
frame_stats.encoded_image_size.bytes();
stats->target_encode_bitrate.AddSample(StatsSample(
frame_stats.target_encode_bitrate, frame_stats.encoded_time));
// Stats sliced on encoded frame type.
if (frame_stats.encoded_frame_type == VideoFrameType::kVideoFrameKey) {
++stats->num_send_key_frames;
}
}
// Next stats can be calculated only if frame was received on remote side.
if (comparison.type != FrameComparisonType::kDroppedFrame) {
@ -447,6 +454,20 @@ void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison(
stats->transport_time_ms.AddSample(StatsSample(
(frame_stats.decode_start_time - frame_stats.encoded_time).ms(),
frame_stats.decode_start_time));
// Stats sliced on decoded frame type.
if (frame_stats.pre_decoded_frame_type ==
VideoFrameType::kVideoFrameKey) {
++stats->num_recv_key_frames;
stats->recv_key_frame_size_bytes.AddSample(
StatsSample(frame_stats.pre_decoded_image_size.bytes(),
frame_stats.decode_start_time));
} else if (frame_stats.pre_decoded_frame_type ==
VideoFrameType::kVideoFrameDelta) {
stats->recv_delta_frame_size_bytes.AddSample(
StatsSample(frame_stats.pre_decoded_image_size.bytes(),
frame_stats.decode_start_time));
}
}
if (frame_stats.decode_end_time.IsFinite()) {
stats->decode_time_ms.AddSample(StatsSample(

7
test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h
View File

@ -16,8 +16,10 @@
#include <utility>
#include "absl/types/optional.h"
#include "api/units/data_size.h"
#include "api/units/timestamp.h"
#include "api/video/video_frame.h"
#include "api/video/video_frame_type.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
namespace webrtc {
@ -53,7 +55,10 @@ struct FrameStats {
Timestamp rendered_time = Timestamp::MinusInfinity();
Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity();
int64_t encoded_image_size = 0;
VideoFrameType encoded_frame_type = VideoFrameType::kEmptyFrame;
DataSize encoded_image_size = DataSize::Bytes(0);
VideoFrameType pre_decoded_frame_type = VideoFrameType::kEmptyFrame;
DataSize pre_decoded_image_size = DataSize::Bytes(0);
uint32_t target_encode_bitrate = 0;
absl::optional<int> rendered_frame_width = absl::nullopt;

13
test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h
View File

@ -96,8 +96,10 @@ struct StreamStats {
// The time when the first frame of this stream was captured.
Timestamp stream_started_time;
// Spatial quality metrics.
SamplesStatsCounter psnr;
SamplesStatsCounter ssim;
// Time from frame encoded (time point on exit from encoder) to the
// encoded image received in decoder (time point on entrance to decoder).
SamplesStatsCounter transport_time_ms;
@ -129,6 +131,14 @@ struct StreamStats {
// Counters on which phase how many frames were dropped.
std::map<FrameDropPhase, int64_t> dropped_by_phase;
// Frame count metrics.
int64_t num_send_key_frames = 0;
int64_t num_recv_key_frames = 0;
// Encoded frame size (in bytes) metrics.
SamplesStatsCounter recv_key_frame_size_bytes;
SamplesStatsCounter recv_delta_frame_size_bytes;
// Vector of encoders used for this stream by sending client.
std::vector<StreamCodecInfo> encoders;
// Vectors of decoders used for this stream by receiving client.
@ -214,6 +224,9 @@ struct DefaultVideoQualityAnalyzerOptions {
bool compute_ssim = true;
// If true, weights the luma plane more than the chroma planes in the PSNR.
bool use_weighted_psnr = false;
// Tells DefaultVideoQualityAnalyzer if detailed frame stats should be
// reported.
bool report_detailed_frame_stats = false;
// If true DefaultVideoQualityAnalyzer will try to adjust frames before
// computing PSNR and SSIM for them. In some cases picture may be shifted by
// a few pixels after the encode/decode step. Those difference is invisible