diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index 09629308c4..ab610a0364 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -662,10 +662,12 @@ if (!build_with_chromium) { "../../../api:array_view", "../../../api:video_quality_analyzer_api", "../../../api/numerics", + "../../../api/units:data_size", "../../../api/units:time_delta", "../../../api/units:timestamp", "../../../api/video:encoded_image", "../../../api/video:video_frame", + "../../../api/video:video_frame_type", "../../../api/video:video_rtp_headers", "../../../common_video", "../../../rtc_base:checks", @@ -706,8 +708,10 @@ if (!build_with_chromium) { "../../../api:array_view", "../../../api:scoped_refptr", "../../../api/numerics:numerics", + "../../../api/units:data_size", "../../../api/units:timestamp", "../../../api/video:video_frame", + "../../../api/video:video_frame_type", "../../../common_video", "../../../rtc_base:checks", "../../../rtc_base:platform_thread", diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc index ecc92f28b9..abb9290105 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -317,7 +317,8 @@ void DefaultVideoQualityAnalyzer::OnFrameEncoded( used_encoder.last_frame_id = frame_id; used_encoder.switched_on_at = now; used_encoder.switched_from_at = now; - it->second.OnFrameEncoded(now, encoded_image.size(), + it->second.OnFrameEncoded(now, encoded_image._frameType, + DataSize::Bytes(encoded_image.size()), stats.target_encode_bitrate, used_encoder); } @@ -363,7 +364,9 @@ void DefaultVideoQualityAnalyzer::OnFramePreDecode( ->receive_time(); it->second.OnFramePreDecode(peer_index, /*received_time=*/last_receive_time, - /*decode_start_time=*/Now()); + /*decode_start_time=*/Now(), + input_image._frameType, + DataSize::Bytes(input_image.size())); } void DefaultVideoQualityAnalyzer::OnFrameDecoded( @@ -867,6 +870,28 @@ void DefaultVideoQualityAnalyzer::ReportResults( static_cast(stats.total_encoded_images_payload) / static_cast(test_duration.us()) * kMicrosPerSecond, "bytesPerSecond", /*important=*/false, ImproveDirection::kNone); + + if (options_.report_detailed_frame_stats) { + test::PrintResult("num_encoded_frames", "", test_case_name, + frame_counters.encoded, "count", + /*important=*/false, ImproveDirection::kBiggerIsBetter); + test::PrintResult("num_decoded_frames", "", test_case_name, + frame_counters.decoded, "count", + /*important=*/false, ImproveDirection::kBiggerIsBetter); + test::PrintResult("num_send_key_frames", "", test_case_name, + stats.num_send_key_frames, "count", + /*important=*/false, ImproveDirection::kBiggerIsBetter); + test::PrintResult("num_recv_key_frames", "", test_case_name, + stats.num_recv_key_frames, "count", + /*important=*/false, ImproveDirection::kBiggerIsBetter); + + ReportResult("recv_key_frame_size_bytes", test_case_name, + stats.recv_key_frame_size_bytes, "count", + ImproveDirection::kBiggerIsBetter); + ReportResult("recv_delta_frame_size_bytes", test_case_name, + stats.recv_delta_frame_size_bytes, "count", + ImproveDirection::kBiggerIsBetter); + } } void DefaultVideoQualityAnalyzer::ReportResult( @@ -1046,10 +1071,12 @@ bool DefaultVideoQualityAnalyzer::FrameInFlight::HaveAllPeersReceived() const { void DefaultVideoQualityAnalyzer::FrameInFlight::OnFrameEncoded( webrtc::Timestamp time, - int64_t encoded_image_size, + VideoFrameType frame_type, + DataSize encoded_image_size, uint32_t target_encode_bitrate, StreamCodecInfo used_encoder) { encoded_time_ = time; + frame_type_ = frame_type; encoded_image_size_ = encoded_image_size; target_encode_bitrate_ += target_encode_bitrate; // Update used encoder info. If simulcast/SVC is used, this method can @@ -1070,9 +1097,13 @@ void DefaultVideoQualityAnalyzer::FrameInFlight::OnFrameEncoded( void DefaultVideoQualityAnalyzer::FrameInFlight::OnFramePreDecode( size_t peer, webrtc::Timestamp received_time, - webrtc::Timestamp decode_start_time) { + webrtc::Timestamp decode_start_time, + VideoFrameType frame_type, + DataSize encoded_image_size) { receiver_stats_[peer].received_time = received_time; receiver_stats_[peer].decode_start_time = decode_start_time; + receiver_stats_[peer].frame_type = frame_type; + receiver_stats_[peer].encoded_image_size = encoded_image_size; } bool DefaultVideoQualityAnalyzer::FrameInFlight::HasReceivedTime( @@ -1134,6 +1165,7 @@ FrameStats DefaultVideoQualityAnalyzer::FrameInFlight::GetStatsForPeer( stats.pre_encode_time = pre_encode_time_; stats.encoded_time = encoded_time_; stats.target_encode_bitrate = target_encode_bitrate_; + stats.encoded_frame_type = frame_type_; stats.encoded_image_size = encoded_image_size_; stats.used_encoder = used_encoder_; @@ -1148,6 +1180,8 @@ FrameStats DefaultVideoQualityAnalyzer::FrameInFlight::GetStatsForPeer( stats.rendered_frame_width = receiver_stats->rendered_frame_width; stats.rendered_frame_height = receiver_stats->rendered_frame_height; stats.used_decoder = receiver_stats->used_decoder; + stats.pre_decoded_frame_type = receiver_stats->frame_type; + stats.pre_decoded_image_size = receiver_stats->encoded_image_size; } return stats; } diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h index bbda276767..ee5d499120 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h @@ -22,9 +22,11 @@ #include "api/array_view.h" #include "api/numerics/samples_stats_counter.h" #include "api/test/video_quality_analyzer_interface.h" +#include "api/units/data_size.h" #include "api/units/timestamp.h" #include "api/video/encoded_image.h" #include "api/video/video_frame.h" +#include "api/video/video_frame_type.h" #include "rtc_base/event.h" #include "rtc_base/platform_thread.h" #include "rtc_base/synchronization/mutex.h" @@ -179,6 +181,10 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { Timestamp rendered_time = Timestamp::MinusInfinity(); Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity(); + // Type and encoded size of received frame. + VideoFrameType frame_type = VideoFrameType::kEmptyFrame; + DataSize encoded_image_size = DataSize::Bytes(0); + absl::optional rendered_frame_width = absl::nullopt; absl::optional rendered_frame_height = absl::nullopt; @@ -217,7 +223,8 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { void SetPreEncodeTime(webrtc::Timestamp time) { pre_encode_time_ = time; } void OnFrameEncoded(webrtc::Timestamp time, - int64_t encoded_image_size, + VideoFrameType frame_type, + DataSize encoded_image_size, uint32_t target_encode_bitrate, StreamCodecInfo used_encoder); @@ -225,7 +232,9 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { void OnFramePreDecode(size_t peer, webrtc::Timestamp received_time, - webrtc::Timestamp decode_start_time); + webrtc::Timestamp decode_start_time, + VideoFrameType frame_type, + DataSize encoded_image_size); bool HasReceivedTime(size_t peer) const; @@ -267,7 +276,9 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { Timestamp captured_time_; Timestamp pre_encode_time_ = Timestamp::MinusInfinity(); Timestamp encoded_time_ = Timestamp::MinusInfinity(); - int64_t encoded_image_size_ = 0; + // Type and encoded size of sent frame. + VideoFrameType frame_type_ = VideoFrameType::kEmptyFrame; + DataSize encoded_image_size_ = DataSize::Bytes(0); uint32_t target_encode_bitrate_ = 0; // Can be not set if frame was dropped by encoder. absl::optional used_encoder_ = absl::nullopt; diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc index 904671001a..cbe1a709f7 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc @@ -18,6 +18,7 @@ #include "api/array_view.h" #include "api/scoped_refptr.h" #include "api/video/i420_buffer.h" +#include "api/video/video_frame_type.h" #include "common_video/libyuv/include/webrtc_libyuv.h" #include "rtc_base/checks.h" #include "rtc_base/platform_thread.h" @@ -425,9 +426,15 @@ void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison( (frame_stats.encoded_time - frame_stats.pre_encode_time).ms(), frame_stats.encoded_time)); stats->encode_frame_rate.AddEvent(frame_stats.encoded_time); - stats->total_encoded_images_payload += frame_stats.encoded_image_size; + stats->total_encoded_images_payload += + frame_stats.encoded_image_size.bytes(); stats->target_encode_bitrate.AddSample(StatsSample( frame_stats.target_encode_bitrate, frame_stats.encoded_time)); + + // Stats sliced on encoded frame type. + if (frame_stats.encoded_frame_type == VideoFrameType::kVideoFrameKey) { + ++stats->num_send_key_frames; + } } // Next stats can be calculated only if frame was received on remote side. if (comparison.type != FrameComparisonType::kDroppedFrame) { @@ -447,6 +454,20 @@ void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison( stats->transport_time_ms.AddSample(StatsSample( (frame_stats.decode_start_time - frame_stats.encoded_time).ms(), frame_stats.decode_start_time)); + + // Stats sliced on decoded frame type. + if (frame_stats.pre_decoded_frame_type == + VideoFrameType::kVideoFrameKey) { + ++stats->num_recv_key_frames; + stats->recv_key_frame_size_bytes.AddSample( + StatsSample(frame_stats.pre_decoded_image_size.bytes(), + frame_stats.decode_start_time)); + } else if (frame_stats.pre_decoded_frame_type == + VideoFrameType::kVideoFrameDelta) { + stats->recv_delta_frame_size_bytes.AddSample( + StatsSample(frame_stats.pre_decoded_image_size.bytes(), + frame_stats.decode_start_time)); + } } if (frame_stats.decode_end_time.IsFinite()) { stats->decode_time_ms.AddSample(StatsSample( diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h index d13eb21c3e..04f653c02b 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h @@ -16,8 +16,10 @@ #include #include "absl/types/optional.h" +#include "api/units/data_size.h" #include "api/units/timestamp.h" #include "api/video/video_frame.h" +#include "api/video/video_frame_type.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" namespace webrtc { @@ -53,7 +55,10 @@ struct FrameStats { Timestamp rendered_time = Timestamp::MinusInfinity(); Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity(); - int64_t encoded_image_size = 0; + VideoFrameType encoded_frame_type = VideoFrameType::kEmptyFrame; + DataSize encoded_image_size = DataSize::Bytes(0); + VideoFrameType pre_decoded_frame_type = VideoFrameType::kEmptyFrame; + DataSize pre_decoded_image_size = DataSize::Bytes(0); uint32_t target_encode_bitrate = 0; absl::optional rendered_frame_width = absl::nullopt; diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h index 3b52a80d35..8f0afd36f7 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h @@ -96,8 +96,10 @@ struct StreamStats { // The time when the first frame of this stream was captured. Timestamp stream_started_time; + // Spatial quality metrics. SamplesStatsCounter psnr; SamplesStatsCounter ssim; + // Time from frame encoded (time point on exit from encoder) to the // encoded image received in decoder (time point on entrance to decoder). SamplesStatsCounter transport_time_ms; @@ -129,6 +131,14 @@ struct StreamStats { // Counters on which phase how many frames were dropped. std::map dropped_by_phase; + // Frame count metrics. + int64_t num_send_key_frames = 0; + int64_t num_recv_key_frames = 0; + + // Encoded frame size (in bytes) metrics. + SamplesStatsCounter recv_key_frame_size_bytes; + SamplesStatsCounter recv_delta_frame_size_bytes; + // Vector of encoders used for this stream by sending client. std::vector encoders; // Vectors of decoders used for this stream by receiving client. @@ -214,6 +224,9 @@ struct DefaultVideoQualityAnalyzerOptions { bool compute_ssim = true; // If true, weights the luma plane more than the chroma planes in the PSNR. bool use_weighted_psnr = false; + // Tells DefaultVideoQualityAnalyzer if detailed frame stats should be + // reported. + bool report_detailed_frame_stats = false; // If true DefaultVideoQualityAnalyzer will try to adjust frames before // computing PSNR and SSIM for them. In some cases picture may be shifted by // a few pixels after the encode/decode step. Those difference is invisible