diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc index d39a06878e..a8381ce128 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -211,6 +211,12 @@ uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured( } StreamState* state = &stream_states_.at(stream_index); state->PushBack(frame_id); + absl::optional time_between_captured_frames = absl::nullopt; + if (state->last_captured_frame_time().has_value()) { + time_between_captured_frames = + captured_time - *state->last_captured_frame_time(); + } + state->SetLastCapturedFrameTime(captured_time); // Update frames in flight info. auto it = captured_frames_in_flight_.find(frame_id); if (it != captured_frames_in_flight_.end()) { @@ -247,6 +253,7 @@ uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured( } captured_frames_in_flight_.emplace( frame_id, FrameInFlight(stream_index, frame_id, captured_time, + time_between_captured_frames, std::move(frame_receivers_indexes))); // Store local copy of the frame with frame_id set. VideoFrame local_frame(frame); @@ -337,6 +344,13 @@ void DefaultVideoQualityAnalyzer::OnFrameEncoded( } } Timestamp now = Now(); + StreamState& state = stream_states_.at(frame_in_flight.stream()); + absl::optional time_between_encoded_frames = absl::nullopt; + if (state.last_encoded_frame_time().has_value()) { + time_between_encoded_frames = now - *state.last_encoded_frame_time(); + } + state.SetLastEncodedFrameTime(now); + StreamCodecInfo used_encoder; used_encoder.codec_name = stats.encoder_name; used_encoder.first_frame_id = frame_id; @@ -350,8 +364,9 @@ void DefaultVideoQualityAnalyzer::OnFrameEncoded( size_t stream_index = encoded_image.SpatialIndex().value_or( encoded_image.SimulcastIndex().value_or(0)); frame_in_flight.OnFrameEncoded( - now, encoded_image._frameType, DataSize::Bytes(encoded_image.size()), - stats.target_encode_bitrate, stream_index, stats.qp, used_encoder); + now, time_between_encoded_frames, encoded_image._frameType, + DataSize::Bytes(encoded_image.size()), stats.target_encode_bitrate, + stream_index, stats.qp, used_encoder); if (options_.report_infra_metrics) { analyzer_stats_.on_frame_encoded_processing_time_ms.AddSample( diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h index 1ea59bdfca..879fc02347 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "api/array_view.h" @@ -178,7 +179,7 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { // 4. There too many frames in flight for current video stream and X is the // oldest frame id in this stream. In such case only the frame content // will be removed, but the map entry will be preserved. - std::map captured_frames_in_flight_ + std::unordered_map captured_frames_in_flight_ RTC_GUARDED_BY(mutex_); // Global frames count for all video streams. FrameCounters frame_counters_ RTC_GUARDED_BY(mutex_); @@ -190,19 +191,19 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { std::map stream_frame_counters_ RTC_GUARDED_BY(mutex_); // Map from stream index in `streams_` to its StreamState. - std::map stream_states_ RTC_GUARDED_BY(mutex_); + std::unordered_map stream_states_ RTC_GUARDED_BY(mutex_); // Map from stream index in `streams_` to sender peer index in `peers_`. - std::map stream_to_sender_ RTC_GUARDED_BY(mutex_); + std::unordered_map stream_to_sender_ RTC_GUARDED_BY(mutex_); // Stores history mapping between stream index in `streams_` and frame ids. // Updated when frame id overlap. It required to properly return stream label // after 1st frame from simulcast streams was already rendered and last is // still encoding. - std::map> stream_to_frame_id_history_ + std::unordered_map> stream_to_frame_id_history_ RTC_GUARDED_BY(mutex_); // Map from stream index to the list of frames as they were met in the stream. - std::map> stream_to_frame_id_full_history_ - RTC_GUARDED_BY(mutex_); + std::unordered_map> + stream_to_frame_id_full_history_ RTC_GUARDED_BY(mutex_); AnalyzerStats analyzer_stats_ RTC_GUARDED_BY(mutex_); DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer_; diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc index b0ed041301..056bd73f88 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc @@ -16,6 +16,7 @@ #include "absl/types/optional.h" #include "api/units/data_size.h" +#include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "api/video/video_frame.h" #include "api/video/video_frame_type.h" @@ -36,14 +37,17 @@ absl::optional MaybeGetValue(const std::unordered_map& map, } // namespace -FrameInFlight::FrameInFlight(size_t stream, - uint16_t frame_id, - Timestamp captured_time, - std::set expected_receivers) +FrameInFlight::FrameInFlight( + size_t stream, + uint16_t frame_id, + Timestamp captured_time, + absl::optional time_between_captured_frames, + std::set expected_receivers) : stream_(stream), expected_receivers_(std::move(expected_receivers)), frame_id_(frame_id), - captured_time_(captured_time) {} + captured_time_(captured_time), + time_between_captured_frames_(time_between_captured_frames) {} std::vector FrameInFlight::GetPeersWhichDidntReceive() const { std::vector out; @@ -73,14 +77,21 @@ bool FrameInFlight::HaveAllPeersReceived() const { return true; } -void FrameInFlight::OnFrameEncoded(webrtc::Timestamp time, - VideoFrameType frame_type, - DataSize encoded_image_size, - uint32_t target_encode_bitrate, - int stream_index, - int qp, - StreamCodecInfo used_encoder) { +void FrameInFlight::OnFrameEncoded( + webrtc::Timestamp time, + absl::optional time_between_encoded_frames, + VideoFrameType frame_type, + DataSize encoded_image_size, + uint32_t target_encode_bitrate, + int stream_index, + int qp, + StreamCodecInfo used_encoder) { encoded_time_ = time; + if (time_between_encoded_frames.has_value()) { + time_between_encoded_frames_ = + time_between_encoded_frames_.value_or(TimeDelta::Zero()) + + *time_between_encoded_frames; + } frame_type_ = frame_type; encoded_image_size_ = encoded_image_size; target_encode_bitrate_ += target_encode_bitrate; @@ -171,6 +182,8 @@ FrameStats FrameInFlight::GetStatsForPeer(size_t peer) const { RTC_DCHECK(!IsSuperfluous(peer)) << "This frame is superfluous for peer " << peer; FrameStats stats(frame_id_, captured_time_); + stats.time_between_captured_frames = time_between_captured_frames_; + stats.time_between_encoded_frames = time_between_encoded_frames_; stats.pre_encode_time = pre_encode_time_; stats.encoded_time = encoded_time_; stats.target_encode_bitrate = target_encode_bitrate_; diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h index 06552c7013..4093558e00 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h @@ -34,9 +34,11 @@ struct ReceiverFrameStats { Timestamp decode_start_time = Timestamp::MinusInfinity(); Timestamp decode_end_time = Timestamp::MinusInfinity(); Timestamp rendered_time = Timestamp::MinusInfinity(); + // Will be finite if there is frame rendered before this one. Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity(); - TimeDelta time_between_rendered_frames = TimeDelta::Zero(); + // Will be set if there is frame rendered before this one. + absl::optional time_between_rendered_frames = absl::nullopt; // Type and encoded size of received frame. VideoFrameType frame_type = VideoFrameType::kEmptyFrame; @@ -68,6 +70,7 @@ class FrameInFlight { FrameInFlight(size_t stream, uint16_t frame_id, Timestamp captured_time, + absl::optional time_between_captured_frames, std::set expected_receivers); size_t stream() const { return stream_; } @@ -89,6 +92,7 @@ class FrameInFlight { void SetPreEncodeTime(Timestamp time) { pre_encode_time_ = time; } void OnFrameEncoded(Timestamp time, + absl::optional time_between_encoded_frames, VideoFrameType frame_type, DataSize encoded_image_size, uint32_t target_encode_bitrate, @@ -156,14 +160,16 @@ class FrameInFlight { // any peer or can be safely deleted. It is responsibility of the user of this // object to decide when it should be deleted. std::set expected_receivers_; - // Store frame id separately because `frame_` can be removed when we have too - // much memory consuption. uint16_t frame_id_ = VideoFrame::kNotSetId; // Frame events timestamp. Timestamp captured_time_; Timestamp pre_encode_time_ = Timestamp::MinusInfinity(); Timestamp encoded_time_ = Timestamp::MinusInfinity(); + + absl::optional time_between_captured_frames_ = absl::nullopt; + absl::optional time_between_encoded_frames_ = absl::nullopt; + // Type and encoded size of sent frame. VideoFrameType frame_type_ = VideoFrameType::kEmptyFrame; DataSize encoded_image_size_ = DataSize::Bytes(0); diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc index 7ea2a2a90c..65b6fcc52b 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc @@ -464,6 +464,11 @@ void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison( StatsSample(ssim, frame_stats.received_time, metadata)); } stats->capture_frame_rate.AddEvent(frame_stats.captured_time); + if (frame_stats.time_between_captured_frames.has_value()) { + stats->time_between_captured_frames_ms.AddSample( + StatsSample(*frame_stats.time_between_captured_frames, + frame_stats.captured_time, metadata)); + } // Compute dropped phase for dropped frame if (comparison.type == FrameComparisonType::kDroppedFrame) { @@ -487,6 +492,11 @@ void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison( StatsSample(frame_stats.encoded_time - frame_stats.pre_encode_time, frame_stats.encoded_time, metadata)); stats->encode_frame_rate.AddEvent(frame_stats.encoded_time); + if (frame_stats.time_between_encoded_frames.has_value()) { + stats->time_between_encoded_frames_ms.AddSample( + StatsSample(*frame_stats.time_between_encoded_frames, + frame_stats.encoded_time, metadata)); + } stats->total_encoded_images_payload += frame_stats.encoded_image_size.bytes(); stats->target_encode_bitrate.AddSample(StatsSample( @@ -546,16 +556,17 @@ void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison( if (frame_stats.prev_frame_rendered_time.IsFinite() && frame_stats.rendered_time.IsFinite()) { + RTC_DCHECK(frame_stats.time_between_rendered_frames.has_value()); stats->time_between_rendered_frames_ms.AddSample( - StatsSample(frame_stats.time_between_rendered_frames, + StatsSample(*frame_stats.time_between_rendered_frames, frame_stats.rendered_time, metadata)); TimeDelta average_time_between_rendered_frames = TimeDelta::Millis( stats->time_between_rendered_frames_ms.GetAverage()); - if (frame_stats.time_between_rendered_frames > + if (*frame_stats.time_between_rendered_frames > std::max(kFreezeThreshold + average_time_between_rendered_frames, 3 * average_time_between_rendered_frames)) { stats->freeze_time_ms.AddSample( - StatsSample(frame_stats.time_between_rendered_frames, + StatsSample(*frame_stats.time_between_rendered_frames, frame_stats.rendered_time, metadata)); auto freeze_end_it = stream_last_freeze_end_time_.find(comparison.stats_key); diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h index b4254805b0..e2e55eb8a7 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h @@ -59,9 +59,10 @@ struct FrameStats { Timestamp rendered_time = Timestamp::MinusInfinity(); Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity(); - // Time between this and previous rendered frame excluding time when related - // stream was paused for related receiver. - TimeDelta time_between_rendered_frames = TimeDelta::Zero(); + // Next timings are set if and only if previous frame exist. + absl::optional time_between_captured_frames = absl::nullopt; + absl::optional time_between_encoded_frames = absl::nullopt; + absl::optional time_between_rendered_frames = absl::nullopt; VideoFrameType encoded_frame_type = VideoFrameType::kEmptyFrame; DataSize encoded_image_size = DataSize::Bytes(0); diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h index ef6ac4bf7a..73cbcc03df 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h @@ -126,6 +126,9 @@ struct StreamStats { SamplesStatsCounter total_delay_incl_transport_ms; // Time between frames out from renderer. SamplesStatsCounter time_between_rendered_frames_ms; + SamplesStatsCounter time_between_captured_frames_ms; + // Time between frames out from encoder. + SamplesStatsCounter time_between_encoded_frames_ms; SamplesRateCounter capture_frame_rate; SamplesRateCounter encode_frame_rate; SamplesStatsCounter encode_time_ms; diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h index 190432cfb6..22fbfd4a40 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h @@ -70,6 +70,20 @@ class StreamState { return frame_ids_.size(kAliveFramesQueueIndex); } + void SetLastCapturedFrameTime(Timestamp time) { + last_captured_frame_time_ = time; + } + absl::optional last_captured_frame_time() const { + return last_captured_frame_time_; + } + + void SetLastEncodedFrameTime(Timestamp time) { + last_encoded_frame_time_ = time; + } + absl::optional last_encoded_frame_time() const { + return last_encoded_frame_time_; + } + void SetLastRenderedFrameTime(size_t peer, Timestamp time); absl::optional last_rendered_frame_time(size_t peer) const; @@ -99,6 +113,8 @@ class StreamState { // frame_id2 and consider those frames as dropped and then compare received // frame with the one from `FrameInFlight` with id frame_id3. MultiReaderQueue frame_ids_; + absl::optional last_captured_frame_time_ = absl::nullopt; + absl::optional last_encoded_frame_time_ = absl::nullopt; std::unordered_map last_rendered_frame_time_; // Mapping from peer's index to pausable state for this receiver. std::unordered_map pausable_state_; diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc index a66d1a9a5b..86144a07dd 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc @@ -2147,9 +2147,11 @@ TEST_F(DefaultVideoQualityAnalyzerSimulatedTimeTest, // TODO(bugs.webrtc.org/14995): value should exclude pause EXPECT_THAT(GetTimeSortedValues(bob_stream_stats.time_between_freezes_ms), ElementsAre(2950.0)); - // TODO(bugs.webrtc.org/14995): Fix capture_frame_rate (has to be ~20.0) + EXPECT_THAT(bob_stream_stats.time_between_captured_frames_ms.GetAverage(), + 50.0); + EXPECT_THAT(bob_stream_stats.time_between_encoded_frames_ms.GetAverage(), + 50.0); ExpectRateIs(bob_stream_stats.capture_frame_rate, 13.559322); - // TODO(bugs.webrtc.org/14995): Fix encode_frame_rate (has to be ~20.0) ExpectRateIs(bob_stream_stats.encode_frame_rate, 13.559322); EXPECT_DOUBLE_EQ(bob_stream_stats.harmonic_framerate_fps, 20); @@ -2208,22 +2210,22 @@ TEST_F(DefaultVideoQualityAnalyzerSimulatedTimeTest, std::map streams_stats = analyzer.GetStats(); std::map frame_counters = analyzer.GetPerStreamCounters(); - StreamStats bob_stream_stats1 = + StreamStats bob_stream_stats_1 = streams_stats.at(StatsKey("alice_video_1", "bob")); FrameCounters bob_frame_counters1 = frame_counters.at(StatsKey("alice_video_1", "bob")); EXPECT_THAT(bob_frame_counters1.dropped, Eq(0)); EXPECT_THAT(bob_frame_counters1.rendered, Eq(40)); - EXPECT_THAT(GetTimeSortedValues(bob_stream_stats1.freeze_time_ms), + EXPECT_THAT(GetTimeSortedValues(bob_stream_stats_1.freeze_time_ms), ElementsAre(0.0)); // TODO(bugs.webrtc.org/14995): value should exclude pause - EXPECT_THAT(GetTimeSortedValues(bob_stream_stats1.time_between_freezes_ms), + EXPECT_THAT(GetTimeSortedValues(bob_stream_stats_1.time_between_freezes_ms), ElementsAre(2950.0)); - // TODO(bugs.webrtc.org/14995): Fix capture_frame_rate (has to be ~20.0) - ExpectRateIs(bob_stream_stats1.capture_frame_rate, 13.559322); - // TODO(bugs.webrtc.org/14995): Fix encode_frame_rate (has to be ~20.0) - ExpectRateIs(bob_stream_stats1.encode_frame_rate, 13.559322); - EXPECT_DOUBLE_EQ(bob_stream_stats1.harmonic_framerate_fps, 20.0); + EXPECT_THAT(bob_stream_stats_1.time_between_captured_frames_ms.GetAverage(), + 50.0); + EXPECT_THAT(bob_stream_stats_1.time_between_encoded_frames_ms.GetAverage(), + 50.0); + EXPECT_DOUBLE_EQ(bob_stream_stats_1.harmonic_framerate_fps, 20.0); // Bob should have 20 fps without freeze on both streams. StreamStats bob_stream_stats_2 = @@ -2237,10 +2239,10 @@ TEST_F(DefaultVideoQualityAnalyzerSimulatedTimeTest, // TODO(bugs.webrtc.org/14995): value should exclude pause EXPECT_THAT(GetTimeSortedValues(bob_stream_stats_2.time_between_freezes_ms), ElementsAre(2950.0)); - // TODO(bugs.webrtc.org/14995): Fix capture_frame_rate (has to be ~20.0) - ExpectRateIs(bob_stream_stats_2.capture_frame_rate, 13.559322); - // TODO(bugs.webrtc.org/14995): Fix encode_frame_rate (has to be ~20.0) - ExpectRateIs(bob_stream_stats_2.encode_frame_rate, 13.559322); + EXPECT_THAT(bob_stream_stats_2.time_between_captured_frames_ms.GetAverage(), + 50.0); + EXPECT_THAT(bob_stream_stats_2.time_between_encoded_frames_ms.GetAverage(), + 50.0); EXPECT_DOUBLE_EQ(bob_stream_stats_2.harmonic_framerate_fps, 20.0); }