diff --git a/api/test/video_quality_analyzer_interface.h b/api/test/video_quality_analyzer_interface.h index d27c9ea015..91ecbc8a6c 100644 --- a/api/test/video_quality_analyzer_interface.h +++ b/api/test/video_quality_analyzer_interface.h @@ -142,6 +142,9 @@ class VideoQualityAnalyzerInterface // Will be called before test adds new participant in the middle of a call. virtual void RegisterParticipantInCall(absl::string_view peer_name) {} + // Will be called after test removed existing participant in the middle of the + // call. + virtual void UnregisterParticipantInCall(absl::string_view peer_name) {} // Tells analyzer that analysis complete and it should calculate final // statistics. diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc index 4426db4317..4355b1e1d7 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -186,7 +186,7 @@ uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured( MutexLock lock(&mutex_); stream_to_sender_[stream_index] = peer_index; frame_counters_.captured++; - for (size_t i = 0; i < peers_->size(); ++i) { + for (size_t i : peers_->GetAllIndexes()) { if (i != peer_index || options_.enable_receive_own_stream) { InternalStatsKey key(stream_index, peer_index, i); stream_frame_counters_[key].captured++; @@ -212,7 +212,7 @@ uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured( // If we overflow uint16_t and hit previous frame id and this frame is // still in flight, it means that this stream wasn't rendered for long // time and we need to process existing frame as dropped. - for (size_t i = 0; i < peers_->size(); ++i) { + for (size_t i : peers_->GetPresentIndexes()) { if (i == peer_index && !options_.enable_receive_own_stream) { continue; } @@ -274,17 +274,18 @@ void DefaultVideoQualityAnalyzer::OnFramePreEncode( << "DefaultVideoQualityAnalyzer has to be started before use"; auto it = captured_frames_in_flight_.find(frame.id()); - RTC_DCHECK(it != captured_frames_in_flight_.end()) + RTC_CHECK(it != captured_frames_in_flight_.end()) << "Frame id=" << frame.id() << " not found"; + FrameInFlight& frame_in_flight = it->second; frame_counters_.pre_encoded++; size_t peer_index = peers_->index(peer_name); - for (size_t i = 0; i < peers_->size(); ++i) { + for (size_t i : peers_->GetAllIndexes()) { if (i != peer_index || options_.enable_receive_own_stream) { - InternalStatsKey key(it->second.stream(), peer_index, i); + InternalStatsKey key(frame_in_flight.stream(), peer_index, i); stream_frame_counters_.at(key).pre_encoded++; } } - it->second.SetPreEncodeTime(Now()); + frame_in_flight.SetPreEncodeTime(Now()); } void DefaultVideoQualityAnalyzer::OnFrameEncoded( @@ -300,21 +301,23 @@ void DefaultVideoQualityAnalyzer::OnFrameEncoded( if (it == captured_frames_in_flight_.end()) { RTC_LOG(LS_WARNING) << "The encoding of video frame with id [" << frame_id << "] for peer [" - << peer_name << "] finished after all receivers rendered this frame. " - << "It can be OK for simulcast/SVC if higher quality stream is not " - << "required, but it may indicate an ERROR for singlecast or if it " - << "happens often."; + << peer_name << "] finished after all receivers rendered this frame or " + << "were removed. It can be OK for simulcast/SVC if higher quality " + << "stream is not required or the last receiver was unregistered " + << "between encoding of different layers, but it may indicate an ERROR " + << "for singlecast or if it happens often."; return; } + FrameInFlight& frame_in_flight = it->second; // For SVC we can receive multiple encoded images for one frame, so to cover // all cases we have to pick the last encode time. - if (!it->second.HasEncodedTime()) { + if (!frame_in_flight.HasEncodedTime()) { // Increase counters only when we meet this frame first time. frame_counters_.encoded++; size_t peer_index = peers_->index(peer_name); - for (size_t i = 0; i < peers_->size(); ++i) { + for (size_t i : peers_->GetAllIndexes()) { if (i != peer_index || options_.enable_receive_own_stream) { - InternalStatsKey key(it->second.stream(), peer_index, i); + InternalStatsKey key(frame_in_flight.stream(), peer_index, i); stream_frame_counters_.at(key).encoded++; } } @@ -326,9 +329,9 @@ void DefaultVideoQualityAnalyzer::OnFrameEncoded( used_encoder.last_frame_id = frame_id; used_encoder.switched_on_at = now; used_encoder.switched_from_at = now; - it->second.OnFrameEncoded(now, encoded_image._frameType, - DataSize::Bytes(encoded_image.size()), - stats.target_encode_bitrate, used_encoder); + frame_in_flight.OnFrameEncoded(now, encoded_image._frameType, + DataSize::Bytes(encoded_image.size()), + stats.target_encode_bitrate, used_encoder); } void DefaultVideoQualityAnalyzer::OnFrameDropped( @@ -558,9 +561,7 @@ void DefaultVideoQualityAnalyzer::RegisterParticipantInCall( // as well. Sending stats (from this peer to others) will be added by // DefaultVideoQualityAnalyzer::OnFrameCaptured. std::vector> stream_started_time; - for (auto& key_val : stream_to_sender_) { - size_t stream_index = key_val.first; - size_t sender_peer_index = key_val.second; + for (auto [stream_index, sender_peer_index] : stream_to_sender_) { InternalStatsKey key(stream_index, sender_peer_index, new_peer_index); // To initiate `FrameCounters` for the stream we should pick frame @@ -568,7 +569,7 @@ void DefaultVideoQualityAnalyzer::RegisterParticipantInCall( // and any receiver's peer index and copy from its sender side // counters. FrameCounters counters; - for (size_t i = 0; i < peers_->size(); ++i) { + for (size_t i : peers_->GetPresentIndexes()) { InternalStatsKey prototype_key(stream_index, sender_peer_index, i); auto it = stream_frame_counters_.find(prototype_key); if (it != stream_frame_counters_.end()) { @@ -589,16 +590,54 @@ void DefaultVideoQualityAnalyzer::RegisterParticipantInCall( start_time_); // Ensure, that frames states are handled correctly // (e.g. dropped frames tracking). - for (auto& key_val : stream_states_) { - key_val.second.AddPeer(new_peer_index); + for (auto& [stream_index, stream_state] : stream_states_) { + stream_state.AddPeer(new_peer_index); } // Register new peer for every frame in flight. // It is guaranteed, that no garbage FrameInFlight objects will stay in // memory because of adding new peer. Even if the new peer won't receive the // frame, the frame will be removed by OnFrameRendered after next frame comes // for the new peer. It is important because FrameInFlight is a large object. - for (auto& key_val : captured_frames_in_flight_) { - key_val.second.AddExpectedReceiver(new_peer_index); + for (auto& [frame_id, frame_in_flight] : captured_frames_in_flight_) { + frame_in_flight.AddExpectedReceiver(new_peer_index); + } +} + +void DefaultVideoQualityAnalyzer::UnregisterParticipantInCall( + absl::string_view peer_name) { + MutexLock lock(&mutex_); + RTC_CHECK(peers_->HasName(peer_name)); + absl::optional peer_index = peers_->RemoveIfPresent(peer_name); + RTC_CHECK(peer_index.has_value()); + + for (auto& [stream_index, stream_state] : stream_states_) { + if (!options_.enable_receive_own_stream && + peer_index == stream_state.sender()) { + continue; + } + stream_state.RemovePeer(*peer_index); + } + + // Remove peer from every frame in flight. If we removed that last expected + // receiver for the frame, then we should removed this frame if it was + // already encoded. If frame wasn't encoded, it still will be used by sender + // side pipeline, so we can't delete it yet. + for (auto it = captured_frames_in_flight_.begin(); + it != captured_frames_in_flight_.end();) { + FrameInFlight& frame_in_flight = it->second; + frame_in_flight.RemoveExpectedReceiver(*peer_index); + // If frame was fully sent and all receivers received it, then erase it. + // It may happen that when we remove FrameInFlight only some Simulcast/SVC + // layers were encoded and frame has encoded time, but more layers might be + // encoded after removal. In such case it's safe to still remove a frame, + // because OnFrameEncoded method will correctly handle the case when there + // is no FrameInFlight for the received encoded image. + if (frame_in_flight.HasEncodedTime() && + frame_in_flight.HaveAllPeersReceived()) { + it = captured_frames_in_flight_.erase(it); + } else { + it++; + } } } @@ -622,28 +661,31 @@ void DefaultVideoQualityAnalyzer::Stop() { for (auto& state_entry : stream_states_) { const size_t stream_index = state_entry.first; StreamState& stream_state = state_entry.second; - for (size_t i = 0; i < peers_->size(); ++i) { - if (i == stream_state.sender() && !options_.enable_receive_own_stream) { + for (size_t peer_index : peers_->GetPresentIndexes()) { + if (peer_index == stream_state.sender() && + !options_.enable_receive_own_stream) { continue; } - InternalStatsKey stats_key(stream_index, stream_state.sender(), i); + InternalStatsKey stats_key(stream_index, stream_state.sender(), + peer_index); // If there are no freezes in the call we have to report // time_between_freezes_ms as call duration and in such case // `stream_last_freeze_end_time` for this stream will be `start_time_`. // If there is freeze, then we need add time from last rendered frame // to last freeze end as time between freezes. - if (stream_state.last_rendered_frame_time(i)) { + if (stream_state.last_rendered_frame_time(peer_index)) { last_rendered_frame_times.emplace( - stats_key, stream_state.last_rendered_frame_time(i).value()); + stats_key, + stream_state.last_rendered_frame_time(peer_index).value()); } // Add frames in flight for this stream into frames comparator. // Frames in flight were not rendered, so they won't affect stream's // last rendered frame time. - while (!stream_state.IsEmpty(i)) { - uint16_t frame_id = stream_state.PopFront(i); + while (!stream_state.IsEmpty(peer_index)) { + uint16_t frame_id = stream_state.PopFront(peer_index); auto it = captured_frames_in_flight_.find(frame_id); RTC_DCHECK(it != captured_frames_in_flight_.end()); FrameInFlight& frame = it->second; @@ -651,7 +693,7 @@ void DefaultVideoQualityAnalyzer::Stop() { frames_comparator_.AddComparison( stats_key, /*captured=*/absl::nullopt, /*rendered=*/absl::nullopt, FrameComparisonType::kFrameInFlight, - frame.GetStatsForPeer(i)); + frame.GetStatsForPeer(peer_index)); if (frame.HaveAllPeersReceived()) { captured_frames_in_flight_.erase(it); diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h index 188588bd5c..5aec7c1e26 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h @@ -79,7 +79,10 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { void OnDecoderError(absl::string_view peer_name, uint16_t frame_id, int32_t error_code) override; + void RegisterParticipantInCall(absl::string_view peer_name) override; + void UnregisterParticipantInCall(absl::string_view peer_name) override; + void Stop() override; std::string GetStreamLabel(uint16_t frame_id) override; void OnStatsReports( diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc index 1d4d24a210..40662fd31d 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc @@ -110,6 +110,31 @@ void FakeCPULoad() { ASSERT_TRUE(std::is_sorted(temp.begin(), temp.end())); } +void PassFramesThroughAnalyzer(DefaultVideoQualityAnalyzer& analyzer, + absl::string_view sender, + absl::string_view stream_label, + std::vector receivers, + int frames_count, + test::FrameGeneratorInterface& frame_generator) { + for (int i = 0; i < frames_count; ++i) { + VideoFrame frame = NextFrame(&frame_generator, /*timestamp_us=*/1); + uint16_t frame_id = + analyzer.OnFrameCaptured(sender, std::string(stream_label), frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode(sender, frame); + analyzer.OnFrameEncoded(sender, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + for (absl::string_view receiver : receivers) { + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(receiver, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(receiver, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(receiver, received_frame); + } + } +} + TEST(DefaultVideoQualityAnalyzerTest, MemoryOverloadedAndThenAllFramesReceived) { std::unique_ptr frame_generator = @@ -1481,5 +1506,475 @@ TEST(DefaultVideoQualityAnalyzerTest, GetStreamFrames) { EXPECT_EQ(analyzer.GetStreamFrames(), stream_to_frame_ids); } +TEST(DefaultVideoQualityAnalyzerTest, ReceiverReceivedFramesWhenSenderRemoved) { + std::unique_ptr frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", std::vector{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + + analyzer.UnregisterParticipantInCall("alice"); + + analyzer.OnFramePreDecode("bob", frame.id(), FakeEncode(frame)); + analyzer.OnFrameDecoded("bob", DeepCopy(frame), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered("bob", DeepCopy(frame)); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 1); + EXPECT_EQ(stream_conters.decoded, 1); + EXPECT_EQ(stream_conters.rendered, 1); +} + +TEST(DefaultVideoQualityAnalyzerTest, + ReceiverReceivedFramesWhenSenderRemovedWithSelfview) { + std::unique_ptr frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.enable_receive_own_stream = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", std::vector{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + + analyzer.UnregisterParticipantInCall("alice"); + + analyzer.OnFramePreDecode("bob", frame.id(), FakeEncode(frame)); + analyzer.OnFrameDecoded("bob", DeepCopy(frame), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered("bob", DeepCopy(frame)); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 1); + EXPECT_EQ(stream_conters.decoded, 1); + EXPECT_EQ(stream_conters.rendered, 1); +} + +TEST(DefaultVideoQualityAnalyzerTest, + SenderReceivedFramesWhenReceiverRemovedWithSelfview) { + std::unique_ptr frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.enable_receive_own_stream = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", std::vector{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + + analyzer.UnregisterParticipantInCall("bob"); + + analyzer.OnFramePreDecode("alice", frame.id(), FakeEncode(frame)); + analyzer.OnFrameDecoded("alice", DeepCopy(frame), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered("alice", DeepCopy(frame)); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "alice")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 1); + EXPECT_EQ(stream_conters.decoded, 1); + EXPECT_EQ(stream_conters.rendered, 1); +} + +TEST(DefaultVideoQualityAnalyzerTest, + SenderAndReceiverReceivedFramesWhenReceiverRemovedWithSelfview) { + std::unique_ptr frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.enable_receive_own_stream = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", std::vector{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + + analyzer.OnFramePreDecode("bob", frame.id(), FakeEncode(frame)); + analyzer.OnFrameDecoded("bob", DeepCopy(frame), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered("bob", DeepCopy(frame)); + + analyzer.UnregisterParticipantInCall("bob"); + + analyzer.OnFramePreDecode("alice", frame.id(), FakeEncode(frame)); + analyzer.OnFrameDecoded("alice", DeepCopy(frame), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered("alice", DeepCopy(frame)); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters alice_alice_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "alice")); + EXPECT_EQ(alice_alice_stream_conters.captured, 1); + EXPECT_EQ(alice_alice_stream_conters.pre_encoded, 1); + EXPECT_EQ(alice_alice_stream_conters.encoded, 1); + EXPECT_EQ(alice_alice_stream_conters.received, 1); + EXPECT_EQ(alice_alice_stream_conters.decoded, 1); + EXPECT_EQ(alice_alice_stream_conters.rendered, 1); + + FrameCounters alice_bob_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(alice_bob_stream_conters.captured, 1); + EXPECT_EQ(alice_bob_stream_conters.pre_encoded, 1); + EXPECT_EQ(alice_bob_stream_conters.encoded, 1); + EXPECT_EQ(alice_bob_stream_conters.received, 1); + EXPECT_EQ(alice_bob_stream_conters.decoded, 1); + EXPECT_EQ(alice_bob_stream_conters.rendered, 1); +} + +TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforeCapturing2ndFrame) { + std::unique_ptr frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", std::vector{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"}, + /*frames_count=*/1, *frame_generator); + analyzer.UnregisterParticipantInCall("bob"); + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {}, + /*frames_count=*/1, *frame_generator); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 2); + EXPECT_EQ(global_stream_conters.pre_encoded, 2); + EXPECT_EQ(global_stream_conters.encoded, 2); + EXPECT_EQ(global_stream_conters.received, 1); + EXPECT_EQ(global_stream_conters.decoded, 1); + EXPECT_EQ(global_stream_conters.rendered, 1); + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 2); + EXPECT_EQ(stream_conters.pre_encoded, 2); + EXPECT_EQ(stream_conters.encoded, 2); + EXPECT_EQ(stream_conters.received, 1); + EXPECT_EQ(stream_conters.decoded, 1); + EXPECT_EQ(stream_conters.rendered, 1); +} + +TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforePreEncoded) { + std::unique_ptr frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", std::vector{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.UnregisterParticipantInCall("bob"); + analyzer.OnFramePreEncode("alice", frame); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 1); + EXPECT_EQ(global_stream_conters.pre_encoded, 1); + EXPECT_EQ(global_stream_conters.encoded, 1); + EXPECT_EQ(global_stream_conters.received, 0); + EXPECT_EQ(global_stream_conters.decoded, 0); + EXPECT_EQ(global_stream_conters.rendered, 0); + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 0); + EXPECT_EQ(stream_conters.decoded, 0); + EXPECT_EQ(stream_conters.rendered, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforeEncoded) { + std::unique_ptr frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", std::vector{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + analyzer.UnregisterParticipantInCall("bob"); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 1); + EXPECT_EQ(global_stream_conters.pre_encoded, 1); + EXPECT_EQ(global_stream_conters.encoded, 1); + EXPECT_EQ(global_stream_conters.received, 0); + EXPECT_EQ(global_stream_conters.decoded, 0); + EXPECT_EQ(global_stream_conters.rendered, 0); + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 0); + EXPECT_EQ(stream_conters.decoded, 0); + EXPECT_EQ(stream_conters.rendered, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, + ReceiverRemovedBetweenSimulcastLayersEncoded) { + std::unique_ptr frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", std::vector{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + // 1st simulcast layer encoded + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + analyzer.UnregisterParticipantInCall("bob"); + // 2nd simulcast layer encoded + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 1); + EXPECT_EQ(global_stream_conters.pre_encoded, 1); + EXPECT_EQ(global_stream_conters.encoded, 1); + EXPECT_EQ(global_stream_conters.received, 0); + EXPECT_EQ(global_stream_conters.decoded, 0); + EXPECT_EQ(global_stream_conters.rendered, 0); + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 0); + EXPECT_EQ(stream_conters.decoded, 0); + EXPECT_EQ(stream_conters.rendered, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, UnregisterOneAndRegisterAnother) { + std::unique_ptr frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", + std::vector{"alice", "bob", "charlie"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"bob", "charlie"}, + /*frames_count=*/2, *frame_generator); + analyzer.UnregisterParticipantInCall("bob"); + analyzer.RegisterParticipantInCall("david"); + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"charlie", "david"}, + /*frames_count=*/4, *frame_generator); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 6); + EXPECT_EQ(global_stream_conters.pre_encoded, 6); + EXPECT_EQ(global_stream_conters.encoded, 6); + EXPECT_EQ(global_stream_conters.received, 12); + EXPECT_EQ(global_stream_conters.decoded, 12); + EXPECT_EQ(global_stream_conters.rendered, 12); + FrameCounters alice_bob_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(alice_bob_stream_conters.captured, 6); + EXPECT_EQ(alice_bob_stream_conters.pre_encoded, 6); + EXPECT_EQ(alice_bob_stream_conters.encoded, 6); + EXPECT_EQ(alice_bob_stream_conters.received, 2); + EXPECT_EQ(alice_bob_stream_conters.decoded, 2); + EXPECT_EQ(alice_bob_stream_conters.rendered, 2); + FrameCounters alice_charlie_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "charlie")); + EXPECT_EQ(alice_charlie_stream_conters.captured, 6); + EXPECT_EQ(alice_charlie_stream_conters.pre_encoded, 6); + EXPECT_EQ(alice_charlie_stream_conters.encoded, 6); + EXPECT_EQ(alice_charlie_stream_conters.received, 6); + EXPECT_EQ(alice_charlie_stream_conters.decoded, 6); + EXPECT_EQ(alice_charlie_stream_conters.rendered, 6); + FrameCounters alice_david_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "david")); + EXPECT_EQ(alice_david_stream_conters.captured, 6); + EXPECT_EQ(alice_david_stream_conters.pre_encoded, 6); + EXPECT_EQ(alice_david_stream_conters.encoded, 6); + EXPECT_EQ(alice_david_stream_conters.received, 4); + EXPECT_EQ(alice_david_stream_conters.decoded, 4); + EXPECT_EQ(alice_david_stream_conters.rendered, 4); +} + +TEST(DefaultVideoQualityAnalyzerTest, + UnregisterOneAndRegisterAnotherRegisterBack) { + std::unique_ptr frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", + std::vector{"alice", "bob", "charlie"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"bob", "charlie"}, + /*frames_count=*/2, *frame_generator); + analyzer.UnregisterParticipantInCall("bob"); + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"charlie"}, + /*frames_count=*/4, *frame_generator); + analyzer.RegisterParticipantInCall("bob"); + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"bob", "charlie"}, + /*frames_count=*/6, *frame_generator); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 12); + EXPECT_EQ(global_stream_conters.pre_encoded, 12); + EXPECT_EQ(global_stream_conters.encoded, 12); + EXPECT_EQ(global_stream_conters.received, 20); + EXPECT_EQ(global_stream_conters.decoded, 20); + EXPECT_EQ(global_stream_conters.rendered, 20); + FrameCounters alice_bob_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(alice_bob_stream_conters.captured, 12); + EXPECT_EQ(alice_bob_stream_conters.pre_encoded, 12); + EXPECT_EQ(alice_bob_stream_conters.encoded, 12); + EXPECT_EQ(alice_bob_stream_conters.received, 8); + EXPECT_EQ(alice_bob_stream_conters.decoded, 8); + EXPECT_EQ(alice_bob_stream_conters.rendered, 8); + FrameCounters alice_charlie_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "charlie")); + EXPECT_EQ(alice_charlie_stream_conters.captured, 12); + EXPECT_EQ(alice_charlie_stream_conters.pre_encoded, 12); + EXPECT_EQ(alice_charlie_stream_conters.encoded, 12); + EXPECT_EQ(alice_charlie_stream_conters.received, 12); + EXPECT_EQ(alice_charlie_stream_conters.decoded, 12); + EXPECT_EQ(alice_charlie_stream_conters.rendered, 12); +} + } // namespace } // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/names_collection.cc b/test/pc/e2e/analyzer/video/names_collection.cc index 6ee2ef06ec..3ccab620f8 100644 --- a/test/pc/e2e/analyzer/video/names_collection.cc +++ b/test/pc/e2e/analyzer/video/names_collection.cc @@ -90,4 +90,12 @@ std::set NamesCollection::GetPresentIndexes() const { return out; } +std::set NamesCollection::GetAllIndexes() const { + std::set out; + for (size_t i = 0; i < names_.size(); ++i) { + out.insert(i); + } + return out; +} + } // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/names_collection.h b/test/pc/e2e/analyzer/video/names_collection.h index 4b4a439e35..e29bad1ad4 100644 --- a/test/pc/e2e/analyzer/video/names_collection.h +++ b/test/pc/e2e/analyzer/video/names_collection.h @@ -75,6 +75,10 @@ class NamesCollection { // collection. std::set GetPresentIndexes() const; + // Returns a set of all indexes known to the collection including indexes for + // names that were removed. + std::set GetAllIndexes() const; + private: std::vector names_; std::vector removed_;