From 8ef7da78b37d2996fad57739c658d0fb00b16dab Mon Sep 17 00:00:00 2001 From: Artem Titov Date: Mon, 22 Nov 2021 12:49:23 +0100 Subject: [PATCH] [DVQA] Remove sender from DVQA StatsKey Sender isn't actually require to identify the stream, so specifying it every time is useless. This CL removes sender from StatsKey object and introduces StreamsInfo object which contains all required metadata about streams that are seen by DVQA. Bug: b/205824594 Change-Id: I5b6be3865a30fd5980ff6e7e50906abe70a632ee Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/238562 Commit-Queue: Artem Titov Reviewed-by: Per Kjellander Reviewed-by: Mirko Bonadei Cr-Commit-Position: refs/heads/main@{#35399} --- .../video/default_video_quality_analyzer.cc | 53 ++++++++++++--- .../video/default_video_quality_analyzer.h | 3 +- ...t_video_quality_analyzer_shared_objects.cc | 64 +++++++++++++++++-- ...lt_video_quality_analyzer_shared_objects.h | 51 +++++++++++++-- .../default_video_quality_analyzer_test.cc | 49 ++++++++------ test/pc/e2e/peer_connection_e2e_smoke_test.cc | 3 +- 6 files changed, 181 insertions(+), 42 deletions(-) diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc index d3a8df49f1..ab5cb5461c 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -676,6 +676,40 @@ std::set DefaultVideoQualityAnalyzer::GetKnownVideoStreams() const { return out; } +VideoStreamsInfo DefaultVideoQualityAnalyzer::GetKnownStreams() const { + MutexLock lock(&mutex_); + std::map stream_to_sender; + std::map> sender_to_streams; + std::map> stream_to_receivers; + + for (auto& item : frames_comparator_.stream_stats()) { + const std::string& stream_label = streams_.name(item.first.stream); + const std::string& sender = peers_->name(item.first.sender); + const std::string& receiver = peers_->name(item.first.receiver); + RTC_LOG(LS_INFO) << item.first.ToString() << " ==> " + << "stream=" << stream_label << "; sender=" << sender + << "; receiver=" << receiver; + stream_to_sender.emplace(stream_label, sender); + auto streams_it = sender_to_streams.find(sender); + if (streams_it != sender_to_streams.end()) { + streams_it->second.emplace(stream_label); + } else { + sender_to_streams.emplace(sender, std::set{stream_label}); + } + auto receivers_it = stream_to_receivers.find(stream_label); + if (receivers_it != stream_to_receivers.end()) { + receivers_it->second.emplace(receiver); + } else { + stream_to_receivers.emplace(stream_label, + std::set{receiver}); + } + } + + return VideoStreamsInfo(std::move(stream_to_sender), + std::move(sender_to_streams), + std::move(stream_to_receivers)); +} + const FrameCounters& DefaultVideoQualityAnalyzer::GetGlobalCounters() const { MutexLock lock(&mutex_); return frame_counters_; @@ -710,8 +744,8 @@ void DefaultVideoQualityAnalyzer::ReportResults() { MutexLock lock(&mutex_); for (auto& item : frames_comparator_.stream_stats()) { - ReportResults(GetTestCaseName(StatsKeyToMetricName(ToStatsKey(item.first))), - item.second, stream_frame_counters_.at(item.first)); + ReportResults(GetTestCaseName(ToMetricName(item.first)), item.second, + stream_frame_counters_.at(item.first)); } test::PrintResult("cpu_usage", "", test_label_.c_str(), GetCpuUsagePercent(), "%", false, ImproveDirection::kSmallerIsBetter); @@ -855,16 +889,19 @@ Timestamp DefaultVideoQualityAnalyzer::Now() { StatsKey DefaultVideoQualityAnalyzer::ToStatsKey( const InternalStatsKey& key) const { - return StatsKey(streams_.name(key.stream), peers_->name(key.sender), - peers_->name(key.receiver)); + return StatsKey(streams_.name(key.stream), peers_->name(key.receiver)); } -std::string DefaultVideoQualityAnalyzer::StatsKeyToMetricName( - const StatsKey& key) const { +std::string DefaultVideoQualityAnalyzer::ToMetricName( + const InternalStatsKey& key) const { + const std::string& stream_label = streams_.name(key.stream); if (peers_->size() <= 2 && key.sender != key.receiver) { - return key.stream_label; + return stream_label; } - return key.ToString(); + rtc::StringBuilder out; + out << stream_label << "_" << peers_->name(key.sender) << "_" + << peers_->name(key.receiver); + return out.str(); } double DefaultVideoQualityAnalyzer::GetCpuUsagePercent() { diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h index 15b753d2e2..a2a885c8e7 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h @@ -82,6 +82,7 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { // Returns set of stream labels, that were met during test call. std::set GetKnownVideoStreams() const; + VideoStreamsInfo GetKnownStreams() const; const FrameCounters& GetGlobalCounters() const; // Returns frame counter per stream label. Valid stream labels can be obtained // by calling GetKnownVideoStreams() @@ -322,7 +323,7 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); // Returns string representation of stats key for metrics naming. Used for // backward compatibility by metrics naming for 2 peers cases. - std::string StatsKeyToMetricName(const StatsKey& key) const + std::string ToMetricName(const InternalStatsKey& key) const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); const DefaultVideoQualityAnalyzerOptions options_; diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc index 04b7296a1a..cc17429072 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc @@ -9,6 +9,9 @@ */ #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" +#include +#include + #include "api/units/timestamp.h" #include "rtc_base/checks.h" #include "rtc_base/strings/string_builder.h" @@ -47,7 +50,7 @@ StreamStats::StreamStats(Timestamp stream_started_time) std::string StatsKey::ToString() const { rtc::StringBuilder out; - out << stream_label << "_" << sender << "_" << receiver; + out << stream_label << "_" << receiver; return out.str(); } @@ -55,15 +58,64 @@ bool operator<(const StatsKey& a, const StatsKey& b) { if (a.stream_label != b.stream_label) { return a.stream_label < b.stream_label; } - if (a.sender != b.sender) { - return a.sender < b.sender; - } return a.receiver < b.receiver; } bool operator==(const StatsKey& a, const StatsKey& b) { - return a.stream_label == b.stream_label && a.sender == b.sender && - a.receiver == b.receiver; + return a.stream_label == b.stream_label && a.receiver == b.receiver; +} + +VideoStreamsInfo::VideoStreamsInfo( + std::map stream_to_sender, + std::map> sender_to_streams, + std::map> stream_to_receivers) + : stream_to_sender_(std::move(stream_to_sender)), + sender_to_streams_(std::move(sender_to_streams)), + stream_to_receivers_(std::move(stream_to_receivers)) {} + +std::set VideoStreamsInfo::GetStatsKeys() const { + std::set out; + for (const std::string& stream_label : GetStreams()) { + for (const std::string& receiver : GetReceivers(stream_label)) { + out.insert(StatsKey(stream_label, receiver)); + } + } + return out; +} + +std::set VideoStreamsInfo::GetStreams() const { + std::set out; + std::transform(stream_to_sender_.begin(), stream_to_sender_.end(), + std::inserter(out, out.end()), + [](auto map_entry) { return map_entry.first; }); + return out; +} + +std::set VideoStreamsInfo::GetStreams( + absl::string_view sender_name) const { + auto it = sender_to_streams_.find(std::string(sender_name)); + if (it == sender_to_streams_.end()) { + return {}; + } + return it->second; +} + +absl::optional VideoStreamsInfo::GetSender( + absl::string_view stream_label) const { + auto it = stream_to_sender_.find(std::string(stream_label)); + if (it == stream_to_sender_.end()) { + return absl::nullopt; + } + return it->second; +} + +std::set VideoStreamsInfo::GetReceivers( + absl::string_view stream_label) const { + auto it = stream_to_receivers_.find(std::string(stream_label)); + if (it == stream_to_receivers_.end()) { + return {}; + } + return it->second; } } // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h index a089efd215..4f12aa729b 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -156,17 +157,19 @@ struct AnalyzerStats { }; struct StatsKey { - StatsKey(std::string stream_label, std::string sender, std::string receiver) - : stream_label(std::move(stream_label)), - sender(std::move(sender)), - receiver(std::move(receiver)) {} + // Keep this constructor for temporary backward compatibility. + StatsKey(std::string stream_label, + std::string /*sender*/, + std::string receiver) + : stream_label(std::move(stream_label)), receiver(std::move(receiver)) {} + + StatsKey(std::string stream_label, std::string receiver) + : stream_label(std::move(stream_label)), receiver(std::move(receiver)) {} std::string ToString() const; // Label of video stream to which stats belongs to. std::string stream_label; - // Name of the peer which send this stream. - std::string sender; // Name of the peer on which stream was received. std::string receiver; }; @@ -175,6 +178,42 @@ struct StatsKey { bool operator<(const StatsKey& a, const StatsKey& b); bool operator==(const StatsKey& a, const StatsKey& b); +// Contains all metadata related to the video streams that were seen by the +// video analyzer. +class VideoStreamsInfo { + public: + std::set GetStatsKeys() const; + + // Returns all stream labels that are known to the video analyzer. + std::set GetStreams() const; + + // Returns set of the stream for specified `sender_name`. If sender didn't + // send any streams or `sender_name` isn't known to the video analyzer + // empty set will be returned. + std::set GetStreams(absl::string_view sender_name) const; + + // Returns sender name for specified `stream_label`. Returns `absl::nullopt` + // if provided `stream_label` isn't known to the video analyzer. + absl::optional GetSender(absl::string_view stream_label) const; + + // Returns set of the receivers for specified `stream_label`. If stream wasn't + // received by any peer or `stream_label` isn't known to the video analyzer + // empty set will be returned. + std::set GetReceivers(absl::string_view stream_label) const; + + protected: + friend class DefaultVideoQualityAnalyzer; + VideoStreamsInfo( + std::map stream_to_sender, + std::map> sender_to_streams, + std::map> stream_to_receivers); + + private: + std::map stream_to_sender_; + std::map> sender_to_streams_; + std::map> stream_to_receivers_; +}; + struct DefaultVideoQualityAnalyzerOptions { // Tells DefaultVideoQualityAnalyzer if heavy metrics like PSNR and SSIM have // to be computed or not. diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc index 792aa33160..d5bc2cd3f2 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc @@ -472,8 +472,17 @@ TEST(DefaultVideoQualityAnalyzerTest, NormalScenario2Receivers) { EXPECT_EQ(frame_counters.decoded, kMaxFramesInFlightPerStream); EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream); EXPECT_EQ(frame_counters.dropped, kMaxFramesInFlightPerStream); - EXPECT_EQ(analyzer.GetKnownVideoStreams().size(), 2lu); - for (auto stream_key : analyzer.GetKnownVideoStreams()) { + + VideoStreamsInfo streams_info = analyzer.GetKnownStreams(); + EXPECT_EQ(streams_info.GetStreams(), std::set{kStreamLabel}); + EXPECT_EQ(streams_info.GetStreams(kAlice), + std::set{kStreamLabel}); + EXPECT_EQ(streams_info.GetSender(kStreamLabel), kAlice); + EXPECT_EQ(streams_info.GetReceivers(kStreamLabel), + (std::set{kBob, kCharlie})); + + EXPECT_EQ(streams_info.GetStatsKeys().size(), 2lu); + for (auto stream_key : streams_info.GetStatsKeys()) { FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(stream_key); // On some devices the pipeline can be too slow, so we actually can't @@ -489,8 +498,8 @@ TEST(DefaultVideoQualityAnalyzerTest, NormalScenario2Receivers) { } std::map stats = analyzer.GetStats(); - const StatsKey kAliceBobStats(kStreamLabel, kAlice, kBob); - const StatsKey kAliceCharlieStats(kStreamLabel, kAlice, kCharlie); + const StatsKey kAliceBobStats(kStreamLabel, kBob); + const StatsKey kAliceCharlieStats(kStreamLabel, kCharlie); EXPECT_EQ(stats.size(), 2lu); { auto it = stats.find(kAliceBobStats); @@ -629,8 +638,7 @@ TEST(DefaultVideoQualityAnalyzerTest, HeavyQualityMetricsFromEqualFrames) { << ToString(frames_in_flight_sizes); std::map stream_stats = analyzer.GetStats(); - const StatsKey kAliceBobStats(kStreamLabel, kSenderPeerName, - kReceiverPeerName); + const StatsKey kAliceBobStats(kStreamLabel, kReceiverPeerName); EXPECT_EQ(stream_stats.size(), 1lu); auto it = stream_stats.find(kAliceBobStats); @@ -694,8 +702,7 @@ TEST(DefaultVideoQualityAnalyzerTest, << ToString(frames_in_flight_sizes); std::map stream_stats = analyzer.GetStats(); - const StatsKey kAliceBobStats(kStreamLabel, kSenderPeerName, - kReceiverPeerName); + const StatsKey kAliceBobStats(kStreamLabel, kReceiverPeerName); EXPECT_EQ(stream_stats.size(), 1lu); auto it = stream_stats.find(kAliceBobStats); @@ -872,10 +879,12 @@ TEST(DefaultVideoQualityAnalyzerTest, RuntimeParticipantsAdding) { EXPECT_EQ(frame_counters.rendered, 2 * kFramesCount); EXPECT_EQ(frame_counters.dropped, kOneThirdFrames); - EXPECT_EQ(analyzer.GetKnownVideoStreams().size(), 3lu); - const StatsKey kAliceBobStats(kStreamLabel, kAlice, kBob); - const StatsKey kAliceCharlieStats(kStreamLabel, kAlice, kCharlie); - const StatsKey kAliceKatieStats(kStreamLabel, kAlice, kKatie); + const StatsKey kAliceBobStats(kStreamLabel, kBob); + const StatsKey kAliceCharlieStats(kStreamLabel, kCharlie); + const StatsKey kAliceKatieStats(kStreamLabel, kKatie); + EXPECT_EQ(analyzer.GetKnownStreams().GetStatsKeys(), + (std::set{kAliceBobStats, kAliceCharlieStats, + kAliceKatieStats})); { FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(kAliceBobStats); @@ -1044,7 +1053,7 @@ TEST(DefaultVideoQualityAnalyzerTest, EXPECT_EQ(analyzer.GetStats().size(), 2lu); { FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( - StatsKey(kStreamLabel, kSenderPeerName, kReceiverPeerName)); + StatsKey(kStreamLabel, kReceiverPeerName)); EXPECT_EQ(stream_conters.captured, 3); EXPECT_EQ(stream_conters.pre_encoded, 3); EXPECT_EQ(stream_conters.encoded, 3); @@ -1054,7 +1063,7 @@ TEST(DefaultVideoQualityAnalyzerTest, } { FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( - StatsKey(kStreamLabel, kSenderPeerName, kSenderPeerName)); + StatsKey(kStreamLabel, kSenderPeerName)); EXPECT_EQ(stream_conters.captured, 3); EXPECT_EQ(stream_conters.pre_encoded, 3); EXPECT_EQ(stream_conters.encoded, 3); @@ -1139,7 +1148,7 @@ TEST(DefaultVideoQualityAnalyzerTest, EXPECT_EQ(analyzer.GetStats().size(), 2lu); { FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( - StatsKey(kStreamLabel, kSenderPeerName, kReceiverPeerName)); + StatsKey(kStreamLabel, kReceiverPeerName)); EXPECT_EQ(stream_conters.captured, 3); EXPECT_EQ(stream_conters.pre_encoded, 3); EXPECT_EQ(stream_conters.encoded, 3); @@ -1149,7 +1158,7 @@ TEST(DefaultVideoQualityAnalyzerTest, } { FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( - StatsKey(kStreamLabel, kSenderPeerName, kSenderPeerName)); + StatsKey(kStreamLabel, kSenderPeerName)); EXPECT_EQ(stream_conters.captured, 3); EXPECT_EQ(stream_conters.pre_encoded, 3); EXPECT_EQ(stream_conters.encoded, 3); @@ -1210,7 +1219,7 @@ TEST(DefaultVideoQualityAnalyzerTest, CodecTrackedCorrectly) { std::map stats = analyzer.GetStats(); ASSERT_EQ(stats.size(), 1lu); const StreamStats& stream_stats = - stats.at(StatsKey(kStreamLabel, kSenderPeerName, kReceiverPeerName)); + stats.at(StatsKey(kStreamLabel, kReceiverPeerName)); ASSERT_EQ(stream_stats.encoders.size(), 2lu); EXPECT_EQ(stream_stats.encoders[0].codec_name, codec_names[0]); EXPECT_EQ(stream_stats.encoders[0].first_frame_id, frames[0].id()); @@ -1308,7 +1317,7 @@ TEST(DefaultVideoQualityAnalyzerTest, EXPECT_EQ(analyzer.GetStats().size(), 1lu); { FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( - StatsKey(kStreamLabel, kSenderPeerName, kReceiverPeerName)); + StatsKey(kStreamLabel, kReceiverPeerName)); EXPECT_EQ(stream_conters.captured, 6); EXPECT_EQ(stream_conters.pre_encoded, 5); EXPECT_EQ(stream_conters.encoded, 4); @@ -1405,7 +1414,7 @@ TEST( EXPECT_EQ(analyzer.GetStats().size(), 2lu); { FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( - StatsKey(kStreamLabel, kSenderPeerName, kReceiverPeerName)); + StatsKey(kStreamLabel, kReceiverPeerName)); EXPECT_EQ(stream_conters.captured, 6); EXPECT_EQ(stream_conters.pre_encoded, 5); EXPECT_EQ(stream_conters.encoded, 4); @@ -1415,7 +1424,7 @@ TEST( } { FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( - StatsKey(kStreamLabel, kSenderPeerName, kSenderPeerName)); + StatsKey(kStreamLabel, kSenderPeerName)); EXPECT_EQ(stream_conters.captured, 6); EXPECT_EQ(stream_conters.pre_encoded, 5); EXPECT_EQ(stream_conters.encoded, 4); diff --git a/test/pc/e2e/peer_connection_e2e_smoke_test.cc b/test/pc/e2e/peer_connection_e2e_smoke_test.cc index cb34fa0ed7..d9c268570c 100644 --- a/test/pc/e2e/peer_connection_e2e_smoke_test.cc +++ b/test/pc/e2e/peer_connection_e2e_smoke_test.cc @@ -93,7 +93,8 @@ class PeerConnectionE2EQualityTestSmokeTest : public ::testing::Test { fixture_->Run(run_params); EXPECT_GE(fixture_->GetRealTestDuration(), run_params.run_duration); - for (auto stream_key : video_quality_analyzer_->GetKnownVideoStreams()) { + VideoStreamsInfo known_streams = video_quality_analyzer_->GetKnownStreams(); + for (const StatsKey& stream_key : known_streams.GetStatsKeys()) { FrameCounters stream_conters = video_quality_analyzer_->GetPerStreamCounters().at(stream_key); // On some devices the pipeline can be too slow, so we actually can't