From 647d3264382979b48e9f82e070627c9cb48956c2 Mon Sep 17 00:00:00 2001 From: Artem Titov Date: Wed, 11 Aug 2021 18:27:36 +0200 Subject: [PATCH] Add tracking of video encoder/decoder used for stream in DVQA Bug: b/196035476 Change-Id: I882f2236c9522f06ad60332ab2a4bb9226b1bd8e Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/228423 Reviewed-by: Mirko Bonadei Reviewed-by: Ilya Nikolaevskiy Commit-Queue: Artem Titov Cr-Commit-Position: refs/heads/master@{#34732} --- api/test/video_quality_analyzer_interface.h | 6 +- .../video/default_video_quality_analyzer.cc | 132 +++++++++++++++--- .../video/default_video_quality_analyzer.h | 53 ++++++- .../default_video_quality_analyzer_test.cc | 69 +++++++++ .../video/quality_analyzing_video_decoder.cc | 27 ++-- .../video/quality_analyzing_video_decoder.h | 12 +- .../video/quality_analyzing_video_encoder.cc | 89 ++++++------ .../video/quality_analyzing_video_encoder.h | 14 +- 8 files changed, 316 insertions(+), 86 deletions(-) diff --git a/api/test/video_quality_analyzer_interface.h b/api/test/video_quality_analyzer_interface.h index c8c7094c25..86e969d57c 100644 --- a/api/test/video_quality_analyzer_interface.h +++ b/api/test/video_quality_analyzer_interface.h @@ -57,16 +57,18 @@ class VideoQualityAnalyzerInterface : public StatsObserverInterface { public: // Contains extra statistic provided by video encoder. struct EncoderStats { + std::string encoder_name = "unknown"; // TODO(hbos) https://crbug.com/webrtc/9547, // https://crbug.com/webrtc/11443: improve stats API to make available // there. - uint32_t target_encode_bitrate; + uint32_t target_encode_bitrate = 0; }; // Contains extra statistic provided by video decoder. struct DecoderStats { + std::string decoder_name = "unknown"; // Decode time provided by decoder itself. If decoder doesn’t produce such // information can be omitted. - absl::optional decode_time_ms; + absl::optional decode_time_ms = absl::nullopt; }; ~VideoQualityAnalyzerInterface() override = default; diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc index ad8ba1fd0d..d382f918bb 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -45,11 +45,47 @@ void LogFrameCounters(const std::string& name, const FrameCounters& counters) { RTC_LOG(INFO) << "[" << name << "] Dropped : " << counters.dropped; } -void LogStreamInternalStats(const std::string& name, const StreamStats& stats) { +void LogStreamInternalStats(const std::string& name, + const StreamStats& stats, + Timestamp start_time) { RTC_LOG(INFO) << "[" << name << "] Dropped by encoder : " << stats.dropped_by_encoder; RTC_LOG(INFO) << "[" << name << "] Dropped before encoder : " << stats.dropped_before_encoder; + Timestamp first_encoded_frame_time = Timestamp::PlusInfinity(); + for (const StreamCodecInfo& encoder : stats.encoders) { + RTC_DCHECK(encoder.switched_on_at.IsFinite()); + RTC_DCHECK(encoder.switched_from_at.IsFinite()); + if (first_encoded_frame_time.IsInfinite()) { + first_encoded_frame_time = encoder.switched_on_at; + } + RTC_LOG(INFO) << "[" << name << "] Used encoder: \"" << encoder.codec_name + << "\" used from (frame_id=" << encoder.first_frame_id + << "; from_stream_start=" + << (encoder.switched_on_at - stats.stream_started_time).ms() + << "ms, from_call_start=" + << (encoder.switched_on_at - start_time).ms() + << "ms) until (frame_id=" << encoder.last_frame_id + << "; from_stream_start=" + << (encoder.switched_from_at - stats.stream_started_time).ms() + << "ms, from_call_start=" + << (encoder.switched_from_at - start_time).ms() << "ms)"; + } + for (const StreamCodecInfo& decoder : stats.decoders) { + RTC_DCHECK(decoder.switched_on_at.IsFinite()); + RTC_DCHECK(decoder.switched_from_at.IsFinite()); + RTC_LOG(INFO) << "[" << name << "] Used decoder: \"" << decoder.codec_name + << "\" used from (frame_id=" << decoder.first_frame_id + << "; from_stream_start=" + << (decoder.switched_on_at - stats.stream_started_time).ms() + << "ms, from_call_start=" + << (decoder.switched_on_at - start_time).ms() + << "ms) until (frame_id=" << decoder.last_frame_id + << "; from_stream_start=" + << (decoder.switched_from_at - stats.stream_started_time).ms() + << "ms, from_call_start=" + << (decoder.switched_from_at - start_time).ms() << "ms)"; + } } template @@ -163,6 +199,7 @@ uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured( const webrtc::VideoFrame& frame) { // `next_frame_id` is atomic, so we needn't lock here. uint16_t frame_id = next_frame_id_++; + Timestamp captured_time = Now(); Timestamp start_time = Timestamp::MinusInfinity(); size_t peer_index = -1; size_t peers_count = -1; @@ -185,7 +222,7 @@ uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured( } InternalStatsKey stats_key(stream_index, peer_index, i); if (stream_stats_.find(stats_key) == stream_stats_.end()) { - stream_stats_.insert({stats_key, StreamStats()}); + stream_stats_.insert({stats_key, StreamStats(captured_time)}); // Assume that the first freeze was before first stream frame captured. // This way time before the first freeze would be counted as time // between freezes. @@ -213,9 +250,10 @@ uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured( auto state_it = stream_states_.find(stream_index); if (state_it == stream_states_.end()) { - stream_states_.emplace(stream_index, - StreamState(peer_index, peers_->size(), - options_.enable_receive_own_stream)); + stream_states_.emplace( + stream_index, + StreamState(peer_index, peers_->size(), + options_.enable_receive_own_stream, captured_time)); } StreamState* state = &stream_states_.at(stream_index); state->PushBack(frame_id); @@ -248,9 +286,8 @@ uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured( } captured_frames_in_flight_.emplace( frame_id, - FrameInFlight(stream_index, frame, - /*captured_time=*/Now(), peer_index, peers_->size(), - options_.enable_receive_own_stream)); + FrameInFlight(stream_index, frame, captured_time, peer_index, + peers_->size(), options_.enable_receive_own_stream)); // Set frame id on local copy of the frame captured_frames_in_flight_.at(frame_id).SetFrameId(frame_id); @@ -326,8 +363,15 @@ void DefaultVideoQualityAnalyzer::OnFrameEncoded( } } } - it->second.OnFrameEncoded(Now(), encoded_image.size(), - stats.target_encode_bitrate); + Timestamp now = Now(); + StreamCodecInfo used_encoder; + used_encoder.codec_name = stats.encoder_name; + used_encoder.first_frame_id = frame_id; + used_encoder.last_frame_id = frame_id; + used_encoder.switched_on_at = now; + used_encoder.switched_from_at = now; + it->second.OnFrameEncoded(now, encoded_image.size(), + stats.target_encode_bitrate, used_encoder); } void DefaultVideoQualityAnalyzer::OnFrameDropped( @@ -393,7 +437,14 @@ void DefaultVideoQualityAnalyzer::OnFrameDecoded( InternalStatsKey key(it->second.stream(), stream_to_sender_.at(it->second.stream()), peer_index); stream_frame_counters_.at(key).decoded++; - it->second.SetDecodeEndTime(peer_index, Now()); + Timestamp now = Now(); + StreamCodecInfo used_decoder; + used_decoder.codec_name = stats.decoder_name; + used_decoder.first_frame_id = frame.id(); + used_decoder.last_frame_id = frame.id(); + used_decoder.switched_on_at = now; + used_decoder.switched_from_at = now; + it->second.OnFrameDecoded(peer_index, now, used_decoder); } void DefaultVideoQualityAnalyzer::OnFrameRendered( @@ -540,7 +591,9 @@ void DefaultVideoQualityAnalyzer::RegisterParticipantInCall( // then `counters` will be empty. In such case empty `counters` are ok. stream_frame_counters_.insert({key, std::move(counters)}); - stream_stats_.insert({key, StreamStats()}); + stream_stats_.insert( + {key, + StreamStats(stream_states_.at(stream_index).stream_started_time())}); stream_last_freeze_end_time_.insert({key, start_time_}); } // Ensure, that frames states are handled correctly @@ -549,7 +602,7 @@ void DefaultVideoQualityAnalyzer::RegisterParticipantInCall( key_val.second.AddPeer(); } // Register new peer for every frame in flight. - // It is guaranteed, that no garbadge FrameInFlight objects will stay in + // It is guaranteed, that no garbage FrameInFlight objects will stay in // memory because of adding new peer. Even if the new peer won't receive the // frame, the frame will be removed by OnFrameRendered after next frame comes // for the new peer. It is important because FrameInFlight is a large object. @@ -594,7 +647,7 @@ void DefaultVideoQualityAnalyzer::Stop() { // If there is freeze, then we need add time from last rendered frame // to last freeze end as time between freezes. if (stream_state.last_rendered_frame_time(i)) { - stream_stats_[stats_key].time_between_freezes_ms.AddSample( + stream_stats_.at(stats_key).time_between_freezes_ms.AddSample( StatsSample( stream_state.last_rendered_frame_time(i).value().ms() - stream_last_freeze_end_time_.at(stats_key).ms(), @@ -825,6 +878,28 @@ void DefaultVideoQualityAnalyzer::ProcessComparison( } } } + // Compute stream codec info. + if (frame_stats.used_encoder.has_value()) { + if (stats->encoders.empty() || stats->encoders.back().codec_name != + frame_stats.used_encoder->codec_name) { + stats->encoders.push_back(*frame_stats.used_encoder); + } + stats->encoders.back().last_frame_id = + frame_stats.used_encoder->last_frame_id; + stats->encoders.back().switched_from_at = + frame_stats.used_encoder->switched_from_at; + } + + if (frame_stats.used_decoder.has_value()) { + if (stats->decoders.empty() || stats->decoders.back().codec_name != + frame_stats.used_decoder->codec_name) { + stats->decoders.push_back(*frame_stats.used_decoder); + } + stats->decoders.back().last_frame_id = + frame_stats.used_decoder->last_frame_id; + stats->decoders.back().switched_from_at = + frame_stats.used_decoder->switched_from_at; + } } void DefaultVideoQualityAnalyzer::ReportResults() { @@ -842,7 +917,8 @@ void DefaultVideoQualityAnalyzer::ReportResults() { for (auto& item : stream_stats_) { LogFrameCounters(ToStatsKey(item.first).ToString(), stream_frame_counters_.at(item.first)); - LogStreamInternalStats(ToStatsKey(item.first).ToString(), item.second); + LogStreamInternalStats(ToStatsKey(item.first).ToString(), item.second, + start_time_); } if (!analyzer_stats_.comparisons_queue_size.IsEmpty()) { RTC_LOG(INFO) << "comparisons_queue_size min=" @@ -1131,10 +1207,24 @@ bool DefaultVideoQualityAnalyzer::FrameInFlight::HaveAllPeersReceived() const { void DefaultVideoQualityAnalyzer::FrameInFlight::OnFrameEncoded( webrtc::Timestamp time, int64_t encoded_image_size, - uint32_t target_encode_bitrate) { + uint32_t target_encode_bitrate, + StreamCodecInfo used_encoder) { encoded_time_ = time; encoded_image_size_ = encoded_image_size; target_encode_bitrate_ += target_encode_bitrate; + // Update used encoder info. If simulcast/SVC is used, this method can + // be called multiple times, in such case we should preserve the value + // of `used_encoder_.switched_on_at` from the first invocation as the + // smallest one. + Timestamp encoder_switched_on_at = used_encoder_.has_value() + ? used_encoder_->switched_on_at + : Timestamp::PlusInfinity(); + RTC_DCHECK(used_encoder.switched_on_at.IsFinite()); + RTC_DCHECK(used_encoder.switched_from_at.IsFinite()); + used_encoder_ = used_encoder; + if (encoder_switched_on_at < used_encoder_->switched_on_at) { + used_encoder_->switched_on_at = encoder_switched_on_at; + } } void DefaultVideoQualityAnalyzer::FrameInFlight::OnFramePreDecode( @@ -1154,6 +1244,14 @@ bool DefaultVideoQualityAnalyzer::FrameInFlight::HasReceivedTime( return it->second.received_time.IsFinite(); } +void DefaultVideoQualityAnalyzer::FrameInFlight::OnFrameDecoded( + size_t peer, + webrtc::Timestamp time, + StreamCodecInfo used_decoder) { + receiver_stats_[peer].decode_end_time = time; + receiver_stats_[peer].used_decoder = used_decoder; +} + bool DefaultVideoQualityAnalyzer::FrameInFlight::HasDecodeEndTime( size_t peer) const { auto it = receiver_stats_.find(peer); @@ -1189,6 +1287,7 @@ DefaultVideoQualityAnalyzer::FrameInFlight::GetStatsForPeer(size_t peer) const { stats.encoded_time = encoded_time_; stats.target_encode_bitrate = target_encode_bitrate_; stats.encoded_image_size = encoded_image_size_; + stats.used_encoder = used_encoder_; absl::optional receiver_stats = MaybeGetValue(receiver_stats_, peer); @@ -1200,6 +1299,7 @@ DefaultVideoQualityAnalyzer::FrameInFlight::GetStatsForPeer(size_t peer) const { stats.prev_frame_rendered_time = receiver_stats->prev_frame_rendered_time; stats.rendered_frame_width = receiver_stats->rendered_frame_width; stats.rendered_frame_height = receiver_stats->rendered_frame_height; + stats.used_decoder = receiver_stats->used_decoder; } return stats; } diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h index 1461dd7375..b853551c7e 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h @@ -75,7 +75,28 @@ struct FrameCounters { int64_t dropped = 0; }; +// Contains information about the codec that was used for encoding or decoding +// the stream. +struct StreamCodecInfo { + // Codec implementation name. + std::string codec_name; + // Id of the first frame for which this codec was used. + uint16_t first_frame_id; + // Id of the last frame for which this codec was used. + uint16_t last_frame_id; + // Timestamp when the first frame was handled by the encode/decoder. + Timestamp switched_on_at = Timestamp::PlusInfinity(); + // Timestamp when this codec was used last time. + Timestamp switched_from_at = Timestamp::PlusInfinity(); +}; + struct StreamStats { + explicit StreamStats(Timestamp stream_started_time) + : stream_started_time(stream_started_time) {} + + // The time when the first frame of this stream was captured. + Timestamp stream_started_time; + SamplesStatsCounter psnr; SamplesStatsCounter ssim; // Time from frame encoded (time point on exit from encoder) to the @@ -108,6 +129,11 @@ struct StreamStats { int64_t total_encoded_images_payload = 0; int64_t dropped_by_encoder = 0; int64_t dropped_before_encoder = 0; + + // Vector of encoders used for this stream by sending client. + std::vector encoders; + // Vectors of decoders used for this stream by receiving client. + std::vector decoders; }; struct AnalyzerStats { @@ -243,6 +269,8 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { double GetCpuUsagePercent(); private: + // Final stats computed for frame after it went through the whole video + // pipeline from capturing to rendering or dropping. struct FrameStats { FrameStats(Timestamp captured_time) : captured_time(captured_time) {} @@ -262,6 +290,11 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { absl::optional rendered_frame_width = absl::nullopt; absl::optional rendered_frame_height = absl::nullopt; + + // Can be not set if frame was dropped by encoder. + absl::optional used_encoder = absl::nullopt; + // Can be not set if frame was dropped in the network. + absl::optional used_decoder = absl::nullopt; }; // Describes why comparison was done in overloaded mode (without calculating @@ -308,12 +341,15 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { public: StreamState(size_t owner, size_t peers_count, - bool enable_receive_own_stream) + bool enable_receive_own_stream, + Timestamp stream_started_time) : owner_(owner), enable_receive_own_stream_(enable_receive_own_stream), + stream_started_time_(stream_started_time), frame_ids_(peers_count) {} size_t owner() const { return owner_; } + Timestamp stream_started_time() const { return stream_started_time_; } void PushBack(uint16_t frame_id) { frame_ids_.PushBack(frame_id); } // Crash if state is empty. Guarantees that there can be no alive frames @@ -338,6 +374,7 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { // Index of the owner. Owner's queue in `frame_ids_` will keep alive frames. const size_t owner_; const bool enable_receive_own_stream_; + const Timestamp stream_started_time_; // To correctly determine dropped frames we have to know sequence of frames // in each stream so we will keep a list of frame ids inside the stream. // This list is represented by multi head queue of frame ids with separate @@ -371,6 +408,9 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { absl::optional rendered_frame_width = absl::nullopt; absl::optional rendered_frame_height = absl::nullopt; + // Can be not set if frame was dropped in the network. + absl::optional used_decoder = absl::nullopt; + bool dropped = false; }; @@ -404,7 +444,8 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { void OnFrameEncoded(webrtc::Timestamp time, int64_t encoded_image_size, - uint32_t target_encode_bitrate); + uint32_t target_encode_bitrate, + StreamCodecInfo used_encoder); bool HasEncodedTime() const { return encoded_time_.IsFinite(); } @@ -414,9 +455,9 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { bool HasReceivedTime(size_t peer) const; - void SetDecodeEndTime(size_t peer, webrtc::Timestamp time) { - receiver_stats_[peer].decode_end_time = time; - } + void OnFrameDecoded(size_t peer, + webrtc::Timestamp time, + StreamCodecInfo used_decoder); bool HasDecodeEndTime(size_t peer) const; @@ -453,6 +494,8 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { Timestamp encoded_time_ = Timestamp::MinusInfinity(); int64_t encoded_image_size_ = 0; uint32_t target_encode_bitrate_ = 0; + // Can be not set if frame was dropped by encoder. + absl::optional used_encoder_ = absl::nullopt; std::map receiver_stats_; }; diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc index d336cd76e8..edf53e7f31 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc @@ -1049,6 +1049,75 @@ TEST(DefaultVideoQualityAnalyzerTest, FrameCanBeReceivedBySender) { } } +TEST(DefaultVideoQualityAnalyzerTest, CodecTrackedCorrectly) { + std::unique_ptr frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + VideoQualityAnalyzerInterface::EncoderStats encoder_stats; + std::vector codec_names = {"codec_1", "codec_2"}; + std::vector frames; + // Send 3 frame for each codec. + for (size_t i = 0; i < codec_names.size(); ++i) { + for (size_t j = 0; j < 3; ++j) { + VideoFrame frame = NextFrame(frame_generator.get(), 3 * i + j); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + encoder_stats.encoder_name = codec_names[i]; + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + encoder_stats); + frames.push_back(std::move(frame)); + } + } + + // Receive 3 frame for each codec. + VideoQualityAnalyzerInterface::DecoderStats decoder_stats; + for (size_t i = 0; i < codec_names.size(); ++i) { + for (size_t j = 0; j < 3; ++j) { + VideoFrame received_frame = DeepCopy(frames[3 * i + j]); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + decoder_stats.decoder_name = codec_names[i]; + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, decoder_stats); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + std::map stats = analyzer.GetStats(); + ASSERT_EQ(stats.size(), 1lu); + const StreamStats& stream_stats = + stats.at(StatsKey(kStreamLabel, kSenderPeerName, kReceiverPeerName)); + ASSERT_EQ(stream_stats.encoders.size(), 2lu); + EXPECT_EQ(stream_stats.encoders[0].codec_name, codec_names[0]); + EXPECT_EQ(stream_stats.encoders[0].first_frame_id, frames[0].id()); + EXPECT_EQ(stream_stats.encoders[0].last_frame_id, frames[2].id()); + EXPECT_EQ(stream_stats.encoders[1].codec_name, codec_names[1]); + EXPECT_EQ(stream_stats.encoders[1].first_frame_id, frames[3].id()); + EXPECT_EQ(stream_stats.encoders[1].last_frame_id, frames[5].id()); + + ASSERT_EQ(stream_stats.decoders.size(), 2lu); + EXPECT_EQ(stream_stats.decoders[0].codec_name, codec_names[0]); + EXPECT_EQ(stream_stats.decoders[0].first_frame_id, frames[0].id()); + EXPECT_EQ(stream_stats.decoders[0].last_frame_id, frames[2].id()); + EXPECT_EQ(stream_stats.decoders[1].codec_name, codec_names[1]); + EXPECT_EQ(stream_stats.decoders[1].first_frame_id, frames[3].id()); + EXPECT_EQ(stream_stats.decoders[1].last_frame_id, frames[5].id()); +} + } // namespace } // namespace webrtc_pc_e2e } // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc index 4c4c19f326..9fb711637e 100644 --- a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc @@ -43,6 +43,12 @@ QualityAnalyzingVideoDecoder::~QualityAnalyzingVideoDecoder() = default; int32_t QualityAnalyzingVideoDecoder::InitDecode( const VideoCodec* codec_settings, int32_t number_of_cores) { + { + MutexLock lock(&mutex_); + codec_name_ = + std::string(CodecTypeToPayloadString(codec_settings->codecType)) + "_" + + delegate_->GetDecoderInfo().implementation_name; + } return delegate_->InitDecode(codec_settings, number_of_cores); } @@ -77,7 +83,7 @@ int32_t QualityAnalyzingVideoDecoder::Decode(const EncodedImage& input_image, EncodedImage* origin_image; { - MutexLock lock(&lock_); + MutexLock lock(&mutex_); // Store id to be able to retrieve it in analyzing callback. timestamp_to_frame_id_.insert({input_image.Timestamp(), out.id}); // Store encoded image to prevent its destruction while it is used in @@ -94,7 +100,7 @@ int32_t QualityAnalyzingVideoDecoder::Decode(const EncodedImage& input_image, if (result != WEBRTC_VIDEO_CODEC_OK) { // If delegate decoder failed, then cleanup data for this image. { - MutexLock lock(&lock_); + MutexLock lock(&mutex_); timestamp_to_frame_id_.erase(input_image.Timestamp()); decoding_images_.erase(out.id); } @@ -114,7 +120,7 @@ int32_t QualityAnalyzingVideoDecoder::Release() { // frames, so we don't take a lock to prevent deadlock. int32_t result = delegate_->Release(); - MutexLock lock(&lock_); + MutexLock lock(&mutex_); analyzing_callback_->SetDelegateCallback(nullptr); timestamp_to_frame_id_.clear(); decoding_images_.clear(); @@ -138,7 +144,7 @@ QualityAnalyzingVideoDecoder::DecoderCallback::~DecoderCallback() = default; void QualityAnalyzingVideoDecoder::DecoderCallback::SetDelegateCallback( DecodedImageCallback* delegate) { - MutexLock lock(&callback_lock_); + MutexLock lock(&callback_mutex_); delegate_callback_ = delegate; } @@ -150,7 +156,7 @@ int32_t QualityAnalyzingVideoDecoder::DecoderCallback::Decoded( decoder_->OnFrameDecoded(&decodedImage, /*decode_time_ms=*/absl::nullopt, /*qp=*/absl::nullopt); - MutexLock lock(&callback_lock_); + MutexLock lock(&callback_mutex_); RTC_DCHECK(delegate_callback_); return delegate_callback_->Decoded(decodedImage); } @@ -160,7 +166,7 @@ int32_t QualityAnalyzingVideoDecoder::DecoderCallback::Decoded( int64_t decode_time_ms) { decoder_->OnFrameDecoded(&decodedImage, decode_time_ms, /*qp=*/absl::nullopt); - MutexLock lock(&callback_lock_); + MutexLock lock(&callback_mutex_); RTC_DCHECK(delegate_callback_); return delegate_callback_->Decoded(decodedImage, decode_time_ms); } @@ -171,7 +177,7 @@ void QualityAnalyzingVideoDecoder::DecoderCallback::Decoded( absl::optional qp) { decoder_->OnFrameDecoded(&decodedImage, decode_time_ms, qp); - MutexLock lock(&callback_lock_); + MutexLock lock(&callback_mutex_); RTC_DCHECK(delegate_callback_); delegate_callback_->Decoded(decodedImage, decode_time_ms, qp); } @@ -186,7 +192,7 @@ QualityAnalyzingVideoDecoder::DecoderCallback::IrrelevantSimulcastStreamDecoded( .set_timestamp_rtp(timestamp_ms) .set_id(frame_id) .build(); - MutexLock lock(&callback_lock_); + MutexLock lock(&callback_mutex_); RTC_DCHECK(delegate_callback_); delegate_callback_->Decoded(dummy_frame, absl::nullopt, absl::nullopt); return WEBRTC_VIDEO_CODEC_OK; @@ -206,8 +212,9 @@ void QualityAnalyzingVideoDecoder::OnFrameDecoded( absl::optional decode_time_ms, absl::optional qp) { uint16_t frame_id; + std::string codec_name; { - MutexLock lock(&lock_); + MutexLock lock(&mutex_); auto it = timestamp_to_frame_id_.find(frame->timestamp()); if (it == timestamp_to_frame_id_.end()) { // Ensure, that we have info about this frame. It can happen that for some @@ -221,11 +228,13 @@ void QualityAnalyzingVideoDecoder::OnFrameDecoded( frame_id = it->second; timestamp_to_frame_id_.erase(it); decoding_images_.erase(frame_id); + codec_name = codec_name_; } // Set frame id to the value, that was extracted from corresponding encoded // image. frame->set_id(frame_id); VideoQualityAnalyzerInterface::DecoderStats stats; + stats.decoder_name = codec_name; stats.decode_time_ms = decode_time_ms; analyzer_->OnFrameDecoded(peer_name_, *frame, stats); } diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h index e150c91cb4..9177b961f6 100644 --- a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h @@ -92,8 +92,8 @@ class QualityAnalyzingVideoDecoder : public VideoDecoder { rtc::scoped_refptr dummy_frame_buffer_; - Mutex callback_lock_; - DecodedImageCallback* delegate_callback_ RTC_GUARDED_BY(callback_lock_); + Mutex callback_mutex_; + DecodedImageCallback* delegate_callback_ RTC_GUARDED_BY(callback_mutex_); }; void OnFrameDecoded(VideoFrame* frame, @@ -110,14 +110,16 @@ class QualityAnalyzingVideoDecoder : public VideoDecoder { // VideoDecoder interface assumes async delivery of decoded video frames. // This lock is used to protect shared state, that have to be propagated // from received EncodedImage to resulted VideoFrame. - Mutex lock_; + Mutex mutex_; - std::map timestamp_to_frame_id_ RTC_GUARDED_BY(lock_); + // Name of the video codec type used. Ex: VP8, VP9, H264 etc. + std::string codec_name_ RTC_GUARDED_BY(mutex_); + std::map timestamp_to_frame_id_ RTC_GUARDED_BY(mutex_); // Stores currently being decoded images by frame id. Because // EncodedImageDataExtractor can create new copy on EncodedImage we need to // ensure, that this image won't be deleted during async decoding. To do it // all images are putted into this map and removed from here inside callback. - std::map decoding_images_ RTC_GUARDED_BY(lock_); + std::map decoding_images_ RTC_GUARDED_BY(mutex_); }; // Produces QualityAnalyzingVideoDecoder, which hold decoders, produced by diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc index bc921ac266..392ebc2563 100644 --- a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc @@ -77,7 +77,7 @@ void QualityAnalyzingVideoEncoder::SetFecControllerOverride( int32_t QualityAnalyzingVideoEncoder::InitEncode( const VideoCodec* codec_settings, const Settings& settings) { - MutexLock lock(&lock_); + MutexLock lock(&mutex_); codec_settings_ = *codec_settings; mode_ = SimulcastMode::kNormal; if (codec_settings->codecType == kVideoCodecVP9) { @@ -108,7 +108,7 @@ int32_t QualityAnalyzingVideoEncoder::RegisterEncodeCompleteCallback( EncodedImageCallback* callback) { // We need to get a lock here because delegate_callback can be hypothetically // accessed from different thread (encoder one) concurrently. - MutexLock lock(&lock_); + MutexLock lock(&mutex_); delegate_callback_ = callback; return delegate_->RegisterEncodeCompleteCallback(this); } @@ -118,7 +118,7 @@ int32_t QualityAnalyzingVideoEncoder::Release() { // frames, so we don't take a lock to prevent deadlock. int32_t result = delegate_->Release(); - MutexLock lock(&lock_); + MutexLock lock(&mutex_); delegate_callback_ = nullptr; return result; } @@ -127,7 +127,7 @@ int32_t QualityAnalyzingVideoEncoder::Encode( const VideoFrame& frame, const std::vector* frame_types) { { - MutexLock lock(&lock_); + MutexLock lock(&mutex_); // Store id to be able to retrieve it in analyzing callback. timestamp_to_frame_id_list_.push_back({frame.timestamp(), frame.id()}); // If this list is growing, it means that we are not receiving new encoded @@ -139,7 +139,7 @@ int32_t QualityAnalyzingVideoEncoder::Encode( if (result != WEBRTC_VIDEO_CODEC_OK) { // If origin encoder failed, then cleanup data for this frame. { - MutexLock lock(&lock_); + MutexLock lock(&mutex_); // The timestamp-frame_id pair can be not the last one, so we need to // find it first and then remove. We will search from the end, because // usually it will be the last or close to the last one. @@ -162,50 +162,50 @@ void QualityAnalyzingVideoEncoder::SetRates( RTC_DCHECK_GT(bitrate_multiplier_, 0.0); if (fabs(bitrate_multiplier_ - kNoMultiplier) < kEps) { { - MutexLock lock(&lock_); + MutexLock lock(&mutex_); bitrate_allocation_ = parameters.bitrate; } return delegate_->SetRates(parameters); } - // Simulating encoder overshooting target bitrate, by configuring actual - // encoder too high. Take care not to adjust past limits of config, - // otherwise encoders may crash on DCHECK. - VideoBitrateAllocation multiplied_allocation; - for (size_t si = 0; si < kMaxSpatialLayers; ++si) { - const uint32_t spatial_layer_bitrate_bps = - parameters.bitrate.GetSpatialLayerSum(si); - if (spatial_layer_bitrate_bps == 0) { - continue; - } + RateControlParameters adjusted_params = parameters; + { + MutexLock lock(&mutex_); + // Simulating encoder overshooting target bitrate, by configuring actual + // encoder too high. Take care not to adjust past limits of config, + // otherwise encoders may crash on DCHECK. + VideoBitrateAllocation multiplied_allocation; + for (size_t si = 0; si < kMaxSpatialLayers; ++si) { + const uint32_t spatial_layer_bitrate_bps = + parameters.bitrate.GetSpatialLayerSum(si); + if (spatial_layer_bitrate_bps == 0) { + continue; + } - uint32_t min_bitrate_bps; - uint32_t max_bitrate_bps; - std::tie(min_bitrate_bps, max_bitrate_bps) = - GetMinMaxBitratesBps(codec_settings_, si); - double bitrate_multiplier = bitrate_multiplier_; - const uint32_t corrected_bitrate = rtc::checked_cast( - bitrate_multiplier * spatial_layer_bitrate_bps); - if (corrected_bitrate < min_bitrate_bps) { - bitrate_multiplier = min_bitrate_bps / spatial_layer_bitrate_bps; - } else if (corrected_bitrate > max_bitrate_bps) { - bitrate_multiplier = max_bitrate_bps / spatial_layer_bitrate_bps; - } + uint32_t min_bitrate_bps; + uint32_t max_bitrate_bps; + std::tie(min_bitrate_bps, max_bitrate_bps) = + GetMinMaxBitratesBps(codec_settings_, si); + double bitrate_multiplier = bitrate_multiplier_; + const uint32_t corrected_bitrate = rtc::checked_cast( + bitrate_multiplier * spatial_layer_bitrate_bps); + if (corrected_bitrate < min_bitrate_bps) { + bitrate_multiplier = min_bitrate_bps / spatial_layer_bitrate_bps; + } else if (corrected_bitrate > max_bitrate_bps) { + bitrate_multiplier = max_bitrate_bps / spatial_layer_bitrate_bps; + } - for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) { - if (parameters.bitrate.HasBitrate(si, ti)) { - multiplied_allocation.SetBitrate( - si, ti, - rtc::checked_cast(bitrate_multiplier * - parameters.bitrate.GetBitrate(si, ti))); + for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) { + if (parameters.bitrate.HasBitrate(si, ti)) { + multiplied_allocation.SetBitrate( + si, ti, + rtc::checked_cast( + bitrate_multiplier * parameters.bitrate.GetBitrate(si, ti))); + } } } - } - RateControlParameters adjusted_params = parameters; - adjusted_params.bitrate = multiplied_allocation; - { - MutexLock lock(&lock_); + adjusted_params.bitrate = multiplied_allocation; bitrate_allocation_ = adjusted_params.bitrate; } return delegate_->SetRates(adjusted_params); @@ -234,8 +234,9 @@ EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage( uint16_t frame_id; bool discard = false; uint32_t target_encode_bitrate = 0; + std::string codec_name; { - MutexLock lock(&lock_); + MutexLock lock(&mutex_); std::pair timestamp_frame_id; while (!timestamp_to_frame_id_list_.empty()) { timestamp_frame_id = timestamp_to_frame_id_list_.front(); @@ -269,12 +270,16 @@ EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage( target_encode_bitrate = bitrate_allocation_.GetSpatialLayerSum( encoded_image.SpatialIndex().value_or(0)); } + codec_name = + std::string(CodecTypeToPayloadString(codec_settings_.codecType)) + "_" + + delegate_->GetEncoderInfo().implementation_name; } if (!discard) { // Analyzer should see only encoded images, that weren't discarded. But all // not discarded layers have to be passed. VideoQualityAnalyzerInterface::EncoderStats stats; + stats.encoder_name = codec_name; stats.target_encode_bitrate = target_encode_bitrate; analyzer_->OnFrameEncoded(peer_name_, frame_id, encoded_image, stats); } @@ -287,7 +292,7 @@ EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage( const EncodedImage& image = injector_->InjectData(frame_id, discard, encoded_image); { - MutexLock lock(&lock_); + MutexLock lock(&mutex_); RTC_DCHECK(delegate_callback_); return delegate_callback_->OnEncodedImage(image, codec_specific_info); } @@ -295,7 +300,7 @@ EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage( void QualityAnalyzingVideoEncoder::OnDroppedFrame( EncodedImageCallback::DropReason reason) { - MutexLock lock(&lock_); + MutexLock lock(&mutex_); analyzer_->OnFrameDropped(peer_name_, reason); RTC_DCHECK(delegate_callback_); delegate_callback_->OnDroppedFrame(reason); diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h index ce00292bf2..11d3be3364 100644 --- a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h @@ -132,7 +132,7 @@ class QualityAnalyzingVideoEncoder : public VideoEncoder, }; bool ShouldDiscard(uint16_t frame_id, const EncodedImage& encoded_image) - RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_); + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); const std::string peer_name_; std::unique_ptr delegate_; @@ -150,14 +150,14 @@ class QualityAnalyzingVideoEncoder : public VideoEncoder, // VideoEncoder interface assumes async delivery of encoded images. // This lock is used to protect shared state, that have to be propagated // from received VideoFrame to resulted EncodedImage. - Mutex lock_; + Mutex mutex_; - VideoCodec codec_settings_; - SimulcastMode mode_ RTC_GUARDED_BY(lock_); - EncodedImageCallback* delegate_callback_ RTC_GUARDED_BY(lock_); + VideoCodec codec_settings_ RTC_GUARDED_BY(mutex_); + SimulcastMode mode_ RTC_GUARDED_BY(mutex_); + EncodedImageCallback* delegate_callback_ RTC_GUARDED_BY(mutex_); std::list> timestamp_to_frame_id_list_ - RTC_GUARDED_BY(lock_); - VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(lock_); + RTC_GUARDED_BY(mutex_); + VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(mutex_); }; // Produces QualityAnalyzingVideoEncoder, which hold decoders, produced by