admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add tracking of video encoder/decoder used for stream in DVQA

Browse Source 
Bug: b/196035476
Change-Id: I882f2236c9522f06ad60332ab2a4bb9226b1bd8e
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/228423
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#34732}
This commit is contained in:
Artem Titov 2021-08-11 18:27:36 +02:00 committed by WebRTC LUCI CQ
parent 9ff59a396b
commit 647d326438
8 changed files with 316 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
api/test/video_quality_analyzer_interface.h
View File

@ -57,16 +57,18 @@ class VideoQualityAnalyzerInterface : public StatsObserverInterface {
public:
// Contains extra statistic provided by video encoder.
struct EncoderStats {
std::string encoder_name = "unknown";
// TODO(hbos) https://crbug.com/webrtc/9547,
// https://crbug.com/webrtc/11443: improve stats API to make available
// there.
uint32_t target_encode_bitrate;
uint32_t target_encode_bitrate = 0;
};
// Contains extra statistic provided by video decoder.
struct DecoderStats {
std::string decoder_name = "unknown";
// Decode time provided by decoder itself. If decoder doesn’t produce such
// information can be omitted.
absl::optional<int32_t> decode_time_ms;
absl::optional<int32_t> decode_time_ms = absl::nullopt;
};
~VideoQualityAnalyzerInterface() override = default;

132
test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
View File

@ -45,11 +45,47 @@ void LogFrameCounters(const std::string& name, const FrameCounters& counters) {
RTC_LOG(INFO) << "[" << name << "] Dropped : " << counters.dropped;
}
void LogStreamInternalStats(const std::string& name, const StreamStats& stats) {
void LogStreamInternalStats(const std::string& name,
const StreamStats& stats,
Timestamp start_time) {
RTC_LOG(INFO) << "[" << name
<< "] Dropped by encoder : " << stats.dropped_by_encoder;
RTC_LOG(INFO) << "[" << name << "] Dropped before encoder : "
<< stats.dropped_before_encoder;
Timestamp first_encoded_frame_time = Timestamp::PlusInfinity();
for (const StreamCodecInfo& encoder : stats.encoders) {
RTC_DCHECK(encoder.switched_on_at.IsFinite());
RTC_DCHECK(encoder.switched_from_at.IsFinite());
if (first_encoded_frame_time.IsInfinite()) {
first_encoded_frame_time = encoder.switched_on_at;
}
RTC_LOG(INFO) << "[" << name << "] Used encoder: \"" << encoder.codec_name
<< "\" used from (frame_id=" << encoder.first_frame_id
<< "; from_stream_start="
<< (encoder.switched_on_at - stats.stream_started_time).ms()
<< "ms, from_call_start="
<< (encoder.switched_on_at - start_time).ms()
<< "ms) until (frame_id=" << encoder.last_frame_id
<< "; from_stream_start="
<< (encoder.switched_from_at - stats.stream_started_time).ms()
<< "ms, from_call_start="
<< (encoder.switched_from_at - start_time).ms() << "ms)";
}
for (const StreamCodecInfo& decoder : stats.decoders) {
RTC_DCHECK(decoder.switched_on_at.IsFinite());
RTC_DCHECK(decoder.switched_from_at.IsFinite());
RTC_LOG(INFO) << "[" << name << "] Used decoder: \"" << decoder.codec_name
<< "\" used from (frame_id=" << decoder.first_frame_id
<< "; from_stream_start="
<< (decoder.switched_on_at - stats.stream_started_time).ms()
<< "ms, from_call_start="
<< (decoder.switched_on_at - start_time).ms()
<< "ms) until (frame_id=" << decoder.last_frame_id
<< "; from_stream_start="
<< (decoder.switched_from_at - stats.stream_started_time).ms()
<< "ms, from_call_start="
<< (decoder.switched_from_at - start_time).ms() << "ms)";
}
}
template <typename T>
@ -163,6 +199,7 @@ uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured(
const webrtc::VideoFrame& frame) {
// `next_frame_id` is atomic, so we needn't lock here.
uint16_t frame_id = next_frame_id_++;
Timestamp captured_time = Now();
Timestamp start_time = Timestamp::MinusInfinity();
size_t peer_index = -1;
size_t peers_count = -1;
@ -185,7 +222,7 @@ uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured(
}
InternalStatsKey stats_key(stream_index, peer_index, i);
if (stream_stats_.find(stats_key) == stream_stats_.end()) {
stream_stats_.insert({stats_key, StreamStats()});
stream_stats_.insert({stats_key, StreamStats(captured_time)});
// Assume that the first freeze was before first stream frame captured.
// This way time before the first freeze would be counted as time
// between freezes.
@ -213,9 +250,10 @@ uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured(
auto state_it = stream_states_.find(stream_index);
if (state_it == stream_states_.end()) {
stream_states_.emplace(stream_index,
StreamState(peer_index, peers_->size(),
options_.enable_receive_own_stream));
stream_states_.emplace(
stream_index,
StreamState(peer_index, peers_->size(),
options_.enable_receive_own_stream, captured_time));
}
StreamState* state = &stream_states_.at(stream_index);
state->PushBack(frame_id);
@ -248,9 +286,8 @@ uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured(
}
captured_frames_in_flight_.emplace(
frame_id,
FrameInFlight(stream_index, frame,
/*captured_time=*/Now(), peer_index, peers_->size(),
options_.enable_receive_own_stream));
FrameInFlight(stream_index, frame, captured_time, peer_index,
peers_->size(), options_.enable_receive_own_stream));
// Set frame id on local copy of the frame
captured_frames_in_flight_.at(frame_id).SetFrameId(frame_id);
@ -326,8 +363,15 @@ void DefaultVideoQualityAnalyzer::OnFrameEncoded(
}
}
}
it->second.OnFrameEncoded(Now(), encoded_image.size(),
stats.target_encode_bitrate);
Timestamp now = Now();
StreamCodecInfo used_encoder;
used_encoder.codec_name = stats.encoder_name;
used_encoder.first_frame_id = frame_id;
used_encoder.last_frame_id = frame_id;
used_encoder.switched_on_at = now;
used_encoder.switched_from_at = now;
it->second.OnFrameEncoded(now, encoded_image.size(),
stats.target_encode_bitrate, used_encoder);
}
void DefaultVideoQualityAnalyzer::OnFrameDropped(
@ -393,7 +437,14 @@ void DefaultVideoQualityAnalyzer::OnFrameDecoded(
InternalStatsKey key(it->second.stream(),
stream_to_sender_.at(it->second.stream()), peer_index);
stream_frame_counters_.at(key).decoded++;
it->second.SetDecodeEndTime(peer_index, Now());
Timestamp now = Now();
StreamCodecInfo used_decoder;
used_decoder.codec_name = stats.decoder_name;
used_decoder.first_frame_id = frame.id();
used_decoder.last_frame_id = frame.id();
used_decoder.switched_on_at = now;
used_decoder.switched_from_at = now;
it->second.OnFrameDecoded(peer_index, now, used_decoder);
}
void DefaultVideoQualityAnalyzer::OnFrameRendered(
@ -540,7 +591,9 @@ void DefaultVideoQualityAnalyzer::RegisterParticipantInCall(
// then `counters` will be empty. In such case empty `counters` are ok.
stream_frame_counters_.insert({key, std::move(counters)});
stream_stats_.insert({key, StreamStats()});
stream_stats_.insert(
{key,
StreamStats(stream_states_.at(stream_index).stream_started_time())});
stream_last_freeze_end_time_.insert({key, start_time_});
}
// Ensure, that frames states are handled correctly
@ -549,7 +602,7 @@ void DefaultVideoQualityAnalyzer::RegisterParticipantInCall(
key_val.second.AddPeer();
}
// Register new peer for every frame in flight.
// It is guaranteed, that no garbadge FrameInFlight objects will stay in
// It is guaranteed, that no garbage FrameInFlight objects will stay in
// memory because of adding new peer. Even if the new peer won't receive the
// frame, the frame will be removed by OnFrameRendered after next frame comes
// for the new peer. It is important because FrameInFlight is a large object.
@ -594,7 +647,7 @@ void DefaultVideoQualityAnalyzer::Stop() {
// If there is freeze, then we need add time from last rendered frame
// to last freeze end as time between freezes.
if (stream_state.last_rendered_frame_time(i)) {
stream_stats_[stats_key].time_between_freezes_ms.AddSample(
stream_stats_.at(stats_key).time_between_freezes_ms.AddSample(
StatsSample(
stream_state.last_rendered_frame_time(i).value().ms() -
stream_last_freeze_end_time_.at(stats_key).ms(),
@ -825,6 +878,28 @@ void DefaultVideoQualityAnalyzer::ProcessComparison(
}
}
}
// Compute stream codec info.
if (frame_stats.used_encoder.has_value()) {
if (stats->encoders.empty() || stats->encoders.back().codec_name !=
frame_stats.used_encoder->codec_name) {
stats->encoders.push_back(*frame_stats.used_encoder);
}
stats->encoders.back().last_frame_id =
frame_stats.used_encoder->last_frame_id;
stats->encoders.back().switched_from_at =
frame_stats.used_encoder->switched_from_at;
}
if (frame_stats.used_decoder.has_value()) {
if (stats->decoders.empty() || stats->decoders.back().codec_name !=
frame_stats.used_decoder->codec_name) {
stats->decoders.push_back(*frame_stats.used_decoder);
}
stats->decoders.back().last_frame_id =
frame_stats.used_decoder->last_frame_id;
stats->decoders.back().switched_from_at =
frame_stats.used_decoder->switched_from_at;
}
}
void DefaultVideoQualityAnalyzer::ReportResults() {
@ -842,7 +917,8 @@ void DefaultVideoQualityAnalyzer::ReportResults() {
for (auto& item : stream_stats_) {
LogFrameCounters(ToStatsKey(item.first).ToString(),
stream_frame_counters_.at(item.first));
LogStreamInternalStats(ToStatsKey(item.first).ToString(), item.second);
LogStreamInternalStats(ToStatsKey(item.first).ToString(), item.second,
start_time_);
}
if (!analyzer_stats_.comparisons_queue_size.IsEmpty()) {
RTC_LOG(INFO) << "comparisons_queue_size min="
@ -1131,10 +1207,24 @@ bool DefaultVideoQualityAnalyzer::FrameInFlight::HaveAllPeersReceived() const {
void DefaultVideoQualityAnalyzer::FrameInFlight::OnFrameEncoded(
webrtc::Timestamp time,
int64_t encoded_image_size,
uint32_t target_encode_bitrate) {
uint32_t target_encode_bitrate,
StreamCodecInfo used_encoder) {
encoded_time_ = time;
encoded_image_size_ = encoded_image_size;
target_encode_bitrate_ += target_encode_bitrate;
// Update used encoder info. If simulcast/SVC is used, this method can
// be called multiple times, in such case we should preserve the value
// of `used_encoder_.switched_on_at` from the first invocation as the
// smallest one.
Timestamp encoder_switched_on_at = used_encoder_.has_value()
? used_encoder_->switched_on_at
: Timestamp::PlusInfinity();
RTC_DCHECK(used_encoder.switched_on_at.IsFinite());
RTC_DCHECK(used_encoder.switched_from_at.IsFinite());
used_encoder_ = used_encoder;
if (encoder_switched_on_at < used_encoder_->switched_on_at) {
used_encoder_->switched_on_at = encoder_switched_on_at;
}
}
void DefaultVideoQualityAnalyzer::FrameInFlight::OnFramePreDecode(
@ -1154,6 +1244,14 @@ bool DefaultVideoQualityAnalyzer::FrameInFlight::HasReceivedTime(
return it->second.received_time.IsFinite();
}
void DefaultVideoQualityAnalyzer::FrameInFlight::OnFrameDecoded(
size_t peer,
webrtc::Timestamp time,
StreamCodecInfo used_decoder) {
receiver_stats_[peer].decode_end_time = time;
receiver_stats_[peer].used_decoder = used_decoder;
}
bool DefaultVideoQualityAnalyzer::FrameInFlight::HasDecodeEndTime(
size_t peer) const {
auto it = receiver_stats_.find(peer);
@ -1189,6 +1287,7 @@ DefaultVideoQualityAnalyzer::FrameInFlight::GetStatsForPeer(size_t peer) const {
stats.encoded_time = encoded_time_;
stats.target_encode_bitrate = target_encode_bitrate_;
stats.encoded_image_size = encoded_image_size_;
stats.used_encoder = used_encoder_;
absl::optional<ReceiverFrameStats> receiver_stats =
MaybeGetValue<ReceiverFrameStats>(receiver_stats_, peer);
@ -1200,6 +1299,7 @@ DefaultVideoQualityAnalyzer::FrameInFlight::GetStatsForPeer(size_t peer) const {
stats.prev_frame_rendered_time = receiver_stats->prev_frame_rendered_time;
stats.rendered_frame_width = receiver_stats->rendered_frame_width;
stats.rendered_frame_height = receiver_stats->rendered_frame_height;
stats.used_decoder = receiver_stats->used_decoder;
}
return stats;
}

53
test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
View File

@ -75,7 +75,28 @@ struct FrameCounters {
int64_t dropped = 0;
};
// Contains information about the codec that was used for encoding or decoding
// the stream.
struct StreamCodecInfo {
// Codec implementation name.
std::string codec_name;
// Id of the first frame for which this codec was used.
uint16_t first_frame_id;
// Id of the last frame for which this codec was used.
uint16_t last_frame_id;
// Timestamp when the first frame was handled by the encode/decoder.
Timestamp switched_on_at = Timestamp::PlusInfinity();
// Timestamp when this codec was used last time.
Timestamp switched_from_at = Timestamp::PlusInfinity();
};
struct StreamStats {
explicit StreamStats(Timestamp stream_started_time)
: stream_started_time(stream_started_time) {}
// The time when the first frame of this stream was captured.
Timestamp stream_started_time;
SamplesStatsCounter psnr;
SamplesStatsCounter ssim;
// Time from frame encoded (time point on exit from encoder) to the
@ -108,6 +129,11 @@ struct StreamStats {
int64_t total_encoded_images_payload = 0;
int64_t dropped_by_encoder = 0;
int64_t dropped_before_encoder = 0;
// Vector of encoders used for this stream by sending client.
std::vector<StreamCodecInfo> encoders;
// Vectors of decoders used for this stream by receiving client.
std::vector<StreamCodecInfo> decoders;
};
struct AnalyzerStats {
@ -243,6 +269,8 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
double GetCpuUsagePercent();
private:
// Final stats computed for frame after it went through the whole video
// pipeline from capturing to rendering or dropping.
struct FrameStats {
FrameStats(Timestamp captured_time) : captured_time(captured_time) {}
@ -262,6 +290,11 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
absl::optional<int> rendered_frame_width = absl::nullopt;
absl::optional<int> rendered_frame_height = absl::nullopt;
// Can be not set if frame was dropped by encoder.
absl::optional<StreamCodecInfo> used_encoder = absl::nullopt;
// Can be not set if frame was dropped in the network.
absl::optional<StreamCodecInfo> used_decoder = absl::nullopt;
};
// Describes why comparison was done in overloaded mode (without calculating
@ -308,12 +341,15 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
public:
StreamState(size_t owner,
size_t peers_count,
bool enable_receive_own_stream)
bool enable_receive_own_stream,
Timestamp stream_started_time)
: owner_(owner),
enable_receive_own_stream_(enable_receive_own_stream),
stream_started_time_(stream_started_time),
frame_ids_(peers_count) {}
size_t owner() const { return owner_; }
Timestamp stream_started_time() const { return stream_started_time_; }
void PushBack(uint16_t frame_id) { frame_ids_.PushBack(frame_id); }
// Crash if state is empty. Guarantees that there can be no alive frames
@ -338,6 +374,7 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
// Index of the owner. Owner's queue in `frame_ids_` will keep alive frames.
const size_t owner_;
const bool enable_receive_own_stream_;
const Timestamp stream_started_time_;
// To correctly determine dropped frames we have to know sequence of frames
// in each stream so we will keep a list of frame ids inside the stream.
// This list is represented by multi head queue of frame ids with separate
@ -371,6 +408,9 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
absl::optional<int> rendered_frame_width = absl::nullopt;
absl::optional<int> rendered_frame_height = absl::nullopt;
// Can be not set if frame was dropped in the network.
absl::optional<StreamCodecInfo> used_decoder = absl::nullopt;
bool dropped = false;
};
@ -404,7 +444,8 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
void OnFrameEncoded(webrtc::Timestamp time,
int64_t encoded_image_size,
uint32_t target_encode_bitrate);
uint32_t target_encode_bitrate,
StreamCodecInfo used_encoder);
bool HasEncodedTime() const { return encoded_time_.IsFinite(); }
@ -414,9 +455,9 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
bool HasReceivedTime(size_t peer) const;
void SetDecodeEndTime(size_t peer, webrtc::Timestamp time) {
receiver_stats_[peer].decode_end_time = time;
}
void OnFrameDecoded(size_t peer,
webrtc::Timestamp time,
StreamCodecInfo used_decoder);
bool HasDecodeEndTime(size_t peer) const;
@ -453,6 +494,8 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
Timestamp encoded_time_ = Timestamp::MinusInfinity();
int64_t encoded_image_size_ = 0;
uint32_t target_encode_bitrate_ = 0;
// Can be not set if frame was dropped by encoder.
absl::optional<StreamCodecInfo> used_encoder_ = absl::nullopt;
std::map<size_t, ReceiverFrameStats> receiver_stats_;
};

69
test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc
View File

@ -1049,6 +1049,75 @@ TEST(DefaultVideoQualityAnalyzerTest, FrameCanBeReceivedBySender) {
}
}
TEST(DefaultVideoQualityAnalyzerTest, CodecTrackedCorrectly) {
std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
/*type=*/absl::nullopt,
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
AnalyzerOptionsForTest());
analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
kAnalyzerMaxThreadsCount);
VideoQualityAnalyzerInterface::EncoderStats encoder_stats;
std::vector<std::string> codec_names = {"codec_1", "codec_2"};
std::vector<VideoFrame> frames;
// Send 3 frame for each codec.
for (size_t i = 0; i < codec_names.size(); ++i) {
for (size_t j = 0; j < 3; ++j) {
VideoFrame frame = NextFrame(frame_generator.get(), 3 * i + j);
frame.set_id(
analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
analyzer.OnFramePreEncode(kSenderPeerName, frame);
encoder_stats.encoder_name = codec_names[i];
analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
encoder_stats);
frames.push_back(std::move(frame));
}
}
// Receive 3 frame for each codec.
VideoQualityAnalyzerInterface::DecoderStats decoder_stats;
for (size_t i = 0; i < codec_names.size(); ++i) {
for (size_t j = 0; j < 3; ++j) {
VideoFrame received_frame = DeepCopy(frames[3 * i + j]);
analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
FakeEncode(received_frame));
decoder_stats.decoder_name = codec_names[i];
analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, decoder_stats);
analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
}
}
// Give analyzer some time to process frames on async thread. The computations
// have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
// means we have an issue!
SleepMs(100);
analyzer.Stop();
std::map<StatsKey, StreamStats> stats = analyzer.GetStats();
ASSERT_EQ(stats.size(), 1lu);
const StreamStats& stream_stats =
stats.at(StatsKey(kStreamLabel, kSenderPeerName, kReceiverPeerName));
ASSERT_EQ(stream_stats.encoders.size(), 2lu);
EXPECT_EQ(stream_stats.encoders[0].codec_name, codec_names[0]);
EXPECT_EQ(stream_stats.encoders[0].first_frame_id, frames[0].id());
EXPECT_EQ(stream_stats.encoders[0].last_frame_id, frames[2].id());
EXPECT_EQ(stream_stats.encoders[1].codec_name, codec_names[1]);
EXPECT_EQ(stream_stats.encoders[1].first_frame_id, frames[3].id());
EXPECT_EQ(stream_stats.encoders[1].last_frame_id, frames[5].id());
ASSERT_EQ(stream_stats.decoders.size(), 2lu);
EXPECT_EQ(stream_stats.decoders[0].codec_name, codec_names[0]);
EXPECT_EQ(stream_stats.decoders[0].first_frame_id, frames[0].id());
EXPECT_EQ(stream_stats.decoders[0].last_frame_id, frames[2].id());
EXPECT_EQ(stream_stats.decoders[1].codec_name, codec_names[1]);
EXPECT_EQ(stream_stats.decoders[1].first_frame_id, frames[3].id());
EXPECT_EQ(stream_stats.decoders[1].last_frame_id, frames[5].id());
}
} // namespace
} // namespace webrtc_pc_e2e
} // namespace webrtc

27
test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc
View File

@ -43,6 +43,12 @@ QualityAnalyzingVideoDecoder::~QualityAnalyzingVideoDecoder() = default;
int32_t QualityAnalyzingVideoDecoder::InitDecode(
const VideoCodec* codec_settings,
int32_t number_of_cores) {
{
MutexLock lock(&mutex_);
codec_name_ =
std::string(CodecTypeToPayloadString(codec_settings->codecType)) + "_" +
delegate_->GetDecoderInfo().implementation_name;
}
return delegate_->InitDecode(codec_settings, number_of_cores);
}
@ -77,7 +83,7 @@ int32_t QualityAnalyzingVideoDecoder::Decode(const EncodedImage& input_image,
EncodedImage* origin_image;
{
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
// Store id to be able to retrieve it in analyzing callback.
timestamp_to_frame_id_.insert({input_image.Timestamp(), out.id});
// Store encoded image to prevent its destruction while it is used in
@ -94,7 +100,7 @@ int32_t QualityAnalyzingVideoDecoder::Decode(const EncodedImage& input_image,
if (result != WEBRTC_VIDEO_CODEC_OK) {
// If delegate decoder failed, then cleanup data for this image.
{
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
timestamp_to_frame_id_.erase(input_image.Timestamp());
decoding_images_.erase(out.id);
}
@ -114,7 +120,7 @@ int32_t QualityAnalyzingVideoDecoder::Release() {
// frames, so we don't take a lock to prevent deadlock.
int32_t result = delegate_->Release();
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
analyzing_callback_->SetDelegateCallback(nullptr);
timestamp_to_frame_id_.clear();
decoding_images_.clear();
@ -138,7 +144,7 @@ QualityAnalyzingVideoDecoder::DecoderCallback::~DecoderCallback() = default;
void QualityAnalyzingVideoDecoder::DecoderCallback::SetDelegateCallback(
DecodedImageCallback* delegate) {
MutexLock lock(&callback_lock_);
MutexLock lock(&callback_mutex_);
delegate_callback_ = delegate;
}
@ -150,7 +156,7 @@ int32_t QualityAnalyzingVideoDecoder::DecoderCallback::Decoded(
decoder_->OnFrameDecoded(&decodedImage, /*decode_time_ms=*/absl::nullopt,
/*qp=*/absl::nullopt);
MutexLock lock(&callback_lock_);
MutexLock lock(&callback_mutex_);
RTC_DCHECK(delegate_callback_);
return delegate_callback_->Decoded(decodedImage);
}
@ -160,7 +166,7 @@ int32_t QualityAnalyzingVideoDecoder::DecoderCallback::Decoded(
int64_t decode_time_ms) {
decoder_->OnFrameDecoded(&decodedImage, decode_time_ms, /*qp=*/absl::nullopt);
MutexLock lock(&callback_lock_);
MutexLock lock(&callback_mutex_);
RTC_DCHECK(delegate_callback_);
return delegate_callback_->Decoded(decodedImage, decode_time_ms);
}
@ -171,7 +177,7 @@ void QualityAnalyzingVideoDecoder::DecoderCallback::Decoded(
absl::optional<uint8_t> qp) {
decoder_->OnFrameDecoded(&decodedImage, decode_time_ms, qp);
MutexLock lock(&callback_lock_);
MutexLock lock(&callback_mutex_);
RTC_DCHECK(delegate_callback_);
delegate_callback_->Decoded(decodedImage, decode_time_ms, qp);
}
@ -186,7 +192,7 @@ QualityAnalyzingVideoDecoder::DecoderCallback::IrrelevantSimulcastStreamDecoded(
.set_timestamp_rtp(timestamp_ms)
.set_id(frame_id)
.build();
MutexLock lock(&callback_lock_);
MutexLock lock(&callback_mutex_);
RTC_DCHECK(delegate_callback_);
delegate_callback_->Decoded(dummy_frame, absl::nullopt, absl::nullopt);
return WEBRTC_VIDEO_CODEC_OK;
@ -206,8 +212,9 @@ void QualityAnalyzingVideoDecoder::OnFrameDecoded(
absl::optional<int32_t> decode_time_ms,
absl::optional<uint8_t> qp) {
uint16_t frame_id;
std::string codec_name;
{
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
auto it = timestamp_to_frame_id_.find(frame->timestamp());
if (it == timestamp_to_frame_id_.end()) {
// Ensure, that we have info about this frame. It can happen that for some
@ -221,11 +228,13 @@ void QualityAnalyzingVideoDecoder::OnFrameDecoded(
frame_id = it->second;
timestamp_to_frame_id_.erase(it);
decoding_images_.erase(frame_id);
codec_name = codec_name_;
}
// Set frame id to the value, that was extracted from corresponding encoded
// image.
frame->set_id(frame_id);
VideoQualityAnalyzerInterface::DecoderStats stats;
stats.decoder_name = codec_name;
stats.decode_time_ms = decode_time_ms;
analyzer_->OnFrameDecoded(peer_name_, *frame, stats);
}

12
test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h
View File

@ -92,8 +92,8 @@ class QualityAnalyzingVideoDecoder : public VideoDecoder {
rtc::scoped_refptr<webrtc::VideoFrameBuffer> dummy_frame_buffer_;
Mutex callback_lock_;
DecodedImageCallback* delegate_callback_ RTC_GUARDED_BY(callback_lock_);
Mutex callback_mutex_;
DecodedImageCallback* delegate_callback_ RTC_GUARDED_BY(callback_mutex_);
};
void OnFrameDecoded(VideoFrame* frame,
@ -110,14 +110,16 @@ class QualityAnalyzingVideoDecoder : public VideoDecoder {
// VideoDecoder interface assumes async delivery of decoded video frames.
// This lock is used to protect shared state, that have to be propagated
// from received EncodedImage to resulted VideoFrame.
Mutex lock_;
Mutex mutex_;
std::map<uint32_t, uint16_t> timestamp_to_frame_id_ RTC_GUARDED_BY(lock_);
// Name of the video codec type used. Ex: VP8, VP9, H264 etc.
std::string codec_name_ RTC_GUARDED_BY(mutex_);
std::map<uint32_t, uint16_t> timestamp_to_frame_id_ RTC_GUARDED_BY(mutex_);
// Stores currently being decoded images by frame id. Because
// EncodedImageDataExtractor can create new copy on EncodedImage we need to
// ensure, that this image won't be deleted during async decoding. To do it
// all images are putted into this map and removed from here inside callback.
std::map<uint16_t, EncodedImage> decoding_images_ RTC_GUARDED_BY(lock_);
std::map<uint16_t, EncodedImage> decoding_images_ RTC_GUARDED_BY(mutex_);
};
// Produces QualityAnalyzingVideoDecoder, which hold decoders, produced by

89
test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc
View File

@ -77,7 +77,7 @@ void QualityAnalyzingVideoEncoder::SetFecControllerOverride(
int32_t QualityAnalyzingVideoEncoder::InitEncode(
const VideoCodec* codec_settings,
const Settings& settings) {
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
codec_settings_ = *codec_settings;
mode_ = SimulcastMode::kNormal;
if (codec_settings->codecType == kVideoCodecVP9) {
@ -108,7 +108,7 @@ int32_t QualityAnalyzingVideoEncoder::RegisterEncodeCompleteCallback(
EncodedImageCallback* callback) {
// We need to get a lock here because delegate_callback can be hypothetically
// accessed from different thread (encoder one) concurrently.
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
delegate_callback_ = callback;
return delegate_->RegisterEncodeCompleteCallback(this);
}
@ -118,7 +118,7 @@ int32_t QualityAnalyzingVideoEncoder::Release() {
// frames, so we don't take a lock to prevent deadlock.
int32_t result = delegate_->Release();
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
delegate_callback_ = nullptr;
return result;
}
@ -127,7 +127,7 @@ int32_t QualityAnalyzingVideoEncoder::Encode(
const VideoFrame& frame,
const std::vector<VideoFrameType>* frame_types) {
{
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
// Store id to be able to retrieve it in analyzing callback.
timestamp_to_frame_id_list_.push_back({frame.timestamp(), frame.id()});
// If this list is growing, it means that we are not receiving new encoded
@ -139,7 +139,7 @@ int32_t QualityAnalyzingVideoEncoder::Encode(
if (result != WEBRTC_VIDEO_CODEC_OK) {
// If origin encoder failed, then cleanup data for this frame.
{
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
// The timestamp-frame_id pair can be not the last one, so we need to
// find it first and then remove. We will search from the end, because
// usually it will be the last or close to the last one.
@ -162,50 +162,50 @@ void QualityAnalyzingVideoEncoder::SetRates(
RTC_DCHECK_GT(bitrate_multiplier_, 0.0);
if (fabs(bitrate_multiplier_ - kNoMultiplier) < kEps) {
{
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
bitrate_allocation_ = parameters.bitrate;
}
return delegate_->SetRates(parameters);
}
// Simulating encoder overshooting target bitrate, by configuring actual
// encoder too high. Take care not to adjust past limits of config,
// otherwise encoders may crash on DCHECK.
VideoBitrateAllocation multiplied_allocation;
for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
const uint32_t spatial_layer_bitrate_bps =
parameters.bitrate.GetSpatialLayerSum(si);
if (spatial_layer_bitrate_bps == 0) {
continue;
}
RateControlParameters adjusted_params = parameters;
{
MutexLock lock(&mutex_);
// Simulating encoder overshooting target bitrate, by configuring actual
// encoder too high. Take care not to adjust past limits of config,
// otherwise encoders may crash on DCHECK.
VideoBitrateAllocation multiplied_allocation;
for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
const uint32_t spatial_layer_bitrate_bps =
parameters.bitrate.GetSpatialLayerSum(si);
if (spatial_layer_bitrate_bps == 0) {
continue;
}
uint32_t min_bitrate_bps;
uint32_t max_bitrate_bps;
std::tie(min_bitrate_bps, max_bitrate_bps) =
GetMinMaxBitratesBps(codec_settings_, si);
double bitrate_multiplier = bitrate_multiplier_;
const uint32_t corrected_bitrate = rtc::checked_cast<uint32_t>(
bitrate_multiplier * spatial_layer_bitrate_bps);
if (corrected_bitrate < min_bitrate_bps) {
bitrate_multiplier = min_bitrate_bps / spatial_layer_bitrate_bps;
} else if (corrected_bitrate > max_bitrate_bps) {
bitrate_multiplier = max_bitrate_bps / spatial_layer_bitrate_bps;
}
uint32_t min_bitrate_bps;
uint32_t max_bitrate_bps;
std::tie(min_bitrate_bps, max_bitrate_bps) =
GetMinMaxBitratesBps(codec_settings_, si);
double bitrate_multiplier = bitrate_multiplier_;
const uint32_t corrected_bitrate = rtc::checked_cast<uint32_t>(
bitrate_multiplier * spatial_layer_bitrate_bps);
if (corrected_bitrate < min_bitrate_bps) {
bitrate_multiplier = min_bitrate_bps / spatial_layer_bitrate_bps;
} else if (corrected_bitrate > max_bitrate_bps) {
bitrate_multiplier = max_bitrate_bps / spatial_layer_bitrate_bps;
}
for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
if (parameters.bitrate.HasBitrate(si, ti)) {
multiplied_allocation.SetBitrate(
si, ti,
rtc::checked_cast<uint32_t>(bitrate_multiplier *
parameters.bitrate.GetBitrate(si, ti)));
for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
if (parameters.bitrate.HasBitrate(si, ti)) {
multiplied_allocation.SetBitrate(
si, ti,
rtc::checked_cast<uint32_t>(
bitrate_multiplier * parameters.bitrate.GetBitrate(si, ti)));
}
}
}
}
RateControlParameters adjusted_params = parameters;
adjusted_params.bitrate = multiplied_allocation;
{
MutexLock lock(&lock_);
adjusted_params.bitrate = multiplied_allocation;
bitrate_allocation_ = adjusted_params.bitrate;
}
return delegate_->SetRates(adjusted_params);
@ -234,8 +234,9 @@ EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage(
uint16_t frame_id;
bool discard = false;
uint32_t target_encode_bitrate = 0;
std::string codec_name;
{
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
std::pair<uint32_t, uint16_t> timestamp_frame_id;
while (!timestamp_to_frame_id_list_.empty()) {
timestamp_frame_id = timestamp_to_frame_id_list_.front();
@ -269,12 +270,16 @@ EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage(
target_encode_bitrate = bitrate_allocation_.GetSpatialLayerSum(
encoded_image.SpatialIndex().value_or(0));
}
codec_name =
std::string(CodecTypeToPayloadString(codec_settings_.codecType)) + "_" +
delegate_->GetEncoderInfo().implementation_name;
}
if (!discard) {
// Analyzer should see only encoded images, that weren't discarded. But all
// not discarded layers have to be passed.
VideoQualityAnalyzerInterface::EncoderStats stats;
stats.encoder_name = codec_name;
stats.target_encode_bitrate = target_encode_bitrate;
analyzer_->OnFrameEncoded(peer_name_, frame_id, encoded_image, stats);
}
@ -287,7 +292,7 @@ EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage(
const EncodedImage& image =
injector_->InjectData(frame_id, discard, encoded_image);
{
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
RTC_DCHECK(delegate_callback_);
return delegate_callback_->OnEncodedImage(image, codec_specific_info);
}
@ -295,7 +300,7 @@ EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage(
void QualityAnalyzingVideoEncoder::OnDroppedFrame(
EncodedImageCallback::DropReason reason) {
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
analyzer_->OnFrameDropped(peer_name_, reason);
RTC_DCHECK(delegate_callback_);
delegate_callback_->OnDroppedFrame(reason);

14
test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h
View File

@ -132,7 +132,7 @@ class QualityAnalyzingVideoEncoder : public VideoEncoder,
};
bool ShouldDiscard(uint16_t frame_id, const EncodedImage& encoded_image)
RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_);
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
const std::string peer_name_;
std::unique_ptr<VideoEncoder> delegate_;
@ -150,14 +150,14 @@ class QualityAnalyzingVideoEncoder : public VideoEncoder,
// VideoEncoder interface assumes async delivery of encoded images.
// This lock is used to protect shared state, that have to be propagated
// from received VideoFrame to resulted EncodedImage.
Mutex lock_;
Mutex mutex_;
VideoCodec codec_settings_;
SimulcastMode mode_ RTC_GUARDED_BY(lock_);
EncodedImageCallback* delegate_callback_ RTC_GUARDED_BY(lock_);
VideoCodec codec_settings_ RTC_GUARDED_BY(mutex_);
SimulcastMode mode_ RTC_GUARDED_BY(mutex_);
EncodedImageCallback* delegate_callback_ RTC_GUARDED_BY(mutex_);
std::list<std::pair<uint32_t, uint16_t>> timestamp_to_frame_id_list_
RTC_GUARDED_BY(lock_);
VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(lock_);
RTC_GUARDED_BY(mutex_);
VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(mutex_);
};
// Produces QualityAnalyzingVideoEncoder, which hold decoders, produced by