admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[DVQA] Use SampleStatsCounter with time between frames instead of SamplesRateCounter

Browse Source 
Bug: webrtc:14995, b/271542055
Change-Id: Ie55924c881c5eb09d987c5e8876cba5baced426b
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/298600
Reviewed-by: Jeremy Leconte <jleconte@google.com>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#39827}
This commit is contained in:
Artem Titov 2023-04-11 12:45:51 +02:00 committed by WebRTC LUCI CQ
parent 2a3942fec1
commit 1db3209e70
9 changed files with 111 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
View File

@ -211,6 +211,12 @@ uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured(
}
StreamState* state = &stream_states_.at(stream_index);
state->PushBack(frame_id);
absl::optional<TimeDelta> time_between_captured_frames = absl::nullopt;
if (state->last_captured_frame_time().has_value()) {
time_between_captured_frames =
captured_time - *state->last_captured_frame_time();
}
state->SetLastCapturedFrameTime(captured_time);
// Update frames in flight info.
auto it = captured_frames_in_flight_.find(frame_id);
if (it != captured_frames_in_flight_.end()) {
@ -247,6 +253,7 @@ uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured(
}
captured_frames_in_flight_.emplace(
frame_id, FrameInFlight(stream_index, frame_id, captured_time,
time_between_captured_frames,
std::move(frame_receivers_indexes)));
// Store local copy of the frame with frame_id set.
VideoFrame local_frame(frame);
@ -337,6 +344,13 @@ void DefaultVideoQualityAnalyzer::OnFrameEncoded(
}
}
Timestamp now = Now();
StreamState& state = stream_states_.at(frame_in_flight.stream());
absl::optional<TimeDelta> time_between_encoded_frames = absl::nullopt;
if (state.last_encoded_frame_time().has_value()) {
time_between_encoded_frames = now - *state.last_encoded_frame_time();
}
state.SetLastEncodedFrameTime(now);
StreamCodecInfo used_encoder;
used_encoder.codec_name = stats.encoder_name;
used_encoder.first_frame_id = frame_id;
@ -350,8 +364,9 @@ void DefaultVideoQualityAnalyzer::OnFrameEncoded(
size_t stream_index = encoded_image.SpatialIndex().value_or(
encoded_image.SimulcastIndex().value_or(0));
frame_in_flight.OnFrameEncoded(
now, encoded_image._frameType, DataSize::Bytes(encoded_image.size()),
stats.target_encode_bitrate, stream_index, stats.qp, used_encoder);
now, time_between_encoded_frames, encoded_image._frameType,
DataSize::Bytes(encoded_image.size()), stats.target_encode_bitrate,
stream_index, stats.qp, used_encoder);
if (options_.report_infra_metrics) {
analyzer_stats_.on_frame_encoded_processing_time_ms.AddSample(

13
test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
View File

@ -18,6 +18,7 @@
#include <memory>
#include <set>
#include <string>
#include <unordered_map>
#include <vector>
#include "api/array_view.h"
@ -178,7 +179,7 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
// 4. There too many frames in flight for current video stream and X is the
// oldest frame id in this stream. In such case only the frame content
// will be removed, but the map entry will be preserved.
std::map<uint16_t, FrameInFlight> captured_frames_in_flight_
std::unordered_map<uint16_t, FrameInFlight> captured_frames_in_flight_
RTC_GUARDED_BY(mutex_);
// Global frames count for all video streams.
FrameCounters frame_counters_ RTC_GUARDED_BY(mutex_);
@ -190,19 +191,19 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
std::map<InternalStatsKey, FrameCounters> stream_frame_counters_
RTC_GUARDED_BY(mutex_);
// Map from stream index in `streams_` to its StreamState.
std::map<size_t, StreamState> stream_states_ RTC_GUARDED_BY(mutex_);
std::unordered_map<size_t, StreamState> stream_states_ RTC_GUARDED_BY(mutex_);
// Map from stream index in `streams_` to sender peer index in `peers_`.
std::map<size_t, size_t> stream_to_sender_ RTC_GUARDED_BY(mutex_);
std::unordered_map<size_t, size_t> stream_to_sender_ RTC_GUARDED_BY(mutex_);
// Stores history mapping between stream index in `streams_` and frame ids.
// Updated when frame id overlap. It required to properly return stream label
// after 1st frame from simulcast streams was already rendered and last is
// still encoding.
std::map<size_t, std::set<uint16_t>> stream_to_frame_id_history_
std::unordered_map<size_t, std::set<uint16_t>> stream_to_frame_id_history_
RTC_GUARDED_BY(mutex_);
// Map from stream index to the list of frames as they were met in the stream.
std::map<size_t, std::vector<uint16_t>> stream_to_frame_id_full_history_
RTC_GUARDED_BY(mutex_);
std::unordered_map<size_t, std::vector<uint16_t>>
stream_to_frame_id_full_history_ RTC_GUARDED_BY(mutex_);
AnalyzerStats analyzer_stats_ RTC_GUARDED_BY(mutex_);
DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer_;

37
test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc
View File

@ -16,6 +16,7 @@
#include "absl/types/optional.h"
#include "api/units/data_size.h"
#include "api/units/time_delta.h"
#include "api/units/timestamp.h"
#include "api/video/video_frame.h"
#include "api/video/video_frame_type.h"
@ -36,14 +37,17 @@ absl::optional<T> MaybeGetValue(const std::unordered_map<size_t, T>& map,
} // namespace
FrameInFlight::FrameInFlight(size_t stream,
uint16_t frame_id,
Timestamp captured_time,
std::set<size_t> expected_receivers)
FrameInFlight::FrameInFlight(
size_t stream,
uint16_t frame_id,
Timestamp captured_time,
absl::optional<TimeDelta> time_between_captured_frames,
std::set<size_t> expected_receivers)
: stream_(stream),
expected_receivers_(std::move(expected_receivers)),
frame_id_(frame_id),
captured_time_(captured_time) {}
captured_time_(captured_time),
time_between_captured_frames_(time_between_captured_frames) {}
std::vector<size_t> FrameInFlight::GetPeersWhichDidntReceive() const {
std::vector<size_t> out;
@ -73,14 +77,21 @@ bool FrameInFlight::HaveAllPeersReceived() const {
return true;
}
void FrameInFlight::OnFrameEncoded(webrtc::Timestamp time,
VideoFrameType frame_type,
DataSize encoded_image_size,
uint32_t target_encode_bitrate,
int stream_index,
int qp,
StreamCodecInfo used_encoder) {
void FrameInFlight::OnFrameEncoded(
webrtc::Timestamp time,
absl::optional<TimeDelta> time_between_encoded_frames,
VideoFrameType frame_type,
DataSize encoded_image_size,
uint32_t target_encode_bitrate,
int stream_index,
int qp,
StreamCodecInfo used_encoder) {
encoded_time_ = time;
if (time_between_encoded_frames.has_value()) {
time_between_encoded_frames_ =
time_between_encoded_frames_.value_or(TimeDelta::Zero()) +
*time_between_encoded_frames;
}
frame_type_ = frame_type;
encoded_image_size_ = encoded_image_size;
target_encode_bitrate_ += target_encode_bitrate;
@ -171,6 +182,8 @@ FrameStats FrameInFlight::GetStatsForPeer(size_t peer) const {
RTC_DCHECK(!IsSuperfluous(peer))
<< "This frame is superfluous for peer " << peer;
FrameStats stats(frame_id_, captured_time_);
stats.time_between_captured_frames = time_between_captured_frames_;
stats.time_between_encoded_frames = time_between_encoded_frames_;
stats.pre_encode_time = pre_encode_time_;
stats.encoded_time = encoded_time_;
stats.target_encode_bitrate = target_encode_bitrate_;

12
test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h
View File

@ -34,9 +34,11 @@ struct ReceiverFrameStats {
Timestamp decode_start_time = Timestamp::MinusInfinity();
Timestamp decode_end_time = Timestamp::MinusInfinity();
Timestamp rendered_time = Timestamp::MinusInfinity();
// Will be finite if there is frame rendered before this one.
Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity();
TimeDelta time_between_rendered_frames = TimeDelta::Zero();
// Will be set if there is frame rendered before this one.
absl::optional<TimeDelta> time_between_rendered_frames = absl::nullopt;
// Type and encoded size of received frame.
VideoFrameType frame_type = VideoFrameType::kEmptyFrame;
@ -68,6 +70,7 @@ class FrameInFlight {
FrameInFlight(size_t stream,
uint16_t frame_id,
Timestamp captured_time,
absl::optional<TimeDelta> time_between_captured_frames,
std::set<size_t> expected_receivers);
size_t stream() const { return stream_; }
@ -89,6 +92,7 @@ class FrameInFlight {
void SetPreEncodeTime(Timestamp time) { pre_encode_time_ = time; }
void OnFrameEncoded(Timestamp time,
absl::optional<TimeDelta> time_between_encoded_frames,
VideoFrameType frame_type,
DataSize encoded_image_size,
uint32_t target_encode_bitrate,
@ -156,14 +160,16 @@ class FrameInFlight {
// any peer or can be safely deleted. It is responsibility of the user of this
// object to decide when it should be deleted.
std::set<size_t> expected_receivers_;
// Store frame id separately because `frame_` can be removed when we have too
// much memory consuption.
uint16_t frame_id_ = VideoFrame::kNotSetId;
// Frame events timestamp.
Timestamp captured_time_;
Timestamp pre_encode_time_ = Timestamp::MinusInfinity();
Timestamp encoded_time_ = Timestamp::MinusInfinity();
absl::optional<TimeDelta> time_between_captured_frames_ = absl::nullopt;
absl::optional<TimeDelta> time_between_encoded_frames_ = absl::nullopt;
// Type and encoded size of sent frame.
VideoFrameType frame_type_ = VideoFrameType::kEmptyFrame;
DataSize encoded_image_size_ = DataSize::Bytes(0);

17
test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc
View File

@ -464,6 +464,11 @@ void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison(
StatsSample(ssim, frame_stats.received_time, metadata));
}
stats->capture_frame_rate.AddEvent(frame_stats.captured_time);
if (frame_stats.time_between_captured_frames.has_value()) {
stats->time_between_captured_frames_ms.AddSample(
StatsSample(*frame_stats.time_between_captured_frames,
frame_stats.captured_time, metadata));
}
// Compute dropped phase for dropped frame
if (comparison.type == FrameComparisonType::kDroppedFrame) {
@ -487,6 +492,11 @@ void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison(
StatsSample(frame_stats.encoded_time - frame_stats.pre_encode_time,
frame_stats.encoded_time, metadata));
stats->encode_frame_rate.AddEvent(frame_stats.encoded_time);
if (frame_stats.time_between_encoded_frames.has_value()) {
stats->time_between_encoded_frames_ms.AddSample(
StatsSample(*frame_stats.time_between_encoded_frames,
frame_stats.encoded_time, metadata));
}
stats->total_encoded_images_payload +=
frame_stats.encoded_image_size.bytes();
stats->target_encode_bitrate.AddSample(StatsSample(
@ -546,16 +556,17 @@ void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison(
if (frame_stats.prev_frame_rendered_time.IsFinite() &&
frame_stats.rendered_time.IsFinite()) {
RTC_DCHECK(frame_stats.time_between_rendered_frames.has_value());
stats->time_between_rendered_frames_ms.AddSample(
StatsSample(frame_stats.time_between_rendered_frames,
StatsSample(*frame_stats.time_between_rendered_frames,
frame_stats.rendered_time, metadata));
TimeDelta average_time_between_rendered_frames = TimeDelta::Millis(
stats->time_between_rendered_frames_ms.GetAverage());
if (frame_stats.time_between_rendered_frames >
if (*frame_stats.time_between_rendered_frames >
std::max(kFreezeThreshold + average_time_between_rendered_frames,
3 * average_time_between_rendered_frames)) {
stats->freeze_time_ms.AddSample(
StatsSample(frame_stats.time_between_rendered_frames,
StatsSample(*frame_stats.time_between_rendered_frames,
frame_stats.rendered_time, metadata));
auto freeze_end_it =
stream_last_freeze_end_time_.find(comparison.stats_key);

7
test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h
View File

@ -59,9 +59,10 @@ struct FrameStats {
Timestamp rendered_time = Timestamp::MinusInfinity();
Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity();
// Time between this and previous rendered frame excluding time when related
// stream was paused for related receiver.
TimeDelta time_between_rendered_frames = TimeDelta::Zero();
// Next timings are set if and only if previous frame exist.
absl::optional<TimeDelta> time_between_captured_frames = absl::nullopt;
absl::optional<TimeDelta> time_between_encoded_frames = absl::nullopt;
absl::optional<TimeDelta> time_between_rendered_frames = absl::nullopt;
VideoFrameType encoded_frame_type = VideoFrameType::kEmptyFrame;
DataSize encoded_image_size = DataSize::Bytes(0);

3
test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h
View File

@ -126,6 +126,9 @@ struct StreamStats {
SamplesStatsCounter total_delay_incl_transport_ms;
// Time between frames out from renderer.
SamplesStatsCounter time_between_rendered_frames_ms;
SamplesStatsCounter time_between_captured_frames_ms;
// Time between frames out from encoder.
SamplesStatsCounter time_between_encoded_frames_ms;
SamplesRateCounter capture_frame_rate;
SamplesRateCounter encode_frame_rate;
SamplesStatsCounter encode_time_ms;

16
test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h
View File

@ -70,6 +70,20 @@ class StreamState {
return frame_ids_.size(kAliveFramesQueueIndex);
}
void SetLastCapturedFrameTime(Timestamp time) {
last_captured_frame_time_ = time;
}
absl::optional<Timestamp> last_captured_frame_time() const {
return last_captured_frame_time_;
}
void SetLastEncodedFrameTime(Timestamp time) {
last_encoded_frame_time_ = time;
}
absl::optional<Timestamp> last_encoded_frame_time() const {
return last_encoded_frame_time_;
}
void SetLastRenderedFrameTime(size_t peer, Timestamp time);
absl::optional<Timestamp> last_rendered_frame_time(size_t peer) const;
@ -99,6 +113,8 @@ class StreamState {
// frame_id2 and consider those frames as dropped and then compare received
// frame with the one from `FrameInFlight` with id frame_id3.
MultiReaderQueue<uint16_t> frame_ids_;
absl::optional<Timestamp> last_captured_frame_time_ = absl::nullopt;
absl::optional<Timestamp> last_encoded_frame_time_ = absl::nullopt;
std::unordered_map<size_t, Timestamp> last_rendered_frame_time_;
// Mapping from peer's index to pausable state for this receiver.
std::unordered_map<size_t, PausableState> pausable_state_;

30
test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc
View File

@ -2147,9 +2147,11 @@ TEST_F(DefaultVideoQualityAnalyzerSimulatedTimeTest,
// TODO(bugs.webrtc.org/14995): value should exclude pause
EXPECT_THAT(GetTimeSortedValues(bob_stream_stats.time_between_freezes_ms),
ElementsAre(2950.0));
// TODO(bugs.webrtc.org/14995): Fix capture_frame_rate (has to be ~20.0)
EXPECT_THAT(bob_stream_stats.time_between_captured_frames_ms.GetAverage(),
50.0);
EXPECT_THAT(bob_stream_stats.time_between_encoded_frames_ms.GetAverage(),
50.0);
ExpectRateIs(bob_stream_stats.capture_frame_rate, 13.559322);
// TODO(bugs.webrtc.org/14995): Fix encode_frame_rate (has to be ~20.0)
ExpectRateIs(bob_stream_stats.encode_frame_rate, 13.559322);
EXPECT_DOUBLE_EQ(bob_stream_stats.harmonic_framerate_fps, 20);
@ -2208,22 +2210,22 @@ TEST_F(DefaultVideoQualityAnalyzerSimulatedTimeTest,
std::map<StatsKey, StreamStats> streams_stats = analyzer.GetStats();
std::map<StatsKey, FrameCounters> frame_counters =
analyzer.GetPerStreamCounters();
StreamStats bob_stream_stats1 =
StreamStats bob_stream_stats_1 =
streams_stats.at(StatsKey("alice_video_1", "bob"));
FrameCounters bob_frame_counters1 =
frame_counters.at(StatsKey("alice_video_1", "bob"));
EXPECT_THAT(bob_frame_counters1.dropped, Eq(0));
EXPECT_THAT(bob_frame_counters1.rendered, Eq(40));
EXPECT_THAT(GetTimeSortedValues(bob_stream_stats1.freeze_time_ms),
EXPECT_THAT(GetTimeSortedValues(bob_stream_stats_1.freeze_time_ms),
ElementsAre(0.0));
// TODO(bugs.webrtc.org/14995): value should exclude pause
EXPECT_THAT(GetTimeSortedValues(bob_stream_stats1.time_between_freezes_ms),
EXPECT_THAT(GetTimeSortedValues(bob_stream_stats_1.time_between_freezes_ms),
ElementsAre(2950.0));
// TODO(bugs.webrtc.org/14995): Fix capture_frame_rate (has to be ~20.0)
ExpectRateIs(bob_stream_stats1.capture_frame_rate, 13.559322);
// TODO(bugs.webrtc.org/14995): Fix encode_frame_rate (has to be ~20.0)
ExpectRateIs(bob_stream_stats1.encode_frame_rate, 13.559322);
EXPECT_DOUBLE_EQ(bob_stream_stats1.harmonic_framerate_fps, 20.0);
EXPECT_THAT(bob_stream_stats_1.time_between_captured_frames_ms.GetAverage(),
50.0);
EXPECT_THAT(bob_stream_stats_1.time_between_encoded_frames_ms.GetAverage(),
50.0);
EXPECT_DOUBLE_EQ(bob_stream_stats_1.harmonic_framerate_fps, 20.0);
// Bob should have 20 fps without freeze on both streams.
StreamStats bob_stream_stats_2 =
@ -2237,10 +2239,10 @@ TEST_F(DefaultVideoQualityAnalyzerSimulatedTimeTest,
// TODO(bugs.webrtc.org/14995): value should exclude pause
EXPECT_THAT(GetTimeSortedValues(bob_stream_stats_2.time_between_freezes_ms),
ElementsAre(2950.0));
// TODO(bugs.webrtc.org/14995): Fix capture_frame_rate (has to be ~20.0)
ExpectRateIs(bob_stream_stats_2.capture_frame_rate, 13.559322);
// TODO(bugs.webrtc.org/14995): Fix encode_frame_rate (has to be ~20.0)
ExpectRateIs(bob_stream_stats_2.encode_frame_rate, 13.559322);
EXPECT_THAT(bob_stream_stats_2.time_between_captured_frames_ms.GetAverage(),
50.0);
EXPECT_THAT(bob_stream_stats_2.time_between_encoded_frames_ms.GetAverage(),
50.0);
EXPECT_DOUBLE_EQ(bob_stream_stats_2.harmonic_framerate_fps, 20.0);
}