admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduce default video quality analyzer

Browse Source 
This implementation won't support spatial layers and simulcast. It will
be added in next CLs.

Bug: webrtc:10138
Change-Id: I08baef36fb15b8d2d2fa222c761d40508de7ff61
Reviewed-on: https://webrtc-review.googlesource.com/c/121944
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Peter Slatala <psla@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26676}
This commit is contained in:
Artem Titov 2019-02-13 22:15:04 +01:00 committed by Commit Bot
parent b1ea48c2e8
commit 6b88a8f161
4 changed files with 867 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
test/pc/e2e/BUILD.gn
View File

@ -261,7 +261,7 @@ if (rtc_include_tests) {
"peer_connection_e2e_smoke_test.cc",
]
deps = [
":example_video_quality_analyzer",
":default_video_quality_analyzer",
"../../../api:callfactory_api",
"../../../api:libjingle_peerconnection_api",
"../../../api:scoped_refptr",
@ -313,3 +313,34 @@ rtc_source_set("example_video_quality_analyzer") {
"api:video_quality_analyzer_api",
]
}
rtc_source_set("default_video_quality_analyzer") {
visibility = [ "*" ]
testonly = true
sources = [
"analyzer/video/default_video_quality_analyzer.cc",
"analyzer/video/default_video_quality_analyzer.h",
]
deps = [
"../..:perf_test",
"../../../api/units:time_delta",
"../../../api/units:timestamp",
"../../../api/video:encoded_image",
"../../../api/video:video_frame",
"../../../common_video:common_video",
"../../../rtc_base:criticalsection",
"../../../rtc_base:logging",
"../../../rtc_base:rtc_base_approved",
"../../../rtc_base:rtc_event",
"../../../rtc_base:rtc_numerics",
"../../../system_wrappers:system_wrappers",
"api:video_quality_analyzer_api",
"//third_party/abseil-cpp/absl/memory:memory",
]
if (!build_with_chromium && is_clang) {
# Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163).
suppressed_configs += [ "//build/config/clang:find_bad_constructs" ]
}
}

542
test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc Normal file
View File

@ -0,0 +1,542 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
#include <algorithm>
#include <utility>
#include "absl/memory/memory.h"
#include "api/units/time_delta.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "rtc_base/logging.h"
#include "test/testsupport/perf_test.h"
namespace webrtc {
namespace test {
namespace {
constexpr int kMaxActiveComparisons = 10;
constexpr int kFreezeThresholdMs = 150;
} // namespace
void RateCounter::AddEvent(Timestamp event_time) {
if (event_first_time_.IsMinusInfinity()) {
event_first_time_ = event_time;
}
event_last_time_ = event_time;
event_count_++;
}
double RateCounter::GetEventsPerSecond() const {
RTC_DCHECK(!IsEmpty());
return static_cast<double>(event_count_) /
(event_last_time_ - event_first_time_).seconds();
}
DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer(std::string test_label)
: test_label_(std::move(test_label)), clock_(Clock::GetRealTimeClock()) {}
DefaultVideoQualityAnalyzer::~DefaultVideoQualityAnalyzer() {
Stop();
}
void DefaultVideoQualityAnalyzer::Start(int max_threads_count) {
for (int i = 0; i < max_threads_count; i++) {
auto thread = absl::make_unique<rtc::PlatformThread>(
&DefaultVideoQualityAnalyzer::ProcessComparisonsThread, this,
("DefaultVideoQualityAnalyzerWorker-" + std::to_string(i)).data(),
rtc::ThreadPriority::kNormalPriority);
thread->Start();
thread_pool_.push_back(std::move(thread));
}
{
rtc::CritScope crit(&lock_);
state_ = State::kActive;
}
}
uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured(
const std::string& stream_label,
const webrtc::VideoFrame& frame) {
// |next_frame_id| is atomic, so we needn't lock here.
uint16_t frame_id = next_frame_id_++;
{
// Ensure stats for this stream exists.
rtc::CritScope crit(&comparison_lock_);
if (stream_stats_.find(stream_label) == stream_stats_.end()) {
stream_stats_.insert({stream_label, StreamStats()});
// Assume that the first freeze was before first stream frame captured.
// This way time before the first freeze would be counted as time between
// freezes.
stream_last_freeze_end_time_.insert({stream_label, Now()});
}
}
{
rtc::CritScope crit(&lock_);
frame_counters_.captured++;
stream_frame_counters_[stream_label].captured++;
StreamState* state = &stream_states_[stream_label];
state->frame_ids.push_back(frame_id);
// Update frames in flight info.
auto it = captured_frames_in_flight_.find(frame_id);
if (it != captured_frames_in_flight_.end()) {
// We overflow uint16_t and hit previous frame id and this frame is still
// in flight. It means that this stream wasn't rendered for long time and
// we need to process existing frame as dropped.
auto stats_it = frame_stats_.find(frame_id);
RTC_DCHECK(stats_it != frame_stats_.end());
RTC_DCHECK(frame_id == state->frame_ids.front());
state->frame_ids.pop_front();
frame_counters_.dropped++;
stream_frame_counters_[stream_label].dropped++;
AddComparison(it->second, state->last_rendered_frame, true,
stats_it->second);
captured_frames_in_flight_.erase(it);
frame_stats_.erase(stats_it);
}
captured_frames_in_flight_.insert(
std::pair<uint16_t, VideoFrame>(frame_id, frame));
// Set frame id on local copy of the frame
captured_frames_in_flight_.at(frame_id).set_id(frame_id);
frame_stats_.insert(std::pair<uint16_t, FrameStats>(
frame_id, FrameStats(stream_label, /*captured_time=*/Now())));
}
return frame_id;
}
void DefaultVideoQualityAnalyzer::OnFramePreEncode(
const webrtc::VideoFrame& frame) {
rtc::CritScope crit(&lock_);
auto it = frame_stats_.find(frame.id());
RTC_DCHECK(it != frame_stats_.end());
frame_counters_.pre_encoded++;
stream_frame_counters_[it->second.stream_label].pre_encoded++;
it->second.pre_encode_time = Now();
}
void DefaultVideoQualityAnalyzer::OnFrameEncoded(
uint16_t frame_id,
const webrtc::EncodedImage& encoded_image) {
rtc::CritScope crit(&lock_);
// TODO(titovartem) we need to pick right spatial index here.
auto it = frame_stats_.find(frame_id);
RTC_DCHECK(it != frame_stats_.end());
RTC_DCHECK(it->second.encoded_time.IsInfinite())
<< "Received multiple spatial layers for stream_label="
<< it->second.stream_label;
frame_counters_.encoded++;
stream_frame_counters_[it->second.stream_label].encoded++;
it->second.encoded_time = Now();
}
void DefaultVideoQualityAnalyzer::OnFrameDropped(
webrtc::EncodedImageCallback::DropReason reason) {
// Here we do nothing, because we will see this drop on renderer side.
}
void DefaultVideoQualityAnalyzer::OnFrameReceived(
uint16_t frame_id,
const webrtc::EncodedImage& input_image) {
// TODO(titovartem) We should always receive only single spatial layer here.
rtc::CritScope crit(&lock_);
auto it = frame_stats_.find(frame_id);
RTC_DCHECK(it != frame_stats_.end());
RTC_DCHECK(it->second.received_time.IsInfinite())
<< "Received multiple spatial layers for stream_label="
<< it->second.stream_label;
frame_counters_.received++;
stream_frame_counters_[it->second.stream_label].received++;
it->second.received_time = Now();
}
void DefaultVideoQualityAnalyzer::OnFrameDecoded(
const webrtc::VideoFrame& frame,
absl::optional<int32_t> decode_time_ms,
absl::optional<uint8_t> qp) {
rtc::CritScope crit(&lock_);
auto it = frame_stats_.find(frame.id());
RTC_DCHECK(it != frame_stats_.end());
frame_counters_.decoded++;
stream_frame_counters_[it->second.stream_label].decoded++;
it->second.decoded_time = Now();
}
void DefaultVideoQualityAnalyzer::OnFrameRendered(
const webrtc::VideoFrame& frame) {
rtc::CritScope crit(&lock_);
auto stats_it = frame_stats_.find(frame.id());
RTC_DCHECK(stats_it != frame_stats_.end());
FrameStats* frame_stats = &stats_it->second;
// Update frames counters.
frame_counters_.rendered++;
stream_frame_counters_[frame_stats->stream_label].rendered++;
// Update current frame stats.
frame_stats->rendered_time = Now();
frame_stats->rendered_frame_width = frame.width();
frame_stats->rendered_frame_height = frame.height();
// Find corresponding captured frame.
auto frame_it = captured_frames_in_flight_.find(frame.id());
RTC_DCHECK(frame_it != captured_frames_in_flight_.end());
const VideoFrame& captured_frame = frame_it->second;
// After we received frame here we need to check if there are any dropped
// frames between this one and last one, that was rendered for this video
// stream.
const std::string& stream_label = frame_stats->stream_label;
StreamState* state = &stream_states_[stream_label];
int dropped_count = 0;
while (!state->frame_ids.empty() && state->frame_ids.front() != frame.id()) {
dropped_count++;
uint16_t dropped_frame_id = state->frame_ids.front();
state->frame_ids.pop_front();
// Frame with id |dropped_frame_id| was dropped. We need:
// 1. Update global and stream frame counters
// 2. Extract corresponding frame from |captured_frames_in_flight_|
// 3. Extract corresponding frame stats from |frame_stats_|
// 4. Send extracted frame to comparison with dropped=true
// 5. Cleanup dropped frame
frame_counters_.dropped++;
stream_frame_counters_[stream_label].dropped++;
auto dropped_frame_stats_it = frame_stats_.find(dropped_frame_id);
RTC_DCHECK(dropped_frame_stats_it != frame_stats_.end());
auto dropped_frame_it = captured_frames_in_flight_.find(dropped_frame_id);
RTC_CHECK(dropped_frame_it != captured_frames_in_flight_.end());
AddComparison(dropped_frame_it->second, state->last_rendered_frame, true,
dropped_frame_stats_it->second);
frame_stats_.erase(dropped_frame_stats_it);
captured_frames_in_flight_.erase(dropped_frame_it);
}
RTC_DCHECK(!state->frame_ids.empty());
state->frame_ids.pop_front();
state->last_rendered_frame = frame;
if (state->last_rendered_frame_time) {
frame_stats->prev_frame_rendered_time =
state->last_rendered_frame_time.value();
}
state->last_rendered_frame_time = frame_stats->rendered_time;
{
rtc::CritScope cr(&comparison_lock_);
stream_stats_[stream_label].skipped_between_rendered.AddSample(
dropped_count);
}
AddComparison(captured_frame, frame, false, *frame_stats);
captured_frames_in_flight_.erase(frame_it);
frame_stats_.erase(stats_it);
}
void DefaultVideoQualityAnalyzer::OnEncoderError(
const webrtc::VideoFrame& frame,
int32_t error_code) {
RTC_LOG(LS_ERROR) << "Encoder error for frame.id=" << frame.id()
<< ", code=" << error_code;
}
void DefaultVideoQualityAnalyzer::OnDecoderError(uint16_t frame_id,
int32_t error_code) {
RTC_LOG(LS_ERROR) << "Decoder error for frame_id=" << frame_id
<< ", code=" << error_code;
}
void DefaultVideoQualityAnalyzer::Stop() {
{
rtc::CritScope crit(&lock_);
if (state_ == State::kStopped) {
return;
}
state_ = State::kStopped;
}
comparison_available_event_.Set();
for (auto& thread : thread_pool_) {
thread->Stop();
}
// PlatformThread have to be deleted on the same thread, where it was created
thread_pool_.clear();
// Perform final Metrics update. On this place analyzer is stopped and no one
// holds any locks.
{
// Time between freezes.
// Count time since the last freeze to the end of the call as time
// between freezes.
rtc::CritScope crit1(&lock_);
rtc::CritScope crit2(&comparison_lock_);
for (auto& item : stream_stats_) {
if (item.second.freeze_time_ms.IsEmpty()) {
continue;
}
const StreamState& state = stream_states_[item.first];
if (state.last_rendered_frame_time) {
item.second.time_between_freezes_ms.AddSample(
(state.last_rendered_frame_time.value() -
stream_last_freeze_end_time_.at(item.first))
.ms());
}
}
}
ReportResults();
}
std::set<std::string> DefaultVideoQualityAnalyzer::GetKnownVideoStreams()
const {
rtc::CritScope crit2(&comparison_lock_);
std::set<std::string> out;
for (auto& item : stream_stats_) {
out.insert(item.first);
}
return out;
}
const FrameCounters& DefaultVideoQualityAnalyzer::GetGlobalCounters() {
rtc::CritScope crit(&lock_);
return frame_counters_;
}
const std::map<std::string, FrameCounters>&
DefaultVideoQualityAnalyzer::GetPerStreamCounters() const {
rtc::CritScope crit(&lock_);
return stream_frame_counters_;
}
const std::map<std::string, StreamStats>&
DefaultVideoQualityAnalyzer::GetStats() const {
rtc::CritScope cri(&comparison_lock_);
return stream_stats_;
}
const AnalyzerStats& DefaultVideoQualityAnalyzer::GetAnalyzerStats() const {
rtc::CritScope crit(&comparison_lock_);
return analyzer_stats_;
}
void DefaultVideoQualityAnalyzer::AddComparison(
absl::optional<VideoFrame> captured,
absl::optional<VideoFrame> rendered,
bool dropped,
FrameStats frame_stats) {
rtc::CritScope crit(&comparison_lock_);
analyzer_stats_.comparisons_queue_size.AddSample(comparisons_.size());
// If there too many computations waiting in the queue, we won't provide
// frames itself to make future computations lighter.
if (comparisons_.size() >= kMaxActiveComparisons) {
comparisons_.emplace_back(dropped, frame_stats);
} else {
comparisons_.emplace_back(std::move(captured), std::move(rendered), dropped,
frame_stats);
}
comparison_available_event_.Set();
}
void DefaultVideoQualityAnalyzer::ProcessComparisonsThread(void* obj) {
static_cast<DefaultVideoQualityAnalyzer*>(obj)->ProcessComparisons();
}
void DefaultVideoQualityAnalyzer::ProcessComparisons() {
while (true) {
// Try to pick next comparison to perform from the queue.
absl::optional<FrameComparison> comparison = absl::nullopt;
{
rtc::CritScope crit(&comparison_lock_);
if (!comparisons_.empty()) {
comparison = comparisons_.front();
comparisons_.pop_front();
if (!comparisons_.empty()) {
comparison_available_event_.Set();
}
}
}
if (!comparison) {
bool more_frames_expected;
{
// If there are no comparisons and state is stopped =>
// no more frames expected.
rtc::CritScope crit(&lock_);
more_frames_expected = state_ != State::kStopped;
}
if (!more_frames_expected) {
comparison_available_event_.Set();
return;
}
comparison_available_event_.Wait(1000);
continue;
}
ProcessComparison(comparison.value());
}
}
void DefaultVideoQualityAnalyzer::ProcessComparison(
const FrameComparison& comparison) {
// Perform expensive psnr and ssim calculations while not holding lock.
double psnr = -1.0;
double ssim = -1.0;
if (comparison.captured && !comparison.dropped) {
psnr = I420PSNR(&*comparison.captured, &*comparison.rendered);
ssim = I420SSIM(&*comparison.captured, &*comparison.rendered);
}
const FrameStats& frame_stats = comparison.frame_stats;
rtc::CritScope crit(&comparison_lock_);
auto stats_it = stream_stats_.find(frame_stats.stream_label);
RTC_CHECK(stats_it != stream_stats_.end());
StreamStats* stats = &stats_it->second;
analyzer_stats_.comparisons_done++;
if (!comparison.captured) {
analyzer_stats_.overloaded_comparisons_done++;
}
if (psnr > 0) {
stats->psnr.AddSample(psnr);
}
if (ssim > 0) {
stats->ssim.AddSample(ssim);
}
if (frame_stats.encoded_time.IsFinite()) {
stats->encode_time_ms.AddSample(
(frame_stats.encoded_time - frame_stats.pre_encode_time).ms());
stats->encode_frame_rate.AddEvent(frame_stats.encoded_time);
} else {
if (frame_stats.pre_encode_time.IsFinite()) {
stats->dropped_by_encoder++;
} else {
stats->dropped_before_encoder++;
}
}
// Next stats can be calculated only if frame was received on remote side.
if (!comparison.dropped) {
stats->resolution_of_encoded_image.AddSample(
*comparison.frame_stats.rendered_frame_width *
*comparison.frame_stats.rendered_frame_height);
stats->transport_time_ms.AddSample(
(frame_stats.received_time - frame_stats.encoded_time).ms());
stats->total_delay_incl_transport_ms.AddSample(
(frame_stats.rendered_time - frame_stats.captured_time).ms());
stats->decode_time_ms.AddSample(
(frame_stats.decoded_time - frame_stats.received_time).ms());
if (frame_stats.prev_frame_rendered_time.IsFinite()) {
TimeDelta time_between_rendered_frames =
frame_stats.rendered_time - frame_stats.prev_frame_rendered_time;
stats->time_between_rendered_frames_ms.AddSample(
time_between_rendered_frames.ms());
double average_time_between_rendered_frames_ms =
stats->time_between_rendered_frames_ms.GetAverage();
if (time_between_rendered_frames.ms() >
std::max(kFreezeThresholdMs + average_time_between_rendered_frames_ms,
3 * average_time_between_rendered_frames_ms)) {
stats->freeze_time_ms.AddSample(time_between_rendered_frames.ms());
auto freeze_end_it =
stream_last_freeze_end_time_.find(frame_stats.stream_label);
RTC_DCHECK(freeze_end_it != stream_last_freeze_end_time_.end());
stats->time_between_freezes_ms.AddSample(
(frame_stats.prev_frame_rendered_time - freeze_end_it->second)
.ms());
freeze_end_it->second = frame_stats.rendered_time;
}
}
}
}
void DefaultVideoQualityAnalyzer::ReportResults() const {
rtc::CritScope crit1(&lock_);
rtc::CritScope crit2(&comparison_lock_);
for (auto& item : stream_stats_) {
ReportResults(GetTestCaseName(item.first), item.second,
stream_frame_counters_.at(item.first));
}
}
void DefaultVideoQualityAnalyzer::ReportResults(std::string test_case_name,
StreamStats stats,
FrameCounters frame_counters) {
ReportResult("psnr", test_case_name, stats.psnr, "dB");
ReportResult("ssim", test_case_name, stats.ssim, "unitless");
ReportResult("transport_time", test_case_name, stats.transport_time_ms, "ms");
ReportResult("total_delay_incl_transport", test_case_name,
stats.total_delay_incl_transport_ms, "ms");
ReportResult("time_between_rendered_frames", test_case_name,
stats.time_between_rendered_frames_ms, "ms");
test::PrintResult("encode_frame_rate", "", test_case_name,
stats.encode_frame_rate.IsEmpty()
? 0
: stats.encode_frame_rate.GetEventsPerSecond(),
"fps", /*important=*/false);
ReportResult("encode_time", test_case_name, stats.encode_time_ms, "ms");
ReportResult("time_between_freezes", test_case_name,
stats.time_between_freezes_ms, "ms");
ReportResult("pixels_per_frame", test_case_name,
stats.resolution_of_encoded_image, "unitless");
test::PrintResult("min_psnr", "", test_case_name,
stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), "dB",
/*important=*/false);
ReportResult("decode_time", test_case_name, stats.decode_time_ms, "ms");
test::PrintResult("dropped_frames", "", test_case_name,
frame_counters.dropped, "unitless",
/*important=*/false);
ReportResult("max_skipped", test_case_name, stats.skipped_between_rendered,
"unitless");
}
void DefaultVideoQualityAnalyzer::ReportResult(
const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit) {
test::PrintResultMeanAndError(
metric_name, /*modifier=*/"", test_case_name,
counter.IsEmpty() ? 0 : counter.GetAverage(),
counter.IsEmpty() ? 0 : counter.GetStandardDeviation(), unit,
/*important=*/false);
}
std::string DefaultVideoQualityAnalyzer::GetTestCaseName(
const std::string& stream_label) const {
return test_label_ + "/" + stream_label;
}
Timestamp DefaultVideoQualityAnalyzer::Now() {
return Timestamp::us(clock_->TimeInMicroseconds());
}
DefaultVideoQualityAnalyzer::FrameStats::FrameStats(std::string stream_label,
Timestamp captured_time)
: stream_label(std::move(stream_label)), captured_time(captured_time) {}
DefaultVideoQualityAnalyzer::FrameComparison::FrameComparison(
absl::optional<VideoFrame> captured,
absl::optional<VideoFrame> rendered,
bool dropped,
FrameStats frame_stats)
: captured(std::move(captured)),
rendered(std::move(rendered)),
dropped(dropped),
frame_stats(std::move(frame_stats)) {}
DefaultVideoQualityAnalyzer::FrameComparison::FrameComparison(
bool dropped,
FrameStats frame_stats)
: captured(absl::nullopt),
rendered(absl::nullopt),
dropped(dropped),
frame_stats(std::move(frame_stats)) {}
} // namespace test
} // namespace webrtc

280
test/pc/e2e/analyzer/video/default_video_quality_analyzer.h Normal file
View File

@ -0,0 +1,280 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_H_
#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_H_
#include <atomic>
#include <deque>
#include <list>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <vector>
#include "api/units/timestamp.h"
#include "api/video/encoded_image.h"
#include "api/video/video_frame.h"
#include "rtc_base/critical_section.h"
#include "rtc_base/event.h"
#include "rtc_base/numerics/samples_stats_counter.h"
#include "rtc_base/platform_thread.h"
#include "system_wrappers/include/clock.h"
#include "test/pc/e2e/api/video_quality_analyzer_interface.h"
namespace webrtc {
namespace test {
class RateCounter {
public:
void AddEvent(Timestamp event_time);
bool IsEmpty() const { return event_first_time_ == event_last_time_; }
double GetEventsPerSecond() const;
private:
Timestamp event_first_time_ = Timestamp::MinusInfinity();
Timestamp event_last_time_ = Timestamp::MinusInfinity();
int64_t event_count_ = 0;
};
struct FrameCounters {
// Count of frames, that were passed into WebRTC pipeline by video stream
// source.
int64_t captured = 0;
// Count of frames that reached video encoder.
int64_t pre_encoded = 0;
// Count of encoded images that were produced by encoder for all requested
// spatial layers and simulcast streams.
int64_t encoded = 0;
// Count of encoded images received in decoder for all requested spatial
// layers and simulcast streams.
int64_t received = 0;
// Count of frames that were produced by decoder.
int64_t decoded = 0;
// Count of frames that went out from WebRTC pipeline to video sink.
int64_t rendered = 0;
// Count of frames that were dropped in any point between capturing and
// rendering.
int64_t dropped = 0;
};
struct StreamStats {
public:
SamplesStatsCounter psnr;
SamplesStatsCounter ssim;
// Time from frame encoded (time point on exit from encoder) to the
// encoded image received in decoder (time point on entrance to decoder).
SamplesStatsCounter transport_time_ms;
// Time from frame was captured on device to time frame was displayed on
// device.
SamplesStatsCounter total_delay_incl_transport_ms;
// Time between frames out from renderer.
SamplesStatsCounter time_between_rendered_frames_ms;
RateCounter encode_frame_rate;
SamplesStatsCounter encode_time_ms;
SamplesStatsCounter decode_time_ms;
// Max frames skipped between two nearest.
SamplesStatsCounter skipped_between_rendered;
// In the next 2 metrics freeze is a pause that is longer, than maximum:
// 1. 150ms
// 2. 3 * average time between two sequential frames.
// Item 1 will cover high fps video and is a duration, that is noticeable by
// human eye. Item 2 will cover low fps video like screen sharing.
// Freeze duration.
SamplesStatsCounter freeze_time_ms;
// Mean time between one freeze end and next freeze start.
SamplesStatsCounter time_between_freezes_ms;
SamplesStatsCounter resolution_of_encoded_image;
int64_t dropped_by_encoder = 0;
int64_t dropped_before_encoder = 0;
};
struct AnalyzerStats {
public:
// Size of analyzer internal comparisons queue, measured when new element
// id added to the queue.
SamplesStatsCounter comparisons_queue_size;
// Amount of performed comparisons of 2 video frames from captured and
// rendered streams.
int64_t comparisons_done = 0;
// Amount of overloaded comparisons. Comparison is overloaded if it is queued
// when there are too many not processed comparisons in the queue. Overloaded
// comparison doesn't include metrics, that require heavy computations like
// SSIM and PSNR.
int64_t overloaded_comparisons_done = 0;
};
class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
public:
explicit DefaultVideoQualityAnalyzer(std::string test_label);
~DefaultVideoQualityAnalyzer() override;
void Start(int max_threads_count) override;
uint16_t OnFrameCaptured(const std::string& stream_label,
const VideoFrame& frame) override;
void OnFramePreEncode(const VideoFrame& frame) override;
void OnFrameEncoded(uint16_t frame_id,
const EncodedImage& encoded_image) override;
void OnFrameDropped(EncodedImageCallback::DropReason reason) override;
void OnFrameReceived(uint16_t frame_id,
const EncodedImage& input_image) override;
void OnFrameDecoded(const VideoFrame& frame,
absl::optional<int32_t> decode_time_ms,
absl::optional<uint8_t> qp) override;
void OnFrameRendered(const VideoFrame& frame) override;
void OnEncoderError(const VideoFrame& frame, int32_t error_code) override;
void OnDecoderError(uint16_t frame_id, int32_t error_code) override;
void Stop() override;
// Returns set of stream labels, that were met during test call.
std::set<std::string> GetKnownVideoStreams() const;
const FrameCounters& GetGlobalCounters();
// Returns frame counter per stream label. Valid stream labels can be obtained
// by calling GetKnownVideoStreams()
const std::map<std::string, FrameCounters>& GetPerStreamCounters() const;
// Returns video quality stats per stream label. Valid stream labels can be
// obtained by calling GetKnownVideoStreams()
const std::map<std::string, StreamStats>& GetStats() const;
const AnalyzerStats& GetAnalyzerStats() const;
private:
struct FrameStats {
FrameStats(std::string stream_label, Timestamp captured_time);
std::string stream_label;
// Frame events timestamp.
Timestamp captured_time;
Timestamp pre_encode_time = Timestamp::MinusInfinity();
Timestamp encoded_time = Timestamp::MinusInfinity();
Timestamp received_time = Timestamp::MinusInfinity();
Timestamp decoded_time = Timestamp::MinusInfinity();
Timestamp rendered_time = Timestamp::MinusInfinity();
Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity();
absl::optional<int> rendered_frame_width = absl::nullopt;
absl::optional<int> rendered_frame_height = absl::nullopt;
};
// Represents comparison between two VideoFrames. Contains video frames itself
// and stats. Can be one of two types:
// 1. Normal - in this case |captured| is presented and either |rendered| is
// presented and |dropped| is false, either |rendered| is omitted and
// |dropped| is true.
// 2. Overloaded - in this case both |captured| and |rendered| are omitted
// because there were too many comparisons in the queue. |dropped| can be
// true or false showing was frame dropped or not.
struct FrameComparison {
FrameComparison(absl::optional<VideoFrame> captured,
absl::optional<VideoFrame> rendered,
bool dropped,
FrameStats frame_stats);
FrameComparison(bool dropped, FrameStats frameStats);
// Frames can be omitted if there too many computations waiting in the
// queue.
absl::optional<VideoFrame> captured;
absl::optional<VideoFrame> rendered;
// If true frame was dropped somewhere from capturing to rendering and
// wasn't rendered on remote peer side. If |dropped| is true, |rendered|
// will be |absl::nullopt|.
bool dropped;
FrameStats frame_stats;
};
// Represents a current state of video stream.
struct StreamState {
// To correctly determine dropped frames we have to know sequence of frames
// in each stream so we will keep a list of frame ids inside the stream.
// When the frame is rendered, we will pop ids from the list for until id
// will match with rendered one. All ids before matched one can be
// considered as dropped:
//
// | frame_id1 |->| frame_id2 |->| frame_id3 |->| frame_id4 |
//
// If we received frame with id frame_id3, then we will pop frame_id1 and
// frame_id2 and consider that frames as dropped and then compare received
// frame with the one from |captured_frames_in_flight_| with id frame_id3.
// Also we will put it into the |last_rendered_frame|.
std::list<uint16_t> frame_ids;
absl::optional<VideoFrame> last_rendered_frame = absl::nullopt;
absl::optional<Timestamp> last_rendered_frame_time = absl::nullopt;
};
enum State { kNew, kActive, kStopped };
// Returns last rendered frame for stream if there is one or nullptr
// otherwise.
VideoFrame* GetLastRenderedFrame(const std::string& stream_label)
RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_);
void SetLastRenderedFrame(const std::string& stream_label,
const VideoFrame& frame)
RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_);
void AddComparison(absl::optional<VideoFrame> captured,
absl::optional<VideoFrame> rendered,
bool dropped,
FrameStats frame_stats);
static void ProcessComparisonsThread(void* obj);
void ProcessComparisons();
void ProcessComparison(const FrameComparison& comparison);
// Report results for all metrics for all streams.
void ReportResults() const;
static void ReportResults(std::string test_case_name,
StreamStats stats,
FrameCounters frame_counters);
// Report result for single metric for specified stream.
static void ReportResult(const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit);
// Returns name of current test case for reporting.
std::string GetTestCaseName(const std::string& stream_label) const;
Timestamp Now();
const std::string test_label_;
webrtc::Clock* const clock_;
std::atomic<uint16_t> next_frame_id_{0};
rtc::CriticalSection lock_;
State state_ RTC_GUARDED_BY(lock_) = State::kNew;
// Frames that were captured by all streams and still aren't rendered by any
// stream or deemed dropped.
std::map<uint16_t, VideoFrame> captured_frames_in_flight_
RTC_GUARDED_BY(lock_);
// Global frames count for all video streams.
FrameCounters frame_counters_ RTC_GUARDED_BY(lock_);
// Frame counters per each stream.
std::map<std::string, FrameCounters> stream_frame_counters_
RTC_GUARDED_BY(lock_);
std::map<uint16_t, FrameStats> frame_stats_ RTC_GUARDED_BY(lock_);
std::map<std::string, StreamState> stream_states_ RTC_GUARDED_BY(lock_);
rtc::CriticalSection comparison_lock_;
std::map<std::string, StreamStats> stream_stats_
RTC_GUARDED_BY(comparison_lock_);
std::map<std::string, Timestamp> stream_last_freeze_end_time_
RTC_GUARDED_BY(comparison_lock_);
std::deque<FrameComparison> comparisons_ RTC_GUARDED_BY(comparison_lock_);
AnalyzerStats analyzer_stats_ RTC_GUARDED_BY(comparison_lock_);
std::vector<std::unique_ptr<rtc::PlatformThread>> thread_pool_;
rtc::Event comparison_available_event_;
};
} // namespace test
} // namespace webrtc
#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_H_

27
test/pc/e2e/peer_connection_e2e_smoke_test.cc
View File

@ -16,7 +16,7 @@
#include "rtc_base/async_invoker.h"
#include "rtc_base/fake_network.h"
#include "test/gtest.h"
#include "test/pc/e2e/analyzer/video/example_video_quality_analyzer.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
#include "test/pc/e2e/api/create_peerconnection_quality_test_fixture.h"
#include "test/pc/e2e/api/peerconnection_quality_test_fixture.h"
#include "test/scenario/network/network_emulation.h"
@ -101,8 +101,8 @@ TEST(PeerConnectionE2EQualityTestSmokeTest, RunWithEmulatedNetwork) {
// Create analyzers.
auto analyzers = absl::make_unique<Analyzers>();
analyzers->video_quality_analyzer =
absl::make_unique<ExampleVideoQualityAnalyzer>();
auto* video_analyzer = static_cast<ExampleVideoQualityAnalyzer*>(
absl::make_unique<DefaultVideoQualityAnalyzer>("smoke_test");
auto* video_analyzer = static_cast<DefaultVideoQualityAnalyzer*>(
analyzers->video_quality_analyzer.get());
auto fixture =
@ -111,17 +111,16 @@ TEST(PeerConnectionE2EQualityTestSmokeTest, RunWithEmulatedNetwork) {
std::move(bob_components), absl::make_unique<Params>(),
RunParams{TimeDelta::seconds(5)});
RTC_LOG(INFO) << "Captured: " << video_analyzer->frames_captured();
RTC_LOG(INFO) << "Sent : " << video_analyzer->frames_sent();
RTC_LOG(INFO) << "Received: " << video_analyzer->frames_received();
RTC_LOG(INFO) << "Rendered: " << video_analyzer->frames_rendered();
RTC_LOG(INFO) << "Dropped : " << video_analyzer->frames_dropped();
// 150 = 30fps * 5s
EXPECT_GE(video_analyzer->frames_captured(), 150lu);
// EXPECT_NEAR(video_analyzer->frames_sent(), 150, 15);
// EXPECT_NEAR(video_analyzer->frames_received(), 150, 15);
// EXPECT_NEAR(video_analyzer->frames_rendered(), 150, 15);
// 150 = 30fps * 5s. On some devices pipeline can be too slow, so it can
// happen, that frames will stuck in the middle, so we actually can't force
// real constraints here, so lets just check, that at least 1 frame passed
// whole pipeline.
EXPECT_GE(video_analyzer->GetGlobalCounters().captured, 150);
EXPECT_GE(video_analyzer->GetGlobalCounters().pre_encoded, 1);
EXPECT_GE(video_analyzer->GetGlobalCounters().encoded, 1);
EXPECT_GE(video_analyzer->GetGlobalCounters().received, 1);
EXPECT_GE(video_analyzer->GetGlobalCounters().decoded, 1);
EXPECT_GE(video_analyzer->GetGlobalCounters().rendered, 1);
}
} // namespace test