admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
webrtc_m130/video/video_analyzer.cc
Markus Handell c89fdd716c Refactor the PlatformThread API. 
PlatformThread's API is using old style function pointers, causes
casting, is unintuitive and forces artificial call sequences, and
is additionally possible to misuse in release mode.

Fix this by an API face lift:
1. The class is turned into a handle, which can be empty.
2. The only way of getting a non-empty PlatformThread is by calling
SpawnJoinable or SpawnDetached, clearly conveying the semantics to the
code reader.
3. Handles can be Finalized, which works differently for joinable and
detached threads:
  a) Handles for detached threads are simply closed where applicable.
  b) Joinable threads are joined before handles are closed.
4. The destructor finalizes handles. No explicit call is needed.

Fixed: webrtc:12727
Change-Id: Id00a0464edf4fc9e552b6a1fbb5d2e1280e88811
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/215075
Commit-Queue: Markus Handell <handellm@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Tommi <tommi@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#33923}
2021-05-05 09:59:07 +00:00

1036 lines
38 KiB
C++
Raw Blame History

/*
* Copyright 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "video/video_analyzer.h"
#include <algorithm>
#include <utility>
#include "absl/algorithm/container.h"
#include "absl/flags/flag.h"
#include "absl/flags/parse.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "modules/rtp_rtcp/source/create_video_rtp_depacketizer.h"
#include "modules/rtp_rtcp/source/rtp_packet.h"
#include "rtc_base/cpu_time.h"
#include "rtc_base/format_macros.h"
#include "rtc_base/memory_usage.h"
#include "rtc_base/task_queue_for_test.h"
#include "rtc_base/task_utils/repeating_task.h"
#include "rtc_base/time_utils.h"
#include "system_wrappers/include/cpu_info.h"
#include "test/call_test.h"
#include "test/testsupport/file_utils.h"
#include "test/testsupport/frame_writer.h"
#include "test/testsupport/perf_test.h"
#include "test/testsupport/test_artifacts.h"
ABSL_FLAG(bool,
save_worst_frame,
false,
"Enable saving a frame with the lowest PSNR to a jpeg file in the "
"test_artifacts_dir");
namespace webrtc {
namespace {
constexpr TimeDelta kSendStatsPollingInterval = TimeDelta::Seconds(1);
constexpr size_t kMaxComparisons = 10;
// How often is keep alive message printed.
constexpr int kKeepAliveIntervalSeconds = 30;
// Interval between checking that the test is over.
constexpr int kProbingIntervalMs = 500;
constexpr int kKeepAliveIntervalIterations =
kKeepAliveIntervalSeconds * 1000 / kProbingIntervalMs;
bool IsFlexfec(int payload_type) {
return payload_type == test::CallTest::kFlexfecPayloadType;
}
} // namespace
VideoAnalyzer::VideoAnalyzer(test::LayerFilteringTransport* transport,
const std::string& test_label,
double avg_psnr_threshold,
double avg_ssim_threshold,
int duration_frames,
TimeDelta test_duration,
FILE* graph_data_output_file,
const std::string& graph_title,
uint32_t ssrc_to_analyze,
uint32_t rtx_ssrc_to_analyze,
size_t selected_stream,
int selected_sl,
int selected_tl,
bool is_quick_test_enabled,
Clock* clock,
std::string rtp_dump_name,
TaskQueueBase* task_queue)
: transport_(transport),
receiver_(nullptr),
call_(nullptr),
send_stream_(nullptr),
receive_stream_(nullptr),
audio_receive_stream_(nullptr),
captured_frame_forwarder_(this, clock, duration_frames, test_duration),
test_label_(test_label),
graph_data_output_file_(graph_data_output_file),
graph_title_(graph_title),
ssrc_to_analyze_(ssrc_to_analyze),
rtx_ssrc_to_analyze_(rtx_ssrc_to_analyze),
selected_stream_(selected_stream),
selected_sl_(selected_sl),
selected_tl_(selected_tl),
mean_decode_time_ms_(0.0),
freeze_count_(0),
total_freezes_duration_ms_(0),
total_frames_duration_ms_(0),
sum_squared_frame_durations_(0),
decode_frame_rate_(0),
render_frame_rate_(0),
last_fec_bytes_(0),
frames_to_process_(duration_frames),
test_end_(clock->CurrentTime() + test_duration),
frames_recorded_(0),
frames_processed_(0),
captured_frames_(0),
dropped_frames_(0),
dropped_frames_before_first_encode_(0),
dropped_frames_before_rendering_(0),
last_render_time_(0),
last_render_delta_ms_(0),
last_unfreeze_time_ms_(0),
rtp_timestamp_delta_(0),
cpu_time_(0),
wallclock_time_(0),
avg_psnr_threshold_(avg_psnr_threshold),
avg_ssim_threshold_(avg_ssim_threshold),
is_quick_test_enabled_(is_quick_test_enabled),
quit_(false),
done_(true, false),
vp8_depacketizer_(CreateVideoRtpDepacketizer(kVideoCodecVP8)),
vp9_depacketizer_(CreateVideoRtpDepacketizer(kVideoCodecVP9)),
clock_(clock),
start_ms_(clock->TimeInMilliseconds()),
task_queue_(task_queue) {
// Create thread pool for CPU-expensive PSNR/SSIM calculations.
// Try to use about as many threads as cores, but leave kMinCoresLeft alone,
// so that we don't accidentally starve "real" worker threads (codec etc).
// Also, don't allocate more than kMaxComparisonThreads, even if there are
// spare cores.
uint32_t num_cores = CpuInfo::DetectNumberOfCores();
RTC_DCHECK_GE(num_cores, 1);
static const uint32_t kMinCoresLeft = 4;
static const uint32_t kMaxComparisonThreads = 8;
if (num_cores <= kMinCoresLeft) {
num_cores = 1;
} else {
num_cores -= kMinCoresLeft;
num_cores = std::min(num_cores, kMaxComparisonThreads);
}
for (uint32_t i = 0; i < num_cores; ++i) {
comparison_thread_pool_.push_back(rtc::PlatformThread::SpawnJoinable(
[this] {
while (CompareFrames()) {
}
},
"Analyzer"));
}
if (!rtp_dump_name.empty()) {
fprintf(stdout, "Writing rtp dump to %s\n", rtp_dump_name.c_str());
rtp_file_writer_.reset(test::RtpFileWriter::Create(
test::RtpFileWriter::kRtpDump, rtp_dump_name));
}
}
VideoAnalyzer::~VideoAnalyzer() {
{
MutexLock lock(&comparison_lock_);
quit_ = true;
}
// Joins all threads.
comparison_thread_pool_.clear();
}
void VideoAnalyzer::SetReceiver(PacketReceiver* receiver) {
receiver_ = receiver;
}
void VideoAnalyzer::SetSource(
rtc::VideoSourceInterface<VideoFrame>* video_source,
bool respect_sink_wants) {
if (respect_sink_wants)
captured_frame_forwarder_.SetSource(video_source);
rtc::VideoSinkWants wants;
video_source->AddOrUpdateSink(InputInterface(), wants);
}
void VideoAnalyzer::SetCall(Call* call) {
MutexLock lock(&lock_);
RTC_DCHECK(!call_);
call_ = call;
}
void VideoAnalyzer::SetSendStream(VideoSendStream* stream) {
MutexLock lock(&lock_);
RTC_DCHECK(!send_stream_);
send_stream_ = stream;
}
void VideoAnalyzer::SetReceiveStream(VideoReceiveStream* stream) {
MutexLock lock(&lock_);
RTC_DCHECK(!receive_stream_);
receive_stream_ = stream;
}
void VideoAnalyzer::SetAudioReceiveStream(AudioReceiveStream* recv_stream) {
MutexLock lock(&lock_);
RTC_CHECK(!audio_receive_stream_);
audio_receive_stream_ = recv_stream;
}
rtc::VideoSinkInterface<VideoFrame>* VideoAnalyzer::InputInterface() {
return &captured_frame_forwarder_;
}
rtc::VideoSourceInterface<VideoFrame>* VideoAnalyzer::OutputInterface() {
return &captured_frame_forwarder_;
}
PacketReceiver::DeliveryStatus VideoAnalyzer::DeliverPacket(
MediaType media_type,
rtc::CopyOnWriteBuffer packet,
int64_t packet_time_us) {
// Ignore timestamps of RTCP packets. They're not synchronized with
// RTP packet timestamps and so they would confuse wrap_handler_.
if (RtpHeaderParser::IsRtcp(packet.cdata(), packet.size())) {
return receiver_->DeliverPacket(media_type, std::move(packet),
packet_time_us);
}
if (rtp_file_writer_) {
test::RtpPacket p;
memcpy(p.data, packet.cdata(), packet.size());
p.length = packet.size();
p.original_length = packet.size();
p.time_ms = clock_->TimeInMilliseconds() - start_ms_;
rtp_file_writer_->WritePacket(&p);
}
RtpPacket rtp_packet;
rtp_packet.Parse(packet);
if (!IsFlexfec(rtp_packet.PayloadType()) &&
(rtp_packet.Ssrc() == ssrc_to_analyze_ ||
rtp_packet.Ssrc() == rtx_ssrc_to_analyze_)) {
// Ignore FlexFEC timestamps, to avoid collisions with media timestamps.
// (FlexFEC and media are sent on different SSRCs, which have different
// timestamps spaces.)
// Also ignore packets from wrong SSRC, but include retransmits.
MutexLock lock(&lock_);
int64_t timestamp =
wrap_handler_.Unwrap(rtp_packet.Timestamp() - rtp_timestamp_delta_);
recv_times_[timestamp] = clock_->CurrentNtpInMilliseconds();
}
return receiver_->DeliverPacket(media_type, std::move(packet),
packet_time_us);
}
void VideoAnalyzer::PreEncodeOnFrame(const VideoFrame& video_frame) {
MutexLock lock(&lock_);
if (!first_encoded_timestamp_) {
while (frames_.front().timestamp() != video_frame.timestamp()) {
++dropped_frames_before_first_encode_;
frames_.pop_front();
RTC_CHECK(!frames_.empty());
}
first_encoded_timestamp_ = video_frame.timestamp();
}
}
void VideoAnalyzer::PostEncodeOnFrame(size_t stream_id, uint32_t timestamp) {
MutexLock lock(&lock_);
if (!first_sent_timestamp_ && stream_id == selected_stream_) {
first_sent_timestamp_ = timestamp;
}
}
bool VideoAnalyzer::SendRtp(const uint8_t* packet,
size_t length,
const PacketOptions& options) {
RtpPacket rtp_packet;
rtp_packet.Parse(packet, length);
int64_t current_time = clock_->CurrentNtpInMilliseconds();
bool result = transport_->SendRtp(packet, length, options);
{
MutexLock lock(&lock_);
if (rtp_timestamp_delta_ == 0 && rtp_packet.Ssrc() == ssrc_to_analyze_) {
RTC_CHECK(static_cast<bool>(first_sent_timestamp_));
rtp_timestamp_delta_ = rtp_packet.Timestamp() - *first_sent_timestamp_;
}
if (!IsFlexfec(rtp_packet.PayloadType()) &&
rtp_packet.Ssrc() == ssrc_to_analyze_) {
// Ignore FlexFEC timestamps, to avoid collisions with media timestamps.
// (FlexFEC and media are sent on different SSRCs, which have different
// timestamps spaces.)
// Also ignore packets from wrong SSRC and retransmits.
int64_t timestamp =
wrap_handler_.Unwrap(rtp_packet.Timestamp() - rtp_timestamp_delta_);
send_times_[timestamp] = current_time;
if (IsInSelectedSpatialAndTemporalLayer(rtp_packet)) {
encoded_frame_sizes_[timestamp] += rtp_packet.payload_size();
}
}
}
return result;
}
bool VideoAnalyzer::SendRtcp(const uint8_t* packet, size_t length) {
return transport_->SendRtcp(packet, length);
}
void VideoAnalyzer::OnFrame(const VideoFrame& video_frame) {
int64_t render_time_ms = clock_->CurrentNtpInMilliseconds();
MutexLock lock(&lock_);
StartExcludingCpuThreadTime();
int64_t send_timestamp =
wrap_handler_.Unwrap(video_frame.timestamp() - rtp_timestamp_delta_);
while (wrap_handler_.Unwrap(frames_.front().timestamp()) < send_timestamp) {
if (!last_rendered_frame_) {
// No previous frame rendered, this one was dropped after sending but
// before rendering.
++dropped_frames_before_rendering_;
} else {
AddFrameComparison(frames_.front(), *last_rendered_frame_, true,
render_time_ms);
}
frames_.pop_front();
RTC_DCHECK(!frames_.empty());
}
VideoFrame reference_frame = frames_.front();
frames_.pop_front();
int64_t reference_timestamp =
wrap_handler_.Unwrap(reference_frame.timestamp());
if (send_timestamp == reference_timestamp - 1) {
// TODO(ivica): Make this work for > 2 streams.
// Look at RTPSender::BuildRTPHeader.
++send_timestamp;
}
ASSERT_EQ(reference_timestamp, send_timestamp);
AddFrameComparison(reference_frame, video_frame, false, render_time_ms);
last_rendered_frame_ = video_frame;
StopExcludingCpuThreadTime();
}
void VideoAnalyzer::Wait() {
// Frame comparisons can be very expensive. Wait for test to be done, but
// at time-out check if frames_processed is going up. If so, give it more
// time, otherwise fail. Hopefully this will reduce test flakiness.
RepeatingTaskHandle stats_polling_task = RepeatingTaskHandle::DelayedStart(
task_queue_, kSendStatsPollingInterval, [this] {
PollStats();
return kSendStatsPollingInterval;
});
int last_frames_processed = -1;
int last_frames_captured = -1;
int iteration = 0;
while (!done_.Wait(kProbingIntervalMs)) {
int frames_processed;
int frames_captured;
{
MutexLock lock(&comparison_lock_);
frames_processed = frames_processed_;
frames_captured = captured_frames_;
}
// Print some output so test infrastructure won't think we've crashed.
const char* kKeepAliveMessages[3] = {
"Uh, I'm-I'm not quite dead, sir.",
"Uh, I-I think uh, I could pull through, sir.",
"Actually, I think I'm all right to come with you--"};
if (++iteration % kKeepAliveIntervalIterations == 0) {
printf("- %s\n", kKeepAliveMessages[iteration % 3]);
}
if (last_frames_processed == -1) {
last_frames_processed = frames_processed;
last_frames_captured = frames_captured;
continue;
}
if (frames_processed == last_frames_processed &&
last_frames_captured == frames_captured &&
clock_->CurrentTime() > test_end_) {
done_.Set();
break;
}
last_frames_processed = frames_processed;
last_frames_captured = frames_captured;
}
if (iteration > 0)
printf("- Farewell, sweet Concorde!\n");
SendTask(RTC_FROM_HERE, task_queue_, [&] { stats_polling_task.Stop(); });
PrintResults();
if (graph_data_output_file_)
PrintSamplesToFile();
}
void VideoAnalyzer::StartMeasuringCpuProcessTime() {
MutexLock lock(&cpu_measurement_lock_);
cpu_time_ -= rtc::GetProcessCpuTimeNanos();
wallclock_time_ -= rtc::SystemTimeNanos();
}
void VideoAnalyzer::StopMeasuringCpuProcessTime() {
MutexLock lock(&cpu_measurement_lock_);
cpu_time_ += rtc::GetProcessCpuTimeNanos();
wallclock_time_ += rtc::SystemTimeNanos();
}
void VideoAnalyzer::StartExcludingCpuThreadTime() {
MutexLock lock(&cpu_measurement_lock_);
cpu_time_ += rtc::GetThreadCpuTimeNanos();
}
void VideoAnalyzer::StopExcludingCpuThreadTime() {
MutexLock lock(&cpu_measurement_lock_);
cpu_time_ -= rtc::GetThreadCpuTimeNanos();
}
double VideoAnalyzer::GetCpuUsagePercent() {
MutexLock lock(&cpu_measurement_lock_);
return static_cast<double>(cpu_time_) / wallclock_time_ * 100.0;
}
bool VideoAnalyzer::IsInSelectedSpatialAndTemporalLayer(
const RtpPacket& rtp_packet) {
if (rtp_packet.PayloadType() == test::CallTest::kPayloadTypeVP8) {
auto parsed_payload = vp8_depacketizer_->Parse(rtp_packet.PayloadBuffer());
RTC_DCHECK(parsed_payload);
const auto& vp8_header = absl::get<RTPVideoHeaderVP8>(
parsed_payload->video_header.video_type_header);
int temporal_idx = vp8_header.temporalIdx;
return selected_tl_ < 0 || temporal_idx == kNoTemporalIdx ||
temporal_idx <= selected_tl_;
}
if (rtp_packet.PayloadType() == test::CallTest::kPayloadTypeVP9) {
auto parsed_payload = vp9_depacketizer_->Parse(rtp_packet.PayloadBuffer());
RTC_DCHECK(parsed_payload);
const auto& vp9_header = absl::get<RTPVideoHeaderVP9>(
parsed_payload->video_header.video_type_header);
int temporal_idx = vp9_header.temporal_idx;
int spatial_idx = vp9_header.spatial_idx;
return (selected_tl_ < 0 || temporal_idx == kNoTemporalIdx ||
temporal_idx <= selected_tl_) &&
(selected_sl_ < 0 || spatial_idx == kNoSpatialIdx ||
spatial_idx <= selected_sl_);
}
return true;
}
void VideoAnalyzer::PollStats() {
// Do not grab |comparison_lock_|, before |GetStats()| completes.
// Otherwise a deadlock may occur:
// 1) |comparison_lock_| is acquired after |lock_|
// 2) |lock_| is acquired after internal pacer lock in SendRtp()
// 3) internal pacer lock is acquired by GetStats().
Call::Stats call_stats = call_->GetStats();
MutexLock lock(&comparison_lock_);
send_bandwidth_bps_.AddSample(call_stats.send_bandwidth_bps);
VideoSendStream::Stats send_stats = send_stream_->GetStats();
// It's not certain that we yet have estimates for any of these stats.
// Check that they are positive before mixing them in.
if (send_stats.encode_frame_rate > 0)
encode_frame_rate_.AddSample(send_stats.encode_frame_rate);
if (send_stats.avg_encode_time_ms > 0)
encode_time_ms_.AddSample(send_stats.avg_encode_time_ms);
if (send_stats.encode_usage_percent > 0)
encode_usage_percent_.AddSample(send_stats.encode_usage_percent);
if (send_stats.media_bitrate_bps > 0)
media_bitrate_bps_.AddSample(send_stats.media_bitrate_bps);
size_t fec_bytes = 0;
for (const auto& kv : send_stats.substreams) {
fec_bytes += kv.second.rtp_stats.fec.payload_bytes +
kv.second.rtp_stats.fec.padding_bytes;
}
fec_bitrate_bps_.AddSample((fec_bytes - last_fec_bytes_) * 8);
last_fec_bytes_ = fec_bytes;
if (receive_stream_ != nullptr) {
VideoReceiveStream::Stats receive_stats = receive_stream_->GetStats();
// |total_decode_time_ms| gives a good estimate of the mean decode time,
// |decode_ms| is used to keep track of the standard deviation.
if (receive_stats.frames_decoded > 0)
mean_decode_time_ms_ =
static_cast<double>(receive_stats.total_decode_time_ms) /
receive_stats.frames_decoded;
if (receive_stats.decode_ms > 0)
decode_time_ms_.AddSample(receive_stats.decode_ms);
if (receive_stats.max_decode_ms > 0)
decode_time_max_ms_.AddSample(receive_stats.max_decode_ms);
if (receive_stats.width > 0 && receive_stats.height > 0) {
pixels_.AddSample(receive_stats.width * receive_stats.height);
}
// |frames_decoded| and |frames_rendered| are used because they are more
// accurate than |decode_frame_rate| and |render_frame_rate|.
// The latter two are calculated on a momentary basis.
const double total_frames_duration_sec_double =
static_cast<double>(receive_stats.total_frames_duration_ms) / 1000.0;
if (total_frames_duration_sec_double > 0) {
decode_frame_rate_ = static_cast<double>(receive_stats.frames_decoded) /
total_frames_duration_sec_double;
render_frame_rate_ = static_cast<double>(receive_stats.frames_rendered) /
total_frames_duration_sec_double;
}
// Freeze metrics.
freeze_count_ = receive_stats.freeze_count;
total_freezes_duration_ms_ = receive_stats.total_freezes_duration_ms;
total_frames_duration_ms_ = receive_stats.total_frames_duration_ms;
sum_squared_frame_durations_ = receive_stats.sum_squared_frame_durations;
}
if (audio_receive_stream_ != nullptr) {
AudioReceiveStream::Stats receive_stats =
audio_receive_stream_->GetStats(/*get_and_clear_legacy_stats=*/true);
audio_expand_rate_.AddSample(receive_stats.expand_rate);
audio_accelerate_rate_.AddSample(receive_stats.accelerate_rate);
audio_jitter_buffer_ms_.AddSample(receive_stats.jitter_buffer_ms);
}
memory_usage_.AddSample(rtc::GetProcessResidentSizeBytes());
}
bool VideoAnalyzer::CompareFrames() {
if (AllFramesRecorded())
return false;
FrameComparison comparison;
if (!PopComparison(&comparison)) {
// Wait until new comparison task is available, or test is done.
// If done, wake up remaining threads waiting.
comparison_available_event_.Wait(1000);
if (AllFramesRecorded()) {
comparison_available_event_.Set();
return false;
}
return true; // Try again.
}
StartExcludingCpuThreadTime();
PerformFrameComparison(comparison);
StopExcludingCpuThreadTime();
if (FrameProcessed()) {
done_.Set();
comparison_available_event_.Set();
return false;
}
return true;
}
bool VideoAnalyzer::PopComparison(VideoAnalyzer::FrameComparison* comparison) {
MutexLock lock(&comparison_lock_);
// If AllFramesRecorded() is true, it means we have already popped
// frames_to_process_ frames from comparisons_, so there is no more work
// for this thread to be done. frames_processed_ might still be lower if
// all comparisons are not done, but those frames are currently being
// worked on by other threads.
if (comparisons_.empty() || AllFramesRecordedLocked())
return false;
*comparison = comparisons_.front();
comparisons_.pop_front();
FrameRecorded();
return true;
}
void VideoAnalyzer::FrameRecorded() {
++frames_recorded_;
}
bool VideoAnalyzer::AllFramesRecorded() {
MutexLock lock(&comparison_lock_);
return AllFramesRecordedLocked();
}
bool VideoAnalyzer::AllFramesRecordedLocked() {
RTC_DCHECK(frames_recorded_ <= frames_to_process_);
return frames_recorded_ == frames_to_process_ ||
(clock_->CurrentTime() > test_end_ && comparisons_.empty()) || quit_;
}
bool VideoAnalyzer::FrameProcessed() {
MutexLock lock(&comparison_lock_);
++frames_processed_;
assert(frames_processed_ <= frames_to_process_);
return frames_processed_ == frames_to_process_ ||
(clock_->CurrentTime() > test_end_ && comparisons_.empty());
}
void VideoAnalyzer::PrintResults() {
using ::webrtc::test::ImproveDirection;
StopMeasuringCpuProcessTime();
int dropped_frames_diff;
{
MutexLock lock(&lock_);
dropped_frames_diff = dropped_frames_before_first_encode_ +
dropped_frames_before_rendering_ + frames_.size();
}
MutexLock lock(&comparison_lock_);
PrintResult("psnr", psnr_, "dB", ImproveDirection::kBiggerIsBetter);
PrintResult("ssim", ssim_, "unitless", ImproveDirection::kBiggerIsBetter);
PrintResult("sender_time", sender_time_, "ms",
ImproveDirection::kSmallerIsBetter);
PrintResult("receiver_time", receiver_time_, "ms",
ImproveDirection::kSmallerIsBetter);
PrintResult("network_time", network_time_, "ms",
ImproveDirection::kSmallerIsBetter);
PrintResult("total_delay_incl_network", end_to_end_, "ms",
ImproveDirection::kSmallerIsBetter);
PrintResult("time_between_rendered_frames", rendered_delta_, "ms",
ImproveDirection::kSmallerIsBetter);
PrintResult("encode_frame_rate", encode_frame_rate_, "fps",
ImproveDirection::kBiggerIsBetter);
PrintResult("encode_time", encode_time_ms_, "ms",
ImproveDirection::kSmallerIsBetter);
PrintResult("media_bitrate", media_bitrate_bps_, "bps",
ImproveDirection::kNone);
PrintResult("fec_bitrate", fec_bitrate_bps_, "bps", ImproveDirection::kNone);
PrintResult("send_bandwidth", send_bandwidth_bps_, "bps",
ImproveDirection::kNone);
PrintResult("pixels_per_frame", pixels_, "count",
ImproveDirection::kBiggerIsBetter);
test::PrintResult("decode_frame_rate", "", test_label_.c_str(),
decode_frame_rate_, "fps", false,
ImproveDirection::kBiggerIsBetter);
test::PrintResult("render_frame_rate", "", test_label_.c_str(),
render_frame_rate_, "fps", false,
ImproveDirection::kBiggerIsBetter);
// Record the time from the last freeze until the last rendered frame to
// ensure we cover the full timespan of the session. Otherwise the metric
// would penalize an early freeze followed by no freezes until the end.
time_between_freezes_.AddSample(last_render_time_ - last_unfreeze_time_ms_);
// Freeze metrics.
PrintResult("time_between_freezes", time_between_freezes_, "ms",
ImproveDirection::kBiggerIsBetter);
const double freeze_count_double = static_cast<double>(freeze_count_);
const double total_freezes_duration_ms_double =
static_cast<double>(total_freezes_duration_ms_);
const double total_frames_duration_ms_double =
static_cast<double>(total_frames_duration_ms_);
if (total_frames_duration_ms_double > 0) {
test::PrintResult(
"freeze_duration_ratio", "", test_label_.c_str(),
total_freezes_duration_ms_double / total_frames_duration_ms_double,
"unitless", false, ImproveDirection::kSmallerIsBetter);
RTC_DCHECK_LE(total_freezes_duration_ms_double,
total_frames_duration_ms_double);
constexpr double ms_per_minute = 60 * 1000;
const double total_frames_duration_min =
total_frames_duration_ms_double / ms_per_minute;
if (total_frames_duration_min > 0) {
test::PrintResult("freeze_count_per_minute", "", test_label_.c_str(),
freeze_count_double / total_frames_duration_min,
"unitless", false, ImproveDirection::kSmallerIsBetter);
}
}
test::PrintResult("freeze_duration_average", "", test_label_.c_str(),
freeze_count_double > 0
? total_freezes_duration_ms_double / freeze_count_double
: 0,
"ms", false, ImproveDirection::kSmallerIsBetter);
if (1000 * sum_squared_frame_durations_ > 0) {
test::PrintResult(
"harmonic_frame_rate", "", test_label_.c_str(),
total_frames_duration_ms_double / (1000 * sum_squared_frame_durations_),
"fps", false, ImproveDirection::kBiggerIsBetter);
}
if (worst_frame_) {
test::PrintResult("min_psnr", "", test_label_.c_str(), worst_frame_->psnr,
"dB", false, ImproveDirection::kBiggerIsBetter);
}
if (receive_stream_ != nullptr) {
PrintResultWithExternalMean("decode_time", mean_decode_time_ms_,
decode_time_ms_, "ms",
ImproveDirection::kSmallerIsBetter);
}
dropped_frames_ += dropped_frames_diff;
test::PrintResult("dropped_frames", "", test_label_.c_str(), dropped_frames_,
"count", false, ImproveDirection::kSmallerIsBetter);
test::PrintResult("cpu_usage", "", test_label_.c_str(), GetCpuUsagePercent(),
"%", false, ImproveDirection::kSmallerIsBetter);
#if defined(WEBRTC_WIN)
// On Linux and Mac in Resident Set some unused pages may be counted.
// Therefore this metric will depend on order in which tests are run and
// will be flaky.
PrintResult("memory_usage", memory_usage_, "sizeInBytes",
ImproveDirection::kSmallerIsBetter);
#endif
// Saving only the worst frame for manual analysis. Intention here is to
// only detect video corruptions and not to track picture quality. Thus,
// jpeg is used here.
if (absl::GetFlag(FLAGS_save_worst_frame) && worst_frame_) {
std::string output_dir;
test::GetTestArtifactsDir(&output_dir);
std::string output_path =
test::JoinFilename(output_dir, test_label_ + ".jpg");
RTC_LOG(LS_INFO) << "Saving worst frame to " << output_path;
test::JpegFrameWriter frame_writer(output_path);
RTC_CHECK(
frame_writer.WriteFrame(worst_frame_->frame, 100 /*best quality*/));
}
if (audio_receive_stream_ != nullptr) {
PrintResult("audio_expand_rate", audio_expand_rate_, "unitless",
ImproveDirection::kSmallerIsBetter);
PrintResult("audio_accelerate_rate", audio_accelerate_rate_, "unitless",
ImproveDirection::kSmallerIsBetter);
PrintResult("audio_jitter_buffer", audio_jitter_buffer_ms_, "ms",
ImproveDirection::kNone);
}
// Disable quality check for quick test, as quality checks may fail
// because too few samples were collected.
if (!is_quick_test_enabled_) {
EXPECT_GT(*psnr_.GetMean(), avg_psnr_threshold_);
EXPECT_GT(*ssim_.GetMean(), avg_ssim_threshold_);
}
}
void VideoAnalyzer::PerformFrameComparison(
const VideoAnalyzer::FrameComparison& comparison) {
// Perform expensive psnr and ssim calculations while not holding lock.
double psnr = -1.0;
double ssim = -1.0;
if (comparison.reference && !comparison.dropped) {
psnr = I420PSNR(&*comparison.reference, &*comparison.render);
ssim = I420SSIM(&*comparison.reference, &*comparison.render);
}
MutexLock lock(&comparison_lock_);
if (psnr >= 0.0 && (!worst_frame_ || worst_frame_->psnr > psnr)) {
worst_frame_.emplace(FrameWithPsnr{psnr, *comparison.render});
}
if (graph_data_output_file_) {
samples_.push_back(Sample(comparison.dropped, comparison.input_time_ms,
comparison.send_time_ms, comparison.recv_time_ms,
comparison.render_time_ms,
comparison.encoded_frame_size, psnr, ssim));
}
if (psnr >= 0.0)
psnr_.AddSample(psnr);
if (ssim >= 0.0)
ssim_.AddSample(ssim);
if (comparison.dropped) {
++dropped_frames_;
return;
}
if (last_unfreeze_time_ms_ == 0)
last_unfreeze_time_ms_ = comparison.render_time_ms;
if (last_render_time_ != 0) {
const int64_t render_delta_ms =
comparison.render_time_ms - last_render_time_;
rendered_delta_.AddSample(render_delta_ms);
if (last_render_delta_ms_ != 0 &&
render_delta_ms - last_render_delta_ms_ > 150) {
time_between_freezes_.AddSample(last_render_time_ -
last_unfreeze_time_ms_);
last_unfreeze_time_ms_ = comparison.render_time_ms;
}
last_render_delta_ms_ = render_delta_ms;
}
last_render_time_ = comparison.render_time_ms;
sender_time_.AddSample(comparison.send_time_ms - comparison.input_time_ms);
if (comparison.recv_time_ms > 0) {
// If recv_time_ms == 0, this frame consisted of a packets which were all
// lost in the transport. Since we were able to render the frame, however,
// the dropped packets were recovered by FlexFEC. The FlexFEC recovery
// happens internally in Call, and we can therefore here not know which
// FEC packets that protected the lost media packets. Consequently, we
// were not able to record a meaningful recv_time_ms. We therefore skip
// this sample.
//
// The reasoning above does not hold for ULPFEC and RTX, as for those
// strategies the timestamp of the received packets is set to the
// timestamp of the protected/retransmitted media packet. I.e., then
// recv_time_ms != 0, even though the media packets were lost.
receiver_time_.AddSample(comparison.render_time_ms -
comparison.recv_time_ms);
network_time_.AddSample(comparison.recv_time_ms - comparison.send_time_ms);
}
end_to_end_.AddSample(comparison.render_time_ms - comparison.input_time_ms);
encoded_frame_size_.AddSample(comparison.encoded_frame_size);
}
void VideoAnalyzer::PrintResult(
const char* result_type,
Statistics stats,
const char* unit,
webrtc::test::ImproveDirection improve_direction) {
test::PrintResultMeanAndError(
result_type, "", test_label_.c_str(), stats.GetMean().value_or(0),
stats.GetStandardDeviation().value_or(0), unit, false, improve_direction);
}
void VideoAnalyzer::PrintResultWithExternalMean(
const char* result_type,
double mean,
Statistics stats,
const char* unit,
webrtc::test::ImproveDirection improve_direction) {
// If the true mean is different than the sample mean, the sample variance is
// too low. The sample variance given a known mean is obtained by adding the
// squared error between the true mean and the sample mean.
double compensated_variance =
stats.Size() > 0
? *stats.GetVariance() + pow(mean - *stats.GetMean(), 2.0)
: 0.0;
test::PrintResultMeanAndError(result_type, "", test_label_.c_str(), mean,
std::sqrt(compensated_variance), unit, false,
improve_direction);
}
void VideoAnalyzer::PrintSamplesToFile() {
FILE* out = graph_data_output_file_;
MutexLock lock(&comparison_lock_);
absl::c_sort(samples_, [](const Sample& A, const Sample& B) -> bool {
return A.input_time_ms < B.input_time_ms;
});
fprintf(out, "%s\n", graph_title_.c_str());
fprintf(out, "%" RTC_PRIuS "\n", samples_.size());
fprintf(out,
"dropped "
"input_time_ms "
"send_time_ms "
"recv_time_ms "
"render_time_ms "
"encoded_frame_size "
"psnr "
"ssim "
"encode_time_ms\n");
for (const Sample& sample : samples_) {
fprintf(out,
"%d %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" RTC_PRIuS
" %lf %lf\n",
sample.dropped, sample.input_time_ms, sample.send_time_ms,
sample.recv_time_ms, sample.render_time_ms,
sample.encoded_frame_size, sample.psnr, sample.ssim);
}
}
void VideoAnalyzer::AddCapturedFrameForComparison(
const VideoFrame& video_frame) {
bool must_capture = false;
{
MutexLock lock(&comparison_lock_);
must_capture = captured_frames_ < frames_to_process_;
if (must_capture) {
++captured_frames_;
}
}
if (must_capture) {
MutexLock lock(&lock_);
frames_.push_back(video_frame);
}
}
void VideoAnalyzer::AddFrameComparison(const VideoFrame& reference,
const VideoFrame& render,
bool dropped,
int64_t render_time_ms) {
int64_t reference_timestamp = wrap_handler_.Unwrap(reference.timestamp());
int64_t send_time_ms = send_times_[reference_timestamp];
send_times_.erase(reference_timestamp);
int64_t recv_time_ms = recv_times_[reference_timestamp];
recv_times_.erase(reference_timestamp);
// TODO(ivica): Make this work for > 2 streams.
auto it = encoded_frame_sizes_.find(reference_timestamp);
if (it == encoded_frame_sizes_.end())
it = encoded_frame_sizes_.find(reference_timestamp - 1);
size_t encoded_size = it == encoded_frame_sizes_.end() ? 0 : it->second;
if (it != encoded_frame_sizes_.end())
encoded_frame_sizes_.erase(it);
MutexLock lock(&comparison_lock_);
if (comparisons_.size() < kMaxComparisons) {
comparisons_.push_back(FrameComparison(
reference, render, dropped, reference.ntp_time_ms(), send_time_ms,
recv_time_ms, render_time_ms, encoded_size));
} else {
comparisons_.push_back(FrameComparison(dropped, reference.ntp_time_ms(),
send_time_ms, recv_time_ms,
render_time_ms, encoded_size));
}
comparison_available_event_.Set();
}
VideoAnalyzer::FrameComparison::FrameComparison()
: dropped(false),
input_time_ms(0),
send_time_ms(0),
recv_time_ms(0),
render_time_ms(0),
encoded_frame_size(0) {}
VideoAnalyzer::FrameComparison::FrameComparison(const VideoFrame& reference,
const VideoFrame& render,
bool dropped,
int64_t input_time_ms,
int64_t send_time_ms,
int64_t recv_time_ms,
int64_t render_time_ms,
size_t encoded_frame_size)
: reference(reference),
render(render),
dropped(dropped),
input_time_ms(input_time_ms),
send_time_ms(send_time_ms),
recv_time_ms(recv_time_ms),
render_time_ms(render_time_ms),
encoded_frame_size(encoded_frame_size) {}
VideoAnalyzer::FrameComparison::FrameComparison(bool dropped,
int64_t input_time_ms,
int64_t send_time_ms,
int64_t recv_time_ms,
int64_t render_time_ms,
size_t encoded_frame_size)
: dropped(dropped),
input_time_ms(input_time_ms),
send_time_ms(send_time_ms),
recv_time_ms(recv_time_ms),
render_time_ms(render_time_ms),
encoded_frame_size(encoded_frame_size) {}
VideoAnalyzer::Sample::Sample(int dropped,
int64_t input_time_ms,
int64_t send_time_ms,
int64_t recv_time_ms,
int64_t render_time_ms,
size_t encoded_frame_size,
double psnr,
double ssim)
: dropped(dropped),
input_time_ms(input_time_ms),
send_time_ms(send_time_ms),
recv_time_ms(recv_time_ms),
render_time_ms(render_time_ms),
encoded_frame_size(encoded_frame_size),
psnr(psnr),
ssim(ssim) {}
VideoAnalyzer::CapturedFrameForwarder::CapturedFrameForwarder(
VideoAnalyzer* analyzer,
Clock* clock,
int frames_to_capture,
TimeDelta test_duration)
: analyzer_(analyzer),
send_stream_input_(nullptr),
video_source_(nullptr),
clock_(clock),
captured_frames_(0),
frames_to_capture_(frames_to_capture),
test_end_(clock->CurrentTime() + test_duration) {}
void VideoAnalyzer::CapturedFrameForwarder::SetSource(
VideoSourceInterface<VideoFrame>* video_source) {
video_source_ = video_source;
}
void VideoAnalyzer::CapturedFrameForwarder::OnFrame(
const VideoFrame& video_frame) {
VideoFrame copy = video_frame;
// Frames from the capturer does not have a rtp timestamp.
// Create one so it can be used for comparison.
RTC_DCHECK_EQ(0, video_frame.timestamp());
if (video_frame.ntp_time_ms() == 0)
copy.set_ntp_time_ms(clock_->CurrentNtpInMilliseconds());
copy.set_timestamp(copy.ntp_time_ms() * 90);
analyzer_->AddCapturedFrameForComparison(copy);
MutexLock lock(&lock_);
++captured_frames_;
if (send_stream_input_ && clock_->CurrentTime() <= test_end_ &&
captured_frames_ <= frames_to_capture_) {
send_stream_input_->OnFrame(copy);
}
}
void VideoAnalyzer::CapturedFrameForwarder::AddOrUpdateSink(
rtc::VideoSinkInterface<VideoFrame>* sink,
const rtc::VideoSinkWants& wants) {
{
MutexLock lock(&lock_);
RTC_DCHECK(!send_stream_input_ || send_stream_input_ == sink);
send_stream_input_ = sink;
}
if (video_source_) {
video_source_->AddOrUpdateSink(this, wants);
}
}
void VideoAnalyzer::CapturedFrameForwarder::RemoveSink(
rtc::VideoSinkInterface<VideoFrame>* sink) {
MutexLock lock(&lock_);
RTC_DCHECK(sink == send_stream_input_);
send_stream_input_ = nullptr;
}
} // namespace webrtc
Reference in New Issue View Git Blame Copy Permalink