admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
webrtc_m130/video/video_quality_test.cc
Elad Alon 4a87e1c211 Remove encoding code from RtcEventLogImpl and use RtcEventLogEncoder instead 
RtcEventLogImpl no longer hard-codes the way encoding is done. It now relies on RtcEventEncoder for it. This gives two benefits:
1. We can decide between the current encoding and the new encoding (which is still WIP) without code duplication (no need for RtcEventLogImplNew).
2. Encoding is done only when the event needs to be written to a file. This both avoids unnecessary encoding of events which don't end up getting written to a file, as well as is useful for the new, delta-based encoding, which is stateful.

BUG=webrtc:8111

Change-Id: I9517132e5f96b8059002a66fde8d42d3a678c3bb
Reviewed-on: https://webrtc-review.googlesource.com/1365
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Reviewed-by: Björn Terelius <terelius@webrtc.org>
Commit-Queue: Stefan Holmer <stefan@webrtc.org>
Commit-Queue: Elad Alon <eladalon@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#20118}
2017-10-03 15:26:56 +00:00

2188 lines
83 KiB
C++
Raw Blame History

/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "video/video_quality_test.h"
#include <stdio.h>
#include <algorithm>
#include <deque>
#include <map>
#include <set>
#include <sstream>
#include <string>
#include <vector>
#include "api/optional.h"
#include "call/call.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "logging/rtc_event_log/rtc_event_log.h"
#include "media/engine/webrtcvideoengine.h"
#include "modules/audio_mixer/audio_mixer_impl.h"
#include "modules/rtp_rtcp/include/rtp_header_parser.h"
#include "modules/rtp_rtcp/source/rtp_format.h"
#include "modules/rtp_rtcp/source/rtp_utility.h"
#include "modules/video_coding/codecs/h264/include/h264.h"
#include "modules/video_coding/codecs/vp8/include/vp8.h"
#include "modules/video_coding/codecs/vp8/include/vp8_common_types.h"
#include "modules/video_coding/codecs/vp9/include/vp9.h"
#include "rtc_base/checks.h"
#include "rtc_base/cpu_time.h"
#include "rtc_base/event.h"
#include "rtc_base/flags.h"
#include "rtc_base/format_macros.h"
#include "rtc_base/logging.h"
#include "rtc_base/memory_usage.h"
#include "rtc_base/pathutils.h"
#include "rtc_base/platform_file.h"
#include "rtc_base/ptr_util.h"
#include "rtc_base/timeutils.h"
#include "system_wrappers/include/cpu_info.h"
#include "system_wrappers/include/field_trial.h"
#include "test/gtest.h"
#include "test/layer_filtering_transport.h"
#include "test/run_loop.h"
#include "test/statistics.h"
#include "test/testsupport/fileutils.h"
#include "test/testsupport/frame_writer.h"
#include "test/testsupport/test_artifacts.h"
#include "test/vcm_capturer.h"
#include "test/video_renderer.h"
#include "voice_engine/include/voe_base.h"
#include "test/rtp_file_writer.h"
DEFINE_bool(save_worst_frame,
false,
"Enable saving a frame with the lowest PSNR to a jpeg file in the "
"test_artifacts_dir");
namespace {
constexpr int kSendStatsPollingIntervalMs = 1000;
constexpr size_t kMaxComparisons = 10;
constexpr char kSyncGroup[] = "av_sync";
constexpr int kOpusMinBitrateBps = 6000;
constexpr int kOpusBitrateFbBps = 32000;
constexpr int kFramesSentInQuickTest = 1;
constexpr uint32_t kThumbnailSendSsrcStart = 0xE0000;
constexpr uint32_t kThumbnailRtxSsrcStart = 0xF0000;
constexpr int kDefaultMaxQp = cricket::WebRtcVideoChannel::kDefaultQpMax;
struct VoiceEngineState {
VoiceEngineState()
: voice_engine(nullptr),
base(nullptr),
send_channel_id(-1),
receive_channel_id(-1) {}
webrtc::VoiceEngine* voice_engine;
webrtc::VoEBase* base;
int send_channel_id;
int receive_channel_id;
};
void CreateVoiceEngine(
VoiceEngineState* voe,
webrtc::AudioProcessing* apm,
rtc::scoped_refptr<webrtc::AudioDecoderFactory> decoder_factory) {
voe->voice_engine = webrtc::VoiceEngine::Create();
voe->base = webrtc::VoEBase::GetInterface(voe->voice_engine);
EXPECT_EQ(0, voe->base->Init(nullptr, apm, decoder_factory));
webrtc::VoEBase::ChannelConfig config;
config.enable_voice_pacing = true;
voe->send_channel_id = voe->base->CreateChannel(config);
EXPECT_GE(voe->send_channel_id, 0);
voe->receive_channel_id = voe->base->CreateChannel();
EXPECT_GE(voe->receive_channel_id, 0);
}
void DestroyVoiceEngine(VoiceEngineState* voe) {
voe->base->DeleteChannel(voe->send_channel_id);
voe->send_channel_id = -1;
voe->base->DeleteChannel(voe->receive_channel_id);
voe->receive_channel_id = -1;
voe->base->Release();
voe->base = nullptr;
webrtc::VoiceEngine::Delete(voe->voice_engine);
voe->voice_engine = nullptr;
}
class VideoStreamFactory
: public webrtc::VideoEncoderConfig::VideoStreamFactoryInterface {
public:
explicit VideoStreamFactory(const std::vector<webrtc::VideoStream>& streams)
: streams_(streams) {}
private:
std::vector<webrtc::VideoStream> CreateEncoderStreams(
int width,
int height,
const webrtc::VideoEncoderConfig& encoder_config) override {
// The highest layer must match the incoming resolution.
std::vector<webrtc::VideoStream> streams = streams_;
streams[streams_.size() - 1].height = height;
streams[streams_.size() - 1].width = width;
return streams;
}
std::vector<webrtc::VideoStream> streams_;
};
bool IsFlexfec(int payload_type) {
return payload_type == webrtc::VideoQualityTest::kFlexfecPayloadType;
}
} // namespace
namespace webrtc {
class VideoAnalyzer : public PacketReceiver,
public Transport,
public rtc::VideoSinkInterface<VideoFrame> {
public:
VideoAnalyzer(test::LayerFilteringTransport* transport,
const std::string& test_label,
double avg_psnr_threshold,
double avg_ssim_threshold,
int duration_frames,
FILE* graph_data_output_file,
const std::string& graph_title,
uint32_t ssrc_to_analyze,
uint32_t rtx_ssrc_to_analyze,
size_t selected_stream,
int selected_sl,
int selected_tl,
bool is_quick_test_enabled,
Clock* clock,
std::string rtp_dump_name)
: transport_(transport),
receiver_(nullptr),
call_(nullptr),
send_stream_(nullptr),
receive_stream_(nullptr),
captured_frame_forwarder_(this, clock),
test_label_(test_label),
graph_data_output_file_(graph_data_output_file),
graph_title_(graph_title),
ssrc_to_analyze_(ssrc_to_analyze),
rtx_ssrc_to_analyze_(rtx_ssrc_to_analyze),
selected_stream_(selected_stream),
selected_sl_(selected_sl),
selected_tl_(selected_tl),
pre_encode_proxy_(this),
encode_timing_proxy_(this),
last_fec_bytes_(0),
frames_to_process_(duration_frames),
frames_recorded_(0),
frames_processed_(0),
dropped_frames_(0),
dropped_frames_before_first_encode_(0),
dropped_frames_before_rendering_(0),
last_render_time_(0),
rtp_timestamp_delta_(0),
total_media_bytes_(0),
first_sending_time_(0),
last_sending_time_(0),
cpu_time_(0),
wallclock_time_(0),
avg_psnr_threshold_(avg_psnr_threshold),
avg_ssim_threshold_(avg_ssim_threshold),
is_quick_test_enabled_(is_quick_test_enabled),
stats_polling_thread_(&PollStatsThread, this, "StatsPoller"),
comparison_available_event_(false, false),
done_(true, false),
clock_(clock),
start_ms_(clock->TimeInMilliseconds()) {
// Create thread pool for CPU-expensive PSNR/SSIM calculations.
// Try to use about as many threads as cores, but leave kMinCoresLeft alone,
// so that we don't accidentally starve "real" worker threads (codec etc).
// Also, don't allocate more than kMaxComparisonThreads, even if there are
// spare cores.
uint32_t num_cores = CpuInfo::DetectNumberOfCores();
RTC_DCHECK_GE(num_cores, 1);
static const uint32_t kMinCoresLeft = 4;
static const uint32_t kMaxComparisonThreads = 8;
if (num_cores <= kMinCoresLeft) {
num_cores = 1;
} else {
num_cores -= kMinCoresLeft;
num_cores = std::min(num_cores, kMaxComparisonThreads);
}
for (uint32_t i = 0; i < num_cores; ++i) {
rtc::PlatformThread* thread =
new rtc::PlatformThread(&FrameComparisonThread, this, "Analyzer");
thread->Start();
comparison_thread_pool_.push_back(thread);
}
if (!rtp_dump_name.empty()) {
fprintf(stdout, "Writing rtp dump to %s\n", rtp_dump_name.c_str());
rtp_file_writer_.reset(test::RtpFileWriter::Create(
test::RtpFileWriter::kRtpDump, rtp_dump_name));
}
}
~VideoAnalyzer() {
for (rtc::PlatformThread* thread : comparison_thread_pool_) {
thread->Stop();
delete thread;
}
}
virtual void SetReceiver(PacketReceiver* receiver) { receiver_ = receiver; }
void SetSource(test::VideoCapturer* video_capturer, bool respect_sink_wants) {
if (respect_sink_wants)
captured_frame_forwarder_.SetSource(video_capturer);
rtc::VideoSinkWants wants;
video_capturer->AddOrUpdateSink(InputInterface(), wants);
}
void SetCall(Call* call) {
rtc::CritScope lock(&crit_);
RTC_DCHECK(!call_);
call_ = call;
}
void SetSendStream(VideoSendStream* stream) {
rtc::CritScope lock(&crit_);
RTC_DCHECK(!send_stream_);
send_stream_ = stream;
}
void SetReceiveStream(VideoReceiveStream* stream) {
rtc::CritScope lock(&crit_);
RTC_DCHECK(!receive_stream_);
receive_stream_ = stream;
}
rtc::VideoSinkInterface<VideoFrame>* InputInterface() {
return &captured_frame_forwarder_;
}
rtc::VideoSourceInterface<VideoFrame>* OutputInterface() {
return &captured_frame_forwarder_;
}
DeliveryStatus DeliverPacket(MediaType media_type,
const uint8_t* packet,
size_t length,
const PacketTime& packet_time) override {
// Ignore timestamps of RTCP packets. They're not synchronized with
// RTP packet timestamps and so they would confuse wrap_handler_.
if (RtpHeaderParser::IsRtcp(packet, length)) {
return receiver_->DeliverPacket(media_type, packet, length, packet_time);
}
if (rtp_file_writer_) {
test::RtpPacket p;
memcpy(p.data, packet, length);
p.length = length;
p.original_length = length;
p.time_ms = clock_->TimeInMilliseconds() - start_ms_;
rtp_file_writer_->WritePacket(&p);
}
RtpUtility::RtpHeaderParser parser(packet, length);
RTPHeader header;
parser.Parse(&header);
if (!IsFlexfec(header.payloadType) &&
(header.ssrc == ssrc_to_analyze_ ||
header.ssrc == rtx_ssrc_to_analyze_)) {
// Ignore FlexFEC timestamps, to avoid collisions with media timestamps.
// (FlexFEC and media are sent on different SSRCs, which have different
// timestamps spaces.)
// Also ignore packets from wrong SSRC, but include retransmits.
rtc::CritScope lock(&crit_);
int64_t timestamp =
wrap_handler_.Unwrap(header.timestamp - rtp_timestamp_delta_);
recv_times_[timestamp] =
Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
}
return receiver_->DeliverPacket(media_type, packet, length, packet_time);
}
void MeasuredEncodeTiming(int64_t ntp_time_ms, int encode_time_ms) {
rtc::CritScope crit(&comparison_lock_);
samples_encode_time_ms_[ntp_time_ms] = encode_time_ms;
}
void PreEncodeOnFrame(const VideoFrame& video_frame) {
rtc::CritScope lock(&crit_);
if (!first_encoded_timestamp_) {
while (frames_.front().timestamp() != video_frame.timestamp()) {
++dropped_frames_before_first_encode_;
frames_.pop_front();
RTC_CHECK(!frames_.empty());
}
first_encoded_timestamp_ =
rtc::Optional<uint32_t>(video_frame.timestamp());
}
}
void PostEncodeFrameCallback(const EncodedFrame& encoded_frame) {
rtc::CritScope lock(&crit_);
if (!first_sent_timestamp_ &&
encoded_frame.stream_id_ == selected_stream_) {
first_sent_timestamp_ = rtc::Optional<uint32_t>(encoded_frame.timestamp_);
}
}
bool SendRtp(const uint8_t* packet,
size_t length,
const PacketOptions& options) override {
RtpUtility::RtpHeaderParser parser(packet, length);
RTPHeader header;
parser.Parse(&header);
int64_t current_time =
Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
bool result = transport_->SendRtp(packet, length, options);
{
rtc::CritScope lock(&crit_);
if (rtp_timestamp_delta_ == 0 && header.ssrc == ssrc_to_analyze_) {
RTC_CHECK(static_cast<bool>(first_sent_timestamp_));
rtp_timestamp_delta_ = header.timestamp - *first_sent_timestamp_;
}
if (!IsFlexfec(header.payloadType) && header.ssrc == ssrc_to_analyze_) {
// Ignore FlexFEC timestamps, to avoid collisions with media timestamps.
// (FlexFEC and media are sent on different SSRCs, which have different
// timestamps spaces.)
// Also ignore packets from wrong SSRC and retransmits.
int64_t timestamp =
wrap_handler_.Unwrap(header.timestamp - rtp_timestamp_delta_);
send_times_[timestamp] = current_time;
if (IsInSelectedSpatialAndTemporalLayer(packet, length, header)) {
encoded_frame_sizes_[timestamp] +=
length - (header.headerLength + header.paddingLength);
total_media_bytes_ +=
length - (header.headerLength + header.paddingLength);
}
if (first_sending_time_ == 0)
first_sending_time_ = current_time;
last_sending_time_ = current_time;
}
}
return result;
}
bool SendRtcp(const uint8_t* packet, size_t length) override {
return transport_->SendRtcp(packet, length);
}
void OnFrame(const VideoFrame& video_frame) override {
int64_t render_time_ms =
Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
rtc::CritScope lock(&crit_);
StartExcludingCpuThreadTime();
int64_t send_timestamp =
wrap_handler_.Unwrap(video_frame.timestamp() - rtp_timestamp_delta_);
while (wrap_handler_.Unwrap(frames_.front().timestamp()) < send_timestamp) {
if (!last_rendered_frame_) {
// No previous frame rendered, this one was dropped after sending but
// before rendering.
++dropped_frames_before_rendering_;
} else {
AddFrameComparison(frames_.front(), *last_rendered_frame_, true,
render_time_ms);
}
frames_.pop_front();
RTC_DCHECK(!frames_.empty());
}
VideoFrame reference_frame = frames_.front();
frames_.pop_front();
int64_t reference_timestamp =
wrap_handler_.Unwrap(reference_frame.timestamp());
if (send_timestamp == reference_timestamp - 1) {
// TODO(ivica): Make this work for > 2 streams.
// Look at RTPSender::BuildRTPHeader.
++send_timestamp;
}
ASSERT_EQ(reference_timestamp, send_timestamp);
AddFrameComparison(reference_frame, video_frame, false, render_time_ms);
last_rendered_frame_ = rtc::Optional<VideoFrame>(video_frame);
StopExcludingCpuThreadTime();
}
void Wait() {
// Frame comparisons can be very expensive. Wait for test to be done, but
// at time-out check if frames_processed is going up. If so, give it more
// time, otherwise fail. Hopefully this will reduce test flakiness.
stats_polling_thread_.Start();
int last_frames_processed = -1;
int iteration = 0;
while (!done_.Wait(VideoQualityTest::kDefaultTimeoutMs)) {
int frames_processed;
{
rtc::CritScope crit(&comparison_lock_);
frames_processed = frames_processed_;
}
// Print some output so test infrastructure won't think we've crashed.
const char* kKeepAliveMessages[3] = {
"Uh, I'm-I'm not quite dead, sir.",
"Uh, I-I think uh, I could pull through, sir.",
"Actually, I think I'm all right to come with you--"};
printf("- %s\n", kKeepAliveMessages[iteration++ % 3]);
if (last_frames_processed == -1) {
last_frames_processed = frames_processed;
continue;
}
if (frames_processed == last_frames_processed) {
EXPECT_GT(frames_processed, last_frames_processed)
<< "Analyzer stalled while waiting for test to finish.";
done_.Set();
break;
}
last_frames_processed = frames_processed;
}
if (iteration > 0)
printf("- Farewell, sweet Concorde!\n");
stats_polling_thread_.Stop();
}
rtc::VideoSinkInterface<VideoFrame>* pre_encode_proxy() {
return &pre_encode_proxy_;
}
EncodedFrameObserver* encode_timing_proxy() { return &encode_timing_proxy_; }
void StartMeasuringCpuProcessTime() {
rtc::CritScope lock(&cpu_measurement_lock_);
cpu_time_ -= rtc::GetProcessCpuTimeNanos();
wallclock_time_ -= rtc::SystemTimeNanos();
}
void StopMeasuringCpuProcessTime() {
rtc::CritScope lock(&cpu_measurement_lock_);
cpu_time_ += rtc::GetProcessCpuTimeNanos();
wallclock_time_ += rtc::SystemTimeNanos();
}
void StartExcludingCpuThreadTime() {
rtc::CritScope lock(&cpu_measurement_lock_);
cpu_time_ += rtc::GetThreadCpuTimeNanos();
}
void StopExcludingCpuThreadTime() {
rtc::CritScope lock(&cpu_measurement_lock_);
cpu_time_ -= rtc::GetThreadCpuTimeNanos();
}
double GetCpuUsagePercent() {
rtc::CritScope lock(&cpu_measurement_lock_);
return static_cast<double>(cpu_time_) / wallclock_time_ * 100.0;
}
test::LayerFilteringTransport* const transport_;
PacketReceiver* receiver_;
private:
struct FrameComparison {
FrameComparison()
: dropped(false),
input_time_ms(0),
send_time_ms(0),
recv_time_ms(0),
render_time_ms(0),
encoded_frame_size(0) {}
FrameComparison(const VideoFrame& reference,
const VideoFrame& render,
bool dropped,
int64_t input_time_ms,
int64_t send_time_ms,
int64_t recv_time_ms,
int64_t render_time_ms,
size_t encoded_frame_size)
: reference(reference),
render(render),
dropped(dropped),
input_time_ms(input_time_ms),
send_time_ms(send_time_ms),
recv_time_ms(recv_time_ms),
render_time_ms(render_time_ms),
encoded_frame_size(encoded_frame_size) {}
FrameComparison(bool dropped,
int64_t input_time_ms,
int64_t send_time_ms,
int64_t recv_time_ms,
int64_t render_time_ms,
size_t encoded_frame_size)
: dropped(dropped),
input_time_ms(input_time_ms),
send_time_ms(send_time_ms),
recv_time_ms(recv_time_ms),
render_time_ms(render_time_ms),
encoded_frame_size(encoded_frame_size) {}
rtc::Optional<VideoFrame> reference;
rtc::Optional<VideoFrame> render;
bool dropped;
int64_t input_time_ms;
int64_t send_time_ms;
int64_t recv_time_ms;
int64_t render_time_ms;
size_t encoded_frame_size;
};
struct Sample {
Sample(int dropped,
int64_t input_time_ms,
int64_t send_time_ms,
int64_t recv_time_ms,
int64_t render_time_ms,
size_t encoded_frame_size,
double psnr,
double ssim)
: dropped(dropped),
input_time_ms(input_time_ms),
send_time_ms(send_time_ms),
recv_time_ms(recv_time_ms),
render_time_ms(render_time_ms),
encoded_frame_size(encoded_frame_size),
psnr(psnr),
ssim(ssim) {}
int dropped;
int64_t input_time_ms;
int64_t send_time_ms;
int64_t recv_time_ms;
int64_t render_time_ms;
size_t encoded_frame_size;
double psnr;
double ssim;
};
// This class receives the send-side OnEncodeTiming and is provided to not
// conflict with the receiver-side pre_decode_callback.
class OnEncodeTimingProxy : public EncodedFrameObserver {
public:
explicit OnEncodeTimingProxy(VideoAnalyzer* parent) : parent_(parent) {}
void OnEncodeTiming(int64_t ntp_time_ms, int encode_time_ms) override {
parent_->MeasuredEncodeTiming(ntp_time_ms, encode_time_ms);
}
void EncodedFrameCallback(const EncodedFrame& frame) override {
parent_->PostEncodeFrameCallback(frame);
}
private:
VideoAnalyzer* const parent_;
};
// This class receives the send-side OnFrame callback and is provided to not
// conflict with the receiver-side renderer callback.
class PreEncodeProxy : public rtc::VideoSinkInterface<VideoFrame> {
public:
explicit PreEncodeProxy(VideoAnalyzer* parent) : parent_(parent) {}
void OnFrame(const VideoFrame& video_frame) override {
parent_->PreEncodeOnFrame(video_frame);
}
private:
VideoAnalyzer* const parent_;
};
bool IsInSelectedSpatialAndTemporalLayer(const uint8_t* packet,
size_t length,
const RTPHeader& header) {
if (header.payloadType != test::CallTest::kPayloadTypeVP9 &&
header.payloadType != test::CallTest::kPayloadTypeVP8) {
return true;
} else {
// Get VP8 and VP9 specific header to check layers indexes.
const uint8_t* payload = packet + header.headerLength;
const size_t payload_length = length - header.headerLength;
const size_t payload_data_length = payload_length - header.paddingLength;
const bool is_vp8 = header.payloadType == test::CallTest::kPayloadTypeVP8;
std::unique_ptr<RtpDepacketizer> depacketizer(
RtpDepacketizer::Create(is_vp8 ? kRtpVideoVp8 : kRtpVideoVp9));
RtpDepacketizer::ParsedPayload parsed_payload;
bool result =
depacketizer->Parse(&parsed_payload, payload, payload_data_length);
RTC_DCHECK(result);
const int temporal_idx = static_cast<int>(
is_vp8 ? parsed_payload.type.Video.codecHeader.VP8.temporalIdx
: parsed_payload.type.Video.codecHeader.VP9.temporal_idx);
const int spatial_idx = static_cast<int>(
is_vp8 ? kNoSpatialIdx
: parsed_payload.type.Video.codecHeader.VP9.spatial_idx);
return (selected_tl_ < 0 || temporal_idx == kNoTemporalIdx ||
temporal_idx <= selected_tl_) &&
(selected_sl_ < 0 || spatial_idx == kNoSpatialIdx ||
spatial_idx <= selected_sl_);
}
}
void AddFrameComparison(const VideoFrame& reference,
const VideoFrame& render,
bool dropped,
int64_t render_time_ms)
RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_) {
int64_t reference_timestamp = wrap_handler_.Unwrap(reference.timestamp());
int64_t send_time_ms = send_times_[reference_timestamp];
send_times_.erase(reference_timestamp);
int64_t recv_time_ms = recv_times_[reference_timestamp];
recv_times_.erase(reference_timestamp);
// TODO(ivica): Make this work for > 2 streams.
auto it = encoded_frame_sizes_.find(reference_timestamp);
if (it == encoded_frame_sizes_.end())
it = encoded_frame_sizes_.find(reference_timestamp - 1);
size_t encoded_size = it == encoded_frame_sizes_.end() ? 0 : it->second;
if (it != encoded_frame_sizes_.end())
encoded_frame_sizes_.erase(it);
rtc::CritScope crit(&comparison_lock_);
if (comparisons_.size() < kMaxComparisons) {
comparisons_.push_back(FrameComparison(reference, render, dropped,
reference.ntp_time_ms(),
send_time_ms, recv_time_ms,
render_time_ms, encoded_size));
} else {
comparisons_.push_back(FrameComparison(dropped,
reference.ntp_time_ms(),
send_time_ms, recv_time_ms,
render_time_ms, encoded_size));
}
comparison_available_event_.Set();
}
static void PollStatsThread(void* obj) {
static_cast<VideoAnalyzer*>(obj)->PollStats();
}
void PollStats() {
while (!done_.Wait(kSendStatsPollingIntervalMs)) {
rtc::CritScope crit(&comparison_lock_);
Call::Stats call_stats = call_->GetStats();
send_bandwidth_bps_.AddSample(call_stats.send_bandwidth_bps);
VideoSendStream::Stats send_stats = send_stream_->GetStats();
// It's not certain that we yet have estimates for any of these stats.
// Check that they are positive before mixing them in.
if (send_stats.encode_frame_rate > 0)
encode_frame_rate_.AddSample(send_stats.encode_frame_rate);
if (send_stats.avg_encode_time_ms > 0)
encode_time_ms_.AddSample(send_stats.avg_encode_time_ms);
if (send_stats.encode_usage_percent > 0)
encode_usage_percent_.AddSample(send_stats.encode_usage_percent);
if (send_stats.media_bitrate_bps > 0)
media_bitrate_bps_.AddSample(send_stats.media_bitrate_bps);
size_t fec_bytes = 0;
for (auto kv : send_stats.substreams) {
fec_bytes += kv.second.rtp_stats.fec.payload_bytes +
kv.second.rtp_stats.fec.padding_bytes;
}
fec_bitrate_bps_.AddSample((fec_bytes - last_fec_bytes_) * 8);
last_fec_bytes_ = fec_bytes;
if (receive_stream_ != nullptr) {
VideoReceiveStream::Stats receive_stats = receive_stream_->GetStats();
if (receive_stats.decode_ms > 0)
decode_time_ms_.AddSample(receive_stats.decode_ms);
if (receive_stats.max_decode_ms > 0)
decode_time_max_ms_.AddSample(receive_stats.max_decode_ms);
}
memory_usage_.AddSample(rtc::GetProcessResidentSizeBytes());
}
}
static bool FrameComparisonThread(void* obj) {
return static_cast<VideoAnalyzer*>(obj)->CompareFrames();
}
bool CompareFrames() {
if (AllFramesRecorded())
return false;
FrameComparison comparison;
if (!PopComparison(&comparison)) {
// Wait until new comparison task is available, or test is done.
// If done, wake up remaining threads waiting.
comparison_available_event_.Wait(1000);
if (AllFramesRecorded()) {
comparison_available_event_.Set();
return false;
}
return true; // Try again.
}
StartExcludingCpuThreadTime();
PerformFrameComparison(comparison);
StopExcludingCpuThreadTime();
if (FrameProcessed()) {
PrintResults();
if (graph_data_output_file_)
PrintSamplesToFile();
done_.Set();
comparison_available_event_.Set();
return false;
}
return true;
}
bool PopComparison(FrameComparison* comparison) {
rtc::CritScope crit(&comparison_lock_);
// If AllFramesRecorded() is true, it means we have already popped
// frames_to_process_ frames from comparisons_, so there is no more work
// for this thread to be done. frames_processed_ might still be lower if
// all comparisons are not done, but those frames are currently being
// worked on by other threads.
if (comparisons_.empty() || AllFramesRecorded())
return false;
*comparison = comparisons_.front();
comparisons_.pop_front();
FrameRecorded();
return true;
}
// Increment counter for number of frames received for comparison.
void FrameRecorded() {
rtc::CritScope crit(&comparison_lock_);
++frames_recorded_;
}
// Returns true if all frames to be compared have been taken from the queue.
bool AllFramesRecorded() {
rtc::CritScope crit(&comparison_lock_);
assert(frames_recorded_ <= frames_to_process_);
return frames_recorded_ == frames_to_process_;
}
// Increase count of number of frames processed. Returns true if this was the
// last frame to be processed.
bool FrameProcessed() {
rtc::CritScope crit(&comparison_lock_);
++frames_processed_;
assert(frames_processed_ <= frames_to_process_);
return frames_processed_ == frames_to_process_;
}
void PrintResults() {
StopMeasuringCpuProcessTime();
rtc::CritScope crit(&comparison_lock_);
PrintResult("psnr", psnr_, " dB");
PrintResult("ssim", ssim_, " score");
PrintResult("sender_time", sender_time_, " ms");
PrintResult("receiver_time", receiver_time_, " ms");
PrintResult("total_delay_incl_network", end_to_end_, " ms");
PrintResult("time_between_rendered_frames", rendered_delta_, " ms");
PrintResult("encode_frame_rate", encode_frame_rate_, " fps");
PrintResult("encode_time", encode_time_ms_, " ms");
PrintResult("media_bitrate", media_bitrate_bps_, " bps");
PrintResult("fec_bitrate", fec_bitrate_bps_, " bps");
PrintResult("send_bandwidth", send_bandwidth_bps_, " bps");
if (worst_frame_) {
printf("RESULT min_psnr: %s = %lf dB\n", test_label_.c_str(),
worst_frame_->psnr);
}
if (receive_stream_ != nullptr) {
PrintResult("decode_time", decode_time_ms_, " ms");
}
printf("RESULT dropped_frames: %s = %d frames\n", test_label_.c_str(),
dropped_frames_);
printf("RESULT cpu_usage: %s = %lf %%\n", test_label_.c_str(),
GetCpuUsagePercent());
#if defined(WEBRTC_WIN)
// On Linux and Mac in Resident Set some unused pages may be counted.
// Therefore this metric will depend on order in which tests are run and
// will be flaky.
PrintResult("memory_usage", memory_usage_, " bytes");
#endif
// Saving only the worst frame for manual analysis. Intention here is to
// only detect video corruptions and not to track picture quality. Thus,
// jpeg is used here.
if (FLAG_save_worst_frame && worst_frame_) {
std::string output_dir;
test::GetTestArtifactsDir(&output_dir);
std::string output_path =
rtc::Pathname(output_dir, test_label_ + ".jpg").pathname();
LOG(LS_INFO) << "Saving worst frame to " << output_path;
test::JpegFrameWriter frame_writer(output_path);
RTC_CHECK(frame_writer.WriteFrame(worst_frame_->frame,
100 /*best quality*/));
}
// Disable quality check for quick test, as quality checks may fail
// because too few samples were collected.
if (!is_quick_test_enabled_) {
EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_);
EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_);
}
}
void PerformFrameComparison(const FrameComparison& comparison) {
// Perform expensive psnr and ssim calculations while not holding lock.
double psnr = -1.0;
double ssim = -1.0;
if (comparison.reference && !comparison.dropped) {
psnr = I420PSNR(&*comparison.reference, &*comparison.render);
ssim = I420SSIM(&*comparison.reference, &*comparison.render);
}
rtc::CritScope crit(&comparison_lock_);
if (psnr >= 0.0 && (!worst_frame_ || worst_frame_->psnr > psnr)) {
worst_frame_.emplace(FrameWithPsnr{psnr, *comparison.render});
}
if (graph_data_output_file_) {
samples_.push_back(Sample(
comparison.dropped, comparison.input_time_ms, comparison.send_time_ms,
comparison.recv_time_ms, comparison.render_time_ms,
comparison.encoded_frame_size, psnr, ssim));
}
if (psnr >= 0.0)
psnr_.AddSample(psnr);
if (ssim >= 0.0)
ssim_.AddSample(ssim);
if (comparison.dropped) {
++dropped_frames_;
return;
}
if (last_render_time_ != 0)
rendered_delta_.AddSample(comparison.render_time_ms - last_render_time_);
last_render_time_ = comparison.render_time_ms;
sender_time_.AddSample(comparison.send_time_ms - comparison.input_time_ms);
if (comparison.recv_time_ms > 0) {
// If recv_time_ms == 0, this frame consisted of a packets which were all
// lost in the transport. Since we were able to render the frame, however,
// the dropped packets were recovered by FlexFEC. The FlexFEC recovery
// happens internally in Call, and we can therefore here not know which
// FEC packets that protected the lost media packets. Consequently, we
// were not able to record a meaningful recv_time_ms. We therefore skip
// this sample.
//
// The reasoning above does not hold for ULPFEC and RTX, as for those
// strategies the timestamp of the received packets is set to the
// timestamp of the protected/retransmitted media packet. I.e., then
// recv_time_ms != 0, even though the media packets were lost.
receiver_time_.AddSample(comparison.render_time_ms -
comparison.recv_time_ms);
}
end_to_end_.AddSample(comparison.render_time_ms - comparison.input_time_ms);
encoded_frame_size_.AddSample(comparison.encoded_frame_size);
}
void PrintResult(const char* result_type,
test::Statistics stats,
const char* unit) {
printf("RESULT %s: %s = {%f, %f}%s\n",
result_type,
test_label_.c_str(),
stats.Mean(),
stats.StandardDeviation(),
unit);
}
void PrintSamplesToFile(void) {
FILE* out = graph_data_output_file_;
rtc::CritScope crit(&comparison_lock_);
std::sort(samples_.begin(), samples_.end(),
[](const Sample& A, const Sample& B) -> bool {
return A.input_time_ms < B.input_time_ms;
});
fprintf(out, "%s\n", graph_title_.c_str());
fprintf(out, "%" PRIuS "\n", samples_.size());
fprintf(out,
"dropped "
"input_time_ms "
"send_time_ms "
"recv_time_ms "
"render_time_ms "
"encoded_frame_size "
"psnr "
"ssim "
"encode_time_ms\n");
int missing_encode_time_samples = 0;
for (const Sample& sample : samples_) {
auto it = samples_encode_time_ms_.find(sample.input_time_ms);
int encode_time_ms;
if (it != samples_encode_time_ms_.end()) {
encode_time_ms = it->second;
} else {
++missing_encode_time_samples;
encode_time_ms = -1;
}
fprintf(out, "%d %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRIuS
" %lf %lf %d\n",
sample.dropped, sample.input_time_ms, sample.send_time_ms,
sample.recv_time_ms, sample.render_time_ms,
sample.encoded_frame_size, sample.psnr, sample.ssim,
encode_time_ms);
}
if (missing_encode_time_samples) {
fprintf(stderr,
"Warning: Missing encode_time_ms samples for %d frame(s).\n",
missing_encode_time_samples);
}
}
double GetAverageMediaBitrateBps() {
if (last_sending_time_ == first_sending_time_) {
return 0;
} else {
return static_cast<double>(total_media_bytes_) * 8 /
(last_sending_time_ - first_sending_time_) *
rtc::kNumMillisecsPerSec;
}
}
// Implements VideoSinkInterface to receive captured frames from a
// FrameGeneratorCapturer. Implements VideoSourceInterface to be able to act
// as a source to VideoSendStream.
// It forwards all input frames to the VideoAnalyzer for later comparison and
// forwards the captured frames to the VideoSendStream.
class CapturedFrameForwarder : public rtc::VideoSinkInterface<VideoFrame>,
public rtc::VideoSourceInterface<VideoFrame> {
public:
explicit CapturedFrameForwarder(VideoAnalyzer* analyzer, Clock* clock)
: analyzer_(analyzer),
send_stream_input_(nullptr),
video_capturer_(nullptr),
clock_(clock) {}
void SetSource(test::VideoCapturer* video_capturer) {
video_capturer_ = video_capturer;
}
private:
void OnFrame(const VideoFrame& video_frame) override {
VideoFrame copy = video_frame;
// Frames from the capturer does not have a rtp timestamp.
// Create one so it can be used for comparison.
RTC_DCHECK_EQ(0, video_frame.timestamp());
if (video_frame.ntp_time_ms() == 0)
copy.set_ntp_time_ms(clock_->CurrentNtpInMilliseconds());
copy.set_timestamp(copy.ntp_time_ms() * 90);
analyzer_->AddCapturedFrameForComparison(copy);
rtc::CritScope lock(&crit_);
if (send_stream_input_)
send_stream_input_->OnFrame(copy);
}
// Called when |send_stream_.SetSource()| is called.
void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
const rtc::VideoSinkWants& wants) override {
{
rtc::CritScope lock(&crit_);
RTC_DCHECK(!send_stream_input_ || send_stream_input_ == sink);
send_stream_input_ = sink;
}
if (video_capturer_) {
video_capturer_->AddOrUpdateSink(this, wants);
}
}
// Called by |send_stream_| when |send_stream_.SetSource()| is called.
void RemoveSink(rtc::VideoSinkInterface<VideoFrame>* sink) override {
rtc::CritScope lock(&crit_);
RTC_DCHECK(sink == send_stream_input_);
send_stream_input_ = nullptr;
}
VideoAnalyzer* const analyzer_;
rtc::CriticalSection crit_;
rtc::VideoSinkInterface<VideoFrame>* send_stream_input_
RTC_GUARDED_BY(crit_);
test::VideoCapturer* video_capturer_;
Clock* clock_;
};
void AddCapturedFrameForComparison(const VideoFrame& video_frame) {
rtc::CritScope lock(&crit_);
frames_.push_back(video_frame);
}
Call* call_;
VideoSendStream* send_stream_;
VideoReceiveStream* receive_stream_;
CapturedFrameForwarder captured_frame_forwarder_;
const std::string test_label_;
FILE* const graph_data_output_file_;
const std::string graph_title_;
const uint32_t ssrc_to_analyze_;
const uint32_t rtx_ssrc_to_analyze_;
const size_t selected_stream_;
const int selected_sl_;
const int selected_tl_;
PreEncodeProxy pre_encode_proxy_;
OnEncodeTimingProxy encode_timing_proxy_;
std::vector<Sample> samples_ RTC_GUARDED_BY(comparison_lock_);
std::map<int64_t, int> samples_encode_time_ms_
RTC_GUARDED_BY(comparison_lock_);
test::Statistics sender_time_ RTC_GUARDED_BY(comparison_lock_);
test::Statistics receiver_time_ RTC_GUARDED_BY(comparison_lock_);
test::Statistics psnr_ RTC_GUARDED_BY(comparison_lock_);
test::Statistics ssim_ RTC_GUARDED_BY(comparison_lock_);
test::Statistics end_to_end_ RTC_GUARDED_BY(comparison_lock_);
test::Statistics rendered_delta_ RTC_GUARDED_BY(comparison_lock_);
test::Statistics encoded_frame_size_ RTC_GUARDED_BY(comparison_lock_);
test::Statistics encode_frame_rate_ RTC_GUARDED_BY(comparison_lock_);
test::Statistics encode_time_ms_ RTC_GUARDED_BY(comparison_lock_);
test::Statistics encode_usage_percent_ RTC_GUARDED_BY(comparison_lock_);
test::Statistics decode_time_ms_ RTC_GUARDED_BY(comparison_lock_);
test::Statistics decode_time_max_ms_ RTC_GUARDED_BY(comparison_lock_);
test::Statistics media_bitrate_bps_ RTC_GUARDED_BY(comparison_lock_);
test::Statistics fec_bitrate_bps_ RTC_GUARDED_BY(comparison_lock_);
test::Statistics send_bandwidth_bps_ RTC_GUARDED_BY(comparison_lock_);
test::Statistics memory_usage_ RTC_GUARDED_BY(comparison_lock_);
struct FrameWithPsnr {
double psnr;
VideoFrame frame;
};
// Rendered frame with worst PSNR is saved for further analysis.
rtc::Optional<FrameWithPsnr> worst_frame_ RTC_GUARDED_BY(comparison_lock_);
size_t last_fec_bytes_;
const int frames_to_process_;
int frames_recorded_;
int frames_processed_;
int dropped_frames_;
int dropped_frames_before_first_encode_;
int dropped_frames_before_rendering_;
int64_t last_render_time_;
uint32_t rtp_timestamp_delta_;
int64_t total_media_bytes_;
int64_t first_sending_time_;
int64_t last_sending_time_;
int64_t cpu_time_ RTC_GUARDED_BY(cpu_measurement_lock_);
int64_t wallclock_time_ RTC_GUARDED_BY(cpu_measurement_lock_);
rtc::CriticalSection cpu_measurement_lock_;
rtc::CriticalSection crit_;
std::deque<VideoFrame> frames_ RTC_GUARDED_BY(crit_);
rtc::Optional<VideoFrame> last_rendered_frame_ RTC_GUARDED_BY(crit_);
rtc::TimestampWrapAroundHandler wrap_handler_ RTC_GUARDED_BY(crit_);
std::map<int64_t, int64_t> send_times_ RTC_GUARDED_BY(crit_);
std::map<int64_t, int64_t> recv_times_ RTC_GUARDED_BY(crit_);
std::map<int64_t, size_t> encoded_frame_sizes_ RTC_GUARDED_BY(crit_);
rtc::Optional<uint32_t> first_encoded_timestamp_ RTC_GUARDED_BY(crit_);
rtc::Optional<uint32_t> first_sent_timestamp_ RTC_GUARDED_BY(crit_);
const double avg_psnr_threshold_;
const double avg_ssim_threshold_;
bool is_quick_test_enabled_;
rtc::CriticalSection comparison_lock_;
std::vector<rtc::PlatformThread*> comparison_thread_pool_;
rtc::PlatformThread stats_polling_thread_;
rtc::Event comparison_available_event_;
std::deque<FrameComparison> comparisons_ RTC_GUARDED_BY(comparison_lock_);
rtc::Event done_;
std::unique_ptr<test::RtpFileWriter> rtp_file_writer_;
Clock* const clock_;
const int64_t start_ms_;
};
class Vp8EncoderFactory : public cricket::WebRtcVideoEncoderFactory {
public:
Vp8EncoderFactory() {
supported_codecs_.push_back(cricket::VideoCodec("VP8"));
}
~Vp8EncoderFactory() override { RTC_CHECK(live_encoders_.empty()); }
const std::vector<cricket::VideoCodec>& supported_codecs() const override {
return supported_codecs_;
}
VideoEncoder* CreateVideoEncoder(const cricket::VideoCodec& codec) override {
VideoEncoder* encoder = VP8Encoder::Create();
live_encoders_.insert(encoder);
return encoder;
}
void DestroyVideoEncoder(VideoEncoder* encoder) override {
auto it = live_encoders_.find(encoder);
RTC_CHECK(it != live_encoders_.end());
live_encoders_.erase(it);
delete encoder;
}
private:
std::vector<cricket::VideoCodec> supported_codecs_;
std::set<VideoEncoder*> live_encoders_;
};
VideoQualityTest::VideoQualityTest()
: clock_(Clock::GetRealTimeClock()), receive_logs_(0), send_logs_(0) {
payload_type_map_ = test::CallTest::payload_type_map_;
RTC_DCHECK(payload_type_map_.find(kPayloadTypeH264) ==
payload_type_map_.end());
RTC_DCHECK(payload_type_map_.find(kPayloadTypeVP8) ==
payload_type_map_.end());
RTC_DCHECK(payload_type_map_.find(kPayloadTypeVP9) ==
payload_type_map_.end());
payload_type_map_[kPayloadTypeH264] = webrtc::MediaType::VIDEO;
payload_type_map_[kPayloadTypeVP8] = webrtc::MediaType::VIDEO;
payload_type_map_[kPayloadTypeVP9] = webrtc::MediaType::VIDEO;
}
VideoQualityTest::Params::Params()
: call({false, Call::Config::BitrateConfig(), 0}),
video({false, 640, 480, 30, 50, 800, 800, false, "VP8", 1, -1, 0, false,
false, ""}),
audio({false, false, false}),
screenshare({false, false, 10, 0}),
analyzer({"", 0.0, 0.0, 0, "", ""}),
pipe(),
ss({std::vector<VideoStream>(), 0, 0, -1, std::vector<SpatialLayer>()}),
logging({false, "", "", ""}) {}
VideoQualityTest::Params::~Params() = default;
void VideoQualityTest::TestBody() {}
std::string VideoQualityTest::GenerateGraphTitle() const {
std::stringstream ss;
ss << params_.video.codec;
ss << " (" << params_.video.target_bitrate_bps / 1000 << "kbps";
ss << ", " << params_.video.fps << " FPS";
if (params_.screenshare.scroll_duration)
ss << ", " << params_.screenshare.scroll_duration << "s scroll";
if (params_.ss.streams.size() > 1)
ss << ", Stream #" << params_.ss.selected_stream;
if (params_.ss.num_spatial_layers > 1)
ss << ", Layer #" << params_.ss.selected_sl;
ss << ")";
return ss.str();
}
void VideoQualityTest::CheckParams() {
if (!params_.video.enabled)
return;
// Add a default stream in none specified.
if (params_.ss.streams.empty())
params_.ss.streams.push_back(VideoQualityTest::DefaultVideoStream(params_));
if (params_.ss.num_spatial_layers == 0)
params_.ss.num_spatial_layers = 1;
if (params_.pipe.loss_percent != 0 ||
params_.pipe.queue_length_packets != 0) {
// Since LayerFilteringTransport changes the sequence numbers, we can't
// use that feature with pack loss, since the NACK request would end up
// retransmitting the wrong packets.
RTC_CHECK(params_.ss.selected_sl == -1 ||
params_.ss.selected_sl == params_.ss.num_spatial_layers - 1);
RTC_CHECK(params_.video.selected_tl == -1 ||
params_.video.selected_tl ==
params_.video.num_temporal_layers - 1);
}
// TODO(ivica): Should max_bitrate_bps == -1 represent inf max bitrate, as it
// does in some parts of the code?
RTC_CHECK_GE(params_.video.max_bitrate_bps, params_.video.target_bitrate_bps);
RTC_CHECK_GE(params_.video.target_bitrate_bps, params_.video.min_bitrate_bps);
RTC_CHECK_LT(params_.video.selected_tl, params_.video.num_temporal_layers);
RTC_CHECK_LE(params_.ss.selected_stream, params_.ss.streams.size());
for (const VideoStream& stream : params_.ss.streams) {
RTC_CHECK_GE(stream.min_bitrate_bps, 0);
RTC_CHECK_GE(stream.target_bitrate_bps, stream.min_bitrate_bps);
RTC_CHECK_GE(stream.max_bitrate_bps, stream.target_bitrate_bps);
}
// TODO(ivica): Should we check if the sum of all streams/layers is equal to
// the total bitrate? We anyway have to update them in the case bitrate
// estimator changes the total bitrates.
RTC_CHECK_GE(params_.ss.num_spatial_layers, 1);
RTC_CHECK_LE(params_.ss.selected_sl, params_.ss.num_spatial_layers);
RTC_CHECK(params_.ss.spatial_layers.empty() ||
params_.ss.spatial_layers.size() ==
static_cast<size_t>(params_.ss.num_spatial_layers));
if (params_.video.codec == "VP8") {
RTC_CHECK_EQ(params_.ss.num_spatial_layers, 1);
} else if (params_.video.codec == "VP9") {
RTC_CHECK_EQ(params_.ss.streams.size(), 1);
}
RTC_CHECK_GE(params_.call.num_thumbnails, 0);
if (params_.call.num_thumbnails > 0) {
RTC_CHECK_EQ(params_.ss.num_spatial_layers, 1);
RTC_CHECK_EQ(params_.ss.streams.size(), 3);
RTC_CHECK_EQ(params_.video.num_temporal_layers, 3);
RTC_CHECK_EQ(params_.video.codec, "VP8");
}
}
// Static.
std::vector<int> VideoQualityTest::ParseCSV(const std::string& str) {
// Parse comma separated nonnegative integers, where some elements may be
// empty. The empty values are replaced with -1.
// E.g. "10,-20,,30,40" --> {10, 20, -1, 30,40}
// E.g. ",,10,,20," --> {-1, -1, 10, -1, 20, -1}
std::vector<int> result;
if (str.empty())
return result;
const char* p = str.c_str();
int value = -1;
int pos;
while (*p) {
if (*p == ',') {
result.push_back(value);
value = -1;
++p;
continue;
}
RTC_CHECK_EQ(sscanf(p, "%d%n", &value, &pos), 1)
<< "Unexpected non-number value.";
p += pos;
}
result.push_back(value);
return result;
}
// Static.
VideoStream VideoQualityTest::DefaultVideoStream(const Params& params) {
VideoStream stream;
stream.width = params.video.width;
stream.height = params.video.height;
stream.max_framerate = params.video.fps;
stream.min_bitrate_bps = params.video.min_bitrate_bps;
stream.target_bitrate_bps = params.video.target_bitrate_bps;
stream.max_bitrate_bps = params.video.max_bitrate_bps;
stream.max_qp = kDefaultMaxQp;
// TODO(sprang): Can we make this less of a hack?
if (params.video.num_temporal_layers == 2) {
stream.temporal_layer_thresholds_bps.push_back(stream.target_bitrate_bps);
} else if (params.video.num_temporal_layers == 3) {
stream.temporal_layer_thresholds_bps.push_back(stream.max_bitrate_bps / 4);
stream.temporal_layer_thresholds_bps.push_back(stream.target_bitrate_bps);
} else {
RTC_CHECK_LE(params.video.num_temporal_layers, kMaxTemporalStreams);
for (int i = 0; i < params.video.num_temporal_layers - 1; ++i) {
stream.temporal_layer_thresholds_bps.push_back(static_cast<int>(
stream.max_bitrate_bps * kVp8LayerRateAlloction[0][i] + 0.5));
}
}
return stream;
}
// Static.
VideoStream VideoQualityTest::DefaultThumbnailStream() {
VideoStream stream;
stream.width = 320;
stream.height = 180;
stream.max_framerate = 7;
stream.min_bitrate_bps = 7500;
stream.target_bitrate_bps = 37500;
stream.max_bitrate_bps = 50000;
stream.max_qp = kDefaultMaxQp;
return stream;
}
// Static.
void VideoQualityTest::FillScalabilitySettings(
Params* params,
const std::vector<std::string>& stream_descriptors,
int num_streams,
size_t selected_stream,
int num_spatial_layers,
int selected_sl,
const std::vector<std::string>& sl_descriptors) {
if (params->ss.streams.empty() && params->ss.infer_streams) {
webrtc::VideoEncoderConfig encoder_config;
encoder_config.content_type =
params->screenshare.enabled
? webrtc::VideoEncoderConfig::ContentType::kScreen
: webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo;
encoder_config.max_bitrate_bps = params->video.max_bitrate_bps;
encoder_config.min_transmit_bitrate_bps = params->video.min_transmit_bps;
encoder_config.number_of_streams = num_streams;
encoder_config.spatial_layers = params->ss.spatial_layers;
encoder_config.video_stream_factory =
new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
params->video.codec, kDefaultMaxQp, params->video.fps,
params->screenshare.enabled, true);
params->ss.streams =
encoder_config.video_stream_factory->CreateEncoderStreams(
static_cast<int>(params->video.width),
static_cast<int>(params->video.height), encoder_config);
} else {
// Read VideoStream and SpatialLayer elements from a list of comma separated
// lists. To use a default value for an element, use -1 or leave empty.
// Validity checks performed in CheckParams.
RTC_CHECK(params->ss.streams.empty());
for (auto descriptor : stream_descriptors) {
if (descriptor.empty())
continue;
VideoStream stream = VideoQualityTest::DefaultVideoStream(*params);
std::vector<int> v = VideoQualityTest::ParseCSV(descriptor);
if (v[0] != -1)
stream.width = static_cast<size_t>(v[0]);
if (v[1] != -1)
stream.height = static_cast<size_t>(v[1]);
if (v[2] != -1)
stream.max_framerate = v[2];
if (v[3] != -1)
stream.min_bitrate_bps = v[3];
if (v[4] != -1)
stream.target_bitrate_bps = v[4];
if (v[5] != -1)
stream.max_bitrate_bps = v[5];
if (v.size() > 6 && v[6] != -1)
stream.max_qp = v[6];
if (v.size() > 7) {
stream.temporal_layer_thresholds_bps.clear();
stream.temporal_layer_thresholds_bps.insert(
stream.temporal_layer_thresholds_bps.end(), v.begin() + 7, v.end());
} else {
// Automatic TL thresholds for more than two layers not supported.
RTC_CHECK_LE(params->video.num_temporal_layers, 2);
}
params->ss.streams.push_back(stream);
}
}
params->ss.num_spatial_layers = std::max(1, num_spatial_layers);
params->ss.selected_stream = selected_stream;
params->ss.selected_sl = selected_sl;
RTC_CHECK(params->ss.spatial_layers.empty());
for (auto descriptor : sl_descriptors) {
if (descriptor.empty())
continue;
std::vector<int> v = VideoQualityTest::ParseCSV(descriptor);
RTC_CHECK_GT(v[2], 0);
SpatialLayer layer;
layer.scaling_factor_num = v[0] == -1 ? 1 : v[0];
layer.scaling_factor_den = v[1] == -1 ? 1 : v[1];
layer.target_bitrate_bps = v[2];
params->ss.spatial_layers.push_back(layer);
}
}
void VideoQualityTest::SetupVideo(Transport* send_transport,
Transport* recv_transport) {
if (params_.logging.logs)
trace_to_stderr_.reset(new test::TraceToStderr);
size_t num_video_streams = params_.ss.streams.size();
size_t num_flexfec_streams = params_.video.flexfec ? 1 : 0;
CreateSendConfig(num_video_streams, 0, num_flexfec_streams, send_transport);
int payload_type;
if (params_.video.codec == "H264") {
video_encoder_.reset(H264Encoder::Create(cricket::VideoCodec("H264")));
payload_type = kPayloadTypeH264;
} else if (params_.video.codec == "VP8") {
if (params_.screenshare.enabled && params_.ss.streams.size() > 1) {
// Simulcast screenshare needs a simulcast encoder adapter to work, since
// encoders usually can't natively do simulcast with different frame rates
// for the different layers.
video_encoder_.reset(
new SimulcastEncoderAdapter(new Vp8EncoderFactory()));
} else {
video_encoder_.reset(VP8Encoder::Create());
}
payload_type = kPayloadTypeVP8;
} else if (params_.video.codec == "VP9") {
video_encoder_.reset(VP9Encoder::Create());
payload_type = kPayloadTypeVP9;
} else {
RTC_NOTREACHED() << "Codec not supported!";
return;
}
video_send_config_.encoder_settings.encoder = video_encoder_.get();
video_send_config_.encoder_settings.payload_name = params_.video.codec;
video_send_config_.encoder_settings.payload_type = payload_type;
video_send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
video_send_config_.rtp.rtx.payload_type = kSendRtxPayloadType;
for (size_t i = 0; i < num_video_streams; ++i)
video_send_config_.rtp.rtx.ssrcs.push_back(kSendRtxSsrcs[i]);
video_send_config_.rtp.extensions.clear();
if (params_.call.send_side_bwe) {
video_send_config_.rtp.extensions.push_back(
RtpExtension(RtpExtension::kTransportSequenceNumberUri,
test::kTransportSequenceNumberExtensionId));
} else {
video_send_config_.rtp.extensions.push_back(RtpExtension(
RtpExtension::kAbsSendTimeUri, test::kAbsSendTimeExtensionId));
}
video_send_config_.rtp.extensions.push_back(RtpExtension(
RtpExtension::kVideoContentTypeUri, test::kVideoContentTypeExtensionId));
video_send_config_.rtp.extensions.push_back(RtpExtension(
RtpExtension::kVideoTimingUri, test::kVideoTimingExtensionId));
video_encoder_config_.min_transmit_bitrate_bps =
params_.video.min_transmit_bps;
video_send_config_.suspend_below_min_bitrate =
params_.video.suspend_below_min_bitrate;
video_encoder_config_.number_of_streams = params_.ss.streams.size();
video_encoder_config_.max_bitrate_bps = 0;
for (size_t i = 0; i < params_.ss.streams.size(); ++i) {
video_encoder_config_.max_bitrate_bps +=
params_.ss.streams[i].max_bitrate_bps;
}
if (params_.ss.infer_streams) {
video_encoder_config_.video_stream_factory =
new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
params_.video.codec, params_.ss.streams[0].max_qp,
params_.video.fps, params_.screenshare.enabled, true);
} else {
video_encoder_config_.video_stream_factory =
new rtc::RefCountedObject<VideoStreamFactory>(params_.ss.streams);
}
video_encoder_config_.spatial_layers = params_.ss.spatial_layers;
CreateMatchingReceiveConfigs(recv_transport);
const bool decode_all_receive_streams =
params_.ss.selected_stream == params_.ss.streams.size();
for (size_t i = 0; i < num_video_streams; ++i) {
video_receive_configs_[i].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
video_receive_configs_[i].rtp.rtx_ssrc = kSendRtxSsrcs[i];
video_receive_configs_[i]
.rtp.rtx_associated_payload_types[kSendRtxPayloadType] = payload_type;
video_receive_configs_[i].rtp.transport_cc = params_.call.send_side_bwe;
video_receive_configs_[i].rtp.remb = !params_.call.send_side_bwe;
// Enable RTT calculation so NTP time estimator will work.
video_receive_configs_[i].rtp.rtcp_xr.receiver_reference_time_report = true;
// Force fake decoders on non-selected simulcast streams.
if (!decode_all_receive_streams && i != params_.ss.selected_stream) {
VideoReceiveStream::Decoder decoder;
decoder.decoder = new test::FakeDecoder();
decoder.payload_type = video_send_config_.encoder_settings.payload_type;
decoder.payload_name = video_send_config_.encoder_settings.payload_name;
video_receive_configs_[i].decoders.clear();
allocated_decoders_.emplace_back(decoder.decoder);
video_receive_configs_[i].decoders.push_back(decoder);
}
}
if (params_.video.flexfec) {
// Override send config constructed by CreateSendConfig.
if (decode_all_receive_streams) {
for (uint32_t media_ssrc : video_send_config_.rtp.ssrcs) {
video_send_config_.rtp.flexfec.protected_media_ssrcs.push_back(
media_ssrc);
}
} else {
video_send_config_.rtp.flexfec.protected_media_ssrcs = {
kVideoSendSsrcs[params_.ss.selected_stream]};
}
// The matching receive config is _not_ created by
// CreateMatchingReceiveConfigs, since VideoQualityTest is not a BaseTest.
// Set up the receive config manually instead.
FlexfecReceiveStream::Config flexfec_receive_config(recv_transport);
flexfec_receive_config.payload_type =
video_send_config_.rtp.flexfec.payload_type;
flexfec_receive_config.remote_ssrc = video_send_config_.rtp.flexfec.ssrc;
flexfec_receive_config.protected_media_ssrcs =
video_send_config_.rtp.flexfec.protected_media_ssrcs;
flexfec_receive_config.local_ssrc = kReceiverLocalVideoSsrc;
flexfec_receive_config.transport_cc = params_.call.send_side_bwe;
if (params_.call.send_side_bwe) {
flexfec_receive_config.rtp_header_extensions.push_back(
RtpExtension(RtpExtension::kTransportSequenceNumberUri,
test::kTransportSequenceNumberExtensionId));
} else {
flexfec_receive_config.rtp_header_extensions.push_back(RtpExtension(
RtpExtension::kAbsSendTimeUri, test::kAbsSendTimeExtensionId));
}
flexfec_receive_configs_.push_back(flexfec_receive_config);
if (num_video_streams > 0) {
video_receive_configs_[0].rtp.protected_by_flexfec = true;
}
}
if (params_.video.ulpfec) {
video_send_config_.rtp.ulpfec.red_payload_type = kRedPayloadType;
video_send_config_.rtp.ulpfec.ulpfec_payload_type = kUlpfecPayloadType;
video_send_config_.rtp.ulpfec.red_rtx_payload_type = kRtxRedPayloadType;
if (decode_all_receive_streams) {
for (auto it = video_receive_configs_.begin();
it != video_receive_configs_.end(); ++it) {
it->rtp.red_payload_type =
video_send_config_.rtp.ulpfec.red_payload_type;
it->rtp.ulpfec_payload_type =
video_send_config_.rtp.ulpfec.ulpfec_payload_type;
it->rtp.rtx_associated_payload_types[video_send_config_.rtp.ulpfec
.red_rtx_payload_type] =
video_send_config_.rtp.ulpfec.red_payload_type;
}
} else {
video_receive_configs_[params_.ss.selected_stream].rtp.red_payload_type =
video_send_config_.rtp.ulpfec.red_payload_type;
video_receive_configs_[params_.ss.selected_stream]
.rtp.ulpfec_payload_type =
video_send_config_.rtp.ulpfec.ulpfec_payload_type;
video_receive_configs_[params_.ss.selected_stream]
.rtp.rtx_associated_payload_types[video_send_config_.rtp.ulpfec
.red_rtx_payload_type] =
video_send_config_.rtp.ulpfec.red_payload_type;
}
}
}
void VideoQualityTest::SetupThumbnails(Transport* send_transport,
Transport* recv_transport) {
for (int i = 0; i < params_.call.num_thumbnails; ++i) {
thumbnail_encoders_.emplace_back(VP8Encoder::Create());
// Thumbnails will be send in the other way: from receiver_call to
// sender_call.
VideoSendStream::Config thumbnail_send_config(recv_transport);
thumbnail_send_config.rtp.ssrcs.push_back(kThumbnailSendSsrcStart + i);
thumbnail_send_config.encoder_settings.encoder =
thumbnail_encoders_.back().get();
thumbnail_send_config.encoder_settings.payload_name = params_.video.codec;
thumbnail_send_config.encoder_settings.payload_type = kPayloadTypeVP8;
thumbnail_send_config.rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
thumbnail_send_config.rtp.rtx.payload_type = kSendRtxPayloadType;
thumbnail_send_config.rtp.rtx.ssrcs.push_back(kThumbnailRtxSsrcStart + i);
thumbnail_send_config.rtp.extensions.clear();
if (params_.call.send_side_bwe) {
thumbnail_send_config.rtp.extensions.push_back(
RtpExtension(RtpExtension::kTransportSequenceNumberUri,
test::kTransportSequenceNumberExtensionId));
} else {
thumbnail_send_config.rtp.extensions.push_back(RtpExtension(
RtpExtension::kAbsSendTimeUri, test::kAbsSendTimeExtensionId));
}
VideoEncoderConfig thumbnail_encoder_config;
thumbnail_encoder_config.min_transmit_bitrate_bps = 7500;
thumbnail_send_config.suspend_below_min_bitrate =
params_.video.suspend_below_min_bitrate;
thumbnail_encoder_config.number_of_streams = 1;
thumbnail_encoder_config.max_bitrate_bps = 50000;
if (params_.ss.infer_streams) {
thumbnail_encoder_config.video_stream_factory =
new rtc::RefCountedObject<VideoStreamFactory>(params_.ss.streams);
} else {
thumbnail_encoder_config.video_stream_factory =
new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
params_.video.codec, params_.ss.streams[0].max_qp,
params_.video.fps, params_.screenshare.enabled, true);
}
thumbnail_encoder_config.spatial_layers = params_.ss.spatial_layers;
VideoReceiveStream::Config thumbnail_receive_config(send_transport);
thumbnail_receive_config.rtp.remb = false;
thumbnail_receive_config.rtp.transport_cc = true;
thumbnail_receive_config.rtp.local_ssrc = kReceiverLocalVideoSsrc;
for (const RtpExtension& extension : thumbnail_send_config.rtp.extensions)
thumbnail_receive_config.rtp.extensions.push_back(extension);
thumbnail_receive_config.renderer = &fake_renderer_;
VideoReceiveStream::Decoder decoder =
test::CreateMatchingDecoder(thumbnail_send_config.encoder_settings);
allocated_decoders_.push_back(
std::unique_ptr<VideoDecoder>(decoder.decoder));
thumbnail_receive_config.decoders.clear();
thumbnail_receive_config.decoders.push_back(decoder);
thumbnail_receive_config.rtp.remote_ssrc =
thumbnail_send_config.rtp.ssrcs[0];
thumbnail_receive_config.rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
thumbnail_receive_config.rtp.rtx_ssrc = kThumbnailRtxSsrcStart + i;
thumbnail_receive_config.rtp
.rtx_associated_payload_types[kSendRtxPayloadType] = kPayloadTypeVP8;
thumbnail_receive_config.rtp.transport_cc = params_.call.send_side_bwe;
thumbnail_receive_config.rtp.remb = !params_.call.send_side_bwe;
thumbnail_encoder_configs_.push_back(thumbnail_encoder_config.Copy());
thumbnail_send_configs_.push_back(thumbnail_send_config.Copy());
thumbnail_receive_configs_.push_back(thumbnail_receive_config.Copy());
}
for (int i = 0; i < params_.call.num_thumbnails; ++i) {
thumbnail_send_streams_.push_back(receiver_call_->CreateVideoSendStream(
thumbnail_send_configs_[i].Copy(),
thumbnail_encoder_configs_[i].Copy()));
thumbnail_receive_streams_.push_back(sender_call_->CreateVideoReceiveStream(
thumbnail_receive_configs_[i].Copy()));
}
}
void VideoQualityTest::DestroyThumbnailStreams() {
for (VideoSendStream* thumbnail_send_stream : thumbnail_send_streams_)
receiver_call_->DestroyVideoSendStream(thumbnail_send_stream);
thumbnail_send_streams_.clear();
for (VideoReceiveStream* thumbnail_receive_stream :
thumbnail_receive_streams_)
sender_call_->DestroyVideoReceiveStream(thumbnail_receive_stream);
thumbnail_send_streams_.clear();
thumbnail_receive_streams_.clear();
for (std::unique_ptr<test::VideoCapturer>& video_caputurer :
thumbnail_capturers_) {
video_caputurer.reset();
}
}
void VideoQualityTest::SetupScreenshareOrSVC() {
if (params_.screenshare.enabled) {
// Fill out codec settings.
video_encoder_config_.content_type =
VideoEncoderConfig::ContentType::kScreen;
degradation_preference_ =
VideoSendStream::DegradationPreference::kMaintainResolution;
if (params_.video.codec == "VP8") {
VideoCodecVP8 vp8_settings = VideoEncoder::GetDefaultVp8Settings();
vp8_settings.denoisingOn = false;
vp8_settings.frameDroppingOn = false;
vp8_settings.numberOfTemporalLayers =
static_cast<unsigned char>(params_.video.num_temporal_layers);
video_encoder_config_.encoder_specific_settings =
new rtc::RefCountedObject<
VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8_settings);
} else if (params_.video.codec == "VP9") {
VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
vp9_settings.denoisingOn = false;
vp9_settings.frameDroppingOn = false;
vp9_settings.numberOfTemporalLayers =
static_cast<unsigned char>(params_.video.num_temporal_layers);
vp9_settings.numberOfSpatialLayers =
static_cast<unsigned char>(params_.ss.num_spatial_layers);
video_encoder_config_.encoder_specific_settings =
new rtc::RefCountedObject<
VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
}
// Setup frame generator.
const size_t kWidth = 1850;
const size_t kHeight = 1110;
if (params_.screenshare.generate_slides) {
frame_generator_ = test::FrameGenerator::CreateSlideGenerator(
kWidth, kHeight,
params_.screenshare.slide_change_interval * params_.video.fps);
} else {
std::vector<std::string> slides = params_.screenshare.slides;
if (slides.size() == 0) {
slides.push_back(test::ResourcePath("web_screenshot_1850_1110", "yuv"));
slides.push_back(test::ResourcePath("presentation_1850_1110", "yuv"));
slides.push_back(test::ResourcePath("photo_1850_1110", "yuv"));
slides.push_back(
test::ResourcePath("difficult_photo_1850_1110", "yuv"));
}
if (params_.screenshare.scroll_duration == 0) {
// Cycle image every slide_change_interval seconds.
frame_generator_ = test::FrameGenerator::CreateFromYuvFile(
slides, kWidth, kHeight,
params_.screenshare.slide_change_interval * params_.video.fps);
} else {
RTC_CHECK_LE(params_.video.width, kWidth);
RTC_CHECK_LE(params_.video.height, kHeight);
RTC_CHECK_GT(params_.screenshare.slide_change_interval, 0);
const int kPauseDurationMs =
(params_.screenshare.slide_change_interval -
params_.screenshare.scroll_duration) *
1000;
RTC_CHECK_LE(params_.screenshare.scroll_duration,
params_.screenshare.slide_change_interval);
frame_generator_ =
test::FrameGenerator::CreateScrollingInputFromYuvFiles(
clock_, slides, kWidth, kHeight, params_.video.width,
params_.video.height,
params_.screenshare.scroll_duration * 1000, kPauseDurationMs);
}
}
} else if (params_.ss.num_spatial_layers > 1) { // For non-screenshare case.
RTC_CHECK(params_.video.codec == "VP9");
VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
vp9_settings.numberOfTemporalLayers =
static_cast<unsigned char>(params_.video.num_temporal_layers);
vp9_settings.numberOfSpatialLayers =
static_cast<unsigned char>(params_.ss.num_spatial_layers);
video_encoder_config_.encoder_specific_settings = new rtc::RefCountedObject<
VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
}
}
void VideoQualityTest::SetupThumbnailCapturers(size_t num_thumbnail_streams) {
VideoStream thumbnail = DefaultThumbnailStream();
for (size_t i = 0; i < num_thumbnail_streams; ++i) {
thumbnail_capturers_.emplace_back(test::FrameGeneratorCapturer::Create(
static_cast<int>(thumbnail.width), static_cast<int>(thumbnail.height),
thumbnail.max_framerate, clock_));
RTC_DCHECK(thumbnail_capturers_.back());
}
}
void VideoQualityTest::CreateCapturer() {
if (params_.screenshare.enabled) {
test::FrameGeneratorCapturer* frame_generator_capturer =
new test::FrameGeneratorCapturer(clock_, std::move(frame_generator_),
params_.video.fps);
EXPECT_TRUE(frame_generator_capturer->Init());
video_capturer_.reset(frame_generator_capturer);
} else {
if (params_.video.clip_name == "Generator") {
video_capturer_.reset(test::FrameGeneratorCapturer::Create(
static_cast<int>(params_.video.width),
static_cast<int>(params_.video.height), params_.video.fps, clock_));
} else if (params_.video.clip_name.empty()) {
video_capturer_.reset(test::VcmCapturer::Create(
params_.video.width, params_.video.height, params_.video.fps,
params_.video.capture_device_index));
if (!video_capturer_) {
// Failed to get actual camera, use chroma generator as backup.
video_capturer_.reset(test::FrameGeneratorCapturer::Create(
static_cast<int>(params_.video.width),
static_cast<int>(params_.video.height), params_.video.fps, clock_));
}
} else {
video_capturer_.reset(test::FrameGeneratorCapturer::CreateFromYuvFile(
test::ResourcePath(params_.video.clip_name, "yuv"),
params_.video.width, params_.video.height, params_.video.fps,
clock_));
ASSERT_TRUE(video_capturer_) << "Could not create capturer for "
<< params_.video.clip_name
<< ".yuv. Is this resource file present?";
}
}
RTC_DCHECK(video_capturer_.get());
}
void VideoQualityTest::RunWithAnalyzer(const Params& params) {
std::unique_ptr<test::LayerFilteringTransport> send_transport;
std::unique_ptr<test::DirectTransport> recv_transport;
FILE* graph_data_output_file = nullptr;
std::unique_ptr<VideoAnalyzer> analyzer;
params_ = params;
RTC_CHECK(!params_.audio.enabled);
// TODO(ivica): Merge with RunWithRenderer and use a flag / argument to
// differentiate between the analyzer and the renderer case.
CheckParams();
if (!params_.analyzer.graph_data_output_filename.empty()) {
graph_data_output_file =
fopen(params_.analyzer.graph_data_output_filename.c_str(), "w");
RTC_CHECK(graph_data_output_file)
<< "Can't open the file " << params_.analyzer.graph_data_output_filename
<< "!";
}
if (!params.logging.rtc_event_log_name.empty()) {
event_log_ = RtcEventLog::Create(clock_, RtcEventLog::EncodingType::Legacy);
bool event_log_started =
event_log_->StartLogging(params.logging.rtc_event_log_name, -1);
RTC_DCHECK(event_log_started);
}
Call::Config call_config(event_log_.get());
call_config.bitrate_config = params.call.call_bitrate_config;
task_queue_.SendTask([this, &call_config, &send_transport,
&recv_transport]() {
CreateCalls(call_config, call_config);
send_transport = rtc::MakeUnique<test::LayerFilteringTransport>(
&task_queue_, params_.pipe, sender_call_.get(), kPayloadTypeVP8,
kPayloadTypeVP9, params_.video.selected_tl, params_.ss.selected_sl,
payload_type_map_);
recv_transport = rtc::MakeUnique<test::DirectTransport>(
&task_queue_, params_.pipe, receiver_call_.get(), payload_type_map_);
});
std::string graph_title = params_.analyzer.graph_title;
if (graph_title.empty())
graph_title = VideoQualityTest::GenerateGraphTitle();
bool is_quick_test_enabled = field_trial::IsEnabled("WebRTC-QuickPerfTest");
analyzer = rtc::MakeUnique<VideoAnalyzer>(
send_transport.get(), params_.analyzer.test_label,
params_.analyzer.avg_psnr_threshold, params_.analyzer.avg_ssim_threshold,
is_quick_test_enabled
? kFramesSentInQuickTest
: params_.analyzer.test_durations_secs * params_.video.fps,
graph_data_output_file, graph_title,
kVideoSendSsrcs[params_.ss.selected_stream],
kSendRtxSsrcs[params_.ss.selected_stream],
static_cast<size_t>(params_.ss.selected_stream), params.ss.selected_sl,
params_.video.selected_tl, is_quick_test_enabled, clock_,
params_.logging.rtp_dump_name);
task_queue_.SendTask([&]() {
analyzer->SetCall(sender_call_.get());
analyzer->SetReceiver(receiver_call_->Receiver());
send_transport->SetReceiver(analyzer.get());
recv_transport->SetReceiver(sender_call_->Receiver());
SetupVideo(analyzer.get(), recv_transport.get());
SetupThumbnails(analyzer.get(), recv_transport.get());
video_receive_configs_[params_.ss.selected_stream].renderer =
analyzer.get();
video_send_config_.pre_encode_callback = analyzer->pre_encode_proxy();
RTC_DCHECK(!video_send_config_.post_encode_callback);
video_send_config_.post_encode_callback = analyzer->encode_timing_proxy();
SetupScreenshareOrSVC();
CreateFlexfecStreams();
CreateVideoStreams();
analyzer->SetSendStream(video_send_stream_);
if (video_receive_streams_.size() == 1)
analyzer->SetReceiveStream(video_receive_streams_[0]);
video_send_stream_->SetSource(analyzer->OutputInterface(),
degradation_preference_);
SetupThumbnailCapturers(params_.call.num_thumbnails);
for (size_t i = 0; i < thumbnail_send_streams_.size(); ++i) {
thumbnail_send_streams_[i]->SetSource(thumbnail_capturers_[i].get(),
degradation_preference_);
}
CreateCapturer();
analyzer->SetSource(video_capturer_.get(), params_.ss.infer_streams);
StartEncodedFrameLogs(video_send_stream_);
StartEncodedFrameLogs(video_receive_streams_[params_.ss.selected_stream]);
video_send_stream_->Start();
for (VideoSendStream* thumbnail_send_stream : thumbnail_send_streams_)
thumbnail_send_stream->Start();
for (VideoReceiveStream* receive_stream : video_receive_streams_)
receive_stream->Start();
for (VideoReceiveStream* thumbnail_receive_stream :
thumbnail_receive_streams_)
thumbnail_receive_stream->Start();
analyzer->StartMeasuringCpuProcessTime();
video_capturer_->Start();
for (std::unique_ptr<test::VideoCapturer>& video_caputurer :
thumbnail_capturers_) {
video_caputurer->Start();
}
});
analyzer->Wait();
event_log_->StopLogging();
task_queue_.SendTask([&]() {
for (std::unique_ptr<test::VideoCapturer>& video_caputurer :
thumbnail_capturers_)
video_caputurer->Stop();
video_capturer_->Stop();
for (VideoReceiveStream* thumbnail_receive_stream :
thumbnail_receive_streams_)
thumbnail_receive_stream->Stop();
for (VideoReceiveStream* receive_stream : video_receive_streams_)
receive_stream->Stop();
for (VideoSendStream* thumbnail_send_stream : thumbnail_send_streams_)
thumbnail_send_stream->Stop();
video_send_stream_->Stop();
DestroyStreams();
DestroyThumbnailStreams();
if (graph_data_output_file)
fclose(graph_data_output_file);
video_capturer_.reset();
send_transport.reset();
recv_transport.reset();
DestroyCalls();
});
}
void VideoQualityTest::SetupAudio(int send_channel_id,
int receive_channel_id,
Transport* transport,
AudioReceiveStream** audio_receive_stream) {
audio_send_config_ = AudioSendStream::Config(transport);
audio_send_config_.voe_channel_id = send_channel_id;
audio_send_config_.rtp.ssrc = kAudioSendSsrc;
// Add extension to enable audio send side BWE, and allow audio bit rate
// adaptation.
audio_send_config_.rtp.extensions.clear();
if (params_.call.send_side_bwe) {
audio_send_config_.rtp.extensions.push_back(
webrtc::RtpExtension(webrtc::RtpExtension::kTransportSequenceNumberUri,
test::kTransportSequenceNumberExtensionId));
audio_send_config_.min_bitrate_bps = kOpusMinBitrateBps;
audio_send_config_.max_bitrate_bps = kOpusBitrateFbBps;
}
audio_send_config_.send_codec_spec =
rtc::Optional<AudioSendStream::Config::SendCodecSpec>(
{kAudioSendPayloadType,
{"OPUS", 48000, 2,
{{"usedtx", (params_.audio.dtx ? "1" : "0")},
{"stereo", "1"}}}});
audio_send_config_.encoder_factory = encoder_factory_;
audio_send_stream_ = sender_call_->CreateAudioSendStream(audio_send_config_);
AudioReceiveStream::Config audio_config;
audio_config.rtp.local_ssrc = kReceiverLocalAudioSsrc;
audio_config.rtcp_send_transport = transport;
audio_config.voe_channel_id = receive_channel_id;
audio_config.rtp.remote_ssrc = audio_send_config_.rtp.ssrc;
audio_config.rtp.transport_cc = params_.call.send_side_bwe;
audio_config.rtp.extensions = audio_send_config_.rtp.extensions;
audio_config.decoder_factory = decoder_factory_;
audio_config.decoder_map = {{kAudioSendPayloadType, {"OPUS", 48000, 2}}};
if (params_.video.enabled && params_.audio.sync_video)
audio_config.sync_group = kSyncGroup;
*audio_receive_stream =
receiver_call_->CreateAudioReceiveStream(audio_config);
}
void VideoQualityTest::RunWithRenderers(const Params& params) {
std::unique_ptr<test::LayerFilteringTransport> send_transport;
std::unique_ptr<test::DirectTransport> recv_transport;
::VoiceEngineState voe;
std::unique_ptr<test::VideoRenderer> local_preview;
std::vector<std::unique_ptr<test::VideoRenderer>> loopback_renderers;
AudioReceiveStream* audio_receive_stream = nullptr;
task_queue_.SendTask([&]() {
params_ = params;
CheckParams();
// TODO(ivica): Remove bitrate_config and use the default Call::Config(), to
// match the full stack tests.
Call::Config call_config(event_log_.get());
call_config.bitrate_config = params_.call.call_bitrate_config;
rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing(
webrtc::AudioProcessing::Create());
if (params_.audio.enabled) {
CreateVoiceEngine(&voe, audio_processing.get(), decoder_factory_);
AudioState::Config audio_state_config;
audio_state_config.voice_engine = voe.voice_engine;
audio_state_config.audio_mixer = AudioMixerImpl::Create();
audio_state_config.audio_processing = audio_processing;
call_config.audio_state = AudioState::Create(audio_state_config);
}
CreateCalls(call_config, call_config);
// TODO(minyue): consider if this is a good transport even for audio only
// calls.
send_transport = rtc::MakeUnique<test::LayerFilteringTransport>(
&task_queue_, params.pipe, sender_call_.get(), kPayloadTypeVP8,
kPayloadTypeVP9, params.video.selected_tl, params_.ss.selected_sl,
payload_type_map_);
recv_transport = rtc::MakeUnique<test::DirectTransport>(
&task_queue_, params_.pipe, receiver_call_.get(), payload_type_map_);
// TODO(ivica): Use two calls to be able to merge with RunWithAnalyzer or at
// least share as much code as possible. That way this test would also match
// the full stack tests better.
send_transport->SetReceiver(receiver_call_->Receiver());
recv_transport->SetReceiver(sender_call_->Receiver());
if (params_.video.enabled) {
// Create video renderers.
local_preview.reset(test::VideoRenderer::Create(
"Local Preview", params_.video.width, params_.video.height));
const size_t selected_stream_id = params_.ss.selected_stream;
const size_t num_streams = params_.ss.streams.size();
if (selected_stream_id == num_streams) {
for (size_t stream_id = 0; stream_id < num_streams; ++stream_id) {
std::ostringstream oss;
oss << "Loopback Video - Stream #" << static_cast<int>(stream_id);
loopback_renderers.emplace_back(test::VideoRenderer::Create(
oss.str().c_str(), params_.ss.streams[stream_id].width,
params_.ss.streams[stream_id].height));
}
} else {
loopback_renderers.emplace_back(test::VideoRenderer::Create(
"Loopback Video", params_.ss.streams[selected_stream_id].width,
params_.ss.streams[selected_stream_id].height));
}
SetupVideo(send_transport.get(), recv_transport.get());
video_send_config_.pre_encode_callback = local_preview.get();
if (selected_stream_id == num_streams) {
for (size_t stream_id = 0; stream_id < num_streams; ++stream_id) {
video_receive_configs_[stream_id].renderer =
loopback_renderers[stream_id].get();
if (params_.audio.enabled && params_.audio.sync_video)
video_receive_configs_[stream_id].sync_group = kSyncGroup;
}
} else {
video_receive_configs_[selected_stream_id].renderer =
loopback_renderers.back().get();
if (params_.audio.enabled && params_.audio.sync_video)
video_receive_configs_[selected_stream_id].sync_group = kSyncGroup;
}
if (params_.screenshare.enabled)
SetupScreenshareOrSVC();
CreateFlexfecStreams();
CreateVideoStreams();
CreateCapturer();
video_send_stream_->SetSource(video_capturer_.get(),
degradation_preference_);
}
if (params_.audio.enabled) {
SetupAudio(voe.send_channel_id, voe.receive_channel_id,
send_transport.get(), &audio_receive_stream);
}
for (VideoReceiveStream* receive_stream : video_receive_streams_)
StartEncodedFrameLogs(receive_stream);
StartEncodedFrameLogs(video_send_stream_);
// Start sending and receiving video.
if (params_.video.enabled) {
for (VideoReceiveStream* video_receive_stream : video_receive_streams_)
video_receive_stream->Start();
video_send_stream_->Start();
video_capturer_->Start();
}
if (params_.audio.enabled) {
// Start receiving audio.
audio_receive_stream->Start();
EXPECT_EQ(0, voe.base->StartPlayout(voe.receive_channel_id));
// Start sending audio.
audio_send_stream_->Start();
EXPECT_EQ(0, voe.base->StartSend(voe.send_channel_id));
}
});
test::PressEnterToContinue();
task_queue_.SendTask([&]() {
if (params_.audio.enabled) {
// Stop sending audio.
EXPECT_EQ(0, voe.base->StopSend(voe.send_channel_id));
audio_send_stream_->Stop();
// Stop receiving audio.
EXPECT_EQ(0, voe.base->StopPlayout(voe.receive_channel_id));
audio_receive_stream->Stop();
sender_call_->DestroyAudioSendStream(audio_send_stream_);
receiver_call_->DestroyAudioReceiveStream(audio_receive_stream);
}
// Stop receiving and sending video.
if (params_.video.enabled) {
video_capturer_->Stop();
video_send_stream_->Stop();
for (FlexfecReceiveStream* flexfec_receive_stream :
flexfec_receive_streams_) {
for (VideoReceiveStream* video_receive_stream :
video_receive_streams_) {
video_receive_stream->RemoveSecondarySink(flexfec_receive_stream);
}
receiver_call_->DestroyFlexfecReceiveStream(flexfec_receive_stream);
}
for (VideoReceiveStream* receive_stream : video_receive_streams_) {
receive_stream->Stop();
receiver_call_->DestroyVideoReceiveStream(receive_stream);
}
sender_call_->DestroyVideoSendStream(video_send_stream_);
}
video_capturer_.reset();
send_transport.reset();
recv_transport.reset();
if (params_.audio.enabled)
DestroyVoiceEngine(&voe);
local_preview.reset();
loopback_renderers.clear();
DestroyCalls();
});
}
void VideoQualityTest::StartEncodedFrameLogs(VideoSendStream* stream) {
if (!params_.logging.encoded_frame_base_path.empty()) {
std::ostringstream str;
str << send_logs_++;
std::string prefix =
params_.logging.encoded_frame_base_path + "." + str.str() + ".send.";
stream->EnableEncodedFrameRecording(
std::vector<rtc::PlatformFile>(
{rtc::CreatePlatformFile(prefix + "1.ivf"),
rtc::CreatePlatformFile(prefix + "2.ivf"),
rtc::CreatePlatformFile(prefix + "3.ivf")}),
100000000);
}
}
void VideoQualityTest::StartEncodedFrameLogs(VideoReceiveStream* stream) {
if (!params_.logging.encoded_frame_base_path.empty()) {
std::ostringstream str;
str << receive_logs_++;
std::string path =
params_.logging.encoded_frame_base_path + "." + str.str() + ".recv.ivf";
stream->EnableEncodedFrameRecording(rtc::CreatePlatformFile(path),
100000000);
}
}
} // namespace webrtc
Reference in New Issue View Git Blame Copy Permalink