admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make use_single_core option configurable in VideoProcessorIntegrationTests.

Browse Source 
plot_webrtc_test_logs.py: Add number of used cores to figure title.

BUG=webrtc:6634

Review-Url: https://codereview.webrtc.org/2706753005
Cr-Commit-Position: refs/heads/master@{#16756}
This commit is contained in:
asapersson 2017-02-21 08:30:04 -08:00 committed by Commit bot
parent 087613c8df
commit 996103a19f
5 changed files with 66 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
webrtc/modules/video_coding/codecs/test/plot_videoprocessor_integrationtest.cc
View File

@ -22,6 +22,7 @@ const bool kFrameDropperOn = true;
const bool kSpatialResizeOn = false;
const VideoCodecType kVideoCodecType[] = {kVideoCodecVP8};
const bool kHwCodec = false;
const bool kUseSingleCore = true;
// Packet loss probability [0.0, 1.0].
const float kPacketLoss = 0.0f;
@ -53,12 +54,12 @@ class PlotVideoProcessorIntegrationTest
rate_profile.num_frames = kNumFramesLong;
// Codec/network settings.
CodecConfigPars process_settings;
SetCodecParameters(&process_settings, codec_type_, kHwCodec, kPacketLoss,
SetCodecParameters(
&process_settings, codec_type_, kHwCodec, kUseSingleCore, kPacketLoss,
-1, // key_frame_interval
1, // num_temporal_layers
kErrorConcealmentOn, kDenoisingOn, kFrameDropperOn,
kSpatialResizeOn, width, height, filename,
kVerboseLogging);
kErrorConcealmentOn, kDenoisingOn, kFrameDropperOn, kSpatialResizeOn,
width, height, filename, kVerboseLogging);
// Metrics for expected quality (PSNR avg, PSNR min, SSIM avg, SSIM min).
QualityMetrics quality_metrics;
SetQualityMetrics(&quality_metrics, 15.0, 10.0, 0.2, 0.1);

18
webrtc/modules/video_coding/codecs/test/plot_webrtc_test_logs.py
View File

@ -30,7 +30,7 @@ CODEC_TYPE = ('Codec type', 'Codec')
ENCODER_IMPLEMENTATION_NAME = ('Encoder implementation name', 'enc name')
DECODER_IMPLEMENTATION_NAME = ('Decoder implementation name', 'dec name')
NUM_FRAMES = ('Total # of frames', 'num frames')
CORES = ('#CPU cores used', 'cores')
CORES = ('#CPU cores used', 'CPU cores used')
DENOISING = ('Denoising', 'denoising')
RESILIENCE = ('Resilience', 'resilience')
ERROR_CONCEALMENT = ('Error concealment', 'error concealment')
@ -146,7 +146,7 @@ def ParseMetrics(filename, setting1, setting2):
Args:
filename: The name of the file.
setting1: First setting for sorting metrics (e.g. width).
setting2: Second setting for sorting metrics (e.g. cores).
setting2: Second setting for sorting metrics (e.g. CPU cores used).
Returns:
A dictionary holding parsed metrics.
@ -156,19 +156,19 @@ def ParseMetrics(filename, setting1, setting2):
metrics = {
"width: 352": {
"cores: 1.0": {
"CPU cores used: 1.0": {
"encode time (us)": [0.718005, 0.806925, 0.909726, 0.931835, 0.953642],
"PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
"bitrate (kbps)": [50, 100, 300, 500, 1000]
},
"cores: 2.0": {
"CPU cores used: 2.0": {
"encode time (us)": [0.718005, 0.806925, 0.909726, 0.931835, 0.953642],
"PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
"bitrate (kbps)": [50, 100, 300, 500, 1000]
},
},
"width: 176": {
"cores: 1.0": {
"CPU cores used: 1.0": {
"encode time (us)": [0.857897, 0.91608, 0.959173, 0.971116, 0.980961],
"PSNR (dB)": [30.243646, 33.375592, 37.574387, 39.42184, 41.437897],
"bitrate (kbps)": [50, 100, 300, 500, 1000]
@ -273,11 +273,11 @@ def Plot(y_metric, x_metric, metrics):
y_metric = 'PSNR (dB)'
x_metric = 'bitrate (kbps)'
metrics = {
"cores: 1.0": {
"CPU cores used: 1.0": {
"PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
"bitrate (kbps)": [50, 100, 300, 500, 1000]
},
"cores: 2.0": {
"CPU cores used: 2.0": {
"PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
"bitrate (kbps)": [50, 100, 300, 500, 1000]
},
@ -339,6 +339,10 @@ def GetTitle(filename):
for i in range(0, len(codec_types)):
title += codec_types[i] + ', '
cores = ParseSetting(filename, CORES[1])
for i in range(0, len(cores)):
title += cores[i].split('.')[0] + ', '
framerate = ParseSetting(filename, FRAMERATE[1])
for i in range(0, len(framerate)):
title += framerate[i].split('.')[0] + ', '

6
webrtc/modules/video_coding/codecs/test/videoprocessor.cc
View File

@ -124,10 +124,8 @@ bool VideoProcessorImpl::Init() {
<< "Failed to register decode complete callback";
// Init the encoder and decoder.
uint32_t num_cores = 1;
if (!config_.use_single_core) {
num_cores = CpuInfo::DetectNumberOfCores();
}
uint32_t num_cores =
config_.use_single_core ? 1 : CpuInfo::DetectNumberOfCores();
RTC_CHECK_EQ(
encoder_->InitEncode(config_.codec_settings, num_cores,
config_.networking_config.max_payload_size_in_bytes),

55
webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc
View File

@ -18,6 +18,9 @@ namespace {
// In these correctness tests, we only consider SW codecs.
const bool kHwCodec = false;
// Only allow encoder/decoder to use single core, for predictability.
const bool kUseSingleCore = true;
} // namespace
#if defined(WEBRTC_VIDEOPROCESSOR_H264_TESTS)
@ -35,8 +38,8 @@ TEST_F(VideoProcessorIntegrationTest, Process0PercentPacketLossH264) {
rate_profile.num_frames = kNumFramesShort;
// Codec/network settings.
CodecConfigPars process_settings;
SetCodecParameters(&process_settings, kVideoCodecH264, kHwCodec, 0.0f, -1, 1,
false, false, true, false);
SetCodecParameters(&process_settings, kVideoCodecH264, kHwCodec,
kUseSingleCore, 0.0f, -1, 1, false, false, true, false);
// Metrics for expected quality.
QualityMetrics quality_metrics;
SetQualityMetrics(&quality_metrics, 35.0, 25.0, 0.93, 0.70);
@ -66,8 +69,8 @@ TEST_F(VideoProcessorIntegrationTest, Process0PercentPacketLossVP9) {
rate_profile.num_frames = kNumFramesShort;
// Codec/network settings.
CodecConfigPars process_settings;
SetCodecParameters(&process_settings, kVideoCodecVP9, kHwCodec, 0.0f, -1, 1,
false, false, true, false);
SetCodecParameters(&process_settings, kVideoCodecVP9, kHwCodec,
kUseSingleCore, 0.0f, -1, 1, false, false, true, false);
// Metrics for expected quality.
QualityMetrics quality_metrics;
SetQualityMetrics(&quality_metrics, 37.0, 36.0, 0.93, 0.92);
@ -88,8 +91,8 @@ TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLossVP9) {
rate_profile.num_frames = kNumFramesShort;
// Codec/network settings.
CodecConfigPars process_settings;
SetCodecParameters(&process_settings, kVideoCodecVP9, kHwCodec, 0.05f, -1, 1,
false, false, true, false);
SetCodecParameters(&process_settings, kVideoCodecVP9, kHwCodec,
kUseSingleCore, 0.05f, -1, 1, false, false, true, false);
// Metrics for expected quality.
QualityMetrics quality_metrics;
SetQualityMetrics(&quality_metrics, 17.0, 14.0, 0.45, 0.36);
@ -114,8 +117,8 @@ TEST_F(VideoProcessorIntegrationTest, ProcessNoLossChangeBitRateVP9) {
rate_profile.num_frames = kNumFramesLong;
// Codec/network settings.
CodecConfigPars process_settings;
SetCodecParameters(&process_settings, kVideoCodecVP9, kHwCodec, 0.0f, -1, 1,
false, false, true, false);
SetCodecParameters(&process_settings, kVideoCodecVP9, kHwCodec,
kUseSingleCore, 0.0f, -1, 1, false, false, true, false);
// Metrics for expected quality.
QualityMetrics quality_metrics;
SetQualityMetrics(&quality_metrics, 35.5, 30.0, 0.90, 0.85);
@ -147,8 +150,8 @@ TEST_F(VideoProcessorIntegrationTest,
rate_profile.num_frames = kNumFramesLong;
// Codec/network settings.
CodecConfigPars process_settings;
SetCodecParameters(&process_settings, kVideoCodecVP9, kHwCodec, 0.0f, -1, 1,
false, false, true, false);
SetCodecParameters(&process_settings, kVideoCodecVP9, kHwCodec,
kUseSingleCore, 0.0f, -1, 1, false, false, true, false);
// Metrics for expected quality.
QualityMetrics quality_metrics;
SetQualityMetrics(&quality_metrics, 31.5, 18.0, 0.80, 0.43);
@ -170,8 +173,8 @@ TEST_F(VideoProcessorIntegrationTest, ProcessNoLossDenoiserOnVP9) {
rate_profile.num_frames = kNumFramesShort;
// Codec/network settings.
CodecConfigPars process_settings;
SetCodecParameters(&process_settings, kVideoCodecVP9, kHwCodec, 0.0f, -1, 1,
false, true, true, false);
SetCodecParameters(&process_settings, kVideoCodecVP9, kHwCodec,
kUseSingleCore, 0.0f, -1, 1, false, true, true, false);
// Metrics for expected quality.
QualityMetrics quality_metrics;
SetQualityMetrics(&quality_metrics, 36.8, 35.8, 0.92, 0.91);
@ -195,8 +198,8 @@ TEST_F(VideoProcessorIntegrationTest,
rate_profile.num_frames = kNumFramesLong;
// Codec/network settings.
CodecConfigPars process_settings;
SetCodecParameters(&process_settings, kVideoCodecVP9, kHwCodec, 0.0f, -1, 1,
false, false, true, true);
SetCodecParameters(&process_settings, kVideoCodecVP9, kHwCodec,
kUseSingleCore, 0.0f, -1, 1, false, false, true, true);
// Metrics for expected quality.
QualityMetrics quality_metrics;
SetQualityMetrics(&quality_metrics, 24.0, 13.0, 0.65, 0.37);
@ -223,8 +226,8 @@ TEST_F(VideoProcessorIntegrationTest, ProcessZeroPacketLoss) {
rate_profile.num_frames = kNumFramesShort;
// Codec/network settings.
CodecConfigPars process_settings;
SetCodecParameters(&process_settings, kVideoCodecVP8, kHwCodec, 0.0f, -1, 1,
false, true, true, false);
SetCodecParameters(&process_settings, kVideoCodecVP8, kHwCodec,
kUseSingleCore, 0.0f, -1, 1, false, true, true, false);
// Metrics for expected quality.
QualityMetrics quality_metrics;
SetQualityMetrics(&quality_metrics, 34.95, 33.0, 0.90, 0.89);
@ -245,8 +248,8 @@ TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLoss) {
rate_profile.num_frames = kNumFramesShort;
// Codec/network settings.
CodecConfigPars process_settings;
SetCodecParameters(&process_settings, kVideoCodecVP8, kHwCodec, 0.05f, -1, 1,
false, true, true, false);
SetCodecParameters(&process_settings, kVideoCodecVP8, kHwCodec,
kUseSingleCore, 0.05f, -1, 1, false, true, true, false);
// Metrics for expected quality.
QualityMetrics quality_metrics;
SetQualityMetrics(&quality_metrics, 20.0, 16.0, 0.60, 0.40);
@ -267,8 +270,8 @@ TEST_F(VideoProcessorIntegrationTest, Process10PercentPacketLoss) {
rate_profile.num_frames = kNumFramesShort;
// Codec/network settings.
CodecConfigPars process_settings;
SetCodecParameters(&process_settings, kVideoCodecVP8, kHwCodec, 0.1f, -1, 1,
false, true, true, false);
SetCodecParameters(&process_settings, kVideoCodecVP8, kHwCodec,
kUseSingleCore, 0.1f, -1, 1, false, true, true, false);
// Metrics for expected quality.
QualityMetrics quality_metrics;
SetQualityMetrics(&quality_metrics, 19.0, 16.0, 0.50, 0.35);
@ -311,8 +314,8 @@ TEST_F(VideoProcessorIntegrationTest, MAYBE_ProcessNoLossChangeBitRateVP8) {
rate_profile.num_frames = kNumFramesLong;
// Codec/network settings.
CodecConfigPars process_settings;
SetCodecParameters(&process_settings, kVideoCodecVP8, kHwCodec, 0.0f, -1, 1,
false, true, true, false);
SetCodecParameters(&process_settings, kVideoCodecVP8, kHwCodec,
kUseSingleCore, 0.0f, -1, 1, false, true, true, false);
// Metrics for expected quality.
QualityMetrics quality_metrics;
SetQualityMetrics(&quality_metrics, 34.0, 32.0, 0.85, 0.80);
@ -352,8 +355,8 @@ TEST_F(VideoProcessorIntegrationTest,
rate_profile.num_frames = kNumFramesLong;
// Codec/network settings.
CodecConfigPars process_settings;
SetCodecParameters(&process_settings, kVideoCodecVP8, kHwCodec, 0.0f, -1, 1,
false, true, true, false);
SetCodecParameters(&process_settings, kVideoCodecVP8, kHwCodec,
kUseSingleCore, 0.0f, -1, 1, false, true, true, false);
// Metrics for expected quality.
QualityMetrics quality_metrics;
SetQualityMetrics(&quality_metrics, 31.0, 22.0, 0.80, 0.65);
@ -388,8 +391,8 @@ TEST_F(VideoProcessorIntegrationTest, MAYBE_ProcessNoLossTemporalLayersVP8) {
rate_profile.num_frames = kNumFramesLong;
// Codec/network settings.
CodecConfigPars process_settings;
SetCodecParameters(&process_settings, kVideoCodecVP8, kHwCodec, 0.0f, -1, 3,
false, true, true, false);
SetCodecParameters(&process_settings, kVideoCodecVP8, kHwCodec,
kUseSingleCore, 0.0f, -1, 3, false, true, true, false);
// Metrics for expected quality.
QualityMetrics quality_metrics;
SetQualityMetrics(&quality_metrics, 32.5, 30.0, 0.85, 0.80);

34
webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h
View File

@ -63,6 +63,7 @@ const char kFilenameForemanCif[] = "foreman_cif";
struct CodecConfigPars {
VideoCodecType codec_type;
bool hw_codec;
bool use_single_core;
float packet_loss;
int num_temporal_layers;
int key_frame_interval;
@ -142,10 +143,7 @@ class VideoProcessorIntegrationTest : public testing::Test {
}
virtual ~VideoProcessorIntegrationTest() = default;
void SetUpCodecConfig(const std::string& filename,
int width,
int height,
bool verbose_logging) {
void SetUpCodecConfig(const CodecConfigPars& process) {
if (hw_codec_) {
#if defined(WEBRTC_VIDEOPROCESSOR_INTEGRATIONTEST_HW_CODECS_ENABLED)
#if defined(WEBRTC_ANDROID)
@ -199,16 +197,16 @@ class VideoProcessorIntegrationTest : public testing::Test {
VideoCodingModule::Codec(codec_type_, &codec_settings_);
// Configure input filename.
config_.input_filename = test::ResourcePath(filename, "yuv");
if (verbose_logging)
printf("Filename: %s\n", filename.c_str());
config_.input_filename = test::ResourcePath(process.filename, "yuv");
if (process.verbose_logging)
printf("Filename: %s\n", process.filename.c_str());
// Generate an output filename in a safe way.
config_.output_filename = test::TempFilename(
test::OutputPath(), "videoprocessor_integrationtest");
config_.frame_length_in_bytes = CalcBufferSize(kI420, width, height);
config_.verbose = verbose_logging;
// Only allow encoder/decoder to use single core, for predictability.
config_.use_single_core = true;
config_.frame_length_in_bytes =
CalcBufferSize(kI420, process.width, process.height);
config_.verbose = process.verbose_logging;
config_.use_single_core = process.use_single_core;
// Key frame interval and packet loss are set for each test.
config_.keyframe_interval = key_frame_interval_;
config_.networking_config.packet_loss_probability = packet_loss_;
@ -216,8 +214,8 @@ class VideoProcessorIntegrationTest : public testing::Test {
// Configure codec settings.
config_.codec_settings = &codec_settings_;
config_.codec_settings->startBitrate = start_bitrate_;
config_.codec_settings->width = width;
config_.codec_settings->height = height;
config_.codec_settings->width = process.width;
config_.codec_settings->height = process.height;
// These features may be set depending on the test.
switch (config_.codec_settings->codecType) {
@ -458,8 +456,7 @@ class VideoProcessorIntegrationTest : public testing::Test {
denoising_on_ = process.denoising_on;
frame_dropper_on_ = process.frame_dropper_on;
spatial_resize_on_ = process.spatial_resize_on;
SetUpCodecConfig(process.filename, process.width, process.height,
process.verbose_logging);
SetUpCodecConfig(process);
// Update the layers and the codec with the initial rates.
bit_rate_ = rate_profile.target_bit_rate[0];
frame_rate_ = rate_profile.input_frame_rate[0];
@ -563,6 +560,7 @@ class VideoProcessorIntegrationTest : public testing::Test {
static void SetCodecParameters(CodecConfigPars* process_settings,
VideoCodecType codec_type,
bool hw_codec,
bool use_single_core,
float packet_loss,
int key_frame_interval,
int num_temporal_layers,
@ -576,6 +574,7 @@ class VideoProcessorIntegrationTest : public testing::Test {
bool verbose_logging) {
process_settings->codec_type = codec_type;
process_settings->hw_codec = hw_codec;
process_settings->use_single_core = use_single_core;
process_settings->packet_loss = packet_loss;
process_settings->key_frame_interval = key_frame_interval;
process_settings->num_temporal_layers = num_temporal_layers,
@ -592,6 +591,7 @@ class VideoProcessorIntegrationTest : public testing::Test {
static void SetCodecParameters(CodecConfigPars* process_settings,
VideoCodecType codec_type,
bool hw_codec,
bool use_single_core,
float packet_loss,
int key_frame_interval,
int num_temporal_layers,
@ -599,8 +599,8 @@ class VideoProcessorIntegrationTest : public testing::Test {
bool denoising_on,
bool frame_dropper_on,
bool spatial_resize_on) {
SetCodecParameters(process_settings, codec_type, hw_codec, packet_loss,
key_frame_interval, num_temporal_layers,
SetCodecParameters(process_settings, codec_type, hw_codec, use_single_core,
packet_loss, key_frame_interval, num_temporal_layers,
error_concealment_on, denoising_on, frame_dropper_on,
spatial_resize_on, kCifWidth, kCifHeight,
kFilenameForemanCif, false /* verbose_logging */);