diff --git a/webrtc/modules/video_coding/BUILD.gn b/webrtc/modules/video_coding/BUILD.gn index d46fd113c8..a3ad4b37c7 100644 --- a/webrtc/modules/video_coding/BUILD.gn +++ b/webrtc/modules/video_coding/BUILD.gn @@ -345,6 +345,7 @@ if (rtc_include_tests) { testonly = true sources = [ "codecs/test/videoprocessor_integrationtest.cc", + "codecs/test/videoprocessor_integrationtest.h", "codecs/vp8/test/vp8_impl_unittest.cc", ] deps = [ diff --git a/webrtc/modules/video_coding/codecs/test/plot_videoprocessor_integrationtest.cc b/webrtc/modules/video_coding/codecs/test/plot_videoprocessor_integrationtest.cc new file mode 100644 index 0000000000..b87286dc6d --- /dev/null +++ b/webrtc/modules/video_coding/codecs/test/plot_videoprocessor_integrationtest.cc @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h" + +namespace webrtc { +namespace test { +namespace { +// Codec settings. +const int kBitrates[] = {30, 50, 100, 200, 300, 500, 1000}; +const int kFps[] = {30}; +const bool kErrorConcealmentOn = false; +const bool kDenoisingOn = false; +const bool kFrameDropperOn = true; +const bool kSpatialResizeOn = false; +const VideoCodecType kVideoCodecType = kVideoCodecVP8; + +// Packet loss probability [0.0, 1.0]. +const float kPacketLoss = 0.0f; + +const bool kVerboseLogging = true; +} // namespace + +// Tests for plotting statistics from logs. +class PlotVideoProcessorIntegrationTest + : public VideoProcessorIntegrationTest, + public ::testing::WithParamInterface<::testing::tuple> { + protected: + PlotVideoProcessorIntegrationTest() + : bitrate_(::testing::get<0>(GetParam())), + framerate_(::testing::get<1>(GetParam())) {} + + virtual ~PlotVideoProcessorIntegrationTest() {} + + void RunTest(int bitrate, + int framerate, + int width, + int height, + const std::string& filename) { + // Bitrate and frame rate profile. + RateProfile rate_profile; + SetRateProfilePars(&rate_profile, + 0, // update_index + bitrate, framerate, + 0); // frame_index_rate_update + rate_profile.frame_index_rate_update[1] = kNbrFramesLong + 1; + rate_profile.num_frames = kNbrFramesLong; + // Codec/network settings. + CodecConfigPars process_settings; + SetCodecParameters(&process_settings, kVideoCodecType, kPacketLoss, + -1, // key_frame_interval + 1, // num_temporal_layers + kErrorConcealmentOn, kDenoisingOn, kFrameDropperOn, + kSpatialResizeOn, width, height, filename, + kVerboseLogging); + // Metrics for expected quality (PSNR avg, PSNR min, SSIM avg, SSIM min). + QualityMetrics quality_metrics; + SetQualityMetrics(&quality_metrics, 15.0, 10.0, 0.2, 0.1); + // Metrics for rate control. + RateControlMetrics rc_metrics[1]; + SetRateControlMetrics(rc_metrics, + 0, // update_index + 300, // max_num_dropped_frames, + 400, // max_key_frame_size_mismatch + 200, // max_delta_frame_size_mismatch + 100, // max_encoding_rate_mismatch + 300, // max_time_hit_target + 0, // num_spatial_resizes + 1); // num_key_frames + ProcessFramesAndVerify(quality_metrics, rate_profile, process_settings, + rc_metrics); + } + const int bitrate_; + const int framerate_; +}; + +INSTANTIATE_TEST_CASE_P(CodecSettings, + PlotVideoProcessorIntegrationTest, + ::testing::Combine(::testing::ValuesIn(kBitrates), + ::testing::ValuesIn(kFps))); + +TEST_P(PlotVideoProcessorIntegrationTest, ProcessSQCif) { + RunTest(bitrate_, framerate_, 128, 96, "foreman_128x96"); +} + +TEST_P(PlotVideoProcessorIntegrationTest, ProcessQQVga) { + RunTest(bitrate_, framerate_, 160, 120, "foreman_160x120"); +} + +TEST_P(PlotVideoProcessorIntegrationTest, ProcessQCif) { + RunTest(bitrate_, framerate_, 176, 144, "foreman_176x144"); +} + +TEST_P(PlotVideoProcessorIntegrationTest, ProcessQVga) { + RunTest(bitrate_, framerate_, 320, 240, "foreman_320x240"); +} + +TEST_P(PlotVideoProcessorIntegrationTest, ProcessCif) { + RunTest(bitrate_, framerate_, 352, 288, "foreman_cif"); +} + +} // namespace test +} // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/test/plot_webrtc_test_logs.py b/webrtc/modules/video_coding/codecs/test/plot_webrtc_test_logs.py new file mode 100755 index 0000000000..85f7afb1e5 --- /dev/null +++ b/webrtc/modules/video_coding/codecs/test/plot_webrtc_test_logs.py @@ -0,0 +1,428 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Plots statistics from WebRTC integration test logs. + +Usage: $ python plot_webrtc_test_logs.py filename.txt +""" + +import numpy +import sys +import re + +import matplotlib.pyplot as plt + +# Log events. +EVENT_START = 'RUN ] CodecSettings/PlotVideoProcessorIntegrationTest.' +EVENT_END = 'OK ] CodecSettings/PlotVideoProcessorIntegrationTest.' + +# Metrics to plot, tuple: (name to parse in file, label to use when plotting). +BITRATE = ('Target Bitrate', 'bitrate (kbps)') +WIDTH = ('Width', 'width') +HEIGHT = ('Height', 'height') +FILENAME = ('Filename', 'clip') +CODEC_TYPE = ('Codec type', 'Codec') +ENCODER_IMPLEMENTATION_NAME = ('Encoder implementation name', 'enc name') +DECODER_IMPLEMENTATION_NAME = ('Decoder implementation name', 'dec name') +NUM_FRAMES = ('Total # of frames', 'num frames') +CORES = ('#CPU cores used', 'cores') +DENOISING = ('Denoising', 'denoising') +RESILIENCE = ('Resilience', 'resilience') +ERROR_CONCEALMENT = ('Error concealment', 'error concealment') +PSNR = ('PSNR avg', 'PSNR (dB)') +SSIM = ('SSIM avg', 'SSIM') +ENC_BITRATE = ('Encoding bitrate', 'encoded bitrate (kbps)') +FRAMERATE = ('Frame rate', 'fps') +NUM_DROPPED_FRAMES = ('Number of dropped frames', 'num dropped frames') +NUM_FRAMES_TO_TARGET = ('Number of frames to approach target rate', + 'frames to reach target rate') +ENCODE_TIME = ('Encoding time', 'encode time (us)') +ENCODE_TIME_AVG = ('Encoding time', 'encode time (us) avg') +DECODE_TIME = ('Decoding time', 'decode time (us)') +DECODE_TIME_AVG = ('Decoding time', 'decode time (us) avg') +FRAME_SIZE = ('Frame sizes', 'frame size (bytes)') +FRAME_SIZE_AVG = ('Frame sizes', 'frame size (bytes) avg') +AVG_KEY_FRAME_SIZE = ('Average key frame size', 'avg key frame size (bytes)') +AVG_NON_KEY_FRAME_SIZE = ('Average non-key frame size', + 'avg non-key frame size (bytes)') + +# Settings. +SETTINGS = [ + WIDTH, + HEIGHT, + FILENAME, + CODEC_TYPE, + NUM_FRAMES, + ENCODER_IMPLEMENTATION_NAME, + DECODER_IMPLEMENTATION_NAME, + ENCODE_TIME, + DECODE_TIME, + FRAME_SIZE, +] + +# Settings, options for x-axis. +X_SETTINGS = [ + CORES, + FRAMERATE, + DENOISING, + RESILIENCE, + ERROR_CONCEALMENT, + BITRATE, # TODO(asapersson): Needs to be last. +] + +# Results. +RESULTS = [ + PSNR, + SSIM, + ENC_BITRATE, + NUM_DROPPED_FRAMES, + NUM_FRAMES_TO_TARGET, + ENCODE_TIME_AVG, + DECODE_TIME_AVG, + AVG_KEY_FRAME_SIZE, + AVG_NON_KEY_FRAME_SIZE, +] + +METRICS_TO_PARSE = SETTINGS + X_SETTINGS + RESULTS + +Y_METRICS = [res[1] for res in RESULTS] + +# Parameters for plotting. +FIG_SIZE_SCALE_FACTOR_X = 2 +FIG_SIZE_SCALE_FACTOR_Y = 2.8 +GRID_COLOR = [0.45, 0.45, 0.45] + + +def ParseSetting(filename, setting): + """Parses setting from file. + + Args: + filename: The name of the file. + setting: Name of setting to parse (e.g. width). + + Returns: + A list holding parsed settings, e.g. ['width: 128.0', 'width: 160.0'] """ + + settings = [] + + f = open(filename) + while True: + line = f.readline() + if not line: + break + if re.search(r'%s' % EVENT_START, line): + # Parse event. + parsed = {} + while True: + line = f.readline() + if not line: + break + if re.search(r'%s' % EVENT_END, line): + # Add parsed setting to list. + if setting in parsed: + s = setting + ': ' + str(parsed[setting]) + if s not in settings: + settings.append(s) + break + + TryFindMetric(parsed, line, f) + + f.close() + return settings + + +def ParseMetrics(filename, setting1, setting2): + """Parses metrics from file. + + Args: + filename: The name of the file. + setting1: First setting for sorting metrics (e.g. width). + setting2: Second setting for sorting metrics (e.g. cores). + + Returns: + A dictionary holding parsed metrics. + + For example: + metrics[key1][key2][measurement] + + metrics = { + "width: 352": { + "cores: 1.0": { + "encode time (us)": [0.718005, 0.806925, 0.909726, 0.931835, 0.953642], + "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551], + "bitrate (kbps)": [50, 100, 300, 500, 1000] + }, + "cores: 2.0": { + "encode time (us)": [0.718005, 0.806925, 0.909726, 0.931835, 0.953642], + "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551], + "bitrate (kbps)": [50, 100, 300, 500, 1000] + }, + }, + "width: 176": { + "cores: 1.0": { + "encode time (us)": [0.857897, 0.91608, 0.959173, 0.971116, 0.980961], + "PSNR (dB)": [30.243646, 33.375592, 37.574387, 39.42184, 41.437897], + "bitrate (kbps)": [50, 100, 300, 500, 1000] + }, + } + } """ + + metrics = {} + + # Parse events. + f = open(filename) + while True: + line = f.readline() + if not line: + break + if re.search(r'%s' % EVENT_START, line): + # Parse event. + parsed = {} + while True: + line = f.readline() + if not line: + break + if re.search(r'%s' % EVENT_END, line): + # Add parsed values to metrics. + key1 = setting1 + ': ' + str(parsed[setting1]) + key2 = setting2 + ': ' + str(parsed[setting2]) + if key1 not in metrics: + metrics[key1] = {} + if key2 not in metrics[key1]: + metrics[key1][key2] = {} + + for label in parsed: + if label not in metrics[key1][key2]: + metrics[key1][key2][label] = [] + metrics[key1][key2][label].append(parsed[label]) + + break + + TryFindMetric(parsed, line, f) + + f.close() + return metrics + + +def TryFindMetric(parsed, line, f): + for metric in METRICS_TO_PARSE: + name = metric[0] + label = metric[1] + if re.search(r'%s' % name, line): + found, value = GetMetric(name, line) + if not found: + # TODO(asapersson): Change format. + # Try find min, max, average stats. + found, minimum = GetMetric("Min", f.readline()) + if not found: + return + found, maximum = GetMetric("Max", f.readline()) + if not found: + return + found, average = GetMetric("Average", f.readline()) + if not found: + return + + parsed[label + ' min'] = minimum + parsed[label + ' max'] = maximum + parsed[label + ' avg'] = average + + parsed[label] = value + return + + +def GetMetric(name, string): + # Float (e.g. bitrate = 98.8253). + pattern = r'%s\s*[:=]\s*([+-]?\d+\.*\d*)' % name + m = re.search(r'%s' % pattern, string) + if m is not None: + return StringToFloat(m.group(1)) + + # Alphanumeric characters (e.g. codec type : VP8). + pattern = r'%s\s*[:=]\s*(\w+)' % name + m = re.search(r'%s' % pattern, string) + if m is not None: + return True, m.group(1) + + return False, -1 + + +def StringToFloat(value): + try: + value = float(value) + except ValueError: + print "Not a float, skipped %s" % value + return False, -1 + + return True, value + + +def Plot(y_metric, x_metric, metrics): + """Plots y_metric vs x_metric per key in metrics. + + For example: + y_metric = 'PSNR (dB)' + x_metric = 'bitrate (kbps)' + metrics = { + "cores: 1.0": { + "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551], + "bitrate (kbps)": [50, 100, 300, 500, 1000] + }, + "cores: 2.0": { + "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551], + "bitrate (kbps)": [50, 100, 300, 500, 1000] + }, + } + """ + for key in metrics: + data = metrics[key] + if y_metric not in data: + print "Failed to find metric: %s" % y_metric + continue + + y = numpy.array(data[y_metric]) + x = numpy.array(data[x_metric]) + if len(y) != len(x): + print "Length mismatch for %s, %s" % (y, x) + continue + + label = y_metric + ' - ' + str(key) + + plt.plot(x, y, label=label, linewidth=1.5, marker='o', markersize=5, + markeredgewidth=0.0) + + +def PlotFigure(settings, y_metrics, x_metric, metrics, title): + """Plots metrics in y_metrics list. One figure is plotted and each entry + in the list is plotted in a subplot (and sorted per settings). + + For example: + settings = ['width: 128.0', 'width: 160.0']. Sort subplot per setting. + y_metrics = ['PSNR (dB)', 'PSNR (dB)']. Metric to plot per subplot. + x_metric = 'bitrate (kbps)' + + """ + + plt.figure() + plt.suptitle(title, fontsize='small', fontweight='bold') + rows = len(settings) + cols = 1 + pos = 1 + while pos <= rows: + plt.rc('grid', color=GRID_COLOR) + ax = plt.subplot(rows, cols, pos) + plt.grid() + plt.setp(ax.get_xticklabels(), visible=(pos == rows), fontsize='small') + plt.setp(ax.get_yticklabels(), fontsize='small') + setting = settings[pos - 1] + Plot(y_metrics[pos - 1], x_metric, metrics[setting]) + plt.title(setting, fontsize='x-small') + plt.legend(fontsize='xx-small') + pos += 1 + + plt.xlabel(x_metric, fontsize='small') + plt.subplots_adjust(left=0.04, right=0.98, bottom=0.04, top=0.96, hspace=0.1) + + +def GetTitle(filename): + title = '' + codec_types = ParseSetting(filename, CODEC_TYPE[1]) + for i in range(0, len(codec_types)): + title += codec_types[i] + ', ' + + framerate = ParseSetting(filename, FRAMERATE[1]) + for i in range(0, len(framerate)): + title += framerate[i].split('.')[0] + ', ' + + enc_names = ParseSetting(filename, ENCODER_IMPLEMENTATION_NAME[1]) + for i in range(0, len(enc_names)): + title += enc_names[i] + ', ' + + dec_names = ParseSetting(filename, DECODER_IMPLEMENTATION_NAME[1]) + for i in range(0, len(dec_names)): + title += dec_names[i] + ', ' + + filenames = ParseSetting(filename, FILENAME[1]) + title += filenames[0].split('_')[0] + + num_frames = ParseSetting(filename, NUM_FRAMES[1]) + for i in range(0, len(num_frames)): + title += ' (' + num_frames[i].split('.')[0] + ')' + + return title + + +def ToString(input_list): + return ToStringWithoutMetric(input_list, ('', '')) + + +def ToStringWithoutMetric(input_list, metric): + i = 1 + output_str = "" + for m in input_list: + if m != metric: + output_str = output_str + ("%s. %s\n" % (i, m[1])) + i += 1 + return output_str + + +def GetIdx(text_list): + return int(raw_input(text_list)) - 1 + + +def main(): + filename = sys.argv[1] + + # Setup. + idx_metric = GetIdx("Choose metric:\n0. All\n%s" % ToString(RESULTS)) + if idx_metric == -1: + # Plot all metrics. One subplot for each metric. + # Per subplot: metric vs bitrate (per resolution). + cores = ParseSetting(filename, CORES[1]) + setting1 = CORES[1] + setting2 = WIDTH[1] + sub_keys = [cores[0]] * len(Y_METRICS) + y_metrics = Y_METRICS + x_metric = BITRATE[1] + else: + resolutions = ParseSetting(filename, WIDTH[1]) + idx = GetIdx("Select metric for x-axis:\n%s" % ToString(X_SETTINGS)) + if X_SETTINGS[idx] == BITRATE: + idx = GetIdx("Plot per:\n%s" % ToStringWithoutMetric(X_SETTINGS, BITRATE)) + idx_setting = METRICS_TO_PARSE.index(X_SETTINGS[idx]) + # Plot one metric. One subplot for each resolution. + # Per subplot: metric vs bitrate (per setting). + setting1 = WIDTH[1] + setting2 = METRICS_TO_PARSE[idx_setting][1] + sub_keys = resolutions + y_metrics = [RESULTS[idx_metric][1]] * len(sub_keys) + x_metric = BITRATE[1] + else: + # Plot one metric. One subplot for each resolution. + # Per subplot: metric vs setting (per bitrate). + setting1 = WIDTH[1] + setting2 = BITRATE[1] + sub_keys = resolutions + y_metrics = [RESULTS[idx_metric][1]] * len(sub_keys) + x_metric = X_SETTINGS[idx][1] + + metrics = ParseMetrics(filename, setting1, setting2) + + # Stretch fig size. + figsize = plt.rcParams["figure.figsize"] + figsize[0] *= FIG_SIZE_SCALE_FACTOR_X + figsize[1] *= FIG_SIZE_SCALE_FACTOR_Y + plt.rcParams["figure.figsize"] = figsize + + PlotFigure(sub_keys, y_metrics, x_metric, metrics, GetTitle(filename)) + + plt.show() + + +if __name__ == '__main__': + main() diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc index e4b20e2c96..88f5b0e3f8 100644 --- a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc +++ b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc @@ -8,578 +8,10 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include - -#include - -#include "webrtc/modules/video_coding/codecs/h264/include/h264.h" -#include "webrtc/modules/video_coding/codecs/test/packet_manipulator.h" -#include "webrtc/modules/video_coding/codecs/test/videoprocessor.h" -#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" -#include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h" -#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" -#include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h" -#include "webrtc/modules/video_coding/include/video_codec_interface.h" -#include "webrtc/modules/video_coding/include/video_coding.h" -#include "webrtc/test/gtest.h" -#include "webrtc/test/testsupport/fileutils.h" -#include "webrtc/test/testsupport/frame_reader.h" -#include "webrtc/test/testsupport/frame_writer.h" -#include "webrtc/test/testsupport/metrics/video_metrics.h" -#include "webrtc/test/testsupport/packet_reader.h" -#include "webrtc/typedefs.h" +#include "webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h" namespace webrtc { -namespace { -// Maximum number of rate updates (i.e., calls to encoder to change bitrate -// and/or frame rate) for the current tests. -const int kMaxNumRateUpdates = 3; - -const int kPercTargetvsActualMismatch = 20; -const int kBaseKeyFrameInterval = 3000; - -// Codec and network settings. -struct CodecConfigPars { - VideoCodecType codec_type; - float packet_loss; - int num_temporal_layers; - int key_frame_interval; - bool error_concealment_on; - bool denoising_on; - bool frame_dropper_on; - bool spatial_resize_on; -}; - -// Quality metrics. -struct QualityMetrics { - double minimum_avg_psnr; - double minimum_min_psnr; - double minimum_avg_ssim; - double minimum_min_ssim; -}; - -// The sequence of bitrate and frame rate changes for the encoder, the frame -// number where the changes are made, and the total number of frames for the -// test. -struct RateProfile { - int target_bit_rate[kMaxNumRateUpdates]; - int input_frame_rate[kMaxNumRateUpdates]; - int frame_index_rate_update[kMaxNumRateUpdates + 1]; - int num_frames; -}; - -// Metrics for the rate control. The rate mismatch metrics are defined as -// percentages.|max_time_hit_target| is defined as number of frames, after a -// rate update is made to the encoder, for the encoder to reach within -// |kPercTargetvsActualMismatch| of new target rate. The metrics are defined for -// each rate update sequence. -struct RateControlMetrics { - int max_num_dropped_frames; - int max_key_frame_size_mismatch; - int max_delta_frame_size_mismatch; - int max_encoding_rate_mismatch; - int max_time_hit_target; - int num_spatial_resizes; - int num_key_frames; -}; - -// Sequence used is foreman (CIF): may be better to use VGA for resize test. -const int kCIFWidth = 352; -const int kCIFHeight = 288; -#if !defined(WEBRTC_IOS) -const int kNbrFramesShort = 100; // Some tests are run for shorter sequence. -#endif -const int kNbrFramesLong = 299; - -// Parameters from VP8 wrapper, which control target size of key frames. -const float kInitialBufferSize = 0.5f; -const float kOptimalBufferSize = 0.6f; -const float kScaleKeyFrameSize = 0.5f; - -void SetRateProfilePars(RateProfile* rate_profile, - int update_index, - int bit_rate, - int frame_rate, - int frame_index_rate_update) { - rate_profile->target_bit_rate[update_index] = bit_rate; - rate_profile->input_frame_rate[update_index] = frame_rate; - rate_profile->frame_index_rate_update[update_index] = frame_index_rate_update; -} - -void SetCodecParameters(CodecConfigPars* process_settings, - VideoCodecType codec_type, - float packet_loss, - int key_frame_interval, - int num_temporal_layers, - bool error_concealment_on, - bool denoising_on, - bool frame_dropper_on, - bool spatial_resize_on) { - process_settings->codec_type = codec_type; - process_settings->packet_loss = packet_loss; - process_settings->key_frame_interval = key_frame_interval; - process_settings->num_temporal_layers = num_temporal_layers, - process_settings->error_concealment_on = error_concealment_on; - process_settings->denoising_on = denoising_on; - process_settings->frame_dropper_on = frame_dropper_on; - process_settings->spatial_resize_on = spatial_resize_on; -} - -void SetQualityMetrics(QualityMetrics* quality_metrics, - double minimum_avg_psnr, - double minimum_min_psnr, - double minimum_avg_ssim, - double minimum_min_ssim) { - quality_metrics->minimum_avg_psnr = minimum_avg_psnr; - quality_metrics->minimum_min_psnr = minimum_min_psnr; - quality_metrics->minimum_avg_ssim = minimum_avg_ssim; - quality_metrics->minimum_min_ssim = minimum_min_ssim; -} - -void SetRateControlMetrics(RateControlMetrics* rc_metrics, - int update_index, - int max_num_dropped_frames, - int max_key_frame_size_mismatch, - int max_delta_frame_size_mismatch, - int max_encoding_rate_mismatch, - int max_time_hit_target, - int num_spatial_resizes, - int num_key_frames) { - rc_metrics[update_index].max_num_dropped_frames = max_num_dropped_frames; - rc_metrics[update_index].max_key_frame_size_mismatch = - max_key_frame_size_mismatch; - rc_metrics[update_index].max_delta_frame_size_mismatch = - max_delta_frame_size_mismatch; - rc_metrics[update_index].max_encoding_rate_mismatch = - max_encoding_rate_mismatch; - rc_metrics[update_index].max_time_hit_target = max_time_hit_target; - rc_metrics[update_index].num_spatial_resizes = num_spatial_resizes; - rc_metrics[update_index].num_key_frames = num_key_frames; -} -} // namespace - -// Integration test for video processor. Encodes+decodes a clip and -// writes it to the output directory. After completion, quality metrics -// (PSNR and SSIM) and rate control metrics are computed to verify that the -// quality and encoder response is acceptable. The rate control tests allow us -// to verify the behavior for changing bitrate, changing frame rate, frame -// dropping/spatial resize, and temporal layers. The limits for the rate -// control metrics are set to be fairly conservative, so failure should only -// happen when some significant regression or breakdown occurs. -class VideoProcessorIntegrationTest : public testing::Test { - protected: - std::unique_ptr encoder_; - std::unique_ptr decoder_; - std::unique_ptr frame_reader_; - std::unique_ptr frame_writer_; - test::PacketReader packet_reader_; - std::unique_ptr packet_manipulator_; - test::Stats stats_; - test::TestConfig config_; - VideoCodec codec_settings_; - std::unique_ptr processor_; - TemporalLayersFactory tl_factory_; - - // Quantities defined/updated for every encoder rate update. - // Some quantities defined per temporal layer (at most 3 layers in this test). - int num_frames_per_update_[3]; - float sum_frame_size_mismatch_[3]; - float sum_encoded_frame_size_[3]; - float encoding_bitrate_[3]; - float per_frame_bandwidth_[3]; - float bit_rate_layer_[3]; - float frame_rate_layer_[3]; - int num_frames_total_; - float sum_encoded_frame_size_total_; - float encoding_bitrate_total_; - float perc_encoding_rate_mismatch_; - int num_frames_to_hit_target_; - bool encoding_rate_within_target_; - int bit_rate_; - int frame_rate_; - int layer_; - float target_size_key_frame_initial_; - float target_size_key_frame_; - float sum_key_frame_size_mismatch_; - int num_key_frames_; - float start_bitrate_; - - // Codec and network settings. - VideoCodecType codec_type_; - float packet_loss_; - int num_temporal_layers_; - int key_frame_interval_; - bool error_concealment_on_; - bool denoising_on_; - bool frame_dropper_on_; - bool spatial_resize_on_; - - VideoProcessorIntegrationTest() {} - virtual ~VideoProcessorIntegrationTest() {} - - void SetUpCodecConfig() { - if (codec_type_ == kVideoCodecH264) { - encoder_.reset(H264Encoder::Create(cricket::VideoCodec("H264"))); - decoder_.reset(H264Decoder::Create()); - VideoCodingModule::Codec(kVideoCodecH264, &codec_settings_); - } else if (codec_type_ == kVideoCodecVP8) { - encoder_.reset(VP8Encoder::Create()); - decoder_.reset(VP8Decoder::Create()); - VideoCodingModule::Codec(kVideoCodecVP8, &codec_settings_); - } else if (codec_type_ == kVideoCodecVP9) { - encoder_.reset(VP9Encoder::Create()); - decoder_.reset(VP9Decoder::Create()); - VideoCodingModule::Codec(kVideoCodecVP9, &codec_settings_); - } - - // CIF is currently used for all tests below. - // Setup the TestConfig struct for processing of a clip in CIF resolution. - config_.input_filename = webrtc::test::ResourcePath("foreman_cif", "yuv"); - - // Generate an output filename in a safe way. - config_.output_filename = webrtc::test::TempFilename( - webrtc::test::OutputPath(), "videoprocessor_integrationtest"); - config_.frame_length_in_bytes = - CalcBufferSize(kI420, kCIFWidth, kCIFHeight); - config_.verbose = false; - // Only allow encoder/decoder to use single core, for predictability. - config_.use_single_core = true; - // Key frame interval and packet loss are set for each test. - config_.keyframe_interval = key_frame_interval_; - config_.networking_config.packet_loss_probability = packet_loss_; - - // Configure codec settings. - config_.codec_settings = &codec_settings_; - config_.codec_settings->startBitrate = start_bitrate_; - config_.codec_settings->width = kCIFWidth; - config_.codec_settings->height = kCIFHeight; - - // These features may be set depending on the test. - switch (config_.codec_settings->codecType) { - case kVideoCodecH264: - config_.codec_settings->H264()->frameDroppingOn = frame_dropper_on_; - config_.codec_settings->H264()->keyFrameInterval = - kBaseKeyFrameInterval; - break; - case kVideoCodecVP8: - config_.codec_settings->VP8()->errorConcealmentOn = - error_concealment_on_; - config_.codec_settings->VP8()->denoisingOn = denoising_on_; - config_.codec_settings->VP8()->numberOfTemporalLayers = - num_temporal_layers_; - config_.codec_settings->VP8()->frameDroppingOn = frame_dropper_on_; - config_.codec_settings->VP8()->automaticResizeOn = spatial_resize_on_; - config_.codec_settings->VP8()->keyFrameInterval = kBaseKeyFrameInterval; - break; - case kVideoCodecVP9: - config_.codec_settings->VP9()->denoisingOn = denoising_on_; - config_.codec_settings->VP9()->numberOfTemporalLayers = - num_temporal_layers_; - config_.codec_settings->VP9()->frameDroppingOn = frame_dropper_on_; - config_.codec_settings->VP9()->automaticResizeOn = spatial_resize_on_; - config_.codec_settings->VP9()->keyFrameInterval = kBaseKeyFrameInterval; - break; - default: - assert(false); - break; - } - frame_reader_.reset(new test::FrameReaderImpl( - config_.input_filename, config_.codec_settings->width, - config_.codec_settings->height)); - frame_writer_.reset(new test::FrameWriterImpl( - config_.output_filename, config_.frame_length_in_bytes)); - ASSERT_TRUE(frame_reader_->Init()); - ASSERT_TRUE(frame_writer_->Init()); - - packet_manipulator_.reset(new test::PacketManipulatorImpl( - &packet_reader_, config_.networking_config, config_.verbose)); - processor_.reset(new test::VideoProcessorImpl( - encoder_.get(), decoder_.get(), frame_reader_.get(), - frame_writer_.get(), packet_manipulator_.get(), config_, &stats_)); - ASSERT_TRUE(processor_->Init()); - } - - // Reset quantities after each encoder update, update the target - // per-frame bandwidth. - void ResetRateControlMetrics(int num_frames) { - for (int i = 0; i < num_temporal_layers_; i++) { - num_frames_per_update_[i] = 0; - sum_frame_size_mismatch_[i] = 0.0f; - sum_encoded_frame_size_[i] = 0.0f; - encoding_bitrate_[i] = 0.0f; - // Update layer per-frame-bandwidth. - per_frame_bandwidth_[i] = static_cast(bit_rate_layer_[i]) / - static_cast(frame_rate_layer_[i]); - } - // Set maximum size of key frames, following setting in the VP8 wrapper. - float max_key_size = kScaleKeyFrameSize * kOptimalBufferSize * frame_rate_; - // We don't know exact target size of the key frames (except for first one), - // but the minimum in libvpx is ~|3 * per_frame_bandwidth| and maximum is - // set by |max_key_size_ * per_frame_bandwidth|. Take middle point/average - // as reference for mismatch. Note key frames always correspond to base - // layer frame in this test. - target_size_key_frame_ = 0.5 * (3 + max_key_size) * per_frame_bandwidth_[0]; - num_frames_total_ = 0; - sum_encoded_frame_size_total_ = 0.0f; - encoding_bitrate_total_ = 0.0f; - perc_encoding_rate_mismatch_ = 0.0f; - num_frames_to_hit_target_ = num_frames; - encoding_rate_within_target_ = false; - sum_key_frame_size_mismatch_ = 0.0; - num_key_frames_ = 0; - } - - // For every encoded frame, update the rate control metrics. - void UpdateRateControlMetrics(int frame_num, FrameType frame_type) { - float encoded_size_kbits = processor_->EncodedFrameSize() * 8.0f / 1000.0f; - // Update layer data. - // Update rate mismatch relative to per-frame bandwidth for delta frames. - if (frame_type == kVideoFrameDelta) { - // TODO(marpan): Should we count dropped (zero size) frames in mismatch? - sum_frame_size_mismatch_[layer_] += - fabs(encoded_size_kbits - per_frame_bandwidth_[layer_]) / - per_frame_bandwidth_[layer_]; - } else { - float target_size = (frame_num == 1) ? target_size_key_frame_initial_ - : target_size_key_frame_; - sum_key_frame_size_mismatch_ += - fabs(encoded_size_kbits - target_size) / target_size; - num_key_frames_ += 1; - } - sum_encoded_frame_size_[layer_] += encoded_size_kbits; - // Encoding bitrate per layer: from the start of the update/run to the - // current frame. - encoding_bitrate_[layer_] = sum_encoded_frame_size_[layer_] * - frame_rate_layer_[layer_] / - num_frames_per_update_[layer_]; - // Total encoding rate: from the start of the update/run to current frame. - sum_encoded_frame_size_total_ += encoded_size_kbits; - encoding_bitrate_total_ = - sum_encoded_frame_size_total_ * frame_rate_ / num_frames_total_; - perc_encoding_rate_mismatch_ = - 100 * fabs(encoding_bitrate_total_ - bit_rate_) / bit_rate_; - if (perc_encoding_rate_mismatch_ < kPercTargetvsActualMismatch && - !encoding_rate_within_target_) { - num_frames_to_hit_target_ = num_frames_total_; - encoding_rate_within_target_ = true; - } - } - - // Verify expected behavior of rate control and print out data. - void VerifyRateControl(int update_index, - int max_key_frame_size_mismatch, - int max_delta_frame_size_mismatch, - int max_encoding_rate_mismatch, - int max_time_hit_target, - int max_num_dropped_frames, - int num_spatial_resizes, - int num_key_frames) { - int num_dropped_frames = processor_->NumberDroppedFrames(); - int num_resize_actions = processor_->NumberSpatialResizes(); - printf( - "For update #: %d,\n " - " Target Bitrate: %d,\n" - " Encoding bitrate: %f,\n" - " Frame rate: %d \n", - update_index, bit_rate_, encoding_bitrate_total_, frame_rate_); - printf( - " Number of frames to approach target rate: %d, \n" - " Number of dropped frames: %d, \n" - " Number of spatial resizes: %d, \n", - num_frames_to_hit_target_, num_dropped_frames, num_resize_actions); - EXPECT_LE(perc_encoding_rate_mismatch_, max_encoding_rate_mismatch); - if (num_key_frames_ > 0) { - int perc_key_frame_size_mismatch = - 100 * sum_key_frame_size_mismatch_ / num_key_frames_; - printf( - " Number of Key frames: %d \n" - " Key frame rate mismatch: %d \n", - num_key_frames_, perc_key_frame_size_mismatch); - EXPECT_LE(perc_key_frame_size_mismatch, max_key_frame_size_mismatch); - } - printf("\n"); - printf("Rates statistics for Layer data \n"); - for (int i = 0; i < num_temporal_layers_; i++) { - printf("Temporal layer #%d \n", i); - int perc_frame_size_mismatch = - 100 * sum_frame_size_mismatch_[i] / num_frames_per_update_[i]; - int perc_encoding_rate_mismatch = - 100 * fabs(encoding_bitrate_[i] - bit_rate_layer_[i]) / - bit_rate_layer_[i]; - printf( - " Target Layer Bit rate: %f \n" - " Layer frame rate: %f, \n" - " Layer per frame bandwidth: %f, \n" - " Layer Encoding bit rate: %f, \n" - " Layer Percent frame size mismatch: %d, \n" - " Layer Percent encoding rate mismatch: %d, \n" - " Number of frame processed per layer: %d \n", - bit_rate_layer_[i], frame_rate_layer_[i], per_frame_bandwidth_[i], - encoding_bitrate_[i], perc_frame_size_mismatch, - perc_encoding_rate_mismatch, num_frames_per_update_[i]); - EXPECT_LE(perc_frame_size_mismatch, max_delta_frame_size_mismatch); - EXPECT_LE(perc_encoding_rate_mismatch, max_encoding_rate_mismatch); - } - printf("\n"); - EXPECT_LE(num_frames_to_hit_target_, max_time_hit_target); - EXPECT_LE(num_dropped_frames, max_num_dropped_frames); - EXPECT_EQ(num_resize_actions, num_spatial_resizes); - EXPECT_EQ(num_key_frames_, num_key_frames); - } - - // Layer index corresponding to frame number, for up to 3 layers. - void LayerIndexForFrame(int frame_number) { - if (num_temporal_layers_ == 1) { - layer_ = 0; - } else if (num_temporal_layers_ == 2) { - // layer 0: 0 2 4 ... - // layer 1: 1 3 - if (frame_number % 2 == 0) { - layer_ = 0; - } else { - layer_ = 1; - } - } else if (num_temporal_layers_ == 3) { - // layer 0: 0 4 8 ... - // layer 1: 2 6 - // layer 2: 1 3 5 7 - if (frame_number % 4 == 0) { - layer_ = 0; - } else if ((frame_number + 2) % 4 == 0) { - layer_ = 1; - } else if ((frame_number + 1) % 2 == 0) { - layer_ = 2; - } - } else { - assert(false); // Only up to 3 layers. - } - } - - // Set the bitrate and frame rate per layer, for up to 3 layers. - void SetLayerRates() { - assert(num_temporal_layers_ <= 3); - for (int i = 0; i < num_temporal_layers_; i++) { - float bit_rate_ratio = - kVp8LayerRateAlloction[num_temporal_layers_ - 1][i]; - if (i > 0) { - float bit_rate_delta_ratio = - kVp8LayerRateAlloction[num_temporal_layers_ - 1][i] - - kVp8LayerRateAlloction[num_temporal_layers_ - 1][i - 1]; - bit_rate_layer_[i] = bit_rate_ * bit_rate_delta_ratio; - } else { - bit_rate_layer_[i] = bit_rate_ * bit_rate_ratio; - } - frame_rate_layer_[i] = - frame_rate_ / static_cast(1 << (num_temporal_layers_ - 1)); - } - if (num_temporal_layers_ == 3) { - frame_rate_layer_[2] = frame_rate_ / 2.0f; - } - } - - // Processes all frames in the clip and verifies the result. - void ProcessFramesAndVerify(QualityMetrics quality_metrics, - RateProfile rate_profile, - CodecConfigPars process, - RateControlMetrics* rc_metrics) { - // Codec/config settings. - codec_type_ = process.codec_type; - start_bitrate_ = rate_profile.target_bit_rate[0]; - packet_loss_ = process.packet_loss; - key_frame_interval_ = process.key_frame_interval; - num_temporal_layers_ = process.num_temporal_layers; - error_concealment_on_ = process.error_concealment_on; - denoising_on_ = process.denoising_on; - frame_dropper_on_ = process.frame_dropper_on; - spatial_resize_on_ = process.spatial_resize_on; - SetUpCodecConfig(); - // Update the layers and the codec with the initial rates. - bit_rate_ = rate_profile.target_bit_rate[0]; - frame_rate_ = rate_profile.input_frame_rate[0]; - SetLayerRates(); - // Set the initial target size for key frame. - target_size_key_frame_initial_ = - 0.5 * kInitialBufferSize * bit_rate_layer_[0]; - processor_->SetRates(bit_rate_, frame_rate_); - // Process each frame, up to |num_frames|. - int num_frames = rate_profile.num_frames; - int update_index = 0; - ResetRateControlMetrics( - rate_profile.frame_index_rate_update[update_index + 1]); - int frame_number = 0; - FrameType frame_type = kVideoFrameDelta; - while (processor_->ProcessFrame(frame_number) && - frame_number < num_frames) { - // Get the layer index for the frame |frame_number|. - LayerIndexForFrame(frame_number); - // Get the frame_type. - frame_type = processor_->EncodedFrameType(); - // Counter for whole sequence run. - ++frame_number; - // Counters for each rate update. - ++num_frames_per_update_[layer_]; - ++num_frames_total_; - UpdateRateControlMetrics(frame_number, frame_type); - // If we hit another/next update, verify stats for current state and - // update layers and codec with new rates. - if (frame_number == - rate_profile.frame_index_rate_update[update_index + 1]) { - VerifyRateControl( - update_index, rc_metrics[update_index].max_key_frame_size_mismatch, - rc_metrics[update_index].max_delta_frame_size_mismatch, - rc_metrics[update_index].max_encoding_rate_mismatch, - rc_metrics[update_index].max_time_hit_target, - rc_metrics[update_index].max_num_dropped_frames, - rc_metrics[update_index].num_spatial_resizes, - rc_metrics[update_index].num_key_frames); - // Update layer rates and the codec with new rates. - ++update_index; - bit_rate_ = rate_profile.target_bit_rate[update_index]; - frame_rate_ = rate_profile.input_frame_rate[update_index]; - SetLayerRates(); - ResetRateControlMetrics( - rate_profile.frame_index_rate_update[update_index + 1]); - processor_->SetRates(bit_rate_, frame_rate_); - } - } - VerifyRateControl(update_index, - rc_metrics[update_index].max_key_frame_size_mismatch, - rc_metrics[update_index].max_delta_frame_size_mismatch, - rc_metrics[update_index].max_encoding_rate_mismatch, - rc_metrics[update_index].max_time_hit_target, - rc_metrics[update_index].max_num_dropped_frames, - rc_metrics[update_index].num_spatial_resizes, - rc_metrics[update_index].num_key_frames); - EXPECT_EQ(num_frames, frame_number); - EXPECT_EQ(num_frames + 1, static_cast(stats_.stats_.size())); - - // Release encoder and decoder to make sure they have finished processing: - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release()); - // Close the files before we start using them for SSIM/PSNR calculations. - frame_reader_->Close(); - frame_writer_->Close(); - - // TODO(marpan): should compute these quality metrics per SetRates update. - test::QualityMetricsResult psnr_result, ssim_result; - EXPECT_EQ(0, test::I420MetricsFromFiles(config_.input_filename.c_str(), - config_.output_filename.c_str(), - config_.codec_settings->width, - config_.codec_settings->height, - &psnr_result, &ssim_result)); - printf("PSNR avg: %f, min: %f\nSSIM avg: %f, min: %f\n", - psnr_result.average, psnr_result.min, ssim_result.average, - ssim_result.min); - stats_.PrintSummary(); - EXPECT_GT(psnr_result.average, quality_metrics.minimum_avg_psnr); - EXPECT_GT(psnr_result.min, quality_metrics.minimum_min_psnr); - EXPECT_GT(ssim_result.average, quality_metrics.minimum_avg_ssim); - EXPECT_GT(ssim_result.min, quality_metrics.minimum_min_ssim); - if (remove(config_.output_filename.c_str()) < 0) { - fprintf(stderr, "Failed to remove temporary file!\n"); - } - } -}; +namespace test { #if defined(WEBRTC_VIDEOPROCESSOR_H264_TESTS) @@ -961,4 +393,5 @@ TEST_F(VideoProcessorIntegrationTest, MAYBE_ProcessNoLossTemporalLayersVP8) { ProcessFramesAndVerify(quality_metrics, rate_profile, process_settings, rc_metrics); } +} // namespace test } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h new file mode 100644 index 0000000000..f244bea183 --- /dev/null +++ b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h @@ -0,0 +1,624 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H_ +#define WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H_ + +#include + +#include +#include + +#include "webrtc/modules/video_coding/codecs/h264/include/h264.h" +#include "webrtc/modules/video_coding/codecs/test/packet_manipulator.h" +#include "webrtc/modules/video_coding/codecs/test/videoprocessor.h" +#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" +#include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h" +#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" +#include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h" +#include "webrtc/modules/video_coding/include/video_codec_interface.h" +#include "webrtc/modules/video_coding/include/video_coding.h" +#include "webrtc/test/gtest.h" +#include "webrtc/test/testsupport/fileutils.h" +#include "webrtc/test/testsupport/frame_reader.h" +#include "webrtc/test/testsupport/frame_writer.h" +#include "webrtc/test/testsupport/metrics/video_metrics.h" +#include "webrtc/test/testsupport/packet_reader.h" +#include "webrtc/typedefs.h" + +namespace webrtc { +namespace test { +// Maximum number of rate updates (i.e., calls to encoder to change bitrate +// and/or frame rate) for the current tests. +const int kMaxNumRateUpdates = 3; + +const int kPercTargetvsActualMismatch = 20; +const int kBaseKeyFrameInterval = 3000; + +// Default sequence is foreman (CIF): may be better to use VGA for resize test. +const int kCifWidth = 352; +const int kCifHeight = 288; +const char kFilenameForemanCif[] = "foreman_cif"; + +// Codec and network settings. +struct CodecConfigPars { + VideoCodecType codec_type; + float packet_loss; + int num_temporal_layers; + int key_frame_interval; + bool error_concealment_on; + bool denoising_on; + bool frame_dropper_on; + bool spatial_resize_on; + int width; + int height; + std::string filename; + bool verbose_logging; +}; + +// Quality metrics. +struct QualityMetrics { + double minimum_avg_psnr; + double minimum_min_psnr; + double minimum_avg_ssim; + double minimum_min_ssim; +}; + +// The sequence of bitrate and frame rate changes for the encoder, the frame +// number where the changes are made, and the total number of frames for the +// test. +struct RateProfile { + int target_bit_rate[kMaxNumRateUpdates]; + int input_frame_rate[kMaxNumRateUpdates]; + int frame_index_rate_update[kMaxNumRateUpdates + 1]; + int num_frames; +}; + +// Metrics for the rate control. The rate mismatch metrics are defined as +// percentages.|max_time_hit_target| is defined as number of frames, after a +// rate update is made to the encoder, for the encoder to reach within +// |kPercTargetvsActualMismatch| of new target rate. The metrics are defined for +// each rate update sequence. +struct RateControlMetrics { + int max_num_dropped_frames; + int max_key_frame_size_mismatch; + int max_delta_frame_size_mismatch; + int max_encoding_rate_mismatch; + int max_time_hit_target; + int num_spatial_resizes; + int num_key_frames; +}; + +#if !defined(WEBRTC_IOS) +const int kNbrFramesShort = 100; // Some tests are run for shorter sequence. +#endif +const int kNbrFramesLong = 299; + +// Parameters from VP8 wrapper, which control target size of key frames. +const float kInitialBufferSize = 0.5f; +const float kOptimalBufferSize = 0.6f; +const float kScaleKeyFrameSize = 0.5f; + +// Integration test for video processor. Encodes+decodes a clip and +// writes it to the output directory. After completion, quality metrics +// (PSNR and SSIM) and rate control metrics are computed to verify that the +// quality and encoder response is acceptable. The rate control tests allow us +// to verify the behavior for changing bitrate, changing frame rate, frame +// dropping/spatial resize, and temporal layers. The limits for the rate +// control metrics are set to be fairly conservative, so failure should only +// happen when some significant regression or breakdown occurs. +class VideoProcessorIntegrationTest : public testing::Test { + protected: + std::unique_ptr encoder_; + std::unique_ptr decoder_; + std::unique_ptr frame_reader_; + std::unique_ptr frame_writer_; + test::PacketReader packet_reader_; + std::unique_ptr packet_manipulator_; + test::Stats stats_; + test::TestConfig config_; + VideoCodec codec_settings_; + std::unique_ptr processor_; + TemporalLayersFactory tl_factory_; + + // Quantities defined/updated for every encoder rate update. + // Some quantities defined per temporal layer (at most 3 layers in this test). + int num_frames_per_update_[3]; + float sum_frame_size_mismatch_[3]; + float sum_encoded_frame_size_[3]; + float encoding_bitrate_[3]; + float per_frame_bandwidth_[3]; + float bit_rate_layer_[3]; + float frame_rate_layer_[3]; + int num_frames_total_; + float sum_encoded_frame_size_total_; + float encoding_bitrate_total_; + float perc_encoding_rate_mismatch_; + int num_frames_to_hit_target_; + bool encoding_rate_within_target_; + int bit_rate_; + int frame_rate_; + int layer_; + float target_size_key_frame_initial_; + float target_size_key_frame_; + float sum_key_frame_size_mismatch_; + int num_key_frames_; + float start_bitrate_; + + // Codec and network settings. + VideoCodecType codec_type_; + float packet_loss_; + int num_temporal_layers_; + int key_frame_interval_; + bool error_concealment_on_; + bool denoising_on_; + bool frame_dropper_on_; + bool spatial_resize_on_; + + VideoProcessorIntegrationTest() {} + virtual ~VideoProcessorIntegrationTest() {} + + void SetUpCodecConfig(const std::string& filename, + int width, + int height, + bool verbose_logging) { + if (codec_type_ == kVideoCodecH264) { + encoder_.reset(H264Encoder::Create(cricket::VideoCodec("H264"))); + decoder_.reset(H264Decoder::Create()); + VideoCodingModule::Codec(kVideoCodecH264, &codec_settings_); + } else if (codec_type_ == kVideoCodecVP8) { + encoder_.reset(VP8Encoder::Create()); + decoder_.reset(VP8Decoder::Create()); + VideoCodingModule::Codec(kVideoCodecVP8, &codec_settings_); + } else if (codec_type_ == kVideoCodecVP9) { + encoder_.reset(VP9Encoder::Create()); + decoder_.reset(VP9Decoder::Create()); + VideoCodingModule::Codec(kVideoCodecVP9, &codec_settings_); + } + + // Configure input filename. + config_.input_filename = test::ResourcePath(filename, "yuv"); + if (verbose_logging) + printf("Filename: %s\n", filename.c_str()); + // Generate an output filename in a safe way. + config_.output_filename = test::TempFilename( + test::OutputPath(), "videoprocessor_integrationtest"); + config_.frame_length_in_bytes = CalcBufferSize(kI420, width, height); + config_.verbose = verbose_logging; + // Only allow encoder/decoder to use single core, for predictability. + config_.use_single_core = true; + // Key frame interval and packet loss are set for each test. + config_.keyframe_interval = key_frame_interval_; + config_.networking_config.packet_loss_probability = packet_loss_; + + // Configure codec settings. + config_.codec_settings = &codec_settings_; + config_.codec_settings->startBitrate = start_bitrate_; + config_.codec_settings->width = width; + config_.codec_settings->height = height; + + // These features may be set depending on the test. + switch (config_.codec_settings->codecType) { + case kVideoCodecH264: + config_.codec_settings->H264()->frameDroppingOn = frame_dropper_on_; + config_.codec_settings->H264()->keyFrameInterval = + kBaseKeyFrameInterval; + break; + case kVideoCodecVP8: + config_.codec_settings->VP8()->errorConcealmentOn = + error_concealment_on_; + config_.codec_settings->VP8()->denoisingOn = denoising_on_; + config_.codec_settings->VP8()->numberOfTemporalLayers = + num_temporal_layers_; + config_.codec_settings->VP8()->frameDroppingOn = frame_dropper_on_; + config_.codec_settings->VP8()->automaticResizeOn = spatial_resize_on_; + config_.codec_settings->VP8()->keyFrameInterval = kBaseKeyFrameInterval; + break; + case kVideoCodecVP9: + config_.codec_settings->VP9()->denoisingOn = denoising_on_; + config_.codec_settings->VP9()->numberOfTemporalLayers = + num_temporal_layers_; + config_.codec_settings->VP9()->frameDroppingOn = frame_dropper_on_; + config_.codec_settings->VP9()->automaticResizeOn = spatial_resize_on_; + config_.codec_settings->VP9()->keyFrameInterval = kBaseKeyFrameInterval; + break; + default: + assert(false); + break; + } + frame_reader_.reset(new test::FrameReaderImpl( + config_.input_filename, config_.codec_settings->width, + config_.codec_settings->height)); + frame_writer_.reset(new test::FrameWriterImpl( + config_.output_filename, config_.frame_length_in_bytes)); + ASSERT_TRUE(frame_reader_->Init()); + ASSERT_TRUE(frame_writer_->Init()); + + packet_manipulator_.reset(new test::PacketManipulatorImpl( + &packet_reader_, config_.networking_config, config_.verbose)); + processor_.reset(new test::VideoProcessorImpl( + encoder_.get(), decoder_.get(), frame_reader_.get(), + frame_writer_.get(), packet_manipulator_.get(), config_, &stats_)); + ASSERT_TRUE(processor_->Init()); + } + + // Reset quantities after each encoder update, update the target + // per-frame bandwidth. + void ResetRateControlMetrics(int num_frames) { + for (int i = 0; i < num_temporal_layers_; i++) { + num_frames_per_update_[i] = 0; + sum_frame_size_mismatch_[i] = 0.0f; + sum_encoded_frame_size_[i] = 0.0f; + encoding_bitrate_[i] = 0.0f; + // Update layer per-frame-bandwidth. + per_frame_bandwidth_[i] = static_cast(bit_rate_layer_[i]) / + static_cast(frame_rate_layer_[i]); + } + // Set maximum size of key frames, following setting in the VP8 wrapper. + float max_key_size = kScaleKeyFrameSize * kOptimalBufferSize * frame_rate_; + // We don't know exact target size of the key frames (except for first one), + // but the minimum in libvpx is ~|3 * per_frame_bandwidth| and maximum is + // set by |max_key_size_ * per_frame_bandwidth|. Take middle point/average + // as reference for mismatch. Note key frames always correspond to base + // layer frame in this test. + target_size_key_frame_ = 0.5 * (3 + max_key_size) * per_frame_bandwidth_[0]; + num_frames_total_ = 0; + sum_encoded_frame_size_total_ = 0.0f; + encoding_bitrate_total_ = 0.0f; + perc_encoding_rate_mismatch_ = 0.0f; + num_frames_to_hit_target_ = num_frames; + encoding_rate_within_target_ = false; + sum_key_frame_size_mismatch_ = 0.0; + num_key_frames_ = 0; + } + + // For every encoded frame, update the rate control metrics. + void UpdateRateControlMetrics(int frame_num, FrameType frame_type) { + float encoded_size_kbits = processor_->EncodedFrameSize() * 8.0f / 1000.0f; + // Update layer data. + // Update rate mismatch relative to per-frame bandwidth for delta frames. + if (frame_type == kVideoFrameDelta) { + // TODO(marpan): Should we count dropped (zero size) frames in mismatch? + sum_frame_size_mismatch_[layer_] += + fabs(encoded_size_kbits - per_frame_bandwidth_[layer_]) / + per_frame_bandwidth_[layer_]; + } else { + float target_size = (frame_num == 1) ? target_size_key_frame_initial_ + : target_size_key_frame_; + sum_key_frame_size_mismatch_ += + fabs(encoded_size_kbits - target_size) / target_size; + num_key_frames_ += 1; + } + sum_encoded_frame_size_[layer_] += encoded_size_kbits; + // Encoding bitrate per layer: from the start of the update/run to the + // current frame. + encoding_bitrate_[layer_] = sum_encoded_frame_size_[layer_] * + frame_rate_layer_[layer_] / + num_frames_per_update_[layer_]; + // Total encoding rate: from the start of the update/run to current frame. + sum_encoded_frame_size_total_ += encoded_size_kbits; + encoding_bitrate_total_ = + sum_encoded_frame_size_total_ * frame_rate_ / num_frames_total_; + perc_encoding_rate_mismatch_ = + 100 * fabs(encoding_bitrate_total_ - bit_rate_) / bit_rate_; + if (perc_encoding_rate_mismatch_ < kPercTargetvsActualMismatch && + !encoding_rate_within_target_) { + num_frames_to_hit_target_ = num_frames_total_; + encoding_rate_within_target_ = true; + } + } + + // Verify expected behavior of rate control and print out data. + void VerifyRateControl(int update_index, + int max_key_frame_size_mismatch, + int max_delta_frame_size_mismatch, + int max_encoding_rate_mismatch, + int max_time_hit_target, + int max_num_dropped_frames, + int num_spatial_resizes, + int num_key_frames) { + int num_dropped_frames = processor_->NumberDroppedFrames(); + int num_resize_actions = processor_->NumberSpatialResizes(); + printf( + "For update #: %d,\n" + " Target Bitrate: %d,\n" + " Encoding bitrate: %f,\n" + " Frame rate: %d \n", + update_index, bit_rate_, encoding_bitrate_total_, frame_rate_); + printf( + " Number of frames to approach target rate: %d, \n" + " Number of dropped frames: %d, \n" + " Number of spatial resizes: %d, \n", + num_frames_to_hit_target_, num_dropped_frames, num_resize_actions); + EXPECT_LE(perc_encoding_rate_mismatch_, max_encoding_rate_mismatch); + if (num_key_frames_ > 0) { + int perc_key_frame_size_mismatch = + 100 * sum_key_frame_size_mismatch_ / num_key_frames_; + printf( + " Number of Key frames: %d \n" + " Key frame rate mismatch: %d \n", + num_key_frames_, perc_key_frame_size_mismatch); + EXPECT_LE(perc_key_frame_size_mismatch, max_key_frame_size_mismatch); + } + printf("\n"); + printf("Rates statistics for Layer data \n"); + for (int i = 0; i < num_temporal_layers_; i++) { + printf("Temporal layer #%d \n", i); + int perc_frame_size_mismatch = + 100 * sum_frame_size_mismatch_[i] / num_frames_per_update_[i]; + int perc_encoding_rate_mismatch = + 100 * fabs(encoding_bitrate_[i] - bit_rate_layer_[i]) / + bit_rate_layer_[i]; + printf( + " Target Layer Bit rate: %f \n" + " Layer frame rate: %f, \n" + " Layer per frame bandwidth: %f, \n" + " Layer Encoding bit rate: %f, \n" + " Layer Percent frame size mismatch: %d, \n" + " Layer Percent encoding rate mismatch: %d, \n" + " Number of frame processed per layer: %d \n", + bit_rate_layer_[i], frame_rate_layer_[i], per_frame_bandwidth_[i], + encoding_bitrate_[i], perc_frame_size_mismatch, + perc_encoding_rate_mismatch, num_frames_per_update_[i]); + EXPECT_LE(perc_frame_size_mismatch, max_delta_frame_size_mismatch); + EXPECT_LE(perc_encoding_rate_mismatch, max_encoding_rate_mismatch); + } + printf("\n"); + EXPECT_LE(num_frames_to_hit_target_, max_time_hit_target); + EXPECT_LE(num_dropped_frames, max_num_dropped_frames); + EXPECT_EQ(num_resize_actions, num_spatial_resizes); + EXPECT_EQ(num_key_frames_, num_key_frames); + } + + // Layer index corresponding to frame number, for up to 3 layers. + void LayerIndexForFrame(int frame_number) { + if (num_temporal_layers_ == 1) { + layer_ = 0; + } else if (num_temporal_layers_ == 2) { + // layer 0: 0 2 4 ... + // layer 1: 1 3 + if (frame_number % 2 == 0) { + layer_ = 0; + } else { + layer_ = 1; + } + } else if (num_temporal_layers_ == 3) { + // layer 0: 0 4 8 ... + // layer 1: 2 6 + // layer 2: 1 3 5 7 + if (frame_number % 4 == 0) { + layer_ = 0; + } else if ((frame_number + 2) % 4 == 0) { + layer_ = 1; + } else if ((frame_number + 1) % 2 == 0) { + layer_ = 2; + } + } else { + assert(false); // Only up to 3 layers. + } + } + + // Set the bitrate and frame rate per layer, for up to 3 layers. + void SetLayerRates() { + assert(num_temporal_layers_ <= 3); + for (int i = 0; i < num_temporal_layers_; i++) { + float bit_rate_ratio = + kVp8LayerRateAlloction[num_temporal_layers_ - 1][i]; + if (i > 0) { + float bit_rate_delta_ratio = + kVp8LayerRateAlloction[num_temporal_layers_ - 1][i] - + kVp8LayerRateAlloction[num_temporal_layers_ - 1][i - 1]; + bit_rate_layer_[i] = bit_rate_ * bit_rate_delta_ratio; + } else { + bit_rate_layer_[i] = bit_rate_ * bit_rate_ratio; + } + frame_rate_layer_[i] = + frame_rate_ / static_cast(1 << (num_temporal_layers_ - 1)); + } + if (num_temporal_layers_ == 3) { + frame_rate_layer_[2] = frame_rate_ / 2.0f; + } + } + + // Processes all frames in the clip and verifies the result. + void ProcessFramesAndVerify(QualityMetrics quality_metrics, + RateProfile rate_profile, + CodecConfigPars process, + RateControlMetrics* rc_metrics) { + // Codec/config settings. + codec_type_ = process.codec_type; + start_bitrate_ = rate_profile.target_bit_rate[0]; + packet_loss_ = process.packet_loss; + key_frame_interval_ = process.key_frame_interval; + num_temporal_layers_ = process.num_temporal_layers; + error_concealment_on_ = process.error_concealment_on; + denoising_on_ = process.denoising_on; + frame_dropper_on_ = process.frame_dropper_on; + spatial_resize_on_ = process.spatial_resize_on; + SetUpCodecConfig(process.filename, process.width, process.height, + process.verbose_logging); + // Update the layers and the codec with the initial rates. + bit_rate_ = rate_profile.target_bit_rate[0]; + frame_rate_ = rate_profile.input_frame_rate[0]; + SetLayerRates(); + // Set the initial target size for key frame. + target_size_key_frame_initial_ = + 0.5 * kInitialBufferSize * bit_rate_layer_[0]; + processor_->SetRates(bit_rate_, frame_rate_); + // Process each frame, up to |num_frames|. + int num_frames = rate_profile.num_frames; + int update_index = 0; + ResetRateControlMetrics( + rate_profile.frame_index_rate_update[update_index + 1]); + int frame_number = 0; + FrameType frame_type = kVideoFrameDelta; + while (processor_->ProcessFrame(frame_number) && + frame_number < num_frames) { + // Get the layer index for the frame |frame_number|. + LayerIndexForFrame(frame_number); + // Get the frame_type. + frame_type = processor_->EncodedFrameType(); + // Counter for whole sequence run. + ++frame_number; + // Counters for each rate update. + ++num_frames_per_update_[layer_]; + ++num_frames_total_; + UpdateRateControlMetrics(frame_number, frame_type); + // If we hit another/next update, verify stats for current state and + // update layers and codec with new rates. + if (frame_number == + rate_profile.frame_index_rate_update[update_index + 1]) { + VerifyRateControl( + update_index, rc_metrics[update_index].max_key_frame_size_mismatch, + rc_metrics[update_index].max_delta_frame_size_mismatch, + rc_metrics[update_index].max_encoding_rate_mismatch, + rc_metrics[update_index].max_time_hit_target, + rc_metrics[update_index].max_num_dropped_frames, + rc_metrics[update_index].num_spatial_resizes, + rc_metrics[update_index].num_key_frames); + // Update layer rates and the codec with new rates. + ++update_index; + bit_rate_ = rate_profile.target_bit_rate[update_index]; + frame_rate_ = rate_profile.input_frame_rate[update_index]; + SetLayerRates(); + ResetRateControlMetrics( + rate_profile.frame_index_rate_update[update_index + 1]); + processor_->SetRates(bit_rate_, frame_rate_); + } + } + VerifyRateControl(update_index, + rc_metrics[update_index].max_key_frame_size_mismatch, + rc_metrics[update_index].max_delta_frame_size_mismatch, + rc_metrics[update_index].max_encoding_rate_mismatch, + rc_metrics[update_index].max_time_hit_target, + rc_metrics[update_index].max_num_dropped_frames, + rc_metrics[update_index].num_spatial_resizes, + rc_metrics[update_index].num_key_frames); + EXPECT_EQ(num_frames, frame_number); + EXPECT_EQ(num_frames + 1, static_cast(stats_.stats_.size())); + + // Release encoder and decoder to make sure they have finished processing: + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release()); + // Close the files before we start using them for SSIM/PSNR calculations. + frame_reader_->Close(); + frame_writer_->Close(); + + // TODO(marpan): should compute these quality metrics per SetRates update. + test::QualityMetricsResult psnr_result, ssim_result; + EXPECT_EQ(0, test::I420MetricsFromFiles(config_.input_filename.c_str(), + config_.output_filename.c_str(), + config_.codec_settings->width, + config_.codec_settings->height, + &psnr_result, &ssim_result)); + printf("PSNR avg: %f, min: %f\nSSIM avg: %f, min: %f\n", + psnr_result.average, psnr_result.min, ssim_result.average, + ssim_result.min); + stats_.PrintSummary(); + EXPECT_GT(psnr_result.average, quality_metrics.minimum_avg_psnr); + EXPECT_GT(psnr_result.min, quality_metrics.minimum_min_psnr); + EXPECT_GT(ssim_result.average, quality_metrics.minimum_avg_ssim); + EXPECT_GT(ssim_result.min, quality_metrics.minimum_min_ssim); + if (remove(config_.output_filename.c_str()) < 0) { + fprintf(stderr, "Failed to remove temporary file!\n"); + } + } + + static void SetRateProfilePars(RateProfile* rate_profile, + int update_index, + int bit_rate, + int frame_rate, + int frame_index_rate_update) { + rate_profile->target_bit_rate[update_index] = bit_rate; + rate_profile->input_frame_rate[update_index] = frame_rate; + rate_profile->frame_index_rate_update[update_index] = + frame_index_rate_update; + } + + static void SetCodecParameters(CodecConfigPars* process_settings, + VideoCodecType codec_type, + float packet_loss, + int key_frame_interval, + int num_temporal_layers, + bool error_concealment_on, + bool denoising_on, + bool frame_dropper_on, + bool spatial_resize_on, + int width, + int height, + const std::string& filename, + bool verbose_logging) { + process_settings->codec_type = codec_type; + process_settings->packet_loss = packet_loss; + process_settings->key_frame_interval = key_frame_interval; + process_settings->num_temporal_layers = num_temporal_layers, + process_settings->error_concealment_on = error_concealment_on; + process_settings->denoising_on = denoising_on; + process_settings->frame_dropper_on = frame_dropper_on; + process_settings->spatial_resize_on = spatial_resize_on; + process_settings->width = width; + process_settings->height = height; + process_settings->filename = filename; + process_settings->verbose_logging = verbose_logging; + } + + static void SetCodecParameters(CodecConfigPars* process_settings, + VideoCodecType codec_type, + float packet_loss, + int key_frame_interval, + int num_temporal_layers, + bool error_concealment_on, + bool denoising_on, + bool frame_dropper_on, + bool spatial_resize_on) { + SetCodecParameters(process_settings, codec_type, packet_loss, + key_frame_interval, num_temporal_layers, + error_concealment_on, denoising_on, frame_dropper_on, + spatial_resize_on, kCifWidth, kCifHeight, + kFilenameForemanCif, false /* verbose_logging */); + } + + static void SetQualityMetrics(QualityMetrics* quality_metrics, + double minimum_avg_psnr, + double minimum_min_psnr, + double minimum_avg_ssim, + double minimum_min_ssim) { + quality_metrics->minimum_avg_psnr = minimum_avg_psnr; + quality_metrics->minimum_min_psnr = minimum_min_psnr; + quality_metrics->minimum_avg_ssim = minimum_avg_ssim; + quality_metrics->minimum_min_ssim = minimum_min_ssim; + } + + static void SetRateControlMetrics(RateControlMetrics* rc_metrics, + int update_index, + int max_num_dropped_frames, + int max_key_frame_size_mismatch, + int max_delta_frame_size_mismatch, + int max_encoding_rate_mismatch, + int max_time_hit_target, + int num_spatial_resizes, + int num_key_frames) { + rc_metrics[update_index].max_num_dropped_frames = max_num_dropped_frames; + rc_metrics[update_index].max_key_frame_size_mismatch = + max_key_frame_size_mismatch; + rc_metrics[update_index].max_delta_frame_size_mismatch = + max_delta_frame_size_mismatch; + rc_metrics[update_index].max_encoding_rate_mismatch = + max_encoding_rate_mismatch; + rc_metrics[update_index].max_time_hit_target = max_time_hit_target; + rc_metrics[update_index].num_spatial_resizes = num_spatial_resizes; + rc_metrics[update_index].num_key_frames = num_key_frames; + } +}; + +} // namespace test +} // namespace webrtc + +#endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H_