From 05cfcd34693d86de7a1a481f071eae561361588a Mon Sep 17 00:00:00 2001 From: ivica Date: Mon, 7 Sep 2015 06:04:16 -0700 Subject: [PATCH] Full stack graphs Updating full stack test to optionally save metadata for each frame and save it to a file with given filename (controlled from the new full_stack_samples executable). Adding a Python script that reads the output generated by full stack test and plots the graph(s). Review URL: https://codereview.webrtc.org/1289933003 Cr-Commit-Position: refs/heads/master@{#9874} --- webrtc/video/full_stack.cc | 158 +++++--- webrtc/video/full_stack.h | 52 +++ webrtc/video/full_stack_plot.py | 414 +++++++++++++++++++++ webrtc/video/full_stack_quality_sampler.cc | 142 +++++++ webrtc/video/video_receive_stream.cc | 4 + webrtc/webrtc_tests.gypi | 26 ++ 6 files changed, 753 insertions(+), 43 deletions(-) create mode 100644 webrtc/video/full_stack.h create mode 100755 webrtc/video/full_stack_plot.py create mode 100644 webrtc/video/full_stack_quality_sampler.cc diff --git a/webrtc/video/full_stack.cc b/webrtc/video/full_stack.cc index 3995f8a0c0..45c28ad623 100644 --- a/webrtc/video/full_stack.cc +++ b/webrtc/video/full_stack.cc @@ -14,6 +14,7 @@ #include "testing/gtest/include/gtest/gtest.h" +#include "webrtc/base/format_macros.h" #include "webrtc/base/scoped_ptr.h" #include "webrtc/base/thread_annotations.h" #include "webrtc/call.h" @@ -25,8 +26,6 @@ #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" #include "webrtc/system_wrappers/interface/event_wrapper.h" #include "webrtc/system_wrappers/interface/sleep.h" -#include "webrtc/test/call_test.h" -#include "webrtc/test/direct_transport.h" #include "webrtc/test/encoder_settings.h" #include "webrtc/test/fake_encoder.h" #include "webrtc/test/frame_generator.h" @@ -34,41 +33,13 @@ #include "webrtc/test/statistics.h" #include "webrtc/test/testsupport/fileutils.h" #include "webrtc/typedefs.h" +#include "webrtc/video/full_stack.h" namespace webrtc { static const int kFullStackTestDurationSecs = 60; static const int kSendStatsPollingIntervalMs = 1000; -enum class ContentMode { - kRealTimeVideo, - kScreensharingStaticImage, - kScreensharingScrollingImage, -}; - -struct FullStackTestParams { - const char* test_label; - struct { - const char* name; - size_t width, height; - int fps; - } clip; - ContentMode mode; - int min_bitrate_bps; - int target_bitrate_bps; - int max_bitrate_bps; - double avg_psnr_threshold; - double avg_ssim_threshold; - int test_durations_secs; - std::string codec; - FakeNetworkPipe::Config link; -}; - -class FullStackTest : public test::CallTest { - protected: - void RunTest(const FullStackTestParams& params); -}; - class VideoAnalyzer : public PacketReceiver, public newapi::Transport, public VideoRenderer, @@ -80,12 +51,14 @@ class VideoAnalyzer : public PacketReceiver, const char* test_label, double avg_psnr_threshold, double avg_ssim_threshold, - int duration_frames) + int duration_frames, + const std::string& graph_data_output_filename) : input_(input), transport_(transport), receiver_(nullptr), send_stream_(nullptr), test_label_(test_label), + graph_data_output_filename_(graph_data_output_filename), frames_to_process_(duration_frames), frames_recorded_(0), frames_processed_(0), @@ -177,8 +150,11 @@ class VideoAnalyzer : public PacketReceiver, header.timestamp - first_send_frame_.timestamp(); first_send_frame_.Reset(); } - send_times_[header.timestamp - rtp_timestamp_delta_] = + uint32_t timestamp = header.timestamp - rtp_timestamp_delta_; + send_times_[timestamp] = Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); + encoded_frame_sizes_[timestamp] += + length - (header.headerLength + header.paddingLength); } return transport_->SendRtp(packet, length); @@ -271,20 +247,26 @@ class VideoAnalyzer : public PacketReceiver, private: struct FrameComparison { FrameComparison() - : dropped(false), send_time_ms(0), recv_time_ms(0), render_time_ms(0) {} + : dropped(false), + send_time_ms(0), + recv_time_ms(0), + render_time_ms(0), + encoded_frame_size(0) {} FrameComparison(const VideoFrame& reference, const VideoFrame& render, bool dropped, int64_t send_time_ms, int64_t recv_time_ms, - int64_t render_time_ms) + int64_t render_time_ms, + size_t encoded_frame_size) : reference(reference), render(render), dropped(dropped), send_time_ms(send_time_ms), recv_time_ms(recv_time_ms), - render_time_ms(render_time_ms) {} + render_time_ms(render_time_ms), + encoded_frame_size(encoded_frame_size) {} VideoFrame reference; VideoFrame render; @@ -292,6 +274,35 @@ class VideoAnalyzer : public PacketReceiver, int64_t send_time_ms; int64_t recv_time_ms; int64_t render_time_ms; + size_t encoded_frame_size; + }; + + struct Sample { + Sample(double dropped, + double input_time_ms, + double send_time_ms, + double recv_time_ms, + double encoded_frame_size, + double psnr, + double ssim, + double render_time_ms) + : dropped(dropped), + input_time_ms(input_time_ms), + send_time_ms(send_time_ms), + recv_time_ms(recv_time_ms), + encoded_frame_size(encoded_frame_size), + psnr(psnr), + ssim(ssim), + render_time_ms(render_time_ms) {} + + double dropped; + double input_time_ms; + double send_time_ms; + double recv_time_ms; + double encoded_frame_size; + double psnr; + double ssim; + double render_time_ms; }; void AddFrameComparison(const VideoFrame& reference, @@ -304,10 +315,18 @@ class VideoAnalyzer : public PacketReceiver, int64_t recv_time_ms = recv_times_[reference.timestamp()]; recv_times_.erase(reference.timestamp()); + size_t encoded_size = encoded_frame_sizes_[reference.timestamp()]; + encoded_frame_sizes_.erase(reference.timestamp()); + + VideoFrame reference_copy; + VideoFrame render_copy; + reference_copy.CopyFrame(reference); + render_copy.CopyFrame(render); + rtc::CritScope crit(&comparison_lock_); - comparisons_.push_back(FrameComparison(reference, render, dropped, + comparisons_.push_back(FrameComparison(reference_copy, render_copy, dropped, send_time_ms, recv_time_ms, - render_time_ms)); + render_time_ms, encoded_size)); comparison_available_event_->Set(); } @@ -365,6 +384,8 @@ class VideoAnalyzer : public PacketReceiver, if (FrameProcessed()) { PrintResults(); + if (!graph_data_output_filename_.empty()) + PrintSamplesToFile(); done_->Set(); comparison_available_event_->Set(); return false; @@ -437,9 +458,18 @@ class VideoAnalyzer : public PacketReceiver, double psnr = I420PSNR(&comparison.reference, &comparison.render); double ssim = I420SSIM(&comparison.reference, &comparison.render); + int64_t input_time_ms = comparison.reference.ntp_time_ms(); + rtc::CritScope crit(&comparison_lock_); + if (!graph_data_output_filename_.empty()) { + samples_.push_back(Sample( + comparison.dropped, input_time_ms, comparison.send_time_ms, + comparison.recv_time_ms, comparison.encoded_frame_size, psnr, ssim, + comparison.render_time_ms)); + } psnr_.AddSample(psnr); ssim_.AddSample(ssim); + if (comparison.dropped) { ++dropped_frames_; return; @@ -448,11 +478,11 @@ class VideoAnalyzer : public PacketReceiver, rendered_delta_.AddSample(comparison.render_time_ms - last_render_time_); last_render_time_ = comparison.render_time_ms; - int64_t input_time_ms = comparison.reference.ntp_time_ms(); sender_time_.AddSample(comparison.send_time_ms - input_time_ms); receiver_time_.AddSample(comparison.render_time_ms - comparison.recv_time_ms); end_to_end_.AddSample(comparison.render_time_ms - input_time_ms); + encoded_frame_size_.AddSample(comparison.encoded_frame_size); } void PrintResult(const char* result_type, @@ -466,7 +496,39 @@ class VideoAnalyzer : public PacketReceiver, unit); } + void PrintSamplesToFile(void) { + FILE* out = fopen(graph_data_output_filename_.c_str(), "w"); + CHECK(out != nullptr) + << "Couldn't open file: " << graph_data_output_filename_; + + rtc::CritScope crit(&comparison_lock_); + std::sort(samples_.begin(), samples_.end(), + [](const Sample& A, const Sample& B) + -> bool { return A.input_time_ms < B.input_time_ms; }); + + fprintf(out, "%s\n", test_label_); + fprintf(out, "%" PRIuS "\n", samples_.size()); + fprintf(out, + "dropped " + "input_time_ms " + "send_time_ms " + "recv_time_ms " + "encoded_frame_size " + "psnr " + "ssim " + "render_time_ms\n"); + for (const Sample& sample : samples_) { + fprintf(out, "%lf %lf %lf %lf %lf %lf %lf %lf\n", sample.dropped, + sample.input_time_ms, sample.send_time_ms, sample.recv_time_ms, + sample.encoded_frame_size, sample.psnr, sample.ssim, + sample.render_time_ms); + } + fclose(out); + } + const char* const test_label_; + std::string graph_data_output_filename_; + std::vector samples_ GUARDED_BY(comparison_lock_); test::Statistics sender_time_ GUARDED_BY(comparison_lock_); test::Statistics receiver_time_ GUARDED_BY(comparison_lock_); test::Statistics psnr_ GUARDED_BY(comparison_lock_); @@ -488,9 +550,11 @@ class VideoAnalyzer : public PacketReceiver, rtc::CriticalSection crit_; std::deque frames_ GUARDED_BY(crit_); + std::deque send_stats_ GUARDED_BY(crit_); VideoFrame last_rendered_frame_ GUARDED_BY(crit_); std::map send_times_ GUARDED_BY(crit_); std::map recv_times_ GUARDED_BY(crit_); + std::map encoded_frame_sizes_ GUARDED_BY(crit_); VideoFrame first_send_frame_ GUARDED_BY(crit_); const double avg_psnr_threshold_; const double avg_ssim_threshold_; @@ -504,11 +568,16 @@ class VideoAnalyzer : public PacketReceiver, }; void FullStackTest::RunTest(const FullStackTestParams& params) { + // TODO(ivica): Add num_temporal_layers as a param. + unsigned char num_temporal_layers = + params.graph_data_output_filename.empty() ? 2 : 1; + test::DirectTransport send_transport(params.link); test::DirectTransport recv_transport(params.link); VideoAnalyzer analyzer(nullptr, &send_transport, params.test_label, params.avg_psnr_threshold, params.avg_ssim_threshold, - params.test_durations_secs * params.clip.fps); + params.test_durations_secs * params.clip.fps, + params.graph_data_output_filename); CreateCalls(Call::Config(), Call::Config()); @@ -557,18 +626,21 @@ void FullStackTest::RunTest(const FullStackTestParams& params) { vp8_settings = VideoEncoder::GetDefaultVp8Settings(); vp8_settings.denoisingOn = false; vp8_settings.frameDroppingOn = false; - vp8_settings.numberOfTemporalLayers = 2; + vp8_settings.numberOfTemporalLayers = num_temporal_layers; encoder_config_.encoder_specific_settings = &vp8_settings; } else if (params.codec == "VP9") { vp9_settings = VideoEncoder::GetDefaultVp9Settings(); vp9_settings.denoisingOn = false; vp9_settings.frameDroppingOn = false; - vp9_settings.numberOfTemporalLayers = 2; + vp9_settings.numberOfTemporalLayers = num_temporal_layers; encoder_config_.encoder_specific_settings = &vp9_settings; } stream->temporal_layer_thresholds_bps.clear(); - stream->temporal_layer_thresholds_bps.push_back(stream->target_bitrate_bps); + if (num_temporal_layers > 1) { + stream->temporal_layer_thresholds_bps.push_back( + stream->target_bitrate_bps); + } } CreateMatchingReceiveConfigs(&recv_transport); diff --git a/webrtc/video/full_stack.h b/webrtc/video/full_stack.h new file mode 100644 index 0000000000..56ef4b329f --- /dev/null +++ b/webrtc/video/full_stack.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef WEBRTC_VIDEO_FULL_STACK_H_ +#define WEBRTC_VIDEO_FULL_STACK_H_ + +#include + +#include "webrtc/test/call_test.h" +#include "webrtc/test/direct_transport.h" + +namespace webrtc { + +enum class ContentMode { + kRealTimeVideo, + kScreensharingStaticImage, + kScreensharingScrollingImage, +}; + +struct FullStackTestParams { + const char* test_label; + struct { + const char* name; + size_t width, height; + int fps; + } clip; + ContentMode mode; + int min_bitrate_bps; + int target_bitrate_bps; + int max_bitrate_bps; + double avg_psnr_threshold; + double avg_ssim_threshold; + int test_durations_secs; + std::string codec; + FakeNetworkPipe::Config link; + std::string graph_data_output_filename; +}; + +class FullStackTest : public test::CallTest { + protected: + void RunTest(const FullStackTestParams& params); +}; + +} // namespace webrtc + +#endif // WEBRTC_VIDEO_FULL_STACK_H_ diff --git a/webrtc/video/full_stack_plot.py b/webrtc/video/full_stack_plot.py new file mode 100755 index 0000000000..92b50a44a1 --- /dev/null +++ b/webrtc/video/full_stack_plot.py @@ -0,0 +1,414 @@ +#!/usr/bin/env python +# Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Generate graphs for data generated by full_stack_quality_sampler.cc. + +Usage examples: + Show end to end time for a single full stack test. + ./full_stack_plot.py -df end_to_end -o 600 --frames 1000 vp9_data.txt + + Show simultaneously PSNR and encoded frame size for two different runs of + full stack test. Averaged over a cycle of 200 frames. Used e.g. for + screenshare slide test. + ./full_stack_plot.py -c 200 -df psnr -drf encoded_frame_size \\ + before.txt after.txt + + Similar to the previous test, but multiple graphs. + ./full_stack_plot.py -c 200 -df psnr vp8.txt vp9.txt --next \\ + -c 200 -df sender_time vp8.txt vp9.txt --next \\ + -c 200 -df end_to_end vp8.txt vp9.txt +""" + +import argparse +from collections import defaultdict +import itertools +import sys +import matplotlib.pyplot as plt +import numpy + +# Fields +DROPPED = 0 +INPUT_TIME = 1 # ms +SEND_TIME = 2 # ms +RECV_TIME = 3 # ms +ENCODED_FRAME_SIZE = 4 # bytes +PSNR = 5 +SSIM = 6 +RENDER_TIME = 7 # ms + +TOTAL_RAW_FIELDS = 8 + +SENDER_TIME = TOTAL_RAW_FIELDS + 0 +RECEIVER_TIME = TOTAL_RAW_FIELDS + 1 +END_TO_END = TOTAL_RAW_FIELDS + 2 +RENDERED_DELTA = TOTAL_RAW_FIELDS + 3 + +FIELD_MASK = 255 + +# Options +HIDE_DROPPED = 256 +RIGHT_Y_AXIS = 512 + +# internal field id, field name, title +_fields = [ + # Raw + (DROPPED, "dropped", "dropped"), + (INPUT_TIME, "input_time_ms", "input time"), + (SEND_TIME, "send_time_ms", "send time"), + (RECV_TIME, "recv_time_ms", "recv time"), + (ENCODED_FRAME_SIZE, "encoded_frame_size", "encoded frame size"), + (PSNR, "psnr", "PSNR"), + (SSIM, "ssim", "SSIM"), + (RENDER_TIME, "render_time_ms", "render time"), + # Auto-generated + (SENDER_TIME, "sender_time", "sender time"), + (RECEIVER_TIME, "receiver_time", "receiver time"), + (END_TO_END, "end_to_end", "end to end"), + (RENDERED_DELTA, "rendered_delta", "rendered delta"), +] + +name_to_id = {field[1]: field[0] for field in _fields} +id_to_title = {field[0]: field[2] for field in _fields} + + +def field_arg_to_id(arg): + if arg == "none": + return None + if arg in name_to_id: + return name_to_id[arg] + if arg + "_ms" in name_to_id: + return name_to_id[arg + "_ms"] + raise Exception("Unrecognized field name \"{}\"".format(arg)) + + +class PlotLine(object): + """Data for a single graph line.""" + + def __init__(self, label, values, flags): + self.label = label + self.values = values + self.flags = flags + + +class Data(object): + """Object representing one full stack test.""" + + def __init__(self, filename): + self.title = "" + self.length = 0 + self.samples = defaultdict(list) + + self._read_samples(filename) + + def _read_samples(self, filename): + """Reads graph data from the given file.""" + f = open(filename) + it = iter(f) + + self.title = it.next().strip() + self.length = int(it.next()) + field_names = [name.strip() for name in it.next().split()] + field_ids = [name_to_id[name] for name in field_names] + + for field_id in field_ids: + self.samples[field_id] = [0.0] * self.length + + for sample_id in xrange(self.length): + for col, value in enumerate(it.next().split()): + self.samples[field_ids[col]][sample_id] = float(value) + + self._subtract_first_input_time() + self._generate_additional_data() + + f.close() + + def _subtract_first_input_time(self): + offset = self.samples[INPUT_TIME][0] + for field in [INPUT_TIME, SEND_TIME, RECV_TIME, RENDER_TIME]: + if field in self.samples: + self.samples[field] = [x - offset for x in self.samples[field]] + + def _generate_additional_data(self): + """Calculates sender time, receiver time etc. from the raw data.""" + s = self.samples + last_render_time = 0 + for field_id in [SENDER_TIME, RECEIVER_TIME, END_TO_END, RENDERED_DELTA]: + s[field_id] = [0] * self.length + + for k in range(self.length): + s[SENDER_TIME][k] = s[SEND_TIME][k] - s[INPUT_TIME][k] + + decoded_time = s[RENDER_TIME][k] + s[RECEIVER_TIME][k] = decoded_time - s[RECV_TIME][k] + s[END_TO_END][k] = decoded_time - s[INPUT_TIME][k] + if not s[DROPPED][k]: + if k > 0: + s[RENDERED_DELTA][k] = decoded_time - last_render_time + last_render_time = decoded_time + + def _hide(self, values): + """ + Replaces values for dropped frames with None. + These values are then skipped by the plot() method. + """ + + return [None if self.samples[DROPPED][k] else values[k] + for k in range(len(values))] + + def add_samples(self, config, target_lines_list): + """Creates graph lines from the current data set with given config.""" + for field in config.fields: + # field is None means the user wants just to skip the color. + if field is None: + target_lines_list.append(None) + continue + + field_id = field & FIELD_MASK + values = self.samples[field_id] + + if field & HIDE_DROPPED: + values = self._hide(values) + + target_lines_list.append(PlotLine( + self.title + " " + id_to_title[field_id], + values, field & ~FIELD_MASK)) + + +def average_over_cycle(values, length): + """ + Returns the list: + [ + avg(values[0], values[length], ...), + avg(values[1], values[length + 1], ...), + ... + avg(values[length - 1], values[2 * length - 1], ...), + ] + + Skips None values when calculating the average value. + """ + + total = [0.0] * length + count = [0] * length + for k in range(len(values)): + if values[k] is not None: + total[k % length] += values[k] + count[k % length] += 1 + + result = [0.0] * length + for k in range(length): + result[k] = total[k] / count[k] if count[k] else None + return result + + +class PlotConfig(object): + """Object representing a single graph.""" + + def __init__(self, fields, data_list, cycle_length=None, frames=None, + offset=0, output_filename=None, title="Graph"): + self.fields = fields + self.data_list = data_list + self.cycle_length = cycle_length + self.frames = frames + self.offset = offset + self.output_filename = output_filename + self.title = title + + def plot(self, ax1): + lines = [] + for data in self.data_list: + if not data: + # Add None lines to skip the colors. + lines.extend([None] * len(self.fields)) + else: + data.add_samples(self, lines) + + def _slice_values(values): + if self.offset: + values = values[self.offset:] + if self.frames: + values = values[:self.frames] + return values + + length = None + for line in lines: + if line is None: + continue + + line.values = _slice_values(line.values) + if self.cycle_length: + line.values = average_over_cycle(line.values, self.cycle_length) + + if length is None: + length = len(line.values) + elif length != len(line.values): + raise Exception("All arrays should have the same length!") + + ax1.set_xlabel("Frame", fontsize="large") + if any(line.flags & RIGHT_Y_AXIS for line in lines if line): + ax2 = ax1.twinx() + ax2.set_xlabel("Frame", fontsize="large") + else: + ax2 = None + + # Have to implement color_cycle manually, due to two scales in a graph. + color_cycle = ["b", "r", "g", "c", "m", "y", "k"] + color_iter = itertools.cycle(color_cycle) + + for line in lines: + if not line: + color_iter.next() + continue + + if self.cycle_length: + x = numpy.array(range(self.cycle_length)) + else: + x = numpy.array(range(self.offset, self.offset + len(line.values))) + y = numpy.array(line.values) + ax = ax2 if line.flags & RIGHT_Y_AXIS else ax1 + ax.plot(x, y, "o-", label=line.label, markersize=3.0, linewidth=1.0, + color=color_iter.next()) + + ax1.grid(True) + if ax2: + ax1.legend(loc="upper left", shadow=True, fontsize="large") + ax2.legend(loc="upper right", shadow=True, fontsize="large") + else: + ax1.legend(loc="best", shadow=True, fontsize="large") + + +def load_files(filenames): + result = [] + for filename in filenames: + if filename in load_files.cache: + result.append(load_files.cache[filename]) + else: + data = Data(filename) + load_files.cache[filename] = data + result.append(data) + return result +load_files.cache = {} + + +def get_parser(): + class CustomAction(argparse.Action): + + def __call__(self, parser, namespace, values, option_string=None): + if "ordered_args" not in namespace: + namespace.ordered_args = [] + namespace.ordered_args.append((self.dest, values)) + + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + + parser.add_argument( + "-c", "--cycle_length", nargs=1, action=CustomAction, + type=int, help="Cycle length over which to average the values.") + parser.add_argument( + "-f", "--field", nargs=1, action=CustomAction, + help="Name of the field to show. Use 'none' to skip a color.") + parser.add_argument("-r", "--right", nargs=0, action=CustomAction, + help="Use right Y axis for given field.") + parser.add_argument("-d", "--drop", nargs=0, action=CustomAction, + help="Hide values for dropped frames.") + parser.add_argument("-o", "--offset", nargs=1, action=CustomAction, type=int, + help="Frame offset.") + parser.add_argument("-n", "--next", nargs=0, action=CustomAction, + help="Separator for multiple graphs.") + parser.add_argument( + "--frames", nargs=1, action=CustomAction, type=int, + help="Frame count to show or take into account while averaging.") + parser.add_argument("-t", "--title", nargs=1, action=CustomAction, + help="Title of the graph.") + parser.add_argument( + "-O", "--output_filename", nargs=1, action=CustomAction, + help="Use to save the graph into a file. " + "Otherwise, a window will be shown.") + parser.add_argument( + "files", nargs="+", action=CustomAction, + help="List of text-based files generated by full_stack.cc") + return parser + + +def _plot_config_from_args(args, graph_num): + # Pylint complains about using kwargs, so have to do it this way. + cycle_length = None + frames = None + offset = 0 + output_filename = None + title = "Graph" + + fields = [] + files = [] + mask = 0 + for key, values in args: + if key == "cycle_length": + cycle_length = values[0] + elif key == "frames": + frames = values[0] + elif key == "offset": + offset = values[0] + elif key == "output_filename": + output_filename = values[0] + elif key == "title": + title = values[0] + elif key == "drop": + mask |= HIDE_DROPPED + elif key == "right": + mask |= RIGHT_Y_AXIS + elif key == "field": + field_id = field_arg_to_id(values[0]) + fields.append(field_id | mask if field_id is not None else None) + mask = 0 # Reset mask after the field argument. + elif key == "files": + files.extend(values) + + if not files: + raise Exception("Missing file argument(s) for graph #{}".format(graph_num)) + if not fields: + raise Exception("Missing field argument(s) for graph #{}".format(graph_num)) + + return PlotConfig(fields, load_files(files), cycle_length=cycle_length, + frames=frames, offset=offset, output_filename=output_filename, + title=title) + + +def plot_configs_from_args(args): + """Generates plot configs for given command line arguments.""" + # The way it works: + # First we detect separators -n/--next and split arguments into groups, one + # for each plot. For each group, we partially parse it with + # argparse.ArgumentParser, modified to remember the order of arguments. + # Then we traverse the argument list and fill the PlotConfig. + args = itertools.groupby(args, lambda x: x in ["-n", "--next"]) + args = list(list(group) for match, group in args if not match) + + parser = get_parser() + plot_configs = [] + for index, raw_args in enumerate(args): + graph_args = parser.parse_args(raw_args).ordered_args + plot_configs.append(_plot_config_from_args(graph_args, index)) + return plot_configs + + +def show_or_save_plots(plot_configs): + for config in plot_configs: + fig = plt.figure(figsize=(14.0, 10.0)) + ax = fig.add_subplot(1, 1, 1) + + plt.title(config.title) + config.plot(ax) + if config.output_filename: + print "Saving to", config.output_filename + fig.savefig(config.output_filename) + plt.close(fig) + + plt.show() + +if __name__ == "__main__": + show_or_save_plots(plot_configs_from_args(sys.argv[1:])) diff --git a/webrtc/video/full_stack_quality_sampler.cc b/webrtc/video/full_stack_quality_sampler.cc new file mode 100644 index 0000000000..a17975699a --- /dev/null +++ b/webrtc/video/full_stack_quality_sampler.cc @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "gflags/gflags.h" +#include "webrtc/test/field_trial.h" +#include "webrtc/test/run_test.h" +#include "webrtc/video/full_stack.h" + +namespace webrtc { +namespace flags { + +DEFINE_string(title, "Full stack graph", "Graph title."); +std::string Title() { + return static_cast(FLAGS_title); +} + +DEFINE_string(filename, "graph_data.txt", "Name of a target graph data file."); +std::string Filename() { + return static_cast(FLAGS_filename); +} + +DEFINE_string(clip_name, "screenshare_slides", "Clip name, resource name."); +std::string ClipName() { + return static_cast(FLAGS_clip_name); +} + +DEFINE_int32(width, 1850, "Video width (crops source)."); +size_t Width() { + return static_cast(FLAGS_width); +} + +DEFINE_int32(height, 1110, "Video height (crops source)."); +size_t Height() { + return static_cast(FLAGS_height); +} + +DEFINE_int32(fps, 5, "Frames per second."); +int Fps() { + return static_cast(FLAGS_fps); +} + +DEFINE_int32( + content_mode, + 1, + "0 - real time video, 1 - screenshare static, 2 - screenshare scrolling."); +ContentMode ContentModeFlag() { + switch (FLAGS_content_mode) { + case 0: + return ContentMode::kRealTimeVideo; + case 1: + return ContentMode::kScreensharingStaticImage; + case 2: + return ContentMode::kScreensharingScrollingImage; + default: + RTC_NOTREACHED() << "Unknown content mode!"; + return ContentMode::kScreensharingStaticImage; + } +} + +DEFINE_int32(test_duration, 60, "Duration of the test in seconds."); +int TestDuration() { + return static_cast(FLAGS_test_duration); +} + +DEFINE_int32(min_bitrate, 50000, "Minimum video bitrate."); +int MinBitrate() { + return static_cast(FLAGS_min_bitrate); +} + +DEFINE_int32(target_bitrate, + 500000, + "Target video bitrate. (Default value here different than in full " + "stack tests!)"); +int TargetBitrate() { + return static_cast(FLAGS_target_bitrate); +} + +DEFINE_int32(max_bitrate, + 500000, + "Maximum video bitrate. (Default value here different than in " + "full stack tests!)"); +int MaxBitrate() { + return static_cast(FLAGS_max_bitrate); +} + +DEFINE_string(codec, "VP9", "Video codec to use."); +std::string Codec() { + return static_cast(FLAGS_codec); +} + +DEFINE_string( + force_fieldtrials, + "", + "Field trials control experimental feature code which can be forced. " + "E.g. running with --force_fieldtrials=WebRTC-FooFeature/Enable/" + " will assign the group Enable to field trial WebRTC-FooFeature. Multiple " + "trials are separated by \"/\""); +} // namespace flags + +class FullStackGenGraph : public FullStackTest { + public: + void TestBody() override { + std::string title = flags::Title(); + std::string clip_name = flags::ClipName(); + FullStackTestParams params = { + title.c_str(), + {clip_name.c_str(), flags::Width(), flags::Height(), flags::Fps()}, + flags::ContentModeFlag(), + flags::MinBitrate(), + flags::TargetBitrate(), + flags::MaxBitrate(), + 0.0, // avg_psnr_threshold + 0.0, // avg_ssim_threshold + flags::TestDuration(), + flags::Codec()}; + params.graph_data_output_filename = flags::Filename(); + + RunTest(params); + } +}; + +void FullStackRun(void) { + FullStackGenGraph full_stack; + full_stack.TestBody(); +} +} // namespace webrtc + +int main(int argc, char* argv[]) { + ::testing::InitGoogleTest(&argc, argv); + google::ParseCommandLineFlags(&argc, &argv, true); + webrtc::test::InitFieldTrialsFromString( + webrtc::flags::FLAGS_force_fieldtrials); + webrtc::test::RunTest(webrtc::FullStackRun); + return 0; +} diff --git a/webrtc/video/video_receive_stream.cc b/webrtc/video/video_receive_stream.cc index 52e37b7b7a..5737ca98a2 100644 --- a/webrtc/video/video_receive_stream.cc +++ b/webrtc/video/video_receive_stream.cc @@ -103,6 +103,8 @@ VideoCodec CreateDecoderVideoCodec(const VideoReceiveStream::Decoder& decoder) { strcpy(codec.plName, decoder.payload_name.c_str()); if (decoder.payload_name == "VP8") { codec.codecType = kVideoCodecVP8; + } else if (decoder.payload_name == "VP9") { + codec.codecType = kVideoCodecVP9; } else if (decoder.payload_name == "H264") { codec.codecType = kVideoCodecH264; } else { @@ -111,6 +113,8 @@ VideoCodec CreateDecoderVideoCodec(const VideoReceiveStream::Decoder& decoder) { if (codec.codecType == kVideoCodecVP8) { codec.codecSpecific.VP8 = VideoEncoder::GetDefaultVp8Settings(); + } else if (codec.codecType == kVideoCodecVP9) { + codec.codecSpecific.VP9 = VideoEncoder::GetDefaultVp9Settings(); } else if (codec.codecType == kVideoCodecH264) { codec.codecSpecific.H264 = VideoEncoder::GetDefaultH264Settings(); } diff --git a/webrtc/webrtc_tests.gypi b/webrtc/webrtc_tests.gypi index 489df750de..8186415b6c 100644 --- a/webrtc/webrtc_tests.gypi +++ b/webrtc/webrtc_tests.gypi @@ -44,6 +44,32 @@ 'webrtc_perf_tests', ], }, + { + 'target_name': 'full_stack_quality_sampler', + 'type': 'executable', + 'sources': [ + 'test/mac/run_test.mm', + 'test/run_test.cc', + 'test/run_test.h', + 'video/full_stack.cc', + 'video/full_stack_quality_sampler.cc', + ], + 'conditions': [ + ['OS=="mac"', { + 'sources!': [ + 'test/run_test.cc', + ], + }], + ], + 'dependencies': [ + '<(DEPTH)/testing/gtest.gyp:gtest', + '<(DEPTH)/third_party/gflags/gflags.gyp:gflags', + '<(webrtc_root)/modules/modules.gyp:video_capture', + 'test/webrtc_test_common.gyp:webrtc_test_common', + 'test/test.gyp:test_main', + 'webrtc', + ], + }, { 'target_name': 'loopback_base', 'type': 'static_library',