admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Comparison of videos with reference frame not starting from zero

Browse Source 
BUG=webrtc:6967

Review-Url: https://codereview.webrtc.org/2553693002
Cr-Commit-Position: refs/heads/master@{#16112}
This commit is contained in:
mandermo 2017-01-17 03:24:57 -08:00 committed by Commit bot
parent 160e4a78e3
commit 7456817d40
5 changed files with 262 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
webrtc/tools/compare_videos.py
View File

@ -48,15 +48,27 @@ def _ParseArgs():
help=('The path to where the zxing executable is located. '
'If omitted, it will be assumed to be present in the '
'PATH with the name zxing[.exe].'))
parser.add_option('--stats_file', type='string', default='stats.txt',
parser.add_option('--stats_file_ref', type='string', default='stats_ref.txt',
help=('Path to the temporary stats file to be created and '
'used. Default: %default'))
'used for the reference video file. '
'Default: %default'))
parser.add_option('--stats_file_test', type='string',
default='stats_test.txt',
help=('Path to the temporary stats file to be created and '
'used for the test video file. Default: %default'))
parser.add_option('--stats_file', type='string',
help=('DEPRECATED'))
parser.add_option('--yuv_frame_width', type='int', default=640,
help='Width of the YUV file\'s frames. Default: %default')
parser.add_option('--yuv_frame_height', type='int', default=480,
help='Height of the YUV file\'s frames. Default: %default')
options, _ = parser.parse_args()
if options.stats_file:
options.stats_file_test = options.stats_file
print ('WARNING: Using deprecated switch --stats_file. '
'The new flag is --stats_file_test.')
if not options.ref_video:
parser.error('You must provide a path to the reference video!')
if not os.path.exists(options.ref_video):
@ -74,6 +86,40 @@ def _ParseArgs():
options.frame_analyzer)
return options
def _DevNull():
"""On Windows, sometimes the inherited stdin handle from the parent process
fails. Workaround this by passing null to stdin to the subprocesses commands.
This function can be used to create the null file handler.
"""
return open(os.devnull, 'r')
def DecodeBarcodesInVideo(options, path_to_decoder, video, stat_file):
# Run barcode decoder on the test video to identify frame numbers.
png_working_directory = tempfile.mkdtemp()
cmd = [
sys.executable,
path_to_decoder,
'--yuv_file=%s' % video,
'--yuv_frame_width=%d' % options.yuv_frame_width,
'--yuv_frame_height=%d' % options.yuv_frame_height,
'--stats_file=%s' % stat_file,
'--png_working_dir=%s' % png_working_directory,
]
if options.zxing_path:
cmd.append('--zxing_path=%s' % options.zxing_path)
if options.ffmpeg_path:
cmd.append('--ffmpeg_path=%s' % options.ffmpeg_path)
barcode_decoder = subprocess.Popen(cmd, stdin=_DevNull(),
stdout=sys.stdout, stderr=sys.stderr)
barcode_decoder.wait()
shutil.rmtree(png_working_directory)
if barcode_decoder.returncode != 0:
print 'Failed to run barcode decoder script.'
return 1
return 0
def main():
"""The main function.
@ -97,32 +143,11 @@ def main():
path_to_decoder = os.path.join(SCRIPT_DIR, 'barcode_tools',
'barcode_decoder.py')
# On Windows, sometimes the inherited stdin handle from the parent process
# fails. Work around this by passing null to stdin to the subprocesses.
null_filehandle = open(os.devnull, 'r')
# Run barcode decoder on the test video to identify frame numbers.
png_working_directory = tempfile.mkdtemp()
cmd = [
sys.executable,
path_to_decoder,
'--yuv_file=%s' % options.test_video,
'--yuv_frame_width=%d' % options.yuv_frame_width,
'--yuv_frame_height=%d' % options.yuv_frame_height,
'--stats_file=%s' % options.stats_file,
'--png_working_dir=%s' % png_working_directory,
]
if options.zxing_path:
cmd.append('--zxing_path=%s' % options.zxing_path)
if options.ffmpeg_path:
cmd.append('--ffmpeg_path=%s' % options.ffmpeg_path)
barcode_decoder = subprocess.Popen(cmd, stdin=null_filehandle,
stdout=sys.stdout, stderr=sys.stderr)
barcode_decoder.wait()
shutil.rmtree(png_working_directory)
if barcode_decoder.returncode != 0:
print 'Failed to run barcode decoder script.'
if DecodeBarcodesInVideo(options, path_to_decoder,
options.ref_video, options.stats_file_ref) != 0:
return 1
if DecodeBarcodesInVideo(options, path_to_decoder,
options.test_video, options.stats_file_test) != 0:
return 1
# Run frame analyzer to compare the videos and print output.
@ -131,11 +156,12 @@ def main():
'--label=%s' % options.label,
'--reference_file=%s' % options.ref_video,
'--test_file=%s' % options.test_video,
'--stats_file=%s' % options.stats_file,
'--stats_file_ref=%s' % options.stats_file_ref,
'--stats_file_test=%s' % options.stats_file_test,
'--width=%d' % options.yuv_frame_width,
'--height=%d' % options.yuv_frame_height,
]
frame_analyzer = subprocess.Popen(cmd, stdin=null_filehandle,
frame_analyzer = subprocess.Popen(cmd, stdin=_DevNull(),
stdout=sys.stdout, stderr=sys.stderr)
frame_analyzer.wait()
if frame_analyzer.returncode != 0:

42
webrtc/tools/frame_analyzer/frame_analyzer.cc
View File

@ -23,12 +23,11 @@
* video. The test video is a record of the reference video which can start at
* an arbitrary point. It is possible that there will be repeated frames or
* skipped frames as well. In order to have a way to compare corresponding
* frames from the two videos, a stats file should be provided. The stats file
* frames from the two videos, two stats files should be provided. One for the
* reference video and one for the test video. The stats file
* is a text file assumed to be in the format:
* frame_xxxx yyyy
* where xxxx is the frame number in the test video and yyyy is the
* corresponding frame number in the original video.
* The video files should be 1420 YUV videos.
* frame_xxxx yyyy where xxxx is the frame number in and yyyy is the
* corresponding barcode. The video files should be 1420 YUV videos.
* The tool prints the result to standard output in the Chromium perf format:
* RESULT <metric>:<label>= <values>
*
@ -36,22 +35,30 @@
*
* Usage:
* frame_analyzer --label=<test_label> --reference_file=<name_of_file>
* --test_file=<name_of_file> --stats_file=<name_of_file> --width=<frame_width>
* --test_file_ref=<name_of_file> --stats_file_test=<name_of_file>
* --stats_file=<name_of_file> --width=<frame_width>
* --height=<frame_height>
*/
int main(int argc, char** argv) {
std::string program_name = argv[0];
std::string usage = "Compares the output video with the initially sent video."
"\nExample usage:\n" + program_name + " --stats_file=stats.txt "
"--reference_file=ref.yuv --test_file=test.yuv --width=320 --height=240\n"
std::string usage =
"Compares the output video with the initially sent video."
"\nExample usage:\n" +
program_name +
" --reference_file=ref.yuv --test_file=test.yuv --width=320 "
"--height=240\n"
"Command line flags:\n"
" - width(int): The width of the reference and test files. Default: -1\n"
" - height(int): The height of the reference and test files. "
" Default: -1\n"
" - label(string): The label to use for the perf output."
" Default: MY_TEST\n"
" - stats_file(string): The full name of the file containing the stats"
" after decoding of the received YUV video. Default: stats.txt\n"
" - stats_file_ref(string): The path to the stats file that will be"
" produced for the reference video file."
" Default: stats_ref.txt\n"
" - stats_file_test(string): The path to the stats file that will be"
" produced for the test video file."
" Default: stats_test.txt\n"
" - reference_file(string): The reference YUV file to compare against."
" Default: ref.yuv\n"
" - test_file(string): The test YUV file to run the analysis for."
@ -66,7 +73,8 @@ int main(int argc, char** argv) {
parser.SetFlag("width", "-1");
parser.SetFlag("height", "-1");
parser.SetFlag("label", "MY_TEST");
parser.SetFlag("stats_file", "stats.txt");
parser.SetFlag("stats_file_ref", "stats_ref.txt");
parser.SetFlag("stats_file_test", "stats_test.txt");
parser.SetFlag("reference_file", "ref.yuv");
parser.SetFlag("test_file", "test.yuv");
parser.SetFlag("help", "false");
@ -90,11 +98,13 @@ int main(int argc, char** argv) {
webrtc::test::RunAnalysis(parser.GetFlag("reference_file").c_str(),
parser.GetFlag("test_file").c_str(),
parser.GetFlag("stats_file").c_str(), width, height,
&results);
parser.GetFlag("stats_file_ref").c_str(),
parser.GetFlag("stats_file_test").c_str(), width,
height, &results);
std::string label = parser.GetFlag("label");
webrtc::test::PrintAnalysisResults(label, &results);
webrtc::test::PrintMaxRepeatedAndSkippedFrames(label,
parser.GetFlag("stats_file"));
webrtc::test::PrintMaxRepeatedAndSkippedFrames(
label, parser.GetFlag("stats_file_ref"),
parser.GetFlag("stats_file_test"));
}

175
webrtc/tools/frame_analyzer/video_quality_analysis.cc
View File

@ -13,7 +13,10 @@
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <algorithm>
#include <string>
#include <map>
#include <utility>
#define STATS_LINE_LENGTH 32
#define Y4M_FILE_HEADER_MAX_SIZE 200
@ -224,8 +227,12 @@ double CalculateMetrics(VideoAnalysisMetricsType video_metrics_type,
return result;
}
void RunAnalysis(const char* reference_file_name, const char* test_file_name,
const char* stats_file_name, int width, int height,
void RunAnalysis(const char* reference_file_name,
const char* test_file_name,
const char* stats_file_reference_name,
const char* stats_file_test_name,
int width,
int height,
ResultsContainer* results) {
// Check if the reference_file_name ends with "y4m".
bool y4m_mode = false;
@ -234,7 +241,8 @@ void RunAnalysis(const char* reference_file_name, const char* test_file_name,
}
int size = GetI420FrameSize(width, height);
FILE* stats_file = fopen(stats_file_name, "r");
FILE* stats_file_ref = fopen(stats_file_reference_name, "r");
FILE* stats_file_test = fopen(stats_file_test_name, "r");
// String buffer for the lines in the stats file.
char line[STATS_LINE_LENGTH];
@ -244,10 +252,29 @@ void RunAnalysis(const char* reference_file_name, const char* test_file_name,
uint8_t* reference_frame = new uint8_t[size];
int previous_frame_number = -1;
// Maps barcode id to the frame id for the reference video.
// In case two frames have same id, then we only save the first one.
std::map<int, int> ref_barcode_to_frame;
// While there are entries in the stats file.
while (GetNextStatsLine(stats_file, line)) {
while (GetNextStatsLine(stats_file_ref, line)) {
int extracted_ref_frame = ExtractFrameSequenceNumber(line);
int decoded_frame_number = ExtractDecodedFrameNumber(line);
// Insert will only add if it is not in map already.
ref_barcode_to_frame.insert(
std::make_pair(decoded_frame_number, extracted_ref_frame));
}
while (GetNextStatsLine(stats_file_test, line)) {
int extracted_test_frame = ExtractFrameSequenceNumber(line);
int decoded_frame_number = ExtractDecodedFrameNumber(line);
auto it = ref_barcode_to_frame.find(decoded_frame_number);
if (it == ref_barcode_to_frame.end()) {
// Not found in the reference video.
// TODO(mandermo) print
continue;
}
int extracted_ref_frame = it->second;
// If there was problem decoding the barcode in this frame or the frame has
// been duplicated, continue.
@ -263,17 +290,17 @@ void RunAnalysis(const char* reference_file_name, const char* test_file_name,
test_frame);
if (y4m_mode) {
ExtractFrameFromY4mFile(reference_file_name, width, height,
decoded_frame_number, reference_frame);
extracted_ref_frame, reference_frame);
} else {
ExtractFrameFromYuvFile(reference_file_name, width, height,
decoded_frame_number, reference_frame);
extracted_ref_frame, reference_frame);
}
// Calculate the PSNR and SSIM.
double result_psnr = CalculateMetrics(kPSNR, reference_frame, test_frame,
width, height);
double result_ssim = CalculateMetrics(kSSIM, reference_frame, test_frame,
width, height);
double result_psnr =
CalculateMetrics(kPSNR, reference_frame, test_frame, width, height);
double result_ssim =
CalculateMetrics(kSSIM, reference_frame, test_frame, width, height);
previous_frame_number = decoded_frame_number;
@ -287,62 +314,124 @@ void RunAnalysis(const char* reference_file_name, const char* test_file_name,
}
// Cleanup.
fclose(stats_file);
fclose(stats_file_ref);
fclose(stats_file_test);
delete[] test_frame;
delete[] reference_frame;
}
void PrintMaxRepeatedAndSkippedFrames(const std::string& label,
const std::string& stats_file_name) {
PrintMaxRepeatedAndSkippedFrames(stdout, label, stats_file_name);
const std::string& stats_file_ref_name,
const std::string& stats_file_test_name) {
PrintMaxRepeatedAndSkippedFrames(stdout, label, stats_file_ref_name,
stats_file_test_name);
}
void PrintMaxRepeatedAndSkippedFrames(FILE* output, const std::string& label,
const std::string& stats_file_name) {
FILE* stats_file = fopen(stats_file_name.c_str(), "r");
if (stats_file == NULL) {
fprintf(stderr, "Couldn't open stats file for reading: %s\n",
stats_file_name.c_str());
return;
}
namespace {
// Clusters the frames in the file. First in the pair is the frame number and
// second is the number
// of frames in that cluster. So if first frame in video has number 100 and it
// is repeated 3 after
// each other, then the first entry in the returned vector has first set to 100
// and second set
// to 3.
std::vector<std::pair<int, int> > CalculateFrameClusters(FILE* file) {
std::vector<std::pair<int, int> > frame_cnt;
char line[STATS_LINE_LENGTH];
int repeated_frames = 1;
int max_repeated_frames = 1;
int max_skipped_frames = 1;
int previous_frame_number = -1;
while (GetNextStatsLine(stats_file, line)) {
while (GetNextStatsLine(file, line)) {
int decoded_frame_number = ExtractDecodedFrameNumber(line);
if (decoded_frame_number == -1) {
continue;
}
// Calculate how many frames a cluster of repeated frames contains.
if (decoded_frame_number == previous_frame_number) {
++repeated_frames;
if (repeated_frames > max_repeated_frames) {
max_repeated_frames = repeated_frames;
}
if (frame_cnt.empty() || frame_cnt.back().first != decoded_frame_number) {
frame_cnt.push_back(std::make_pair(decoded_frame_number, 1));
} else {
repeated_frames = 1;
++frame_cnt.back().second;
}
}
return frame_cnt;
}
} // namespace
void PrintMaxRepeatedAndSkippedFrames(FILE* output,
const std::string& label,
const std::string& stats_file_ref_name,
const std::string& stats_file_test_name) {
FILE* stats_file_ref = fopen(stats_file_ref_name.c_str(), "r");
FILE* stats_file_test = fopen(stats_file_test_name.c_str(), "r");
if (stats_file_ref == NULL) {
fprintf(stderr, "Couldn't open reference stats file for reading: %s\n",
stats_file_ref_name.c_str());
return;
}
if (stats_file_test == NULL) {
fprintf(stderr, "Couldn't open test stats file for reading: %s\n",
stats_file_test_name.c_str());
fclose(stats_file_ref);
return;
}
// Calculate how much frames have been skipped.
if (decoded_frame_number != 0 && previous_frame_number != -1) {
int skipped_frames = decoded_frame_number - previous_frame_number - 1;
int max_repeated_frames = 1;
int max_skipped_frames = 1;
std::vector<std::pair<int, int> > frame_cnt_ref =
CalculateFrameClusters(stats_file_ref);
std::vector<std::pair<int, int> > frame_cnt_test =
CalculateFrameClusters(stats_file_test);
fclose(stats_file_ref);
fclose(stats_file_test);
auto it_ref = frame_cnt_ref.begin();
auto it_test = frame_cnt_test.begin();
auto end_ref = frame_cnt_ref.end();
auto end_test = frame_cnt_test.end();
if (it_test == end_test || it_ref == end_ref) {
fprintf(stderr, "Either test or ref file is empty, nothing to print\n");
return;
}
// Find the first frame in the reference video that match the first frame in
// the test video.
while (it_ref != end_ref && it_ref->first != it_test->first) {
++it_ref;
}
if (it_ref == end_ref) {
fprintf(stderr,
"The barcode in the test video's first frame is not in the "
"reference video.\n");
return;
}
for (;;) {
max_repeated_frames =
std::max(max_repeated_frames, it_test->second - it_ref->second + 1);
++it_test;
if (it_test == end_test) {
break;
}
int skipped_frames = 0;
++it_ref;
while (it_ref != end_ref && it_ref->first != it_test->first) {
skipped_frames += it_ref->second;
++it_ref;
}
if (it_ref == end_ref) {
fprintf(stderr,
"The barcode in the test video is not in the reference video.\n");
return;
}
if (skipped_frames > max_skipped_frames) {
max_skipped_frames = skipped_frames;
}
}
previous_frame_number = decoded_frame_number;
}
fprintf(output, "RESULT Max_repeated: %s= %d\n", label.c_str(),
max_repeated_frames);
fprintf(output, "RESULT Max_skipped: %s= %d\n", label.c_str(),
max_skipped_frames);
fclose(stats_file);
}
void PrintAnalysisResults(const std::string& label, ResultsContainer* results) {

29
webrtc/tools/frame_analyzer/video_quality_analysis.h
View File

@ -45,16 +45,22 @@ enum VideoAnalysisMetricsType {kPSNR, kSSIM};
// There may be missing or duplicate frames. Also the frames start at a random
// position in the original video. We should provide a statistics file along
// with the test video. The stats file contains the connection between the
// actual frames in the test file and their position in the reference video, so
// that the analysis could run with the right frames from both videos. The stats
// file should be in the form 'frame_xxxx yyyy', where xxxx is the consecutive
// number of the frame in the test video, and yyyy is the equivalent frame in
// the reference video. The stats file could be produced by
// actual frames in the test file and their bar code number. There is one file
// for the reference video and one for the test video. The stats file should
// be in the form 'frame_xxxx yyyy', where xxxx is the consecutive
// number of the frame in the test video, and yyyy is the barcode number.
// The stats file could be produced by
// tools/barcode_tools/barcode_decoder.py. This script decodes the barcodes
// integrated in every video and generates the stats file. If three was some
// problem with the decoding there would be 'Barcode error' instead of yyyy.
void RunAnalysis(const char* reference_file_name, const char* test_file_name,
const char* stats_file_name, int width, int height,
// The stat files are used to compare the right frames with each other and
// to calculate statistics.
void RunAnalysis(const char* reference_file_name,
const char* test_file_name,
const char* stats_file_reference_name,
const char* stats_file_test_name,
int width,
int height,
ResultsContainer* results);
// Compute PSNR or SSIM for an I420 frame (all planes). When we are calculating
@ -79,11 +85,14 @@ void PrintAnalysisResults(FILE* output, const std::string& label,
// Calculates max repeated and skipped frames and prints them to stdout in a
// format that is compatible with Chromium performance numbers.
void PrintMaxRepeatedAndSkippedFrames(const std::string& label,
const std::string& stats_file_name);
const std::string& stats_file_ref_name,
const std::string& stats_file_test_name);
// Similar to the above, but will print to the specified file handle.
void PrintMaxRepeatedAndSkippedFrames(FILE* output, const std::string& label,
const std::string& stats_file_name);
void PrintMaxRepeatedAndSkippedFrames(FILE* output,
const std::string& label,
const std::string& stats_file_ref_name,
const std::string& stats_file_test_name);
// Gets the next line from an open stats file.
bool GetNextStatsLine(FILE* stats_file, char* line);

31
webrtc/tools/frame_analyzer/video_quality_analysis_unittest.cc
View File

@ -85,24 +85,42 @@ TEST_F(VideoQualityAnalysisTest, PrintAnalysisResultsThreeFrames) {
}
TEST_F(VideoQualityAnalysisTest, PrintMaxRepeatedAndSkippedFramesInvalidFile) {
std::string stats_filename = OutputPath() + "non-existing-stats-file.txt";
std::string stats_filename_ref =
OutputPath() + "non-existing-stats-file-1.txt";
std::string stats_filename = OutputPath() + "non-existing-stats-file-2.txt";
remove(stats_filename.c_str());
PrintMaxRepeatedAndSkippedFrames(logfile_, "NonExistingStatsFile",
stats_filename);
stats_filename_ref, stats_filename);
}
TEST_F(VideoQualityAnalysisTest,
PrintMaxRepeatedAndSkippedFramesEmptyStatsFile) {
std::string stats_filename = OutputPath() + "empty-stats.txt";
std::string stats_filename_ref = OutputPath() + "empty-stats-1.txt";
std::string stats_filename = OutputPath() + "empty-stats-2.txt";
std::ofstream stats_file;
stats_file.open(stats_filename_ref.c_str());
stats_file.close();
stats_file.open(stats_filename.c_str());
stats_file.close();
PrintMaxRepeatedAndSkippedFrames(logfile_, "EmptyStatsFile", stats_filename);
PrintMaxRepeatedAndSkippedFrames(logfile_, "EmptyStatsFile",
stats_filename_ref, stats_filename);
}
TEST_F(VideoQualityAnalysisTest, PrintMaxRepeatedAndSkippedFramesNormalFile) {
std::string stats_filename = OutputPath() + "stats.txt";
std::string stats_filename_ref = OutputPath() + "stats-1.txt";
std::string stats_filename = OutputPath() + "stats-2.txt";
std::ofstream stats_file;
stats_file.open(stats_filename_ref.c_str());
stats_file << "frame_0001 0100\n";
stats_file << "frame_0002 0101\n";
stats_file << "frame_0003 0102\n";
stats_file << "frame_0004 0103\n";
stats_file << "frame_0005 0106\n";
stats_file << "frame_0006 0107\n";
stats_file << "frame_0007 0108\n";
stats_file.close();
stats_file.open(stats_filename.c_str());
stats_file << "frame_0001 0100\n";
stats_file << "frame_0002 0101\n";
@ -110,7 +128,8 @@ TEST_F(VideoQualityAnalysisTest, PrintMaxRepeatedAndSkippedFramesNormalFile) {
stats_file << "frame_0004 0106\n";
stats_file.close();
PrintMaxRepeatedAndSkippedFrames(logfile_, "NormalStatsFile", stats_filename);
PrintMaxRepeatedAndSkippedFrames(logfile_, "NormalStatsFile",
stats_filename_ref, stats_filename);
}