diff --git a/webrtc/tools/frame_analyzer/video_quality_analysis.cc b/webrtc/tools/frame_analyzer/video_quality_analysis.cc index d1b0e314e1..66ca469ecd 100644 --- a/webrtc/tools/frame_analyzer/video_quality_analysis.cc +++ b/webrtc/tools/frame_analyzer/video_quality_analysis.cc @@ -325,23 +325,32 @@ void PrintMaxRepeatedAndSkippedFrames(const std::string& label, stats_file_test_name); } -namespace { -// Clusters the frames in the file. First in the pair is the frame number and -// second is the number -// of frames in that cluster. So if first frame in video has number 100 and it -// is repeated 3 after -// each other, then the first entry in the returned vector has first set to 100 -// and second set -// to 3. -std::vector > CalculateFrameClusters(FILE* file) { +std::vector > CalculateFrameClusters( + FILE* file, + int* num_decode_errors) { + if (num_decode_errors) { + *num_decode_errors = 0; + } std::vector > frame_cnt; char line[STATS_LINE_LENGTH]; while (GetNextStatsLine(file, line)) { - int decoded_frame_number = ExtractDecodedFrameNumber(line); - if (decoded_frame_number == -1) { - continue; + int decoded_frame_number; + if (IsThereBarcodeError(line)) { + decoded_frame_number = DECODE_ERROR; + if (num_decode_errors) { + ++*num_decode_errors; + } + } else { + decoded_frame_number = ExtractDecodedFrameNumber(line); } - if (frame_cnt.empty() || frame_cnt.back().first != decoded_frame_number) { + if (frame_cnt.size() >= 2 && decoded_frame_number != DECODE_ERROR && + frame_cnt.back().first == DECODE_ERROR && + frame_cnt[frame_cnt.size() - 2].first == decoded_frame_number) { + // Handle when there is a decoding error inside a cluster of frames. + frame_cnt[frame_cnt.size() - 2].second += frame_cnt.back().second + 1; + frame_cnt.pop_back(); + } else if (frame_cnt.empty() || + frame_cnt.back().first != decoded_frame_number) { frame_cnt.push_back(std::make_pair(decoded_frame_number, 1)); } else { ++frame_cnt.back().second; @@ -349,7 +358,6 @@ std::vector > CalculateFrameClusters(FILE* file) { } return frame_cnt; } -} // namespace void PrintMaxRepeatedAndSkippedFrames(FILE* output, const std::string& label, @@ -370,13 +378,16 @@ void PrintMaxRepeatedAndSkippedFrames(FILE* output, } int max_repeated_frames = 1; - int max_skipped_frames = 1; + int max_skipped_frames = 0; + + int decode_errors_ref = 0; + int decode_errors_test = 0; std::vector > frame_cnt_ref = - CalculateFrameClusters(stats_file_ref); + CalculateFrameClusters(stats_file_ref, &decode_errors_ref); std::vector > frame_cnt_test = - CalculateFrameClusters(stats_file_test); + CalculateFrameClusters(stats_file_test, &decode_errors_test); fclose(stats_file_ref); fclose(stats_file_test); @@ -391,9 +402,19 @@ void PrintMaxRepeatedAndSkippedFrames(FILE* output, return; } + while (it_test != end_test && it_test->first == DECODE_ERROR) { + ++it_test; + } + + if (it_test == end_test) { + fprintf(stderr, "Test video only has barcode decode errors\n"); + return; + } + // Find the first frame in the reference video that match the first frame in // the test video. - while (it_ref != end_ref && it_ref->first != it_test->first) { + while (it_ref != end_ref && + (it_ref->first == DECODE_ERROR || it_ref->first != it_test->first)) { ++it_ref; } if (it_ref == end_ref) { @@ -403,33 +424,58 @@ void PrintMaxRepeatedAndSkippedFrames(FILE* output, return; } + int total_skipped_frames = 0; for (;;) { max_repeated_frames = std::max(max_repeated_frames, it_test->second - it_ref->second + 1); + + bool passed_error = false; + ++it_test; + while (it_test != end_test && it_test->first == DECODE_ERROR) { + ++it_test; + passed_error = true; + } if (it_test == end_test) { break; } + int skipped_frames = 0; ++it_ref; - while (it_ref != end_ref && it_ref->first != it_test->first) { - skipped_frames += it_ref->second; - ++it_ref; + for (; it_ref != end_ref; ++it_ref) { + if (it_ref->first != DECODE_ERROR && it_ref->first >= it_test->first) { + break; + } + ++skipped_frames; } - if (it_ref == end_ref) { - fprintf(stderr, - "The barcode in the test video is not in the reference video.\n"); - return; + if (passed_error) { + // If we pass an error in the test video, then we are conservative + // and will not calculate skipped frames for that part. + skipped_frames = 0; } - if (skipped_frames > max_skipped_frames) { - max_skipped_frames = skipped_frames; + if (it_ref != end_ref && it_ref->first == it_test->first) { + total_skipped_frames += skipped_frames; + if (skipped_frames > max_skipped_frames) { + max_skipped_frames = skipped_frames; + } + continue; } + fprintf(stderr, + "Found barcode %d in test video, which is not in reference video", + it_test->first); + return; } fprintf(output, "RESULT Max_repeated: %s= %d\n", label.c_str(), max_repeated_frames); fprintf(output, "RESULT Max_skipped: %s= %d\n", label.c_str(), max_skipped_frames); + fprintf(output, "RESULT Total_skipped: %s= %d\n", label.c_str(), + total_skipped_frames); + fprintf(output, "RESULT Decode_errors_reference: %s= %d\n", label.c_str(), + decode_errors_ref); + fprintf(output, "RESULT Decode_errors_test: %s= %d\n", label.c_str(), + decode_errors_test); } void PrintAnalysisResults(const std::string& label, ResultsContainer* results) { diff --git a/webrtc/tools/frame_analyzer/video_quality_analysis.h b/webrtc/tools/frame_analyzer/video_quality_analysis.h index 93d3605410..b72ad9ab21 100644 --- a/webrtc/tools/frame_analyzer/video_quality_analysis.h +++ b/webrtc/tools/frame_analyzer/video_quality_analysis.h @@ -13,6 +13,7 @@ #include #include +#include #include "libyuv/compare.h" // NOLINT #include "libyuv/convert.h" // NOLINT @@ -82,6 +83,23 @@ void PrintAnalysisResults(const std::string& label, ResultsContainer* results); void PrintAnalysisResults(FILE* output, const std::string& label, ResultsContainer* results); +// The barcode number that means that the barcode could not be decoded. +const int DECODE_ERROR = -1; + +// Clusters the frames in the file. First in the pair is the frame number and +// second is the number of frames in that cluster. So if first frame in video +// has number 100 and it is repeated 3 after each other, then the first entry +// in the returned vector has first set to 100 and second set to 3. +// Decode errors between two frames with same barcode, then it interprets +// the frame with the decode error as having the same id as the two frames +// around it. Eg. [400, DECODE_ERROR, DECODE_ERROR, 400] is becomes an entry +// in return vector with first==400 and second==4. In other cases with decode +// errors like [400, DECODE_ERROR, 401] becomes three entries, each with +// second==1 and the middle has first==DECODE_ERROR. +std::vector > CalculateFrameClusters( + FILE* file, + int* num_decode_errors); + // Calculates max repeated and skipped frames and prints them to stdout in a // format that is compatible with Chromium performance numbers. void PrintMaxRepeatedAndSkippedFrames(const std::string& label, diff --git a/webrtc/tools/frame_analyzer/video_quality_analysis_unittest.cc b/webrtc/tools/frame_analyzer/video_quality_analysis_unittest.cc index 85d0b46e79..aba849d2b3 100644 --- a/webrtc/tools/frame_analyzer/video_quality_analysis_unittest.cc +++ b/webrtc/tools/frame_analyzer/video_quality_analysis_unittest.cc @@ -26,18 +26,20 @@ namespace test { // want those numbers to be picked up as perf numbers. class VideoQualityAnalysisTest : public ::testing::Test { protected: - static void SetUpTestCase() { - std::string log_filename = webrtc::test::OutputPath() + - "VideoQualityAnalysisTest.log"; + void SetUp() { + std::string log_filename = TempFilename(webrtc::test::OutputPath(), + "VideoQualityAnalysisTest.log"); logfile_ = fopen(log_filename.c_str(), "w"); ASSERT_TRUE(logfile_ != NULL); + + stats_filename_ref_ = TempFilename(OutputPath(), "stats-1.txt"); + stats_filename_ = TempFilename(OutputPath(), "stats-2.txt"); } - static void TearDownTestCase() { - ASSERT_EQ(0, fclose(logfile_)); - } - static FILE* logfile_; + void TearDown() { ASSERT_EQ(0, fclose(logfile_)); } + FILE* logfile_; + std::string stats_filename_ref_; + std::string stats_filename_; }; -FILE* VideoQualityAnalysisTest::logfile_ = NULL; TEST_F(VideoQualityAnalysisTest, MatchExtractedY4mFrame) { std::string video_file = @@ -85,33 +87,26 @@ TEST_F(VideoQualityAnalysisTest, PrintAnalysisResultsThreeFrames) { } TEST_F(VideoQualityAnalysisTest, PrintMaxRepeatedAndSkippedFramesInvalidFile) { - std::string stats_filename_ref = - OutputPath() + "non-existing-stats-file-1.txt"; - std::string stats_filename = OutputPath() + "non-existing-stats-file-2.txt"; - remove(stats_filename.c_str()); + remove(stats_filename_.c_str()); PrintMaxRepeatedAndSkippedFrames(logfile_, "NonExistingStatsFile", - stats_filename_ref, stats_filename); + stats_filename_ref_, stats_filename_); } TEST_F(VideoQualityAnalysisTest, PrintMaxRepeatedAndSkippedFramesEmptyStatsFile) { - std::string stats_filename_ref = OutputPath() + "empty-stats-1.txt"; - std::string stats_filename = OutputPath() + "empty-stats-2.txt"; std::ofstream stats_file; - stats_file.open(stats_filename_ref.c_str()); + stats_file.open(stats_filename_ref_.c_str()); stats_file.close(); - stats_file.open(stats_filename.c_str()); + stats_file.open(stats_filename_.c_str()); stats_file.close(); PrintMaxRepeatedAndSkippedFrames(logfile_, "EmptyStatsFile", - stats_filename_ref, stats_filename); + stats_filename_ref_, stats_filename_); } TEST_F(VideoQualityAnalysisTest, PrintMaxRepeatedAndSkippedFramesNormalFile) { - std::string stats_filename_ref = OutputPath() + "stats-1.txt"; - std::string stats_filename = OutputPath() + "stats-2.txt"; std::ofstream stats_file; - stats_file.open(stats_filename_ref.c_str()); + stats_file.open(stats_filename_ref_.c_str()); stats_file << "frame_0001 0100\n"; stats_file << "frame_0002 0101\n"; stats_file << "frame_0003 0102\n"; @@ -121,7 +116,7 @@ TEST_F(VideoQualityAnalysisTest, PrintMaxRepeatedAndSkippedFramesNormalFile) { stats_file << "frame_0007 0108\n"; stats_file.close(); - stats_file.open(stats_filename.c_str()); + stats_file.open(stats_filename_.c_str()); stats_file << "frame_0001 0100\n"; stats_file << "frame_0002 0101\n"; stats_file << "frame_0003 0101\n"; @@ -129,9 +124,217 @@ TEST_F(VideoQualityAnalysisTest, PrintMaxRepeatedAndSkippedFramesNormalFile) { stats_file.close(); PrintMaxRepeatedAndSkippedFrames(logfile_, "NormalStatsFile", - stats_filename_ref, stats_filename); + stats_filename_ref_, stats_filename_); } +namespace { +void VerifyLogOutput(const std::string& log_filename, + const std::vector& expected_out) { + std::ifstream logf(log_filename); + std::string line; + std::size_t i; + for (i = 0; i < expected_out.size() && getline(logf, line); ++i) { + ASSERT_EQ(expected_out.at(i), line); + } + ASSERT_TRUE(i == expected_out.size()) << "Not enough input data"; +} +} // unnamed namespace + +TEST_F(VideoQualityAnalysisTest, + PrintMaxRepeatedAndSkippedFramesSkippedFrames) { + std::ofstream stats_file; + + std::string log_filename = + TempFilename(webrtc::test::OutputPath(), "log.log"); + FILE* logfile = fopen(log_filename.c_str(), "w"); + ASSERT_TRUE(logfile != NULL); + stats_file.open(stats_filename_ref_.c_str()); + stats_file << "frame_0001 0100\n"; + stats_file << "frame_0002 0101\n"; + stats_file << "frame_0002 0101\n"; + stats_file << "frame_0003 0103\n"; + stats_file << "frame_0004 0103\n"; + stats_file << "frame_0005 0106\n"; + stats_file << "frame_0006 0106\n"; + stats_file << "frame_0007 0108\n"; + stats_file << "frame_0008 0110\n"; + stats_file << "frame_0009 0112\n"; + stats_file.close(); + + stats_file.open(stats_filename_.c_str()); + stats_file << "frame_0001 0101\n"; + stats_file << "frame_0002 0101\n"; + stats_file << "frame_0003 0101\n"; + stats_file << "frame_0004 0108\n"; + stats_file << "frame_0005 0108\n"; + stats_file << "frame_0006 0112\n"; + stats_file.close(); + + PrintMaxRepeatedAndSkippedFrames(logfile, "NormalStatsFile", + stats_filename_ref_, stats_filename_); + ASSERT_EQ(0, fclose(logfile)); + + std::vector expected_out = { + "RESULT Max_repeated: NormalStatsFile= 2", + "RESULT Max_skipped: NormalStatsFile= 2", + "RESULT Total_skipped: NormalStatsFile= 3", + "RESULT Decode_errors_reference: NormalStatsFile= 0", + "RESULT Decode_errors_test: NormalStatsFile= 0"}; + VerifyLogOutput(log_filename, expected_out); +} + +TEST_F(VideoQualityAnalysisTest, + PrintMaxRepeatedAndSkippedFramesDecodeErrorInTest) { + std::ofstream stats_file; + + std::string log_filename = + TempFilename(webrtc::test::OutputPath(), "log.log"); + FILE* logfile = fopen(log_filename.c_str(), "w"); + ASSERT_TRUE(logfile != NULL); + stats_file.open(stats_filename_ref_.c_str()); + stats_file << "frame_0001 0100\n"; + stats_file << "frame_0002 0100\n"; + stats_file << "frame_0002 0101\n"; + stats_file << "frame_0003 0103\n"; + stats_file << "frame_0004 0103\n"; + stats_file << "frame_0005 0106\n"; + stats_file << "frame_0006 0107\n"; + stats_file << "frame_0007 0107\n"; + stats_file << "frame_0008 0110\n"; + stats_file << "frame_0009 0112\n"; + stats_file.close(); + + stats_file.open(stats_filename_.c_str()); + stats_file << "frame_0001 0101\n"; + stats_file << "frame_0002 Barcode error\n"; + stats_file << "frame_0003 Barcode error\n"; + stats_file << "frame_0004 Barcode error\n"; + stats_file << "frame_0005 0107\n"; + stats_file << "frame_0006 0110\n"; + stats_file.close(); + + PrintMaxRepeatedAndSkippedFrames(logfile, "NormalStatsFile", + stats_filename_ref_, stats_filename_); + ASSERT_EQ(0, fclose(logfile)); + + std::vector expected_out = { + "RESULT Max_repeated: NormalStatsFile= 1", + "RESULT Max_skipped: NormalStatsFile= 0", + "RESULT Total_skipped: NormalStatsFile= 0", + "RESULT Decode_errors_reference: NormalStatsFile= 0", + "RESULT Decode_errors_test: NormalStatsFile= 3"}; + VerifyLogOutput(log_filename, expected_out); +} + +TEST_F(VideoQualityAnalysisTest, CalculateFrameClustersOneValue) { + std::ofstream stats_file; + + stats_file.open(stats_filename_.c_str()); + stats_file << "frame_0001 0101\n"; + stats_file.close(); + + FILE* stats_filef = fopen(stats_filename_.c_str(), "r"); + ASSERT_TRUE(stats_filef != NULL); + + auto clusters = CalculateFrameClusters(stats_filef, nullptr); + ASSERT_EQ(0, fclose(stats_filef)); + decltype(clusters) expected = {std::make_pair(101, 1)}; + ASSERT_EQ(expected, clusters); +} + +TEST_F(VideoQualityAnalysisTest, CalculateFrameClustersOneOneTwo) { + std::ofstream stats_file; + + stats_file.open(stats_filename_.c_str()); + stats_file << "frame_0001 0101\n"; + stats_file << "frame_0002 0101\n"; + stats_file << "frame_0003 0102\n"; + stats_file.close(); + + FILE* stats_filef = fopen(stats_filename_.c_str(), "r"); + ASSERT_TRUE(stats_filef != NULL); + + auto clusters = CalculateFrameClusters(stats_filef, nullptr); + ASSERT_EQ(0, fclose(stats_filef)); + decltype(clusters) expected = {std::make_pair(101, 2), + std::make_pair(102, 1)}; + ASSERT_EQ(expected, clusters); +} + +TEST_F(VideoQualityAnalysisTest, CalculateFrameClustersOneOneErrErrThree) { + std::ofstream stats_file; + + stats_file.open(stats_filename_.c_str()); + stats_file << "frame_0001 0101\n"; + stats_file << "frame_0002 0101\n"; + stats_file << "frame_0003 Barcode error\n"; + stats_file << "frame_0004 Barcode error\n"; + stats_file << "frame_0005 0103\n"; + stats_file.close(); + + FILE* stats_filef = fopen(stats_filename_.c_str(), "r"); + ASSERT_TRUE(stats_filef != NULL); + + auto clusters = CalculateFrameClusters(stats_filef, nullptr); + ASSERT_EQ(0, fclose(stats_filef)); + decltype(clusters) expected = {std::make_pair(101, 2), + std::make_pair(DECODE_ERROR, 2), + std::make_pair(103, 1)}; + ASSERT_EQ(expected, clusters); +} + +TEST_F(VideoQualityAnalysisTest, CalculateFrameClustersErrErr) { + std::ofstream stats_file; + + stats_file.open(stats_filename_.c_str()); + stats_file << "frame_0001 Barcode error\n"; + stats_file << "frame_0002 Barcode error\n"; + stats_file.close(); + + FILE* stats_filef = fopen(stats_filename_.c_str(), "r"); + ASSERT_TRUE(stats_filef != NULL); + + auto clusters = CalculateFrameClusters(stats_filef, nullptr); + ASSERT_EQ(0, fclose(stats_filef)); + decltype(clusters) expected = {std::make_pair(DECODE_ERROR, 2)}; + ASSERT_EQ(expected, clusters); +} + +TEST_F(VideoQualityAnalysisTest, CalculateFrameClustersOneOneErrErrOneOne) { + std::ofstream stats_file; + + stats_file.open(stats_filename_.c_str()); + stats_file << "frame_0001 0101\n"; + stats_file << "frame_0002 0101\n"; + stats_file << "frame_0003 Barcode error\n"; + stats_file << "frame_0004 Barcode error\n"; + stats_file << "frame_0005 0101\n"; + stats_file << "frame_0006 0101\n"; + stats_file.close(); + + FILE* stats_filef = fopen(stats_filename_.c_str(), "r"); + ASSERT_TRUE(stats_filef != NULL); + + auto clusters = CalculateFrameClusters(stats_filef, nullptr); + ASSERT_EQ(0, fclose(stats_filef)); + decltype(clusters) expected = {std::make_pair(101, 6)}; + ASSERT_EQ(expected, clusters); +} + +TEST_F(VideoQualityAnalysisTest, CalculateFrameClustersEmpty) { + std::ofstream stats_file; + + stats_file.open(stats_filename_.c_str()); + stats_file.close(); + + FILE* stats_filef = fopen(stats_filename_.c_str(), "r"); + ASSERT_TRUE(stats_filef != NULL); + + auto clusters = CalculateFrameClusters(stats_filef, nullptr); + ASSERT_EQ(0, fclose(stats_filef)); + decltype(clusters) expected; + ASSERT_EQ(expected, clusters); +} } // namespace test } // namespace webrtc