diff --git a/src/modules/video_coding/codecs/test/video_codecs_test_framework.gypi b/src/modules/video_coding/codecs/test/video_codecs_test_framework.gypi index c38a5c8c00..6bc42807a7 100644 --- a/src/modules/video_coding/codecs/test/video_codecs_test_framework.gypi +++ b/src/modules/video_coding/codecs/test/video_codecs_test_framework.gypi @@ -50,6 +50,7 @@ 'webrtc_video_coding', 'webrtc_vp8', '<(webrtc_root)/../testing/gtest.gyp:gtest', + '<(webrtc_root)/../test/metrics.gyp:metrics', '<(webrtc_root)/../test/test.gyp:test_support_main', ], 'sources': [ diff --git a/src/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc b/src/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc index 6cc710f186..26435d829c 100644 --- a/src/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc +++ b/src/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc @@ -117,20 +117,18 @@ class VideoProcessorIntegrationTest: public testing::Test { frame_reader_->Close(); frame_writer_->Close(); - QualityMetricsResult result; - EXPECT_EQ(0, PsnrFromFiles(config_.input_filename.c_str(), - config_.output_filename.c_str(), - config_.codec_settings->width, - config_.codec_settings->height, &result)); - EXPECT_GT(result.average, minimum_psnr + 2.0); - EXPECT_GT(result.min, minimum_psnr); - - EXPECT_EQ(0, SsimFromFiles(config_.input_filename.c_str(), - config_.output_filename.c_str(), - config_.codec_settings->width, - config_.codec_settings->height, &result)); - EXPECT_GT(result.average, minimum_ssim + 0.1); - EXPECT_GT(result.min, minimum_ssim); + webrtc::test::QualityMetricsResult psnr_result, ssim_result; + EXPECT_EQ(0, webrtc::test::I420MetricsFromFiles( + config_.input_filename.c_str(), + config_.output_filename.c_str(), + config_.codec_settings->width, + config_.codec_settings->height, + &psnr_result, + &ssim_result)); + EXPECT_GT(psnr_result.average, minimum_psnr + 2.0); + EXPECT_GT(psnr_result.min, minimum_psnr); + EXPECT_GT(ssim_result.average, minimum_ssim + 0.1); + EXPECT_GT(ssim_result.min, minimum_ssim); } }; diff --git a/src/modules/video_coding/codecs/test_framework/benchmark.cc b/src/modules/video_coding/codecs/test_framework/benchmark.cc index 5c7364494b..883330c2a8 100644 --- a/src/modules/video_coding/codecs/test_framework/benchmark.cc +++ b/src/modules/video_coding/codecs/test_framework/benchmark.cc @@ -11,7 +11,6 @@ #include "benchmark.h" #include -#include #include #include #include @@ -19,12 +18,11 @@ #include #endif -#include "event_wrapper.h" -#include "testsupport/fileutils.h" #include "common_video/libyuv/include/libyuv.h" -#include "video_codec_interface.h" -#include "video_source.h" - +#include "system_wrappers/interface/event_wrapper.h" +#include "modules/video_coding/codecs/test_framework/video_source.h" +#include "testsupport/fileutils.h" +#include "testsupport/metrics/video_metrics.h" #define SSIM_CALC 0 // by default, don't compute SSIM @@ -80,8 +78,8 @@ Benchmark::Perform() const int nBitrates = sizeof(bitRate)/sizeof(*bitRate); int testIterations = 10; - double psnr[nBitrates]; - double ssim[nBitrates]; + webrtc::test::QualityMetricsResult psnr[nBitrates]; + webrtc::test::QualityMetricsResult ssim[nBitrates]; double fps[nBitrates]; double totalEncodeTime[nBitrates]; double totalDecodeTime[nBitrates]; @@ -153,12 +151,14 @@ Benchmark::Perform() double actualBitRate = ActualBitRate(_framecnt) / 1000.0; std::cout << " " << actualBitRate; _results << "," << actualBitRate; - PSNRfromFiles(_inname.c_str(), _outname.c_str(), _inst.width, - _inst.height, &psnr[k]); + webrtc::test::QualityMetricsResult psnr_result; + I420PSNRFromFiles(_inname.c_str(), _outname.c_str(), + _inst.width, _inst.height, &psnr[k]); if (SSIM_CALC) { - SSIMfromFiles(_inname.c_str(), _outname.c_str(), _inst.width, - _inst.height, &ssim[k]); + webrtc::test::QualityMetricsResult ssim_result; + I420SSIMFromFiles(_inname.c_str(), _outname.c_str(), + _inst.width, _inst.height, &ssim[k]); } fps[k] = avgFps; @@ -167,8 +167,8 @@ Benchmark::Perform() _results << std::endl << "Y-PSNR [dB]"; for (int k = 0; k < nBitrates; k++) { - std::cout << " " << psnr[k]; - _results << "," << psnr[k]; + std::cout << " " << psnr[k].average; + _results << "," << psnr[k].average; } if (SSIM_CALC) @@ -177,8 +177,8 @@ Benchmark::Perform() _results << std::endl << "SSIM "; for (int k = 0; k < nBitrates; k++) { - std::cout << " " << ssim[k]; - _results << "," << ssim[k]; + std::cout << " " << ssim[k].average; + _results << "," << ssim[k].average; } } diff --git a/src/modules/video_coding/codecs/test_framework/test.cc b/src/modules/video_coding/codecs/test_framework/test.cc index ae882493f7..3eb6935092 100644 --- a/src/modules/video_coding/codecs/test_framework/test.cc +++ b/src/modules/video_coding/codecs/test_framework/test.cc @@ -9,41 +9,16 @@ */ #include "test.h" -#include "video_source.h" -#include "common_video/libyuv/include/libyuv.h" -#include "event_wrapper.h" -#include "thread_wrapper.h" + +#include #include -#include -#include -#include -#include -#include -#include + +#include "testsupport/metrics/video_metrics.h" using namespace webrtc; long filesize(const char *filename); // local function defined at end of file -struct SSIMcontext -{ - SSIMcontext() : - refFileName(NULL), testFileName(NULL), width(0), height(0), - SSIMptr(NULL), startFrame(-1), endFrame(-1), evnt(NULL) {}; - SSIMcontext(const char *ref, const char *test, int w, int h, double *Sptr, - int start, int end, EventWrapper* ev) : - refFileName(ref), testFileName(test), width(w), height(h), - SSIMptr(Sptr), startFrame(start), endFrame(end), evnt(ev) {}; - const char *refFileName; - const char *testFileName; - int width; - int height; - double *SSIMptr; - int startFrame; - int endFrame; - EventWrapper* evnt; -}; - Test::Test(std::string name, std::string description) : _bitRate(0), @@ -80,14 +55,17 @@ Test::Print() (*_log) << _description << std::endl; (*_log) << "Input file: " << _inname << std::endl; (*_log) << "Output file: " << _outname << std::endl; - double psnr = -1.0, ssim = -1.0; - PSNRfromFiles(_inname.c_str(), _outname.c_str(), _inst.width, _inst.height, &psnr); - ssim = SSIMfromFilesMT(4 /* number of threads*/); + webrtc::test::QualityMetricsResult psnr; + webrtc::test::QualityMetricsResult ssim; + I420PSNRFromFiles(_inname.c_str(), _outname.c_str(), _inst.width, + _inst.height, &psnr); + I420SSIMFromFiles(_inname.c_str(), _outname.c_str(), _inst.width, + _inst.height, &ssim); - (*_log) << "PSNR: " << psnr << std::endl; - std::cout << "PSNR: " << psnr << std::endl << std::endl; - (*_log) << "SSIM: " << ssim << std::endl; - std::cout << "SSIM: " << ssim << std::endl << std::endl; + (*_log) << "PSNR: " << psnr.average << std::endl; + std::cout << "PSNR: " << psnr.average << std::endl << std::endl; + (*_log) << "SSIM: " << ssim.average << std::endl; + std::cout << "SSIM: " << ssim.average << std::endl << std::endl; (*_log) << std::endl; } @@ -143,354 +121,6 @@ Test::SetLog(std::fstream* log) _log = log; } -int -Test::PSNRfromFiles(const char *refFileName, const char *testFileName, int width, int height, double *YPSNRptr) -{ - FILE *refFp = fopen(refFileName, "rb"); - if( refFp == NULL ) { - // cannot open reference file - fprintf(stderr, "Cannot open file %s\n", refFileName); - return -1; - } - - FILE *testFp = fopen(testFileName, "rb"); - if( testFp == NULL ) { - // cannot open test file - fprintf(stderr, "Cannot open file %s\n", testFileName); - return -2; - } - - double mse = 0.0; - double mseLogSum = 0.0; - int frames = 0; - - int frameBytes = 3*width*height/2; // bytes in one frame I420 - unsigned char *ref = new unsigned char[frameBytes]; // space for one frame I420 - unsigned char *test = new unsigned char[frameBytes]; // space for one frame I420 - - int refBytes = (int) fread(ref, 1, frameBytes, refFp); - int testBytes = (int) fread(test, 1, frameBytes, testFp); - - while( refBytes == frameBytes && testBytes == frameBytes ) - { - mse = 0.0; - - // calculate Y sum-square-difference - for( int k = 0; k < width * height; k++ ) - { - mse += (test[k] - ref[k]) * (test[k] - ref[k]); - } - - // divide by number of pixels - mse /= (double) (width * height); - - // accumulate for total average - mseLogSum += std::log10( mse ); - frames++; - - refBytes = (int) fread(ref, 1, frameBytes, refFp); - testBytes = (int) fread(test, 1, frameBytes, testFp); - } - - // ypsnrAvg = sum( 10 log (255^2 / MSE) ) / frames - // = 20 * log(255) - 10 * mseLogSum / frames - *YPSNRptr = 20.0 * std::log10(255.0) - 10.0 * mseLogSum / frames; - - delete [] ref; - delete [] test; - - fclose(refFp); - fclose(testFp); - - return 0; -} -int -Test::SSIMfromFiles(const char *refFileName, const char *testFileName, int width, int height, double *SSIMptr, - int startFrame /*= -1*/, int endFrame /*= -1*/) -{ - FILE *refFp = fopen(refFileName, "rb"); - if( refFp == NULL ) { - // cannot open reference file - fprintf(stderr, "Cannot open file %s\n", refFileName); - return -1; - } - - FILE *testFp = fopen(testFileName, "rb"); - if( testFp == NULL ) { - // cannot open test file - fprintf(stderr, "Cannot open file %s\n", testFileName); - return -2; - } - - int frames = 0; - - int frameBytes = 3*width*height/2; // bytes in one frame I420 - unsigned char *ref = new unsigned char[frameBytes]; // space for one frame I420 - unsigned char *test = new unsigned char[frameBytes]; // space for one frame I420 - - if (startFrame >= 0) - { - if (fseek(refFp, frameBytes * startFrame, SEEK_SET) != 0){ - fprintf(stderr, "Cannot go to frame %i in %s\n", startFrame, refFileName); - return -1; - } - if (fseek(testFp, frameBytes * startFrame, SEEK_SET) != 0){ - fprintf(stderr, "Cannot go to frame %i in %s\n", startFrame, testFileName); - return -1; - } - } - - int refBytes = (int) fread(ref, 1, frameBytes, refFp); - int testBytes = (int) fread(test, 1, frameBytes, testFp); - - // - // SSIM: variable definition, window function, initialization - int window = 10; - int flag_window = 0; //0 for uniform window filter, 1 for gaussian symmetric window - float variance_window = 2.0; //variance for window function - float ssimFilter[121]; //2d window filter: typically 11x11 = (window+1)*(window+1) - //statistics per column of window (#columns = window+1), 0 element for avg over all columns - float avgTest[12]; - float avgRef[12]; - float contrastTest[12]; - float contrastRef[12]; - float crossCorr[12]; - // - //offsets for stability - float offset1 = 0.1f; - float offset2 = 0.1f; - float offset3 = offset2/2; - // - //define window for SSIM: take uniform filter for now - float sumfil = 0.0; - int nn=-1; - for(int j=-window/2;j<=window/2;j++) - for(int i=-window/2;i<=window/2;i++) - { - nn+=1; - if (flag_window == 0) - ssimFilter[nn] = 1.0; - else - { - float dist = (float)(i*i) + (float)(j*j); - float tmp = 0.5f*dist/variance_window; - ssimFilter[nn] = exp(-tmp); - } - sumfil +=ssimFilter[nn]; - } - //normalize window - nn=-1; - for(int j=-window/2;j<=window/2;j++) - for(int i=-window/2;i<=window/2;i++) - { - nn+=1; - ssimFilter[nn] = ssimFilter[nn]/((float)sumfil); - } - // - float ssimScene = 0.0; //avgerage SSIM for sequence - // - //SSIM: done with variables and defintion - // - - while( refBytes == frameBytes && testBytes == frameBytes && - !(endFrame >= 0 && frames > endFrame - startFrame)) - { - float ssimFrame = 0.0; - int sh = window/2+1; - int numPixels = 0; - for(int i=sh;irefFileName, ctx->testFileName, ctx->width, ctx->height, ctx->SSIMptr, ctx->startFrame, ctx->endFrame); - ctx->evnt->Set(); - return false; -} - -double Test::SSIMfromFilesMT(const int numThreads) -{ - int numFrames = filesize(_inname.c_str()) / _lengthSourceFrame; - std::vector nFramesVec(numThreads); - std::vector ssimVec(numThreads); - int framesPerCore = (numFrames + numThreads - 1) / numThreads; // rounding up - int i = 0; - int nFrames; - for (nFrames = numFrames; nFrames >= framesPerCore; nFrames -= framesPerCore) - { - nFramesVec[i++] = framesPerCore; - } - if (nFrames > 0) - { - assert(i == numThreads - 1); - nFramesVec[i] = nFrames; // remainder - } - - int frameIx = 0; - std::vector eventVec(numThreads); - std::vector threadVec(numThreads); - std::vector ctxVec(numThreads); - for (i = 0; i < numThreads; i++) - { - eventVec[i] = EventWrapper::Create(); - ctxVec[i] = SSIMcontext(_inname.c_str(), _outname.c_str(), _inst.width, _inst.height, &ssimVec[i], frameIx, frameIx + nFramesVec[i] - 1, eventVec[i]); - threadVec[i] = ThreadWrapper::CreateThread(SSIMthread, &(ctxVec[i]), kLowPriority); - unsigned int id; - threadVec[i]->Start(id); - frameIx += nFramesVec[i]; - } - - // wait for all events - for (i = 0; i < numThreads; i++) { - eventVec[i]->Wait(100000 /* ms*/); - threadVec[i]->Stop(); - delete threadVec[i]; - delete eventVec[i]; - } - - double avgSsim = 0; - for (i = 0; i < numThreads; i++) - { - avgSsim += (ssimVec[i] * nFramesVec[i]); - } - - avgSsim /= numFrames; - return avgSsim; -} - - double Test::ActualBitRate(int nFrames) { return 8.0 * _sumEncBytes / (nFrames / _inst.maxFramerate); @@ -526,11 +156,10 @@ Test::VideoEncodedBufferToEncodedImage(TestVideoEncodedBuffer& videoBuffer, Enco long filesize(const char *filename) { -FILE *f = fopen(filename,"rb"); /* open the file in read only */ - -long size = 0; - if (fseek(f,0,SEEK_END)==0) /* seek was successful */ - size = ftell(f); - fclose(f); - return size; + FILE *f = fopen(filename,"rb"); /* open the file in read only */ + long size = 0; + if (fseek(f,0,SEEK_END)==0) /* seek was successful */ + size = ftell(f); + fclose(f); + return size; } diff --git a/src/modules/video_coding/codecs/test_framework/test.h b/src/modules/video_coding/codecs/test_framework/test.h index 1ac692399b..a31868b5c2 100644 --- a/src/modules/video_coding/codecs/test_framework/test.h +++ b/src/modules/video_coding/codecs/test_framework/test.h @@ -36,20 +36,6 @@ protected: WebRtc_UWord32 frameRate=30, WebRtc_UWord32 bitRate=0); virtual void Teardown(); - static int PSNRfromFiles(const char *refFileName, - const char *testFileName, - int width, - int height, - double *YPSNRptr); - static int SSIMfromFiles(const char *refFileName, - const char *testFileName, - int width, - int height, - double *SSIMptr, - int startByte = -1, int endByte = -1); - double SSIMfromFilesMT(int numThreads); - static bool SSIMthread(void *ctx); - double ActualBitRate(int nFrames); virtual bool PacketLoss(double lossRate, int /*thrown*/); static double RandUniform() { return (std::rand() + 1.0)/(RAND_MAX + 1.0); } diff --git a/src/modules/video_coding/codecs/test_framework/test_framework.gypi b/src/modules/video_coding/codecs/test_framework/test_framework.gypi index 6de3e46990..3639e9a370 100644 --- a/src/modules/video_coding/codecs/test_framework/test_framework.gypi +++ b/src/modules/video_coding/codecs/test_framework/test_framework.gypi @@ -16,6 +16,7 @@ 'type': '<(library)', 'dependencies': [ + '<(webrtc_root)/../test/metrics.gyp:metrics', '<(webrtc_root)/../test/test.gyp:test_support', '<(webrtc_root)/../testing/gtest.gyp:gtest', '<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers', diff --git a/src/modules/video_coding/codecs/tools/video_codecs_tools.gypi b/src/modules/video_coding/codecs/tools/video_codecs_tools.gypi index 2c72b27f0f..f43f285e66 100644 --- a/src/modules/video_coding/codecs/tools/video_codecs_tools.gypi +++ b/src/modules/video_coding/codecs/tools/video_codecs_tools.gypi @@ -18,6 +18,7 @@ 'video_codecs_test_framework', 'webrtc_video_coding', 'webrtc_vp8', + '<(webrtc_root)/../test/metrics.gyp:metrics', '<(webrtc_root)/../third_party/google-gflags/google-gflags.gyp:google-gflags', ], 'sources': [ diff --git a/src/modules/video_coding/codecs/tools/video_quality_measurement.cc b/src/modules/video_coding/codecs/tools/video_quality_measurement.cc index 59f042c0df..b48fa239d2 100644 --- a/src/modules/video_coding/codecs/tools/video_quality_measurement.cc +++ b/src/modules/video_coding/codecs/tools/video_quality_measurement.cc @@ -265,29 +265,27 @@ int HandleCommandLineFlags(webrtc::test::TestConfig* config) { } void CalculateSsimVideoMetrics(webrtc::test::TestConfig* config, - QualityMetricsResult* ssimResult) { + webrtc::test::QualityMetricsResult* result) { Log("Calculating SSIM...\n"); - SsimFromFiles(config->input_filename.c_str(), config->output_filename.c_str(), - config->codec_settings->width, - config->codec_settings->height, ssimResult); - Log(" Average: %3.2f\n", ssimResult->average); - Log(" Min : %3.2f (frame %d)\n", ssimResult->min, - ssimResult->min_frame_number); - Log(" Max : %3.2f (frame %d)\n", ssimResult->max, - ssimResult->max_frame_number); + I420SSIMFromFiles(config->input_filename.c_str(), + config->output_filename.c_str(), + config->codec_settings->width, + config->codec_settings->height, result); + Log(" Average: %3.2f\n", result->average); + Log(" Min : %3.2f (frame %d)\n", result->min, result->min_frame_number); + Log(" Max : %3.2f (frame %d)\n", result->max, result->max_frame_number); } void CalculatePsnrVideoMetrics(webrtc::test::TestConfig* config, - QualityMetricsResult* psnrResult) { + webrtc::test::QualityMetricsResult* result) { Log("Calculating PSNR...\n"); - PsnrFromFiles(config->input_filename.c_str(), config->output_filename.c_str(), + I420PSNRFromFiles(config->input_filename.c_str(), + config->output_filename.c_str(), config->codec_settings->width, - config->codec_settings->height, psnrResult); - Log(" Average: %3.2f\n", psnrResult->average); - Log(" Min : %3.2f (frame %d)\n", psnrResult->min, - psnrResult->min_frame_number); - Log(" Max : %3.2f (frame %d)\n", psnrResult->max, - psnrResult->max_frame_number); + config->codec_settings->height, result); + Log(" Average: %3.2f\n", result->average); + Log(" Min : %3.2f (frame %d)\n", result->min, result->min_frame_number); + Log(" Max : %3.2f (frame %d)\n", result->max, result->max_frame_number); } void PrintConfigurationSummary(const webrtc::test::TestConfig& config) { @@ -312,8 +310,8 @@ void PrintConfigurationSummary(const webrtc::test::TestConfig& config) { } void PrintCsvOutput(const webrtc::test::Stats& stats, - const QualityMetricsResult& ssimResult, - const QualityMetricsResult& psnrResult) { + const webrtc::test::QualityMetricsResult& ssim_result, + const webrtc::test::QualityMetricsResult& psnr_result) { Log("\nCSV output (recommended to run with --noverbose to skip the " "above output)\n"); printf("frame_number encoding_successful decoding_successful " @@ -325,8 +323,8 @@ void PrintCsvOutput(const webrtc::test::Stats& stats, for (unsigned int i = 0; i < stats.stats_.size(); ++i) { const webrtc::test::FrameStatistic& f = stats.stats_[i]; - const FrameResult& ssim = ssimResult.frames[i]; - const FrameResult& psnr = psnrResult.frames[i]; + const webrtc::test::FrameResult& ssim = ssim_result.frames[i]; + const webrtc::test::FrameResult& psnr = psnr_result.frames[i]; printf("%4d, %d, %d, %2d, %2d, %6d, %6d, %5d, %7d, %d, %2d, %2d, " "%5.3f, %5.2f\n", f.frame_number, @@ -348,8 +346,8 @@ void PrintCsvOutput(const webrtc::test::Stats& stats, void PrintPythonOutput(const webrtc::test::TestConfig& config, const webrtc::test::Stats& stats, - const QualityMetricsResult& ssimResult, - const QualityMetricsResult& psnrResult) { + const webrtc::test::QualityMetricsResult& ssim_result, + const webrtc::test::QualityMetricsResult& psnr_result) { Log("\nPython output (recommended to run with --noverbose to skip the " "above output)\n"); printf("test_configuration = [" @@ -412,8 +410,8 @@ void PrintPythonOutput(const webrtc::test::TestConfig& config, printf("frame_data = ["); for (unsigned int i = 0; i < stats.stats_.size(); ++i) { const webrtc::test::FrameStatistic& f = stats.stats_[i]; - const FrameResult& ssim = ssimResult.frames[i]; - const FrameResult& psnr = psnrResult.frames[i]; + const webrtc::test::FrameResult& ssim = ssim_result.frames[i]; + const webrtc::test::FrameResult& psnr = psnr_result.frames[i]; printf("{'frame_number': %d, " "'encoding_successful': %s, 'decoding_successful': %s, " "'encode_time': %d, 'decode_time': %d, " @@ -508,9 +506,9 @@ int main(int argc, char* argv[]) { stats.PrintSummary(); - QualityMetricsResult ssim_result; + webrtc::test::QualityMetricsResult ssim_result; CalculateSsimVideoMetrics(&config, &ssim_result); - QualityMetricsResult psnr_result; + webrtc::test::QualityMetricsResult psnr_result; CalculatePsnrVideoMetrics(&config, &psnr_result); if (FLAGS_csv) { diff --git a/src/modules/video_coding/main/source/video_coding_test.gypi b/src/modules/video_coding/main/source/video_coding_test.gypi index dd47850b9c..52e218142f 100644 --- a/src/modules/video_coding/main/source/video_coding_test.gypi +++ b/src/modules/video_coding/main/source/video_coding_test.gypi @@ -13,6 +13,7 @@ 'dependencies': [ '<(webrtc_root)/../testing/gtest.gyp:gtest', '<(webrtc_root)/../test/test.gyp:test_support', + '<(webrtc_root)/../test/metrics.gyp:metrics', 'webrtc_video_coding', 'rtp_rtcp', 'webrtc_utility', diff --git a/src/modules/video_coding/main/test/codec_database_test.cc b/src/modules/video_coding/main/test/codec_database_test.cc index 7341965b2c..c8829e82ba 100644 --- a/src/modules/video_coding/main/test/codec_database_test.cc +++ b/src/modules/video_coding/main/test/codec_database_test.cc @@ -368,8 +368,9 @@ CodecDataBaseTest::Perform(CmdArgs& args) _vcm->Decode(); // Don't measure PSNR for I420 since it will be perfect. if (sendCodec.codecType != kVideoCodecI420) { - QualityMetricsResult psnr; - PsnrFromFiles(_inname.c_str(), _outname.c_str(), _width, _height, &psnr); + webrtc::test::QualityMetricsResult psnr; + I420PSNRFromFiles(_inname.c_str(), _outname.c_str(), _width, + _height, &psnr); printf("\n @ %d KBPS: ", sendCodec.startBitrate); printf("PSNR from encoder-decoder send-receive control test" "is %f\n\n", psnr.average); diff --git a/src/modules/video_coding/main/test/media_opt_test.cc b/src/modules/video_coding/main/test/media_opt_test.cc index fa7a330643..f7679197b5 100644 --- a/src/modules/video_coding/main/test/media_opt_test.cc +++ b/src/modules/video_coding/main/test/media_opt_test.cc @@ -480,8 +480,9 @@ MediaOptTest::Print(int mode) { double ActualBitRate = 8.0 *( _sumEncBytes / (_frameCnt / _frameRate)); double actualBitRate = ActualBitRate / 1000.0; - QualityMetricsResult psnr; - PsnrFromFiles(_actualSourcename.c_str(), _outname.c_str(), _width, _height, &psnr); + webrtc::test::QualityMetricsResult psnr; + I420PSNRFromFiles(_actualSourcename.c_str(), _outname.c_str(), _width, + _height, &psnr); (_log) << "VCM: Media Optimization Test Cycle Completed!" << std::endl; (_log) << "Input file: " << _inname << std::endl; diff --git a/src/modules/video_coding/main/test/normal_test.cc b/src/modules/video_coding/main/test/normal_test.cc index 8b9cb6ff16..80a14c1e66 100644 --- a/src/modules/video_coding/main/test/normal_test.cc +++ b/src/modules/video_coding/main/test/normal_test.cc @@ -367,9 +367,11 @@ NormalTest::Print() double actualBitRate = ActualBitRate / 1000.0; double avgEncTime = _totalEncodeTime / _frameCnt; double avgDecTime = _totalDecodeTime / _frameCnt; - QualityMetricsResult psnr, ssim; - PsnrFromFiles(_inname.c_str(), _outname.c_str(), _width, _height, &psnr); - SsimFromFiles(_inname.c_str(), _outname.c_str(), _width, _height, &ssim); + webrtc::test::QualityMetricsResult psnr, ssim; + I420PSNRFromFiles(_inname.c_str(), _outname.c_str(), _width, _height, + &psnr); + I420SSIMFromFiles(_inname.c_str(), _outname.c_str(), _width, _height, + &ssim); printf("Actual bitrate: %f kbps\n", actualBitRate); printf("Target bitrate: %f kbps\n", _bitRate); ( _log) << "Actual bitrate: " << actualBitRate<< " kbps\tTarget: " << _bitRate << " kbps" << std::endl; diff --git a/src/modules/video_coding/main/test/quality_modes_test.cc b/src/modules/video_coding/main/test/quality_modes_test.cc index 164e1a5c0e..26eee9b120 100644 --- a/src/modules/video_coding/main/test/quality_modes_test.cc +++ b/src/modules/video_coding/main/test/quality_modes_test.cc @@ -113,8 +113,9 @@ QualityModesTest::Print() double actualBitRate = ActualBitRate / 1000.0; double avgEncTime = _totalEncodeTime / _frameCnt; double avgDecTime = _totalDecodeTime / _frameCnt; - QualityMetricsResult psnr,ssim; - PsnrFromFiles(_inname.c_str(), _outname.c_str(), _nativeWidth, _nativeHeight, &psnr); + webrtc::test::QualityMetricsResult psnr,ssim; + I420PSNRFromFiles(_inname.c_str(), _outname.c_str(), _nativeWidth, + _nativeHeight, &psnr); printf("Actual bitrate: %f kbps\n", actualBitRate); printf("Target bitrate: %f kbps\n", _bitRate); ( _log) << "Actual bitrate: " << actualBitRate<< " kbps\tTarget: " << _bitRate << " kbps" << std::endl; @@ -128,7 +129,8 @@ QualityModesTest::Print() if (_flagSSIM == 1) { printf("***computing SSIM***\n"); - SsimFromFiles(_inname.c_str(), _outname.c_str(), _nativeWidth, _nativeHeight, &ssim); + I420SSIMFromFiles(_inname.c_str(), _outname.c_str(), _nativeWidth, + _nativeHeight, &ssim); printf("SSIM: %f \n", ssim.average); } (_log) << std::endl; diff --git a/src/video_engine/test/auto_test/automated/vie_video_verification_test.cc b/src/video_engine/test/auto_test/automated/vie_video_verification_test.cc index 7fe1a5c13a..885c9640f1 100644 --- a/src/video_engine/test/auto_test/automated/vie_video_verification_test.cc +++ b/src/video_engine/test/auto_test/automated/vie_video_verification_test.cc @@ -8,15 +8,15 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "framedrop_primitives.h" #include "gflags/gflags.h" #include "gtest/gtest.h" #include "testsupport/fileutils.h" #include "testsupport/metrics/video_metrics.h" -#include "vie_autotest.h" -#include "vie_file_based_comparison_tests.h" -#include "vie_integration_test_base.h" -#include "vie_to_file_renderer.h" +#include "video_engine/test/auto_test/automated/vie_integration_test_base.h" +#include "video_engine/test/auto_test/helpers/vie_to_file_renderer.h" +#include "video_engine/test/auto_test/interface/vie_autotest.h" +#include "video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h" +#include "video_engine/test/auto_test/primitives/framedrop_primitives.h" namespace { @@ -79,17 +79,17 @@ class ViEVideoVerificationTest : public testing::Test { void CompareFiles(const std::string& reference_file, const std::string& test_file, double minimum_psnr, double minimum_ssim) { - QualityMetricsResult psnr; - int psnr_error = PsnrFromFiles(reference_file.c_str(), test_file.c_str(), - kInputWidth, kInputHeight, &psnr); - ASSERT_EQ(0, psnr_error) << "PSNR routine failed - output files missing?"; - ASSERT_GT(psnr.average, minimum_psnr); + webrtc::test::QualityMetricsResult psnr; + int error = I420PSNRFromFiles(reference_file.c_str(), test_file.c_str(), + kInputWidth, kInputHeight, &psnr); + EXPECT_EQ(0, error) << "PSNR routine failed - output files missing?"; + EXPECT_GT(psnr.average, minimum_psnr); - QualityMetricsResult ssim; - int ssim_error = SsimFromFiles(reference_file.c_str(), test_file.c_str(), - kInputWidth, kInputHeight, &ssim); - ASSERT_EQ(0, ssim_error) << "SSIM routine failed - output files missing?"; - ASSERT_GT(ssim.average, minimum_ssim); // 1 = perfect, -1 = terrible + webrtc::test::QualityMetricsResult ssim; + error = I420SSIMFromFiles(reference_file.c_str(), test_file.c_str(), + kInputWidth, kInputHeight, &ssim); + EXPECT_EQ(0, error) << "SSIM routine failed - output files missing?"; + EXPECT_GT(ssim.average, minimum_ssim); // 1 = perfect, -1 = terrible ViETest::Log("Results: PSNR: %f (db) SSIM: %f", psnr.average, ssim.average); diff --git a/src/video_engine/test/auto_test/vie_auto_test.gypi b/src/video_engine/test/auto_test/vie_auto_test.gypi index d3b699ed10..28d8f53499 100644 --- a/src/video_engine/test/auto_test/vie_auto_test.gypi +++ b/src/video_engine/test/auto_test/vie_auto_test.gypi @@ -18,6 +18,7 @@ '<(webrtc_root)/voice_engine/voice_engine.gyp:voice_engine_core', '<(webrtc_root)/../testing/gtest.gyp:gtest', '<(webrtc_root)/../third_party/google-gflags/google-gflags.gyp:google-gflags', + '<(webrtc_root)/../test/metrics.gyp:metrics', '<(webrtc_root)/../test/test.gyp:test_support', 'video_engine_core', ], diff --git a/test/metrics.gyp b/test/metrics.gyp new file mode 100644 index 0000000000..70483f9f98 --- /dev/null +++ b/test/metrics.gyp @@ -0,0 +1,46 @@ +# Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +{ + 'includes': [ + '../src/build/common.gypi', + ], + 'targets': [ + { + # The metrics code must be kept in its own GYP file in order to + # avoid a circular dependency error due to the dependency on libyuv. + # If the code would be put in test.gyp a circular dependency error during + # GYP generation would occur, because the libyuv.gypi unittest target + # depends on test_support_main. See issue #160 for more info. + 'target_name': 'metrics', + 'type': '<(library)', + 'dependencies': [ + '<(webrtc_root)/common_video/common_video.gyp:webrtc_libyuv', + ], + 'include_dirs': [ + '.', + ], + 'sources': [ + 'testsupport/metrics/video_metrics.h', + 'testsupport/metrics/video_metrics.cc', + ], + }, + { + 'target_name': 'metrics_unittests', + 'type': 'executable', + 'dependencies': [ + 'metrics', + '<(webrtc_root)/../test/test.gyp:test_support_main', + '<(webrtc_root)/../testing/gtest.gyp:gtest', + ], + 'sources': [ + 'testsupport/metrics/video_metrics_unittest.cc', + ], + }, + ], +} diff --git a/test/test.gyp b/test/test.gyp index 33da3f5b2b..86a57ffebc 100644 --- a/test/test.gyp +++ b/test/test.gyp @@ -43,8 +43,6 @@ 'testsupport/frame_writer.cc', 'testsupport/packet_reader.h', 'testsupport/packet_reader.cc', - 'testsupport/metrics/video_metrics.h', - 'testsupport/metrics/video_metrics.cc', 'testsupport/mock/mock_frame_reader.h', 'testsupport/mock/mock_frame_writer.h', ], @@ -74,7 +72,6 @@ 'testsupport/frame_reader_unittest.cc', 'testsupport/frame_writer_unittest.cc', 'testsupport/packet_reader_unittest.cc', - 'testsupport/metrics/video_metrics_unittest.cc', ], }, ], diff --git a/test/testsupport/metrics/video_metrics.cc b/test/testsupport/metrics/video_metrics.cc index 6a63e1a1ad..9e61ec8acb 100644 --- a/test/testsupport/metrics/video_metrics.cc +++ b/test/testsupport/metrics/video_metrics.cc @@ -11,14 +11,13 @@ #include "testsupport/metrics/video_metrics.h" #include // min_element, max_element -#include -#include +#include +#include -// Calculates PSNR from MSE -static inline double CalcPsnr(double mse) { - // Formula: PSNR = 10 log (255^2 / MSE) = 20 log(255) - 10 log(MSE) - return 20.0 * std::log10(255.0) - 10.0 * std::log10(mse); -} +#include "common_video/libyuv/include/libyuv.h" + +namespace webrtc { +namespace test { // Used for calculating min and max values static bool LessForFrameResultValue (const FrameResult& s1, @@ -26,342 +25,163 @@ static bool LessForFrameResultValue (const FrameResult& s1, return s1.value < s2.value; } -int -PsnrFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName, - WebRtc_Word32 width, WebRtc_Word32 height, QualityMetricsResult *result) -{ - FILE *refFp = fopen(refFileName, "rb"); - if (refFp == NULL ) - { - // cannot open reference file - fprintf(stderr, "Cannot open file %s\n", refFileName); - return -1; - } +enum VideoMetricsType { kPSNR, kSSIM, kBoth }; - FILE *testFp = fopen(testFileName, "rb"); - if (testFp == NULL ) - { - // cannot open test file - fprintf(stderr, "Cannot open file %s\n", testFileName); - fclose(refFp); - return -2; - } - - double mse = 0.0; - double mseSum = 0.0; - WebRtc_Word32 frames = 0; - - // Allocating size for one I420 frame. - WebRtc_Word32 frameBytes = 3 * width * height >> 1; - WebRtc_UWord8 *ref = new WebRtc_UWord8[frameBytes]; - WebRtc_UWord8 *test = new WebRtc_UWord8[frameBytes]; - - WebRtc_Word32 refBytes = (WebRtc_Word32) fread(ref, 1, frameBytes, refFp); - WebRtc_Word32 testBytes = (WebRtc_Word32) fread(test, 1, frameBytes, testFp); - - while (refBytes == frameBytes && testBytes == frameBytes) - { - mse = 0.0; - - WebRtc_Word32 sh = 8; //boundary offset - for (WebRtc_Word32 k2 = sh; k2 < height - sh; k2++) - for (WebRtc_Word32 k = sh; k < width - sh; k++) - { - WebRtc_Word32 kk = k2 * width + k; - mse += (test[kk] - ref[kk]) * (test[kk] - ref[kk]); - } - - // divide by number of pixels - mse /= (double) (width * height); - - // Save statistics for this specific frame - FrameResult frame_result; - frame_result.value = CalcPsnr(mse); - frame_result.frame_number = frames; - result->frames.push_back(frame_result); - - // accumulate for total average - mseSum += mse; - frames++; - - refBytes = (WebRtc_Word32) fread(ref, 1, frameBytes, refFp); - testBytes = (WebRtc_Word32) fread(test, 1, frameBytes, testFp); - } - - if (mseSum == 0) - { - // The PSNR value is undefined in this case. - // This value effectively means that the files are equal. - result->average = std::numeric_limits::max(); - } - else - { - result->average = CalcPsnr(mseSum / frames); - } - int return_code = 0; - if (result->frames.size() == 0) - { - fprintf(stderr, "Tried to measure SSIM from empty files (reference " - "file: %s test file: %s\n", refFileName, testFileName); - return_code = -3; - } - else - { - // Calculate min/max statistics - std::vector::iterator element; - element = min_element(result->frames.begin(), - result->frames.end(), LessForFrameResultValue); - result->min = element->value; - result->min_frame_number = element->frame_number; - element = max_element(result->frames.begin(), - result->frames.end(), LessForFrameResultValue); - result->max = element->value; - result->max_frame_number = element->frame_number; - } - delete [] ref; - delete [] test; - fclose(refFp); - fclose(testFp); - return return_code; +// Calculates metrics for a frame and adds statistics to the result for it. +void CalculateFrame(VideoMetricsType video_metrics_type, + uint8_t* ref, + uint8_t* test, + int width, + int height, + int frame_number, + QualityMetricsResult* result) { + FrameResult frame_result; + frame_result.frame_number = frame_number; + switch (video_metrics_type) { + case kPSNR: + frame_result.value = I420PSNR(ref, test, width, height); + break; + case kSSIM: + frame_result.value = I420SSIM(ref, test, width, height); + break; + default: + assert(false); + } + result->frames.push_back(frame_result); } -static double -Similarity(WebRtc_UWord64 sum_s, WebRtc_UWord64 sum_r, WebRtc_UWord64 sum_sq_s, - WebRtc_UWord64 sum_sq_r, WebRtc_UWord64 sum_sxr, WebRtc_Word32 count) -{ - WebRtc_Word64 ssim_n, ssim_d; - WebRtc_Word64 c1, c2; - const WebRtc_Word64 cc1 = 26634; // (64^2*(.01*255)^2 - const WebRtc_Word64 cc2 = 239708; // (64^2*(.03*255)^2 +// Calculates average, min and max values for the supplied struct, if non-NULL. +void CalculateStats(QualityMetricsResult* result) { + if (result == NULL || result->frames.size() == 0) { + return; + } + // Calculate average + std::vector::iterator iter; + double metrics_values_sum = 0.0; + for (iter = result->frames.begin(); iter != result->frames.end(); ++iter) { + metrics_values_sum += iter->value; + } + result->average = metrics_values_sum / result->frames.size(); - // Scale the constants by number of pixels - c1 = (cc1 * count * count) >> 12; - c2 = (cc2 * count * count) >> 12; - - ssim_n = (2 * sum_s * sum_r + c1) * ((WebRtc_Word64) 2 * count * sum_sxr- - (WebRtc_Word64) 2 * sum_s * sum_r + c2); - - ssim_d = (sum_s * sum_s + sum_r * sum_r + c1)* - ((WebRtc_Word64)count * sum_sq_s - (WebRtc_Word64)sum_s * sum_s + - (WebRtc_Word64)count * sum_sq_r - (WebRtc_Word64) sum_r * sum_r + c2); - - return ssim_n * 1.0 / ssim_d; + // Calculate min/max statistics + iter = min_element(result->frames.begin(), result->frames.end(), + LessForFrameResultValue); + result->min = iter->value; + result->min_frame_number = iter->frame_number; + iter = max_element(result->frames.begin(), result->frames.end(), + LessForFrameResultValue); + result->max = iter->value; + result->max_frame_number = iter->frame_number; } -#if !defined(WEBRTC_USE_SSE2) -static double -Ssim8x8C(WebRtc_UWord8 *s, WebRtc_Word32 sp, - WebRtc_UWord8 *r, WebRtc_Word32 rp) -{ - WebRtc_UWord64 sum_s = 0; - WebRtc_UWord64 sum_r = 0; - WebRtc_UWord64 sum_sq_s = 0; - WebRtc_UWord64 sum_sq_r = 0; - WebRtc_UWord64 sum_sxr = 0; +// Single method that handles all combinations of video metrics calculation, to +// minimize code duplication. Either psnr_result or ssim_result may be NULL, +// depending on which VideoMetricsType is targeted. +int CalculateMetrics(VideoMetricsType video_metrics_type, + const char* ref_filename, + const char* test_filename, + int width, + int height, + QualityMetricsResult* psnr_result, + QualityMetricsResult* ssim_result) { + assert(ref_filename != NULL); + assert(test_filename != NULL); + assert(width > 0); + assert(height > 0); - WebRtc_Word32 i, j; - for (i = 0; i < 8; i++, s += sp,r += rp) - { - for (j = 0; j < 8; j++) - { - sum_s += s[j]; - sum_r += r[j]; - sum_sq_s += s[j] * s[j]; - sum_sq_r += r[j] * r[j]; - sum_sxr += s[j] * r[j]; - } + FILE* ref_fp = fopen(ref_filename, "rb"); + if (ref_fp == NULL) { + // cannot open reference file + fprintf(stderr, "Cannot open file %s\n", ref_filename); + return -1; + } + FILE* test_fp = fopen(test_filename, "rb"); + if (test_fp == NULL) { + // cannot open test file + fprintf(stderr, "Cannot open file %s\n", test_filename); + fclose(ref_fp); + return -2; + } + int frame_number = 0; + + // Allocating size for one I420 frame. + const int frame_length = 3 * width * height >> 1; + uint8_t* ref = new uint8_t[frame_length]; + uint8_t* test = new uint8_t[frame_length]; + + int ref_bytes = fread(ref, 1, frame_length, ref_fp); + int test_bytes = fread(test, 1, frame_length, test_fp); + while (ref_bytes == frame_length && test_bytes == frame_length) { + switch (video_metrics_type) { + case kPSNR: + CalculateFrame(kPSNR, ref, test, width, height, frame_number, + psnr_result); + break; + case kSSIM: + CalculateFrame(kSSIM, ref, test, width, height, frame_number, + ssim_result); + break; + case kBoth: + CalculateFrame(kPSNR, ref, test, width, height, frame_number, + psnr_result); + CalculateFrame(kSSIM, ref, test, width, height, frame_number, + ssim_result); + break; + default: + assert(false); } - return Similarity(sum_s, sum_r, sum_sq_s, sum_sq_r, sum_sxr, 64); -} -#endif - -#if defined(WEBRTC_USE_SSE2) -#include -#include -static double -Ssim8x8Sse2(WebRtc_UWord8 *s, WebRtc_Word32 sp, - WebRtc_UWord8 *r, WebRtc_Word32 rp) -{ - WebRtc_Word32 i; - const __m128i z = _mm_setzero_si128(); - __m128i sum_s_16 = _mm_setzero_si128(); - __m128i sum_r_16 = _mm_setzero_si128(); - __m128i sum_sq_s_32 = _mm_setzero_si128(); - __m128i sum_sq_r_32 = _mm_setzero_si128(); - __m128i sum_sxr_32 = _mm_setzero_si128(); - - for (i = 0; i < 8; i++, s += sp,r += rp) - { - const __m128i s_8 = _mm_loadl_epi64((__m128i*)(s)); - const __m128i r_8 = _mm_loadl_epi64((__m128i*)(r)); - - const __m128i s_16 = _mm_unpacklo_epi8(s_8,z); - const __m128i r_16 = _mm_unpacklo_epi8(r_8,z); - - sum_s_16 = _mm_adds_epu16(sum_s_16, s_16); - sum_r_16 = _mm_adds_epu16(sum_r_16, r_16); - const __m128i sq_s_32 = _mm_madd_epi16(s_16, s_16); - sum_sq_s_32 = _mm_add_epi32(sum_sq_s_32, sq_s_32); - const __m128i sq_r_32 = _mm_madd_epi16(r_16, r_16); - sum_sq_r_32 = _mm_add_epi32(sum_sq_r_32, sq_r_32); - const __m128i sxr_32 = _mm_madd_epi16(s_16, r_16); - sum_sxr_32 = _mm_add_epi32(sum_sxr_32, sxr_32); - } - - const __m128i sum_s_32 = _mm_add_epi32(_mm_unpackhi_epi16(sum_s_16, z), - _mm_unpacklo_epi16(sum_s_16, z)); - const __m128i sum_r_32 = _mm_add_epi32(_mm_unpackhi_epi16(sum_r_16, z), - _mm_unpacklo_epi16(sum_r_16, z)); - - __m128i sum_s_128; - __m128i sum_r_128; - __m128i sum_sq_s_128; - __m128i sum_sq_r_128; - __m128i sum_sxr_128; - - _mm_store_si128 (&sum_s_128, - _mm_add_epi64(_mm_unpackhi_epi32(sum_s_32, z), - _mm_unpacklo_epi32(sum_s_32, z))); - _mm_store_si128 (&sum_r_128, - _mm_add_epi64(_mm_unpackhi_epi32(sum_r_32, z), - _mm_unpacklo_epi32(sum_r_32, z))); - _mm_store_si128 (&sum_sq_s_128, - _mm_add_epi64(_mm_unpackhi_epi32(sum_sq_s_32, z), - _mm_unpacklo_epi32(sum_sq_s_32, z))); - _mm_store_si128 (&sum_sq_r_128, - _mm_add_epi64(_mm_unpackhi_epi32(sum_sq_r_32, z), - _mm_unpacklo_epi32(sum_sq_r_32, z))); - _mm_store_si128 (&sum_sxr_128, - _mm_add_epi64(_mm_unpackhi_epi32(sum_sxr_32, z), - _mm_unpacklo_epi32(sum_sxr_32, z))); - - const WebRtc_UWord64 *sum_s_64 = - reinterpret_cast(&sum_s_128); - const WebRtc_UWord64 *sum_r_64 = - reinterpret_cast(&sum_r_128); - const WebRtc_UWord64 *sum_sq_s_64 = - reinterpret_cast(&sum_sq_s_128); - const WebRtc_UWord64 *sum_sq_r_64 = - reinterpret_cast(&sum_sq_r_128); - const WebRtc_UWord64 *sum_sxr_64 = - reinterpret_cast(&sum_sxr_128); - - const WebRtc_UWord64 sum_s = sum_s_64[0] + sum_s_64[1]; - const WebRtc_UWord64 sum_r = sum_r_64[0] + sum_r_64[1]; - const WebRtc_UWord64 sum_sq_s = sum_sq_s_64[0] + sum_sq_s_64[1]; - const WebRtc_UWord64 sum_sq_r = sum_sq_r_64[0] + sum_sq_r_64[1]; - const WebRtc_UWord64 sum_sxr = sum_sxr_64[0] + sum_sxr_64[1]; - - return Similarity(sum_s, sum_r, sum_sq_s, sum_sq_r, sum_sxr, 64); -} -#endif - -double -SsimFrame(WebRtc_UWord8 *img1, WebRtc_UWord8 *img2, WebRtc_Word32 stride_img1, - WebRtc_Word32 stride_img2, WebRtc_Word32 width, WebRtc_Word32 height) -{ - WebRtc_Word32 i,j; - WebRtc_UWord32 samples = 0; - double ssim_total = 0; - double (*ssim_8x8)(WebRtc_UWord8*, WebRtc_Word32, - WebRtc_UWord8*, WebRtc_Word32 rp); - -#if defined(WEBRTC_USE_SSE2) - ssim_8x8 = Ssim8x8Sse2; -#else - ssim_8x8 = Ssim8x8C; -#endif - - // Sample point start with each 4x4 location - for (i = 0; i < height - 8; i += 4, img1 += stride_img1 * 4, - img2 += stride_img2 * 4) - { - for (j = 0; j < width - 8; j += 4 ) - { - double v = ssim_8x8(img1 + j, stride_img1, img2 + j, stride_img2); - ssim_total += v; - samples++; - } - } - ssim_total /= samples; - return ssim_total; + frame_number++; + ref_bytes = fread(ref, 1, frame_length, ref_fp); + test_bytes = fread(test, 1, frame_length, test_fp); + } + int return_code = 0; + if (frame_number == 0) { + fprintf(stderr, "Tried to measure video metrics from empty files " + "(reference file: %s test file: %s)\n", ref_filename, + test_filename); + return_code = -3; + } else { + CalculateStats(psnr_result); + CalculateStats(ssim_result); + } + delete [] ref; + delete [] test; + fclose(ref_fp); + fclose(test_fp); + return return_code; } -int -SsimFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName, - WebRtc_Word32 width, WebRtc_Word32 height, QualityMetricsResult *result) -{ - FILE *refFp = fopen(refFileName, "rb"); - if (refFp == NULL) - { - // cannot open reference file - fprintf(stderr, "Cannot open file %s\n", refFileName); - return -1; - } - - FILE *testFp = fopen(testFileName, "rb"); - if (testFp == NULL) - { - // cannot open test file - fprintf(stderr, "Cannot open file %s\n", testFileName); - fclose(refFp); - return -2; - } - - WebRtc_Word32 frames = 0; - - // Bytes in one frame I420 - const WebRtc_Word32 frameBytes = 3 * width * height / 2; - WebRtc_UWord8 *ref = new WebRtc_UWord8[frameBytes]; - WebRtc_UWord8 *test = new WebRtc_UWord8[frameBytes]; - - WebRtc_Word32 refBytes = (WebRtc_Word32) fread(ref, 1, frameBytes, refFp); - WebRtc_Word32 testBytes = (WebRtc_Word32) fread(test, 1, frameBytes, testFp); - - double ssimScene = 0.0; //average SSIM for sequence - - while (refBytes == frameBytes && testBytes == frameBytes ) - { - double ssimFrameValue = SsimFrame(ref, test, width, width, width, height); - // Save statistics for this specific frame - FrameResult frame_result; - frame_result.value = ssimFrameValue; - frame_result.frame_number = frames; - result->frames.push_back(frame_result); - - ssimScene += ssimFrameValue; - frames++; - - refBytes = (WebRtc_Word32) fread(ref, 1, frameBytes, refFp); - testBytes = (WebRtc_Word32) fread(test, 1, frameBytes, testFp); - } - int return_code = 0; - if (result->frames.size() == 0) - { - fprintf(stderr, "Tried to measure SSIM from empty files (reference " - "file: %s test file: %s\n", refFileName, testFileName); - return_code = -3; - } - else - { - // SSIM: normalize/average for sequence - ssimScene = ssimScene / frames; - result->average = ssimScene; - - // Calculate min/max statistics - std::vector::iterator element; - element = min_element(result->frames.begin(), - result->frames.end(), LessForFrameResultValue); - result->min = element->value; - result->min_frame_number = element->frame_number; - element = max_element(result->frames.begin(), - result->frames.end(), LessForFrameResultValue); - result->max = element->value; - result->max_frame_number = element->frame_number; - } - delete [] ref; - delete [] test; - fclose(refFp); - fclose(testFp); - return return_code; +int I420MetricsFromFiles(const char* ref_filename, + const char* test_filename, + int width, + int height, + QualityMetricsResult* psnr_result, + QualityMetricsResult* ssim_result) { + assert(psnr_result != NULL); + assert(ssim_result != NULL); + return CalculateMetrics(kBoth, ref_filename, test_filename, width, height, + psnr_result, ssim_result); } + +int I420PSNRFromFiles(const char* ref_filename, + const char* test_filename, + int width, + int height, + QualityMetricsResult* result) { + assert(result != NULL); + return CalculateMetrics(kPSNR, ref_filename, test_filename, width, height, + result, NULL); +} + +int I420SSIMFromFiles(const char* ref_filename, + const char* test_filename, + int width, + int height, + QualityMetricsResult* result) { + assert(result != NULL); + return CalculateMetrics(kSSIM, ref_filename, test_filename, width, height, + NULL, result); +} + +} // namespace test +} // namespace webrtc diff --git a/test/testsupport/metrics/video_metrics.h b/test/testsupport/metrics/video_metrics.h index 43663d6e87..df11a49db2 100644 --- a/test/testsupport/metrics/video_metrics.h +++ b/test/testsupport/metrics/video_metrics.h @@ -8,17 +8,18 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef WEBRTC_MODULES_VIDEO_CODING_TEST_VIDEO_METRICS_H_ -#define WEBRTC_MODULES_VIDEO_CODING_TEST_VIDEO_METRICS_H_ +#ifndef WEBRTC_TESTSUPPORT_METRICS_VIDEO_METRICS_H_ +#define WEBRTC_TESTSUPPORT_METRICS_VIDEO_METRICS_H_ #include #include -#include "typedefs.h" +namespace webrtc { +namespace test { // Contains video quality metrics result for a single frame. struct FrameResult { - WebRtc_Word32 frame_number; + int frame_number; double value; }; @@ -35,46 +36,77 @@ struct QualityMetricsResult { double average; double min; double max; - WebRtc_Word32 min_frame_number; - WebRtc_Word32 max_frame_number; + int min_frame_number; + int max_frame_number; std::vector frames; }; -// PSNR & SSIM calculations - -// PSNR values are filled into the QualityMetricsResult struct. -// If the result is std::numerical_limits::max() the videos were -// equal. Otherwise, PSNR values are in decibel (higher is better). This -// algorithm only compares up to the point when the shortest video ends. -// By definition of PSNR, the result value is undefined if the reference file -// and the test file are identical. In that case the max value for double -// will be set in the result struct. -// -// Returns 0 if successful, negative on errors: +// Calculates PSNR and SSIM values for the reference and test video files +// (must be in I420 format). All calculated values are filled into the +// QualityMetricsResult stucts. +// PSNR values have the unit decibel (dB) where a high value means the test file +// is similar to the reference file. The higher value, the more similar. +// For more info about PSNR, see http://en.wikipedia.org/wiki/PSNR +// SSIM values range between -1.0 and 1.0, where 1.0 means the files are +// identical. For more info about SSIM, see http://en.wikipedia.org/wiki/SSIM +// This function only compares video frames up to the point when the shortest +// video ends. +// Return value: +// 0 if successful, negative on errors: // -1 if the source file cannot be opened // -2 if the test file cannot be opened // -3 if any of the files are empty -int PsnrFromFiles(const WebRtc_Word8 *refFileName, - const WebRtc_Word8 *testFileName, WebRtc_Word32 width, - WebRtc_Word32 height, QualityMetricsResult *result); +// -4 if any arguments are invalid. +int I420MetricsFromFiles(const char* ref_filename, + const char* test_filename, + int width, + int height, + QualityMetricsResult* psnr_result, + QualityMetricsResult* ssim_result); -// SSIM values are filled into the QualityMetricsResult struct. -// Values range between -1 and 1, where 1 means the files were identical. This -// algorithm only compares up to the point when the shortest video ends. -// By definition, SSIM values varies from -1.0, when everything is different -// between the reference file and the test file, up to 1.0 for two identical -// files. +// Calculates PSNR values for the reference and test video files (must be in +// I420 format). All calculated values are filled into the QualityMetricsResult +// struct. +// PSNR values have the unit decibel (dB) where a high value means the test file +// is similar to the reference file. The higher value, the more similar. +// This function only compares video frames up to the point when the shortest +// video ends. +// For more info about PSNR, see http://en.wikipedia.org/wiki/PSNR // -// Returns 0 if successful, negative on errors: +// Return value: +// 0 if successful, negative on errors: // -1 if the source file cannot be opened // -2 if the test file cannot be opened // -3 if any of the files are empty -int SsimFromFiles(const WebRtc_Word8 *refFileName, - const WebRtc_Word8 *testFileName, WebRtc_Word32 width, - WebRtc_Word32 height, QualityMetricsResult *result); +// -4 if any arguments are invalid. +int I420PSNRFromFiles(const char* ref_filename, + const char* test_filename, + int width, + int height, + QualityMetricsResult* result); -double SsimFrame(WebRtc_UWord8 *img1, WebRtc_UWord8 *img2, - WebRtc_Word32 stride_img1, WebRtc_Word32 stride_img2, - WebRtc_Word32 width, WebRtc_Word32 height); +// Calculates SSIM values for the reference and test video files (must be in +// I420 format). All calculated values are filled into the QualityMetricsResult +// struct. +// SSIM values range between -1.0 and 1.0, where 1.0 means the files are +// identical. +// This function only compares video frames up to the point when the shortest +// video ends. +// For more info about SSIM, see http://en.wikipedia.org/wiki/SSIM +// +// Return value: +// 0 if successful, negative on errors: +// -1 if the source file cannot be opened +// -2 if the test file cannot be opened +// -3 if any of the files are empty +// -4 if any arguments are invalid. +int I420SSIMFromFiles(const char* ref_filename, + const char* test_filename, + int width, + int height, + QualityMetricsResult* result); -#endif // WEBRTC_MODULES_VIDEO_CODING_TEST_VIDEO_METRICS_H_ +} // namespace test +} // namespace webrtc + +#endif // WEBRTC_TESTSUPPORT_METRICS_VIDEO_METRICS_H_ diff --git a/test/testsupport/metrics/video_metrics_unittest.cc b/test/testsupport/metrics/video_metrics_unittest.cc index c7d42ea9f8..e77dbff8c9 100644 --- a/test/testsupport/metrics/video_metrics_unittest.cc +++ b/test/testsupport/metrics/video_metrics_unittest.cc @@ -10,8 +10,6 @@ #include "testsupport/metrics/video_metrics.h" -#include - #include "gtest/gtest.h" #include "testsupport/fileutils.h" @@ -25,13 +23,13 @@ static const int kHeight = 288; static const int kMissingReferenceFileReturnCode = -1; static const int kMissingTestFileReturnCode = -2; static const int kEmptyFileReturnCode = -3; -static const double kPsnrPerfectResult = std::numeric_limits::max(); +static const double kPsnrPerfectResult = 48.0; static const double kSsimPerfectResult = 1.0; class VideoMetricsTest: public testing::Test { protected: VideoMetricsTest() { - video_file = webrtc::test::ProjectRootPath() + "resources/foreman_cif.yuv"; + video_file_ = webrtc::test::ResourcePath("foreman_cif_short", "yuv"); } virtual ~VideoMetricsTest() {} void SetUp() { @@ -42,56 +40,100 @@ class VideoMetricsTest: public testing::Test { void TearDown() { std::remove(kEmptyFileName); } - QualityMetricsResult result_; - std::string video_file; + webrtc::test::QualityMetricsResult psnr_result_; + webrtc::test::QualityMetricsResult ssim_result_; + std::string video_file_; }; // Tests that it is possible to run with the same reference as test file -TEST_F(VideoMetricsTest, ReturnsPerfectResultForIdenticalFiles) { - EXPECT_EQ(0, PsnrFromFiles(video_file.c_str(), video_file.c_str(), - kWidth, kHeight, &result_)); - EXPECT_EQ(kPsnrPerfectResult, result_.average); - EXPECT_EQ(SsimFromFiles(video_file.c_str(), video_file.c_str(), kWidth, kHeight, - &result_), 0); - EXPECT_EQ(kSsimPerfectResult, result_.average); +TEST_F(VideoMetricsTest, ReturnsPerfectResultForIdenticalFilesPSNR) { + EXPECT_EQ(0, I420PSNRFromFiles(video_file_.c_str(), video_file_.c_str(), + kWidth, kHeight, &psnr_result_)); + EXPECT_EQ(kPsnrPerfectResult, psnr_result_.average); +} + +TEST_F(VideoMetricsTest, ReturnsPerfectResultForIdenticalFilesSSIM) { + EXPECT_EQ(0, I420SSIMFromFiles(video_file_.c_str(), video_file_.c_str(), + kWidth, kHeight, &ssim_result_)); + EXPECT_EQ(kSsimPerfectResult, ssim_result_.average); +} + +TEST_F(VideoMetricsTest, ReturnsPerfectResultForIdenticalFilesBothMetrics) { + EXPECT_EQ(0, I420MetricsFromFiles(video_file_.c_str(), video_file_.c_str(), + kWidth, kHeight, &psnr_result_, + &ssim_result_)); + EXPECT_EQ(kPsnrPerfectResult, psnr_result_.average); + EXPECT_EQ(kSsimPerfectResult, ssim_result_.average); } // Tests that the right return code is given when the reference file is missing. -TEST_F(VideoMetricsTest, MissingReferenceFile) { +TEST_F(VideoMetricsTest, MissingReferenceFilePSNR) { EXPECT_EQ(kMissingReferenceFileReturnCode, - PsnrFromFiles(kNonExistingFileName, video_file.c_str(), kWidth, - kHeight, &result_)); + I420PSNRFromFiles(kNonExistingFileName, video_file_.c_str(), + kWidth, kHeight, &ssim_result_)); +} + +TEST_F(VideoMetricsTest, MissingReferenceFileSSIM) { EXPECT_EQ(kMissingReferenceFileReturnCode, - SsimFromFiles(kNonExistingFileName, video_file.c_str(), kWidth, - kHeight, &result_)); + I420SSIMFromFiles(kNonExistingFileName, video_file_.c_str(), + kWidth, kHeight, &ssim_result_)); +} + +TEST_F(VideoMetricsTest, MissingReferenceFileBothMetrics) { + EXPECT_EQ(kMissingReferenceFileReturnCode, + I420MetricsFromFiles(kNonExistingFileName, video_file_.c_str(), + kWidth, kHeight, + &psnr_result_, &ssim_result_)); } // Tests that the right return code is given when the test file is missing. -TEST_F(VideoMetricsTest, MissingTestFile) { +TEST_F(VideoMetricsTest, MissingTestFilePSNR) { EXPECT_EQ(kMissingTestFileReturnCode, - PsnrFromFiles(video_file.c_str(), kNonExistingFileName, kWidth, - kHeight, &result_)); + I420PSNRFromFiles(video_file_.c_str(), kNonExistingFileName, + kWidth, kHeight, &ssim_result_)); +} + +TEST_F(VideoMetricsTest, MissingTestFileSSIM) { EXPECT_EQ(kMissingTestFileReturnCode, - SsimFromFiles(video_file.c_str(), kNonExistingFileName, kWidth, - kHeight, &result_)); + I420SSIMFromFiles(video_file_.c_str(), kNonExistingFileName, + kWidth, kHeight, &ssim_result_)); +} + +TEST_F(VideoMetricsTest, MissingTestFileBothMetrics) { + EXPECT_EQ(kMissingTestFileReturnCode, + I420MetricsFromFiles(video_file_.c_str(), kNonExistingFileName, + kWidth, kHeight, + &psnr_result_, &ssim_result_)); } // Tests that the method can be executed with empty files. -TEST_F(VideoMetricsTest, EmptyFiles) { +TEST_F(VideoMetricsTest, EmptyFilesPSNR) { EXPECT_EQ(kEmptyFileReturnCode, - PsnrFromFiles(kEmptyFileName, video_file.c_str(), kWidth, kHeight, - &result_)); + I420PSNRFromFiles(kEmptyFileName, video_file_.c_str(), + kWidth, kHeight, &ssim_result_)); EXPECT_EQ(kEmptyFileReturnCode, - SsimFromFiles(kEmptyFileName, video_file.c_str(), kWidth, kHeight, - &result_)); - // Run the same again with the empty file switched. + I420PSNRFromFiles(video_file_.c_str(), kEmptyFileName, + kWidth, kHeight, &ssim_result_)); +} + +TEST_F(VideoMetricsTest, EmptyFilesSSIM) { EXPECT_EQ(kEmptyFileReturnCode, - PsnrFromFiles(video_file.c_str(), kEmptyFileName, kWidth, kHeight, - &result_)); + I420SSIMFromFiles(kEmptyFileName, video_file_.c_str(), + kWidth, kHeight, &ssim_result_)); EXPECT_EQ(kEmptyFileReturnCode, - SsimFromFiles(video_file.c_str(), kEmptyFileName, kWidth, kHeight, - &result_)); + I420SSIMFromFiles(video_file_.c_str(), kEmptyFileName, + kWidth, kHeight, &ssim_result_)); +} + +TEST_F(VideoMetricsTest, EmptyFilesBothMetrics) { + EXPECT_EQ(kEmptyFileReturnCode, + I420MetricsFromFiles(kEmptyFileName, video_file_.c_str(), + kWidth, kHeight, + &psnr_result_, &ssim_result_)); + EXPECT_EQ(kEmptyFileReturnCode, + I420MetricsFromFiles(video_file_.c_str(), kEmptyFileName, + kWidth, kHeight, + &psnr_result_, &ssim_result_)); } } // namespace webrtc - diff --git a/webrtc.gyp b/webrtc.gyp index eaecbdfd74..4a53e66172 100644 --- a/webrtc.gyp +++ b/webrtc.gyp @@ -20,6 +20,7 @@ 'src/system_wrappers/source/system_wrappers.gyp:*', 'src/video_engine/video_engine.gyp:*', 'src/voice_engine/voice_engine.gyp:*', + 'test/metrics.gyp:*', 'test/test.gyp:*', ], },