From 849fb8ebd28986e7111190348f3f08183258b092 Mon Sep 17 00:00:00 2001 From: "phoglund@webrtc.org" Date: Fri, 23 Nov 2012 09:21:44 +0000 Subject: [PATCH] Removed codec comparison test: it didn't work and probably never will. The central problem is that we cannot sync the frames in the input video with the output video, which makes PSNR/SSIM go crazy. The test only appeared to succeed earlier due to a bug in the test. We can consider this a failed experiment, but we did learn a lot from it :) BUG=550 Review URL: https://webrtc-codereview.appspot.com/969006 git-svn-id: http://webrtc.googlecode.com/svn/trunk@3155 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../automated/vie_video_verification_test.cc | 35 ---- .../vie_file_based_comparison_tests.h | 10 - .../auto_test/primitives/codec_primitives.cc | 184 ------------------ .../auto_test/primitives/codec_primitives.h | 126 ------------ .../source/vie_file_based_comparison_tests.cc | 38 ---- .../test/auto_test/vie_auto_test.gypi | 2 - 6 files changed, 395 deletions(-) delete mode 100644 webrtc/video_engine/test/auto_test/primitives/codec_primitives.cc delete mode 100644 webrtc/video_engine/test/auto_test/primitives/codec_primitives.h diff --git a/webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc b/webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc index 825afe650c..1dbe419c8d 100644 --- a/webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc +++ b/webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc @@ -154,41 +154,6 @@ TEST_F(ViEVideoVerificationTest, DISABLED_RunsBaseStandardTestWithoutErrors) { kVerifyingTestMaxNumAttempts << " attempts."; } -TEST_F(ViEVideoVerificationTest, RunsCodecTestWithoutErrors) { - // We compare the local and remote here instead of with the original. - // The reason is that it is hard to say when the three consecutive tests - // switch over into each other, at which point we would have to restart the - // original to get a fair comparison. - // - // The PSNR and SSIM values are quite low here, and they have to be since - // the codec switches will lead to lag in the output. This is considered - // acceptable, but it probably shouldn't get worse than this. - const double kExpectedMinimumPSNR = 20; - const double kExpectedMinimumSSIM = 0.7; - - for (int attempt = 0; attempt < kVerifyingTestMaxNumAttempts; attempt++) { - InitializeFileRenderers(); - ASSERT_TRUE(tests_.TestCodecs(input_file_, kInputWidth, kInputHeight, - local_file_renderer_, - remote_file_renderer_)); - std::string reference_file = local_file_renderer_->GetFullOutputPath(); - std::string output_file = remote_file_renderer_->GetFullOutputPath(); - StopRenderers(); - - double actual_psnr = 0; - double actual_ssim = 0; - CompareFiles(reference_file, output_file, &actual_psnr, &actual_ssim); - - TearDownFileRenderers(); - - if (actual_psnr >= kExpectedMinimumPSNR && - actual_ssim >= kExpectedMinimumSSIM) { - // Test succeeded! - return; - } - } -} - // Runs a whole stack processing with tracking of which frames are dropped // in the encoder. The local and remote file will not be of equal size because // of unknown reasons. Tests show that they start at the same frame, which is diff --git a/webrtc/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h b/webrtc/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h index a9560bcf08..b73aefbbae 100644 --- a/webrtc/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h +++ b/webrtc/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h @@ -41,16 +41,6 @@ class ViEFileBasedComparisonTests { ViEToFileRenderer* local_file_renderer, ViEToFileRenderer* remote_file_renderer); - // Tries testing the I420 and VP8 codecs in turn. Returns false if the - // input file could not be opened; reports errors using googletest macros - // otherwise. - bool TestCodecs( - const std::string& i420_video_file, - int width, - int height, - ViEToFileRenderer* local_file_renderer, - ViEToFileRenderer* remote_file_renderer); - // Runs a full stack test using the VP8 codec. Tests the full stack and uses // RTP timestamps to sync frames between the endpoints. void TestFullStack( diff --git a/webrtc/video_engine/test/auto_test/primitives/codec_primitives.cc b/webrtc/video_engine/test/auto_test/primitives/codec_primitives.cc deleted file mode 100644 index 0ed38804a5..0000000000 --- a/webrtc/video_engine/test/auto_test/primitives/codec_primitives.cc +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "codec_primitives.h" - -#include "general_primitives.h" -#include "vie_autotest.h" -#include "vie_autotest_defines.h" -#include "vie_to_file_renderer.h" -#include "webrtc/modules/video_capture/include/video_capture_factory.h" -#include "tb_interfaces.h" - -// Helper functions. - -void TestCodecImageProcess(webrtc::VideoCodec video_codec, - webrtc::ViECodec* codec_interface, - int video_channel, - webrtc::ViEImageProcess* image_process) { - - EXPECT_EQ(0, codec_interface->SetSendCodec(video_channel, video_codec)); - FrameCounterEffectFilter frame_counter; - EXPECT_EQ(0, image_process->RegisterRenderEffectFilter(video_channel, - frame_counter)); - AutoTestSleep (KAutoTestSleepTimeMs); - - int max_number_of_rendered_frames = video_codec.maxFramerate * - KAutoTestSleepTimeMs / 1000; - - if (video_codec.codecType == webrtc::kVideoCodecI420) { - // Due to that I420 needs a huge bandwidth, rate control can set - // frame rate very low. This happen since we use the same channel - // as we just tested with vp8. - EXPECT_GT(frame_counter.numFrames, 0); - } else { -#ifdef WEBRTC_ANDROID - // Special case to get the autotest to pass on some slow devices - EXPECT_GT(frame_counter.numFrames, max_number_of_rendered_frames / 6); -#else - EXPECT_GT(frame_counter.numFrames, max_number_of_rendered_frames / 4); -#endif - } - EXPECT_EQ(0, image_process->DeregisterRenderEffectFilter(video_channel)); -} - -// Test switching from i420 to VP8 as send codec and make sure that -// the codec observer gets called after the switch. -void TestCodecCallbacks(webrtc::ViEBase *& base_interface, - webrtc::ViECodec *codec_interface, - int video_channel, - int forced_codec_width, - int forced_codec_height) { - - // Set I420 as send codec so we don't make any assumptions about what - // we currently have as send codec: - SetSendCodec(webrtc::kVideoCodecI420, codec_interface, video_channel, - forced_codec_width, forced_codec_height); - - // Register the observer: - ViEAutotestCodecObserver codec_observer; - EXPECT_EQ(0, codec_interface->RegisterEncoderObserver(video_channel, - codec_observer)); - EXPECT_EQ(0, codec_interface->RegisterDecoderObserver(video_channel, - codec_observer)); - - // Make the switch. - ViETest::Log("Testing codec callbacks..."); - - SetSendCodec(webrtc::kVideoCodecVP8, codec_interface, video_channel, - forced_codec_width, forced_codec_height); - - AutoTestSleep (KAutoTestSleepTimeMs); - - // Verify that we got the right codec. - EXPECT_EQ(webrtc::kVideoCodecVP8, codec_observer.incomingCodec.codecType); - - // Clean up. - EXPECT_EQ(0, codec_interface->DeregisterEncoderObserver(video_channel)); - EXPECT_EQ(0, codec_interface->DeregisterDecoderObserver(video_channel)); - - // Verify results. - EXPECT_GT(codec_observer.incomingCodecCalled, 0); - EXPECT_GT(codec_observer.incomingRatecalled, 0); - EXPECT_GT(codec_observer.outgoingRatecalled, 0); -} - -void TestCodecs(const TbInterfaces& interfaces, - int capture_id, - int video_channel, - int forced_codec_width, - int forced_codec_height) { - webrtc::VideoEngine *video_engine_interface = interfaces.video_engine; - webrtc::ViEBase *base_interface = interfaces.base; - webrtc::ViECapture *capture_interface = interfaces.capture; - webrtc::ViERender *render_interface = interfaces.render; - webrtc::ViECodec *codec_interface = interfaces.codec; - webrtc::ViENetwork *network_interface = interfaces.network; - - // *************************************************************** - // Engine ready. Begin testing class - // *************************************************************** - webrtc::VideoCodec video_codec; - memset(&video_codec, 0, sizeof (webrtc::VideoCodec)); - - // Set up all receive codecs. This basically trains the codec interface - // to be able to recognize all receive codecs based on payload type. - for (int idx = 0; idx < codec_interface->NumberOfCodecs(); idx++) { - EXPECT_EQ(0, codec_interface->GetCodec(idx, video_codec)); - SetSuitableResolution(&video_codec, - forced_codec_width, - forced_codec_height); - - EXPECT_EQ(0, codec_interface->SetReceiveCodec(video_channel, video_codec)); - } - const char *ip_address = "127.0.0.1"; - const unsigned short rtp_port = 6000; - EXPECT_EQ(0, network_interface->SetLocalReceiver(video_channel, rtp_port)); - EXPECT_EQ(0, base_interface->StartReceive(video_channel)); - EXPECT_EQ(0, network_interface->SetSendDestination(video_channel, ip_address, - rtp_port)); - EXPECT_EQ(0, base_interface->StartSend(video_channel)); - - // Run all found codecs - webrtc::ViEImageProcess *image_process = - webrtc::ViEImageProcess::GetInterface(video_engine_interface); - EXPECT_TRUE(image_process != NULL); - - ViETest::Log("Loop through all codecs for %d seconds", - KAutoTestSleepTimeMs / 1000); - for (int i = 0; i < codec_interface->NumberOfCodecs(); i++) { - EXPECT_EQ(0, codec_interface->GetCodec(i, video_codec)); - - if (video_codec.codecType == webrtc::kVideoCodecRED || - video_codec.codecType == webrtc::kVideoCodecULPFEC) { - ViETest::Log("\t %d. %s not tested", i, video_codec.plName); - } else { - ViETest::Log("\t %d. %s", i, video_codec.plName); - SetSuitableResolution(&video_codec, forced_codec_width, - forced_codec_height); - TestCodecImageProcess(video_codec, codec_interface, video_channel, - image_process); - } - } - image_process->Release(); - - TestCodecCallbacks(base_interface, codec_interface, video_channel, - forced_codec_width, forced_codec_height); - - ViETest::Log("Done!"); - - // *************************************************************** - // Testing finished. Tear down Video Engine - // *************************************************************** - EXPECT_EQ(0, base_interface->StopSend(video_channel)); - EXPECT_EQ(0, base_interface->StopReceive(video_channel)); - EXPECT_EQ(0, render_interface->StopRender(capture_id)); - EXPECT_EQ(0, render_interface->StopRender(video_channel)); - EXPECT_EQ(0, render_interface->RemoveRenderer(capture_id)); - EXPECT_EQ(0, render_interface->RemoveRenderer(video_channel)); - EXPECT_EQ(0, capture_interface->DisconnectCaptureDevice(video_channel)); - EXPECT_EQ(0, base_interface->DeleteChannel(video_channel)); -} - -void SetSendCodec(webrtc::VideoCodecType of_type, - webrtc::ViECodec* codec_interface, - int video_channel, - int forced_codec_width, - int forced_codec_height) { - webrtc::VideoCodec codec; - bool ok; - EXPECT_TRUE(ok = FindSpecificCodec(of_type, codec_interface, &codec)); - if (!ok) { - return; - } - - SetSuitableResolution(&codec, forced_codec_width, forced_codec_height); - EXPECT_EQ(0, codec_interface->SetSendCodec(video_channel, codec)); -} diff --git a/webrtc/video_engine/test/auto_test/primitives/codec_primitives.h b/webrtc/video_engine/test/auto_test/primitives/codec_primitives.h deleted file mode 100644 index e7780447c4..0000000000 --- a/webrtc/video_engine/test/auto_test/primitives/codec_primitives.h +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef WEBRTC_VIDEO_ENGINE_TEST_AUTO_TEST_PRIMITIVES_CODEC_PRIMITIVES_H_ -#define WEBRTC_VIDEO_ENGINE_TEST_AUTO_TEST_PRIMITIVES_CODEC_PRIMITIVES_H_ - -#include "video_engine/include/vie_codec.h" -#include "video_engine/include/vie_image_process.h" -#include "video_engine/test/auto_test/interface/vie_autotest_defines.h" -#include "video_engine/test/auto_test/primitives/general_primitives.h" - -class TbInterfaces; - -// Tests that a codec actually renders frames by registering a basic -// render effect filter on the codec and then running it. This test is -// quite lenient on the number of frames that get rendered, so it should not -// be seen as a end-user-visible quality measure - it is more a sanity check -// that the codec at least gets some frames through. - -// The codec resolution can be forced by specifying the forced* variables -// (pass in kDoNotForceResolution if you don't care). -void TestCodecs(const TbInterfaces& interfaces, - int capture_id, - int video_channel, - int forced_codec_width, - int forced_codec_height); - -// This helper function will set the send codec in the codec interface to a -// codec of the specified type. It will generate a test failure if we do not -// support the provided codec type. - -// The codec resolution can be forced by specifying the forced* variables -// (pass in kDoNotForceResolution if you don't care). -void SetSendCodec(webrtc::VideoCodecType of_type, - webrtc::ViECodec* codec_interface, - int video_channel, - int forced_codec_width, - int forced_codec_height); - -class ViEAutotestCodecObserver: public webrtc::ViEEncoderObserver, - public webrtc::ViEDecoderObserver { - public: - int incomingCodecCalled; - int incomingRatecalled; - int outgoingRatecalled; - - unsigned char lastPayloadType; - unsigned short lastWidth; - unsigned short lastHeight; - - unsigned int lastOutgoingFramerate; - unsigned int lastOutgoingBitrate; - unsigned int lastIncomingFramerate; - unsigned int lastIncomingBitrate; - - webrtc::VideoCodec incomingCodec; - - ViEAutotestCodecObserver() { - incomingCodecCalled = 0; - incomingRatecalled = 0; - outgoingRatecalled = 0; - lastPayloadType = 0; - lastWidth = 0; - lastHeight = 0; - lastOutgoingFramerate = 0; - lastOutgoingBitrate = 0; - lastIncomingFramerate = 0; - lastIncomingBitrate = 0; - memset(&incomingCodec, 0, sizeof(incomingCodec)); - } - virtual void IncomingCodecChanged(const int videoChannel, - const webrtc::VideoCodec& videoCodec) { - incomingCodecCalled++; - lastPayloadType = videoCodec.plType; - lastWidth = videoCodec.width; - lastHeight = videoCodec.height; - - memcpy(&incomingCodec, &videoCodec, sizeof(videoCodec)); - } - - virtual void IncomingRate(const int videoChannel, - const unsigned int framerate, - const unsigned int bitrate) { - incomingRatecalled++; - lastIncomingFramerate += framerate; - lastIncomingBitrate += bitrate; - } - - virtual void OutgoingRate(const int videoChannel, - const unsigned int framerate, - const unsigned int bitrate) { - outgoingRatecalled++; - lastOutgoingFramerate += framerate; - lastOutgoingBitrate += bitrate; - } - - virtual void RequestNewKeyFrame(const int videoChannel) { - } -}; - -class FrameCounterEffectFilter : public webrtc::ViEEffectFilter -{ - public: - int numFrames; - FrameCounterEffectFilter() { - numFrames = 0; - } - virtual ~FrameCounterEffectFilter() { - } - - virtual int Transform(int size, unsigned char* frameBuffer, - unsigned int timeStamp90KHz, unsigned int width, - unsigned int height) { - numFrames++; - return 0; - } -}; - -#endif // WEBRTC_VIDEO_ENGINE_TEST_AUTO_TEST_PRIMITIVES_CODEC_PRIMITIVES_H_ diff --git a/webrtc/video_engine/test/auto_test/source/vie_file_based_comparison_tests.cc b/webrtc/video_engine/test/auto_test/source/vie_file_based_comparison_tests.cc index 2f01381dab..354a9457d3 100644 --- a/webrtc/video_engine/test/auto_test/source/vie_file_based_comparison_tests.cc +++ b/webrtc/video_engine/test/auto_test/source/vie_file_based_comparison_tests.cc @@ -12,7 +12,6 @@ #include "video_engine/test/auto_test/interface/vie_autotest_defines.h" #include "video_engine/test/auto_test/primitives/base_primitives.h" -#include "video_engine/test/auto_test/primitives/codec_primitives.h" #include "video_engine/test/auto_test/primitives/framedrop_primitives.h" #include "video_engine/test/auto_test/primitives/general_primitives.h" #include "video_engine/test/libvietest/include/tb_interfaces.h" @@ -80,43 +79,6 @@ bool ViEFileBasedComparisonTests::TestCallSetup( return true; } -bool ViEFileBasedComparisonTests::TestCodecs( - const std::string& i420_video_file, - int width, - int height, - ViEToFileRenderer* local_file_renderer, - ViEToFileRenderer* remote_file_renderer) { - - TbInterfaces interfaces("TestCodecs"); - - ViEFakeCamera fake_camera(interfaces.capture); - if (!fake_camera.StartCameraInNewThread(i420_video_file, width, height)) { - // No point in continuing if we have no proper video source - ADD_FAILURE() << "Could not open input video " << i420_video_file << - ": aborting test..."; - return false; - } - - int video_channel = -1; - int capture_id = fake_camera.capture_id(); - - EXPECT_EQ(0, interfaces.base->CreateChannel(video_channel)); - EXPECT_EQ(0, interfaces.capture->ConnectCaptureDevice( - capture_id, video_channel)); - - ConfigureRtpRtcp(interfaces.rtp_rtcp, video_channel); - - RenderToFile(interfaces.render, capture_id, local_file_renderer); - RenderToFile(interfaces.render, video_channel, remote_file_renderer); - - // Force the codec resolution to what our input video is so we can make - // comparisons later. Our comparison algorithms wouldn't like scaling. - ::TestCodecs(interfaces, capture_id, video_channel, width, height); - - fake_camera.StopCamera(); - return true; -} - void ViEFileBasedComparisonTests::TestFullStack( const std::string& i420_video_file, int width, diff --git a/webrtc/video_engine/test/auto_test/vie_auto_test.gypi b/webrtc/video_engine/test/auto_test/vie_auto_test.gypi index e967d8a7a3..df6ad1b1d9 100644 --- a/webrtc/video_engine/test/auto_test/vie_auto_test.gypi +++ b/webrtc/video_engine/test/auto_test/vie_auto_test.gypi @@ -60,8 +60,6 @@ 'primitives/choice_helpers.cc', 'primitives/choice_helpers.h', 'primitives/choice_helpers_unittest.cc', - 'primitives/codec_primitives.cc', - 'primitives/codec_primitives.h', 'primitives/fake_stdin.h', 'primitives/fake_stdin.cc', 'primitives/framedrop_primitives.h',