diff --git a/src/modules/video_coding/codecs/test/packet_manipulator.cc b/src/modules/video_coding/codecs/test/packet_manipulator.cc index 9fefc73ca0..acdb2e5ccf 100644 --- a/src/modules/video_coding/codecs/test/packet_manipulator.cc +++ b/src/modules/video_coding/codecs/test/packet_manipulator.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -22,11 +22,14 @@ PacketManipulatorImpl::PacketManipulatorImpl(PacketReader* packet_reader, : packet_reader_(packet_reader), config_(config), active_burst_packets_(0), + critsect_(CriticalSectionWrapper::CreateCriticalSection()), + random_seed_(1), verbose_(verbose) { assert(packet_reader); } PacketManipulatorImpl::~PacketManipulatorImpl() { + delete critsect_; } int PacketManipulatorImpl::ManipulatePackets( @@ -77,8 +80,19 @@ int PacketManipulatorImpl::ManipulatePackets( return nbr_packets_dropped; } +void PacketManipulatorImpl::InitializeRandomSeed(unsigned int seed) { + random_seed_ = seed; +} + inline double PacketManipulatorImpl::RandomUniform() { - return (std::rand() + 1.0)/(RAND_MAX + 1.0); + // Use the previous result as new seed before each rand() call. Doing this + // it doesn't matter if other threads are calling rand() since we'll always + // get the same behavior as long as we're using a fixed initial seed. + critsect_->Enter(); + srand(random_seed_); + random_seed_ = std::rand(); + critsect_->Leave(); + return (random_seed_ + 1.0)/(RAND_MAX + 1.0); } const char* PacketLossModeToStr(PacketLossMode e) { diff --git a/src/modules/video_coding/codecs/test/packet_manipulator.h b/src/modules/video_coding/codecs/test/packet_manipulator.h index 418f8c1e6f..e3891cbee3 100644 --- a/src/modules/video_coding/codecs/test/packet_manipulator.h +++ b/src/modules/video_coding/codecs/test/packet_manipulator.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -14,6 +14,7 @@ #include #include "modules/video_coding/codecs/interface/video_codec_interface.h" +#include "system_wrappers/interface/critical_section_wrapper.h" #include "testsupport/packet_reader.h" namespace webrtc { @@ -70,8 +71,8 @@ struct NetworkingConfig { // when CL 172001 has been submitted. This also requires a correct // fragmentation header to be passed to the decoder. // -// To get a deterministic behavior of the packet dropping, initialize the -// random generator with a fixed value before using this class, e.g. srand(0); +// To get a repeatable packet drop pattern, re-initialize the random seed +// using InitializeRandomSeed before each test run. class PacketManipulator { public: virtual ~PacketManipulator() {} @@ -88,9 +89,11 @@ class PacketManipulator { class PacketManipulatorImpl : public PacketManipulator { public: PacketManipulatorImpl(PacketReader* packet_reader, - const NetworkingConfig& config, bool verbose); + const NetworkingConfig& config, + bool verbose); virtual ~PacketManipulatorImpl(); virtual int ManipulatePackets(webrtc::EncodedImage* encoded_image); + virtual void InitializeRandomSeed(unsigned int seed); protected: // Returns a uniformly distributed random value between 0.0 and 1.0 virtual double RandomUniform(); @@ -99,6 +102,8 @@ class PacketManipulatorImpl : public PacketManipulator { const NetworkingConfig& config_; // Used to simulate a burst over several frames. int active_burst_packets_; + CriticalSectionWrapper* critsect_; + unsigned int random_seed_; bool verbose_; }; diff --git a/src/modules/video_coding/codecs/test/packet_manipulator_unittest.cc b/src/modules/video_coding/codecs/test/packet_manipulator_unittest.cc index a4a48d9962..a5d8bc3bd1 100644 --- a/src/modules/video_coding/codecs/test/packet_manipulator_unittest.cc +++ b/src/modules/video_coding/codecs/test/packet_manipulator_unittest.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -14,6 +14,7 @@ #include "gtest/gtest.h" #include "modules/video_coding/codecs/interface/video_codec_interface.h" +#include "modules/video_coding/codecs/test/predictive_packet_manipulator.h" #include "testsupport/unittest_utils.h" #include "typedefs.h" @@ -69,35 +70,6 @@ class PacketManipulatorTest: public PacketRelatedTest { } }; -// Predictive packet manipulator that allows for setup of the result of -// the random invocations. -class PredictivePacketManipulatorImpl : public PacketManipulatorImpl { - public: - PredictivePacketManipulatorImpl(PacketReader* packet_reader, - const NetworkingConfig& config) - : PacketManipulatorImpl(packet_reader, config, false) { - } - // Adds a result. You must add at least the same number of results as the - // expected calls to the RandomUniform method. The results are added to a - // FIFO queue so they will be returned in the same order they were added. - void AddRandomResult(double result) { - ASSERT_TRUE(result >= 0.0 || result <= 1.0) - << "Cannot add results outside the range 0.0 - 1.0, was:" << result; - random_results_.push(result); - } - protected: - double RandomUniform() { - EXPECT_GT(random_results_.size(), 0u) << "No more stored results, please " - "make sure AddRandomResult() is called same amount of times you're " - "going to invoke the RandomUniform() function, i.e. once per packet."; - double result = random_results_.front(); - random_results_.pop(); - return result; - } - private: - std::queue random_results_; -}; - TEST_F(PacketManipulatorTest, Constructor) { PacketManipulatorImpl manipulator(&packet_reader_, no_drop_config_, false); } @@ -129,7 +101,7 @@ TEST_F(PacketManipulatorTest, UniformDropAll) { // Use our customized test class to make the second packet being lost TEST_F(PacketManipulatorTest, UniformDropSinglePacket) { drop_config_.packet_loss_probability = 0.5; - PredictivePacketManipulatorImpl manipulator(&packet_reader_, drop_config_); + PredictivePacketManipulator manipulator(&packet_reader_, drop_config_); manipulator.AddRandomResult(1.0); manipulator.AddRandomResult(0.3); // less than 0.5 will cause packet loss manipulator.AddRandomResult(1.0); @@ -163,7 +135,7 @@ TEST_F(PacketManipulatorTest, BurstDropNinePackets) { drop_config_.packet_loss_probability = 0.5; drop_config_.packet_loss_burst_length = 5; drop_config_.packet_loss_mode = kBurst; - PredictivePacketManipulatorImpl manipulator(&packet_reader_, drop_config_); + PredictivePacketManipulator manipulator(&packet_reader_, drop_config_); manipulator.AddRandomResult(1.0); manipulator.AddRandomResult(0.3); // less than 0.5 will cause packet loss for (int i = 0; i < kNbrPackets - 2; ++i) { diff --git a/src/modules/video_coding/codecs/test/predictive_packet_manipulator.cc b/src/modules/video_coding/codecs/test/predictive_packet_manipulator.cc new file mode 100644 index 0000000000..5668378418 --- /dev/null +++ b/src/modules/video_coding/codecs/test/predictive_packet_manipulator.cc @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/test/predictive_packet_manipulator.h" + +#include +#include + +#include "testsupport/packet_reader.h" + +namespace webrtc { +namespace test { + +PredictivePacketManipulator::PredictivePacketManipulator( + PacketReader* packet_reader, const NetworkingConfig& config) + : PacketManipulatorImpl(packet_reader, config, false) { +} + +PredictivePacketManipulator::~PredictivePacketManipulator() { +} + + +void PredictivePacketManipulator::AddRandomResult(double result) { + assert(result >= 0.0 && result <= 1.0); + random_results_.push(result); +} + +double PredictivePacketManipulator::RandomUniform() { + if(random_results_.size() == 0u) { + fprintf(stderr, "No more stored results, please make sure AddRandomResult()" + "is called same amount of times you're going to invoke the " + "RandomUniform() function, i.e. once per packet.\n"); + assert(false); + } + double result = random_results_.front(); + random_results_.pop(); + return result; +} + +} // namespace test +} // namespace webrtcc diff --git a/src/modules/video_coding/codecs/test/predictive_packet_manipulator.h b/src/modules/video_coding/codecs/test/predictive_packet_manipulator.h new file mode 100644 index 0000000000..22a0ce94f7 --- /dev/null +++ b/src/modules/video_coding/codecs/test/predictive_packet_manipulator.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_PREDICTIVE_PACKET_MANIPULATOR_H_ +#define WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_PREDICTIVE_PACKET_MANIPULATOR_H_ + +#include + +#include "modules/video_coding/codecs/test/packet_manipulator.h" +#include "testsupport/packet_reader.h" + +namespace webrtc { +namespace test { + +// Predictive packet manipulator that allows for setup of the result of +// the random invocations. +class PredictivePacketManipulator : public PacketManipulatorImpl { + public: + PredictivePacketManipulator(PacketReader* packet_reader, + const NetworkingConfig& config); + virtual ~PredictivePacketManipulator(); + // Adds a result. You must add at least the same number of results as the + // expected calls to the RandomUniform method. The results are added to a + // FIFO queue so they will be returned in the same order they were added. + // Result parameter must be 0.0 to 1.0. + void AddRandomResult(double result); + protected: + // Returns a uniformly distributed random value between 0.0 and 1.0 + virtual double RandomUniform(); + + private: + std::queue random_results_; +}; + +} // namespace test +} // namespace webrtc + +#endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_PREDICTIVE_PACKET_MANIPULATOR_H_ diff --git a/src/modules/video_coding/codecs/test/video_codecs_test_framework.gypi b/src/modules/video_coding/codecs/test/video_codecs_test_framework.gypi index 28f3051459..662d09d644 100644 --- a/src/modules/video_coding/codecs/test/video_codecs_test_framework.gypi +++ b/src/modules/video_coding/codecs/test/video_codecs_test_framework.gypi @@ -21,6 +21,8 @@ 'mock/mock_packet_manipulator.h', 'packet_manipulator.h', 'packet_manipulator.cc', + 'predictive_packet_manipulator.h', + 'predictive_packet_manipulator.cc', 'stats.h', 'stats.cc', 'videoprocessor.h', diff --git a/src/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc b/src/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc index 81a6e38401..19c40ad0d1 100644 --- a/src/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc +++ b/src/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc @@ -61,6 +61,8 @@ class VideoProcessorIntegrationTest: public testing::Test { "foreman_cif_short_video_codecs_test_framework_integrationtests.yuv"; config_.frame_length_in_bytes = 3 * kCIFWidth * kCIFHeight / 2; config_.verbose = false; + // Only allow encoder/decoder to use single core, for predictability. + config_.use_single_core = true; // Get a codec configuration struct and configure it. VideoCodingModule::Codec(kVideoCodecVP8, &codec_settings_); @@ -98,11 +100,10 @@ class VideoProcessorIntegrationTest: public testing::Test { } // Processes all frames in the clip and verifies the result. - // The average PSNR for all frames is required to be 2.0 higher than the - // minimum_psnr parameter. - // The minimum SSIM for all frames is required to be 0.1 higher than the - // minimum_ssim parameter. - void ProcessFramesAndVerify(double minimum_psnr, double minimum_ssim) { + void ProcessFramesAndVerify(double minimum_avg_psnr, + double minimum_min_psnr, + double minimum_avg_ssim, + double minimum_min_ssim) { int frame_number = 0; while (processor_->ProcessFrame(frame_number)) { frame_number++; @@ -125,36 +126,48 @@ class VideoProcessorIntegrationTest: public testing::Test { config_.codec_settings->height, &psnr_result, &ssim_result)); - EXPECT_GT(psnr_result.average, minimum_psnr + 2.0); - EXPECT_GT(psnr_result.min, minimum_psnr); - EXPECT_GT(ssim_result.average, minimum_ssim + 0.1); - EXPECT_GT(ssim_result.min, minimum_ssim); + printf("PSNR avg: %f, min: %f SSIM avg: %f, min: %f\n", + psnr_result.average, psnr_result.min, + ssim_result.average, ssim_result.min); + EXPECT_GT(psnr_result.average, minimum_avg_psnr); + EXPECT_GT(psnr_result.min, minimum_min_psnr); + EXPECT_GT(ssim_result.average, minimum_avg_ssim); + EXPECT_GT(ssim_result.min, minimum_min_ssim); } }; // Run with no packet loss. Quality should be very high. TEST_F(VideoProcessorIntegrationTest, ProcessZeroPacketLoss) { config_.networking_config.packet_loss_probability = 0; - double minimum_psnr = 30; - double minimum_ssim = 0.7; - ProcessFramesAndVerify(minimum_psnr, minimum_ssim); + double minimum_avg_psnr = 36; + double minimum_min_psnr = 34; + double minimum_avg_ssim = 0.9; + double minimum_min_ssim = 0.9; + ProcessFramesAndVerify(minimum_avg_psnr, minimum_min_psnr, + minimum_avg_ssim, minimum_min_ssim); } // Run with 5% packet loss. Quality should be a bit lower. // TODO(mflodman): Reenable this once it's not flaky. -TEST_F(VideoProcessorIntegrationTest, DISABLED_Process5PercentPacketLoss) { +TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLoss) { config_.networking_config.packet_loss_probability = 0.05; - double minimum_psnr = 14; - double minimum_ssim = 0.3; - ProcessFramesAndVerify(minimum_psnr, minimum_ssim); + double minimum_avg_psnr = 21; + double minimum_min_psnr = 17; + double minimum_avg_ssim = 0.6; + double minimum_min_ssim = 0.4; + ProcessFramesAndVerify(minimum_avg_psnr, minimum_min_psnr, + minimum_avg_ssim, minimum_min_ssim); } // Run with 10% packet loss. Quality should be even lower. TEST_F(VideoProcessorIntegrationTest, Process10PercentPacketLoss) { config_.networking_config.packet_loss_probability = 0.10; - double minimum_psnr = 12; - double minimum_ssim = 0.2; - ProcessFramesAndVerify(minimum_psnr, minimum_ssim); + double minimum_avg_psnr = 19; + double minimum_min_psnr = 16; + double minimum_avg_ssim = 0.6; + double minimum_min_ssim = 0.4; + ProcessFramesAndVerify(minimum_avg_psnr, minimum_min_psnr, + minimum_avg_ssim, minimum_min_ssim); } } // namespace webrtc diff --git a/src/modules/video_coding/codecs/test/videoprocessor_unittest.cc b/src/modules/video_coding/codecs/test/videoprocessor_unittest.cc index 042660519d..d51ef6bd9b 100644 --- a/src/modules/video_coding/codecs/test/videoprocessor_unittest.cc +++ b/src/modules/video_coding/codecs/test/videoprocessor_unittest.cc @@ -75,7 +75,7 @@ TEST_F(VideoProcessorTest, Init) { &frame_writer_mock_, &packet_manipulator_mock_, config_, &stats_); - video_processor.Init(); + ASSERT_TRUE(video_processor.Init()); } TEST_F(VideoProcessorTest, ProcessFrame) { @@ -91,7 +91,7 @@ TEST_F(VideoProcessorTest, ProcessFrame) { &frame_writer_mock_, &packet_manipulator_mock_, config_, &stats_); - video_processor.Init(); + ASSERT_TRUE(video_processor.Init()); video_processor.ProcessFrame(0); } diff --git a/src/modules/video_coding/codecs/tools/video_quality_measurement.cc b/src/modules/video_coding/codecs/tools/video_quality_measurement.cc index dd00ef28f0..953e3b4b1e 100644 --- a/src/modules/video_coding/codecs/tools/video_quality_measurement.cc +++ b/src/modules/video_coding/codecs/tools/video_quality_measurement.cc @@ -13,6 +13,7 @@ #include #include +#include #ifndef S_ISDIR // Not defined in stat.h on Windows. #define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) @@ -168,12 +169,6 @@ int HandleCommandLineFlags(webrtc::test::TestConfig* config) { // Check single core flag. config->use_single_core = FLAGS_use_single_core; - // Seed our random function if that flag is enabled. This will force - // repeatable behaviour between runs. - if (!FLAGS_disable_fixed_random_seed) { - srand(0); - } - // Get codec specific configuration. webrtc::VideoCodingModule::Codec(webrtc::kVideoCodecVP8, config->codec_settings); @@ -475,15 +470,21 @@ int main(int argc, char* argv[]) { webrtc::test::PacketManipulatorImpl packet_manipulator( &packet_reader, config.networking_config, config.verbose); - webrtc::test::VideoProcessorImpl processor(encoder, decoder, - &frame_reader, - &frame_writer, - &packet_manipulator, - config, &stats); - processor.Init(); + // By default the packet manipulator is seeded with a fixed random. + // If disabled we must generate a new seed. + if (FLAGS_disable_fixed_random_seed) { + packet_manipulator.InitializeRandomSeed(time(NULL)); + } + webrtc::test::VideoProcessor* processor = + new webrtc::test::VideoProcessorImpl(encoder, decoder, + &frame_reader, + &frame_writer, + &packet_manipulator, + config, &stats); + processor->Init(); int frame_number = 0; - while (processor.ProcessFrame(frame_number)) { + while (processor->ProcessFrame(frame_number)) { if (frame_number % 80 == 0) { Log("\n"); // make the output a bit nicer. } @@ -517,6 +518,7 @@ int main(int argc, char* argv[]) { if (FLAGS_python) { PrintPythonOutput(config, stats, ssim_result, psnr_result); } + delete processor; delete encoder; delete decoder; Log("Quality test finished!"); diff --git a/test/testsupport/unittest_utils.h b/test/testsupport/unittest_utils.h index 963a5d3fd1..30464dead7 100644 --- a/test/testsupport/unittest_utils.h +++ b/test/testsupport/unittest_utils.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -46,10 +46,7 @@ class PacketRelatedTest: public testing::Test { memcpy(packet_data_pointer_ + kPacketSizeInBytes * 2, packet3_, 1); } virtual ~PacketRelatedTest() {} - void SetUp() { - // Initialize the random generator with 0 to get deterministic behavior - srand(0); - } + void SetUp() {} void TearDown() {} };