From 35a1756502707ffd88069b9ccdc97a80c68c9812 Mon Sep 17 00:00:00 2001 From: "kjellander@webrtc.org" Date: Thu, 6 Oct 2011 06:44:54 +0000 Subject: [PATCH] First version of video quality measurement program and test framework. See https://docs.google.com/a/google.com/document/d/1w6Nrxw6yTg_sDu18Ux8oZPEMo5F_R-zt62udrmmTeOc/edit?hl=en_US for background, details and additional instructions on usage. BUG= TEST= Review URL: http://webrtc-codereview.appspot.com/175001 git-svn-id: http://webrtc.googlecode.com/svn/trunk@700 4adac7df-926f-26a2-2b94-8c16560cd09d --- src/modules/modules.gyp | 2 + .../video_coding/codecs/test/file_handler.cc | 113 ++++++ .../video_coding/codecs/test/file_handler.h | 82 +++++ .../codecs/test/file_handler_unittest.cc | 115 ++++++ src/modules/video_coding/codecs/test/mocks.h | 104 ++++++ .../codecs/test/packet_manipulator.cc | 77 ++++ .../codecs/test/packet_manipulator.h | 106 ++++++ .../test/packet_manipulator_unittest.cc | 148 ++++++++ .../video_coding/codecs/test/packet_reader.cc | 57 +++ .../video_coding/codecs/test/packet_reader.h | 52 +++ .../codecs/test/packet_reader_unittest.cc | 148 ++++++++ .../codecs/test/run_all_unittests.cc | 17 + src/modules/video_coding/codecs/test/stats.cc | 176 +++++++++ src/modules/video_coding/codecs/test/stats.h | 74 ++++ .../codecs/test/stats_unittest.cc | 62 ++++ .../video_coding/codecs/test/unittest_utils.h | 58 +++ src/modules/video_coding/codecs/test/util.cc | 31 ++ src/modules/video_coding/codecs/test/util.h | 17 + .../test/video_codecs_test_framework.gypi | 88 +++++ .../codecs/test/videoprocessor.cc | 243 +++++++++++++ .../video_coding/codecs/test/videoprocessor.h | 201 +++++++++++ .../codecs/test/videoprocessor_unittest.cc | 149 ++++++++ .../codecs/tools/video_codecs_tools.gypi | 46 +++ .../codecs/tools/video_quality_measurement.cc | 337 ++++++++++++++++++ 24 files changed, 2503 insertions(+) create mode 100644 src/modules/video_coding/codecs/test/file_handler.cc create mode 100644 src/modules/video_coding/codecs/test/file_handler.h create mode 100644 src/modules/video_coding/codecs/test/file_handler_unittest.cc create mode 100644 src/modules/video_coding/codecs/test/mocks.h create mode 100644 src/modules/video_coding/codecs/test/packet_manipulator.cc create mode 100644 src/modules/video_coding/codecs/test/packet_manipulator.h create mode 100644 src/modules/video_coding/codecs/test/packet_manipulator_unittest.cc create mode 100644 src/modules/video_coding/codecs/test/packet_reader.cc create mode 100644 src/modules/video_coding/codecs/test/packet_reader.h create mode 100644 src/modules/video_coding/codecs/test/packet_reader_unittest.cc create mode 100644 src/modules/video_coding/codecs/test/run_all_unittests.cc create mode 100644 src/modules/video_coding/codecs/test/stats.cc create mode 100644 src/modules/video_coding/codecs/test/stats.h create mode 100644 src/modules/video_coding/codecs/test/stats_unittest.cc create mode 100644 src/modules/video_coding/codecs/test/unittest_utils.h create mode 100644 src/modules/video_coding/codecs/test/util.cc create mode 100644 src/modules/video_coding/codecs/test/util.h create mode 100644 src/modules/video_coding/codecs/test/video_codecs_test_framework.gypi create mode 100644 src/modules/video_coding/codecs/test/videoprocessor.cc create mode 100644 src/modules/video_coding/codecs/test/videoprocessor.h create mode 100644 src/modules/video_coding/codecs/test/videoprocessor_unittest.cc create mode 100644 src/modules/video_coding/codecs/tools/video_codecs_tools.gypi create mode 100644 src/modules/video_coding/codecs/tools/video_quality_measurement.cc diff --git a/src/modules/modules.gyp b/src/modules/modules.gyp index 690e95fa58..248a17fd10 100644 --- a/src/modules/modules.gyp +++ b/src/modules/modules.gyp @@ -52,6 +52,8 @@ 'rtp_rtcp/test/test_bwe/test_bwe.gypi', 'rtp_rtcp/test/testFec/test_fec.gypi', 'video_coding/main/source/video_coding_test.gypi', + 'video_coding/codecs/test/video_codecs_test_framework.gypi', + 'video_coding/codecs/tools/video_codecs_tools.gypi', 'video_processing/main/test/vpm_tests.gypi', ], # includes }], # build_with_chromium diff --git a/src/modules/video_coding/codecs/test/file_handler.cc b/src/modules/video_coding/codecs/test/file_handler.cc new file mode 100644 index 0000000000..ef46d00b38 --- /dev/null +++ b/src/modules/video_coding/codecs/test/file_handler.cc @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "file_handler.h" + +#include + +namespace webrtc { +namespace test { + +FileHandlerImpl::FileHandlerImpl(std::string input_filename, + std::string output_filename, + int frame_length_in_bytes) + : input_filename_(input_filename), + output_filename_(output_filename), + frame_length_in_bytes_(frame_length_in_bytes), + input_file_(NULL), + output_file_(NULL) { +} + +FileHandlerImpl::~FileHandlerImpl() { + Close(); +} + +bool FileHandlerImpl::Init() { + if (frame_length_in_bytes_ <= 0) { + fprintf(stderr, "Frame length must be >0, was %d\n", + frame_length_in_bytes_); + return false; + } + + input_file_ = fopen(input_filename_.c_str(), "rb"); + if (input_file_ == NULL) { + fprintf(stderr, "Couldn't open input file for reading: %s\n", + input_filename_.c_str()); + return false; + } + output_file_ = fopen(output_filename_.c_str(), "wb"); + if (output_file_ == NULL) { + fprintf(stderr, "Couldn't open output file for writing: %s\n", + output_filename_.c_str()); + return false; + } + // Calculate total number of frames: + WebRtc_UWord64 source_file_size = GetFileSize(input_filename_); + if (source_file_size <= 0u) { + fprintf(stderr, "Found empty file: %s\n", input_filename_.c_str()); + return false; + } + number_of_frames_ = source_file_size / frame_length_in_bytes_; + return true; +} + +void FileHandlerImpl::Close() { + if (input_file_ != NULL) { + fclose(input_file_); + input_file_ = NULL; + } + if (output_file_ != NULL) { + fclose(output_file_); + output_file_ = NULL; + } +} + +bool FileHandlerImpl::ReadFrame(WebRtc_UWord8* source_buffer) { + assert(source_buffer); + if (input_file_ == NULL) { + fprintf(stderr, "FileHandler is not initialized (input file is NULL)\n"); + return false; + } + fread(source_buffer, 1, frame_length_in_bytes_, input_file_); + if (feof(input_file_) != 0) { + return false; // no more frames to process + } + return true; +} + +WebRtc_UWord64 FileHandlerImpl::GetFileSize(std::string filename) { + FILE* f = fopen(filename.c_str(), "rb"); + WebRtc_UWord64 size = 0; + if (f != NULL) { + if (fseek(f, 0, SEEK_END) == 0) { + size = ftell(f); + } + fclose(f); + } + return size; +} + +bool FileHandlerImpl::WriteFrame(WebRtc_UWord8* frame_buffer) { + assert(frame_buffer); + if (output_file_ == NULL) { + fprintf(stderr, "FileHandler is not initialized (output file is NULL)\n"); + return false; + } + int bytes_written = fwrite(frame_buffer, 1, frame_length_in_bytes_, + output_file_); + if (bytes_written != frame_length_in_bytes_) { + fprintf(stderr, "Failed to write %d bytes to file %s\n", + frame_length_in_bytes_, output_filename_.c_str()); + return false; + } + return true; +} + +} // namespace test +} // namespace webrtc diff --git a/src/modules/video_coding/codecs/test/file_handler.h b/src/modules/video_coding/codecs/test/file_handler.h new file mode 100644 index 0000000000..b12d7b3600 --- /dev/null +++ b/src/modules/video_coding/codecs/test/file_handler.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef SRC_MODULES_VIDEO_CODING_CODECS_TEST_FILE_HANDLER_H_ +#define SRC_MODULES_VIDEO_CODING_CODECS_TEST_FILE_HANDLER_H_ + +#include +#include + +#include "typedefs.h" + +namespace webrtc { +namespace test { + +// Handles reading and writing video files for the test framework's needs. +class FileHandler { + public: + virtual ~FileHandler() {} + + // Initializes the file handler, i.e. opens the input and output files etc. + // This must be called before reading or writing frames has started. + // Returns false if an error has occurred, in addition to printing to stderr. + virtual bool Init() = 0; + + // Reads a frame into the supplied buffer, which must contain enough space + // for the frame size. + // Returns true if there are more frames to read, false if we've already + // read the last frame (in the previous call). + virtual bool ReadFrame(WebRtc_UWord8* source_buffer) = 0; + + // Writes a frame of the configured frame length to the output file. + // Returns true if the write was successful, false otherwise. + virtual bool WriteFrame(WebRtc_UWord8* frame_buffer) = 0; + + // Closes the input and output files. Essentially makes this class impossible + // to use anymore. + virtual void Close() = 0; + + // File size of the supplied file in bytes. Will return 0 if the file is + // empty or if the file does not exist/is readable. + virtual WebRtc_UWord64 GetFileSize(std::string filename) = 0; + // Frame length in bytes of a single frame image. + virtual int GetFrameLength() = 0; + // Total number of frames in the input video source. + virtual int GetNumberOfFrames() = 0; +}; + +class FileHandlerImpl : public FileHandler { + public: + // Creates a file handler. The input file is assumed to exist and be readable + // and the output file must be writable. + FileHandlerImpl(std::string input_filename, + std::string output_filename, + int frame_length_in_bytes); + virtual ~FileHandlerImpl(); + bool Init(); + bool ReadFrame(WebRtc_UWord8* source_buffer); + bool WriteFrame(WebRtc_UWord8* frame_buffer); + void Close(); + WebRtc_UWord64 GetFileSize(std::string filename); + int GetFrameLength() { return frame_length_in_bytes_; } + int GetNumberOfFrames() { return number_of_frames_; } + + private: + std::string input_filename_; + std::string output_filename_; + int frame_length_in_bytes_; + int number_of_frames_; + FILE* input_file_; + FILE* output_file_; +}; + +} // namespace test +} // namespace webrtc + +#endif // SRC_MODULES_VIDEO_CODING_CODECS_TEST_FILE_HANDLER_H_ diff --git a/src/modules/video_coding/codecs/test/file_handler_unittest.cc b/src/modules/video_coding/codecs/test/file_handler_unittest.cc new file mode 100644 index 0000000000..70b1e85ed4 --- /dev/null +++ b/src/modules/video_coding/codecs/test/file_handler_unittest.cc @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "file_handler.h" +#include "gtest/gtest.h" +#include "unittest_utils.h" + +namespace webrtc { +namespace test { + +const std::string kInputFilename = "temp_inputfile.tmp"; +const std::string kOutputFilename = "temp_outputfile.tmp"; +const std::string kInputFileContents = "baz"; +const int kFrameLength = 1e5; // 100 kB + +// Boilerplate code for proper unit tests for FileHandler. +class FileHandlerTest: public testing::Test { + protected: + FileHandler* file_handler_; + + FileHandlerTest() { + // To avoid warnings when using ASSERT_DEATH + ::testing::FLAGS_gtest_death_test_style = "threadsafe"; + } + + virtual ~FileHandlerTest() { + } + + void SetUp() { + // Cleanup any existing files: + std::remove(kInputFilename.c_str()); + std::remove(kOutputFilename.c_str()); + + // Create a dummy input file: + FILE* dummy = fopen(kInputFilename.c_str(), "wb"); + fprintf(dummy, "%s", kInputFileContents.c_str()); + fclose(dummy); + + file_handler_ = new FileHandlerImpl(kInputFilename, kOutputFilename, + kFrameLength); + ASSERT_TRUE(file_handler_->Init()); + } + + void TearDown() { + delete file_handler_; + // Cleanup the temporary file: + std::remove(kInputFilename.c_str()); + std::remove(kOutputFilename.c_str()); + } +}; + +TEST_F(FileHandlerTest, InitSuccess) { + FileHandlerImpl file_handler(kInputFilename, kOutputFilename, kFrameLength); + ASSERT_TRUE(file_handler.Init()); + ASSERT_EQ(kFrameLength, file_handler.GetFrameLength()); + ASSERT_EQ(0, file_handler.GetNumberOfFrames()); +} + +TEST_F(FileHandlerTest, ReadFrame) { + WebRtc_UWord8 buffer[3]; + bool result = file_handler_->ReadFrame(buffer); + ASSERT_FALSE(result); // no more files to read + ASSERT_EQ(kInputFileContents[0], buffer[0]); + ASSERT_EQ(kInputFileContents[1], buffer[1]); + ASSERT_EQ(kInputFileContents[2], buffer[2]); +} + +TEST_F(FileHandlerTest, ReadFrameUninitialized) { + WebRtc_UWord8 buffer[3]; + FileHandlerImpl file_handler(kInputFilename, kOutputFilename, kFrameLength); + ASSERT_FALSE(file_handler.ReadFrame(buffer)); +} + +TEST_F(FileHandlerTest, ReadFrameNullArgument) { + ASSERT_DEATH(file_handler_->ReadFrame(NULL), ""); +} + +TEST_F(FileHandlerTest, WriteFrame) { + WebRtc_UWord8 buffer[kFrameLength]; + memset(buffer, 9, kFrameLength); // Write lots of 9s to the buffer + bool result = file_handler_->WriteFrame(buffer); + ASSERT_TRUE(result); // success + // Close the file and verify the size: + file_handler_->Close(); + ASSERT_EQ(kFrameLength, + static_cast(file_handler_->GetFileSize(kOutputFilename))); +} + +TEST_F(FileHandlerTest, WriteFrameUninitialized) { + WebRtc_UWord8 buffer[3]; + FileHandlerImpl file_handler(kInputFilename, kOutputFilename, kFrameLength); + ASSERT_FALSE(file_handler.WriteFrame(buffer)); +} + +TEST_F(FileHandlerTest, WriteFrameNullArgument) { + ASSERT_DEATH(file_handler_->WriteFrame(NULL), ""); +} + +TEST_F(FileHandlerTest, GetFileSizeExistingFile) { + ASSERT_EQ(kInputFileContents.length(), + file_handler_->GetFileSize(kInputFilename)); +} + +TEST_F(FileHandlerTest, GetFileSizeNonExistingFile) { + ASSERT_EQ(0u, file_handler_->GetFileSize("non-existing-file.tmp")); +} + +} // namespace test +} // namespace webrtc diff --git a/src/modules/video_coding/codecs/test/mocks.h b/src/modules/video_coding/codecs/test/mocks.h new file mode 100644 index 0000000000..1ff77a6650 --- /dev/null +++ b/src/modules/video_coding/codecs/test/mocks.h @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef SRC_MODULES_VIDEO_CODING_CODECS_TEST_MOCKS_H_ +#define SRC_MODULES_VIDEO_CODING_CODECS_TEST_MOCKS_H_ + +#include + +#include "file_handler.h" +#include "gmock/gmock.h" +#include "packet_manipulator.h" +#include "typedefs.h" + +// This file contains mocks that are used by the unit tests. + +namespace webrtc { + +class MockVideoEncoder : public VideoEncoder { + public: + MOCK_CONST_METHOD2(Version, + WebRtc_Word32(WebRtc_Word8 *version, WebRtc_Word32 length)); + MOCK_METHOD3(InitEncode, + WebRtc_Word32(const VideoCodec* codecSettings, + WebRtc_Word32 numberOfCores, + WebRtc_UWord32 maxPayloadSize)); + MOCK_METHOD3(Encode, + WebRtc_Word32(const RawImage& inputImage, + const CodecSpecificInfo* codecSpecificInfo, + VideoFrameType frameType)); + MOCK_METHOD1(RegisterEncodeCompleteCallback, + WebRtc_Word32(EncodedImageCallback* callback)); + MOCK_METHOD0(Release, + WebRtc_Word32()); + MOCK_METHOD0(Reset, + WebRtc_Word32()); + MOCK_METHOD1(SetPacketLoss, + WebRtc_Word32(WebRtc_UWord32 packetLoss)); + MOCK_METHOD2(SetRates, + WebRtc_Word32(WebRtc_UWord32 newBitRate, WebRtc_UWord32 frameRate)); + MOCK_METHOD1(SetPeriodicKeyFrames, + WebRtc_Word32(bool enable)); + MOCK_METHOD2(CodecConfigParameters, + WebRtc_Word32(WebRtc_UWord8* /*buffer*/, WebRtc_Word32)); +}; + +class MockVideoDecoder : public VideoDecoder { + public: + MOCK_METHOD2(InitDecode, + WebRtc_Word32(const VideoCodec* codecSettings, + WebRtc_Word32 numberOfCores)); + MOCK_METHOD5(Decode, + WebRtc_Word32(const EncodedImage& inputImage, + bool missingFrames, + const RTPFragmentationHeader* fragmentation, + const CodecSpecificInfo* codecSpecificInfo, + WebRtc_Word64 renderTimeMs)); + MOCK_METHOD1(RegisterDecodeCompleteCallback, + WebRtc_Word32(DecodedImageCallback* callback)); + MOCK_METHOD0(Release, + WebRtc_Word32()); + MOCK_METHOD0(Reset, + WebRtc_Word32()); + MOCK_METHOD2(SetCodecConfigParameters, + WebRtc_Word32(const WebRtc_UWord8* /*buffer*/, WebRtc_Word32)); + MOCK_METHOD0(Copy, + VideoDecoder*()); +}; + +namespace test { + +class MockFileHandler : public FileHandler { + public: + MOCK_METHOD0(Init, + bool()); + MOCK_METHOD1(ReadFrame, + bool(WebRtc_UWord8* source_buffer)); + MOCK_METHOD1(WriteFrame, + bool(WebRtc_UWord8* frame_buffer)); + MOCK_METHOD0(Close, + void()); + MOCK_METHOD1(GetFileSize, + WebRtc_UWord64(std::string filename)); + MOCK_METHOD0(GetFrameLength, + int()); + MOCK_METHOD0(GetNumberOfFrames, + int()); +}; + +class MockPacketManipulator : public PacketManipulator { + public: + MOCK_METHOD1(ManipulatePackets, + int(webrtc::EncodedImage* encoded_image)); +}; + +} // namespace test +} // namespace webrtc + +#endif // SRC_MODULES_VIDEO_CODING_CODECS_TEST_MOCKS_H_ diff --git a/src/modules/video_coding/codecs/test/packet_manipulator.cc b/src/modules/video_coding/codecs/test/packet_manipulator.cc new file mode 100644 index 0000000000..6eabd6fa21 --- /dev/null +++ b/src/modules/video_coding/codecs/test/packet_manipulator.cc @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "packet_manipulator.h" + +#include + +#include "util.h" + +namespace webrtc { +namespace test { + +PacketManipulatorImpl::PacketManipulatorImpl(PacketReader* packet_reader, + const NetworkingConfig& config) + : packet_reader_(packet_reader), + config_(config), + active_burst_packets_(0) { + assert(packet_reader); +} + +PacketManipulatorImpl::~PacketManipulatorImpl() { +} + +int PacketManipulatorImpl::ManipulatePackets( + webrtc::EncodedImage* encoded_image) { + assert(encoded_image); + int nbr_packets_dropped = 0; + // There's no need to build a copy of the image data since viewing an + // EncodedImage object, setting the length to a new lower value represents + // that everything is dropped after that position in the byte array. + // EncodedImage._size is the allocated bytes. + // EncodedImage._length is how many that are filled with data. + int new_length = 0; + packet_reader_->InitializeReading(encoded_image->_buffer, + encoded_image->_length, + config_.packet_size_in_bytes); + WebRtc_UWord8* packet = NULL; + int nbr_bytes_to_read; + // keep track of if we've lost any packets, since then we shall loose + // the remains of the current frame: + bool packet_loss_has_occurred = false; + while ((nbr_bytes_to_read = packet_reader_->NextPacket(&packet)) > 0) { + // Check if we're currently in a packet loss burst that is not completed: + if (active_burst_packets_ > 0) { + active_burst_packets_--; + nbr_packets_dropped++; + } else if (RandomUniform() < config_.packet_loss_probability || + packet_loss_has_occurred) { + packet_loss_has_occurred = true; + nbr_packets_dropped++; + if (config_.packet_loss_mode == kBurst) { + // Initiate a new burst + active_burst_packets_ = config_.packet_loss_burst_length - 1; + } + } else { + new_length += nbr_bytes_to_read; + } + } + encoded_image->_length = new_length; + if (nbr_packets_dropped > 0) { + // Must set completeFrame to false to inform the decoder about this: + encoded_image->_completeFrame = false; + log("Dropped %d packets for frame %d (frame length: %d)\n", + nbr_packets_dropped, encoded_image->_timeStamp, + encoded_image->_length); + } + return nbr_packets_dropped; +} + +} // namespace test +} // namespace webrtcc diff --git a/src/modules/video_coding/codecs/test/packet_manipulator.h b/src/modules/video_coding/codecs/test/packet_manipulator.h new file mode 100644 index 0000000000..76405ffe54 --- /dev/null +++ b/src/modules/video_coding/codecs/test/packet_manipulator.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef SRC_MODULES_VIDEO_CODING_CODECS_TEST_PACKET_MANIPULATOR_H_ +#define SRC_MODULES_VIDEO_CODING_CODECS_TEST_PACKET_MANIPULATOR_H_ + +#include + +#include "packet_reader.h" +#include "video_codec_interface.h" + +namespace webrtc { +namespace test { + +// Which mode the packet loss shall be performed according to. +enum PacketLossMode { + // Drops packets with a configured probability independently for each packet + kUniform, + // Drops packets similar to uniform but when a packet is being dropped, + // the number of lost packets in a row is equal to the configured burst + // length. + kBurst +}; + +// Contains configurations related to networking and simulation of +// scenarios caused by network interference. +struct NetworkingConfig { + NetworkingConfig() + : packet_size_in_bytes(1500), max_payload_size_in_bytes(1440), + packet_loss_mode(kUniform), packet_loss_probability(0.0), + packet_loss_burst_length(1) { + } + + // Packet size in bytes. Default: 1500 bytes. + int packet_size_in_bytes; + + // Encoder specific setting of maximum size in bytes of each payload. + // Default: 1440 bytes. + int max_payload_size_in_bytes; + + // Packet loss mode. Two different packet loss models are supported: + // uniform or burst. This setting has no effect unless + // packet_loss_probability is >0. + // Default: uniform. + PacketLossMode packet_loss_mode; + + // Packet loss probability. A value between 0.0 and 1.0 that defines the + // probability of a packet being lost. 0.1 means 10% and so on. + // Default: 0 (no loss). + double packet_loss_probability; + + // Packet loss burst length. Defines how many packets will be lost in a burst + // when a packet has been decided to be lost. Must be >=1. Default: 1. + int packet_loss_burst_length; +}; + +// Class for simulating packet loss on the encoded frame data. +// When a packet loss has occurred in a frame, the remaining data in that +// frame is lost (even if burst length is only a single packet). +// TODO(kjellander): Support discarding only individual packets in the frame +// when CL 172001 has been submitted. This also requires a correct +// fragmentation header to be passed to the decoder. +// +// To get a deterministic behavior of the packet dropping, initialize the +// random generator with a fixed value before using this class, e.g. srand(0); +class PacketManipulator { + public: + virtual ~PacketManipulator() {} + + // Manipulates the data of the encoded_image to simulate parts being lost + // during transport. + // If packets are dropped from frame data, the completedFrame field will be + // set to false. + // Returns the number of packets being dropped. + virtual int + ManipulatePackets(webrtc::EncodedImage* encoded_image) = 0; +}; + +class PacketManipulatorImpl : public PacketManipulator { + public: + PacketManipulatorImpl(PacketReader* packet_reader, + const NetworkingConfig& config); + virtual ~PacketManipulatorImpl(); + virtual int ManipulatePackets(webrtc::EncodedImage* encoded_image); + + private: + // Returns a uniformly distributed random value between 0.0 and 1.0 + inline double RandomUniform() { + return (std::rand() + 1.0)/(RAND_MAX + 1.0); + } + PacketReader* packet_reader_; + const NetworkingConfig& config_; + // Used to simulate a burst over several frames. + int active_burst_packets_; +}; + +} // namespace test +} // namespace webrtc + +#endif // SRC_MODULES_VIDEO_CODING_CODECS_TEST_PACKET_MANIPULATOR_H_ diff --git a/src/modules/video_coding/codecs/test/packet_manipulator_unittest.cc b/src/modules/video_coding/codecs/test/packet_manipulator_unittest.cc new file mode 100644 index 0000000000..c2ece40c54 --- /dev/null +++ b/src/modules/video_coding/codecs/test/packet_manipulator_unittest.cc @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "gtest/gtest.h" +#include "packet_manipulator.h" +#include "typedefs.h" +#include "unittest_utils.h" +#include "video_codec_interface.h" + +namespace webrtc { +namespace test { + +class PacketManipulatorTest: public PacketRelatedTest { + protected: + PacketReader packet_reader_; + EncodedImage image_; + static const double kNeverDropProbability = 0.0; + static const double kAlwaysDropProbability = 1.0; + static const int kBurstLength = 1; + NetworkingConfig drop_config_; + NetworkingConfig no_drop_config_; + + PacketManipulatorTest() { + // To avoid warnings when using ASSERT_DEATH + ::testing::FLAGS_gtest_death_test_style = "threadsafe"; + + image_._buffer = packet_data_; + image_._length = kPacketDataLength; + image_._size = kPacketDataLength; + + drop_config_.packet_size_in_bytes = kPacketSizeInBytes; + drop_config_.packet_loss_probability = kAlwaysDropProbability; + drop_config_.packet_loss_burst_length = kBurstLength; + drop_config_.packet_loss_mode = kUniform; + + no_drop_config_.packet_size_in_bytes = kPacketSizeInBytes; + no_drop_config_.packet_loss_probability = kNeverDropProbability; + no_drop_config_.packet_loss_burst_length = kBurstLength; + no_drop_config_.packet_loss_mode = kUniform; + } + + virtual ~PacketManipulatorTest() { + } + + void SetUp() { + PacketRelatedTest::SetUp(); + } + + void TearDown() { + PacketRelatedTest::TearDown(); + } + + void VerifyPacketLoss(int expected_nbr_packets_dropped, + int actual_nbr_packets_dropped, + int expected_packet_data_length, + WebRtc_UWord8* expected_packet_data, + EncodedImage& actual_image) { + EXPECT_EQ(expected_nbr_packets_dropped, actual_nbr_packets_dropped); + EXPECT_EQ(expected_packet_data_length, static_cast(image_._length)); + EXPECT_EQ(0, memcmp(expected_packet_data, actual_image._buffer, + expected_packet_data_length)); + } +}; + +TEST_F(PacketManipulatorTest, Constructor) { + PacketManipulatorImpl manipulator(&packet_reader_, no_drop_config_); +} + +TEST_F(PacketManipulatorTest, ConstructorNullArgument) { + ASSERT_DEATH(PacketManipulatorImpl manipulator(NULL, no_drop_config_), ""); +} + +TEST_F(PacketManipulatorTest, NullImageArgument) { + PacketManipulatorImpl manipulator(&packet_reader_, no_drop_config_); + ASSERT_DEATH(manipulator.ManipulatePackets(NULL), ""); +} + +TEST_F(PacketManipulatorTest, DropNone) { + PacketManipulatorImpl manipulator(&packet_reader_, no_drop_config_); + int nbr_packets_dropped = manipulator.ManipulatePackets(&image_); + VerifyPacketLoss(0, nbr_packets_dropped, kPacketDataLength, + packet_data_, image_); +} + +TEST_F(PacketManipulatorTest, UniformDropNoneSmallFrame) { + int data_length = 400; // smaller than the packet size + image_._length = data_length; + PacketManipulatorImpl manipulator(&packet_reader_, no_drop_config_); + int nbr_packets_dropped = manipulator.ManipulatePackets(&image_); + + VerifyPacketLoss(0, nbr_packets_dropped, data_length, + packet_data_, image_); +} + +TEST_F(PacketManipulatorTest, UniformDropAll) { + PacketManipulatorImpl manipulator(&packet_reader_, drop_config_); + int nbr_packets_dropped = manipulator.ManipulatePackets(&image_); + VerifyPacketLoss(kPacketDataNumberOfPackets, nbr_packets_dropped, + 0, packet_data_, image_); +} + +TEST_F(PacketManipulatorTest, UniformDropSinglePacket) { + drop_config_.packet_loss_probability = 0.5; + PacketManipulatorImpl manipulator(&packet_reader_, drop_config_); + // Execute the test target method: + int nbr_packets_dropped = manipulator.ManipulatePackets(&image_); + + // The deterministic behavior (since we've set srand) will + // make the packet manipulator to throw away the second packet. + // The third packet is lost because when we have lost one, the remains shall + // also be discarded. + VerifyPacketLoss(2, nbr_packets_dropped, kPacketSizeInBytes, packet1_, + image_); +} + +TEST_F(PacketManipulatorTest, BurstDropNinePackets) { + // Create a longer packet data structure + const int kDataLength = kPacketSizeInBytes * 10; + WebRtc_UWord8 data[kDataLength]; + WebRtc_UWord8* data_pointer = data; + // Fill with 0s, 1s and so on to be able to easily verify which were dropped: + for (int i = 0; i < 10; ++i) { + memset(data_pointer + i * kPacketSizeInBytes, i, kPacketSizeInBytes); + } + // Overwrite the defaults from the test fixture: + image_._buffer = data; + image_._length = kDataLength; + image_._size = kDataLength; + + drop_config_.packet_loss_probability = 0.4; + drop_config_.packet_loss_burst_length = 5; + drop_config_.packet_loss_mode = kBurst; + PacketManipulatorImpl manipulator(&packet_reader_, drop_config_); + // Execute the test target method: + int nbr_packets_dropped = manipulator.ManipulatePackets(&image_); + + // Should discard every packet after the first one. + VerifyPacketLoss(9, nbr_packets_dropped, kPacketSizeInBytes, data, image_); +} + +} // namespace test +} // namespace webrtc diff --git a/src/modules/video_coding/codecs/test/packet_reader.cc b/src/modules/video_coding/codecs/test/packet_reader.cc new file mode 100644 index 0000000000..827f665583 --- /dev/null +++ b/src/modules/video_coding/codecs/test/packet_reader.cc @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "packet_reader.h" + +#include +#include + +namespace webrtc { +namespace test { + +PacketReader::PacketReader() + : initialized_(false) { +} + +PacketReader::~PacketReader() { +} + +void PacketReader::InitializeReading(WebRtc_UWord8* data, + int data_length_in_bytes, + int packet_size_in_bytes) { + assert(data); + assert(data_length_in_bytes >= 0); + assert(packet_size_in_bytes > 0); + data_ = data; + data_length_ = data_length_in_bytes; + packet_size_ = packet_size_in_bytes; + currentIndex_ = 0; + initialized_ = true; +} + +int PacketReader::NextPacket(WebRtc_UWord8** packet_pointer) { + if (!initialized_) { + fprintf(stderr, "Attempting to use uninitialized PacketReader!\n"); + return -1; + } + *packet_pointer = data_ + currentIndex_; + // Check if we're about to read the last packet: + if (data_length_ - currentIndex_ <= packet_size_) { + int size = data_length_ - currentIndex_; + currentIndex_ = data_length_; + assert(size >= 0); + return size; + } + currentIndex_ += packet_size_; + assert(packet_size_ >= 0); + return packet_size_; +} + +} // namespace test +} // namespace webrtc diff --git a/src/modules/video_coding/codecs/test/packet_reader.h b/src/modules/video_coding/codecs/test/packet_reader.h new file mode 100644 index 0000000000..7e813cb75e --- /dev/null +++ b/src/modules/video_coding/codecs/test/packet_reader.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef SRC_MODULES_VIDEO_CODING_CODECS_TEST_PACKET_READER_H_ +#define SRC_MODULES_VIDEO_CODING_CODECS_TEST_PACKET_READER_H_ + +#include "typedefs.h" + +namespace webrtc { +namespace test { + +// Reads chunks of data to simulate network packets from a byte array. +class PacketReader { + public: + PacketReader(); + virtual ~PacketReader(); + + // Inizializes a new reading operation. Must be done before invoking the + // NextPacket method. + // * data_length_in_bytes is the length of the data byte array. Must be >= 0. + // 0 length will result in no packets are read. + // * packet_size_in_bytes is the number of bytes to read in each NextPacket + // method call. Must be > 0 + virtual void InitializeReading(WebRtc_UWord8* data, int data_length_in_bytes, + int packet_size_in_bytes); + + // Moves the supplied pointer to the beginning of the next packet. + // Returns: + // * The size of the packet ready to read (lower than the packet size for + // the last packet) + // * 0 if there are no more packets to read + // * -1 if InitializeReading has not been called (also prints to stderr). + virtual int NextPacket(WebRtc_UWord8** packet_pointer); + + private: + WebRtc_UWord8* data_; + int data_length_; + int packet_size_; + int currentIndex_; + bool initialized_; +}; + +} // namespace test +} // namespace webrtc + +#endif // SRC_MODULES_VIDEO_CODING_CODECS_TEST_PACKET_READER_H_ diff --git a/src/modules/video_coding/codecs/test/packet_reader_unittest.cc b/src/modules/video_coding/codecs/test/packet_reader_unittest.cc new file mode 100644 index 0000000000..f7343815b3 --- /dev/null +++ b/src/modules/video_coding/codecs/test/packet_reader_unittest.cc @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "gtest/gtest.h" +#include "packet_reader.h" +#include "typedefs.h" +#include "unittest_utils.h" + +namespace webrtc { +namespace test { + +class PacketReaderTest: public PacketRelatedTest { + protected: + PacketReader* reader_; + + PacketReaderTest() { + // To avoid warnings when using ASSERT_DEATH + ::testing::FLAGS_gtest_death_test_style = "threadsafe"; + } + + virtual ~PacketReaderTest() { + } + + void SetUp() { + reader_ = new PacketReader(); + } + + void TearDown() { + delete reader_; + } + + void VerifyPacketData(int expected_length, + int actual_length, + WebRtc_UWord8* original_data_pointer, + WebRtc_UWord8* new_data_pointer) { + EXPECT_EQ(expected_length, actual_length); + EXPECT_EQ(*original_data_pointer, *new_data_pointer); + EXPECT_EQ(0, memcmp(original_data_pointer, new_data_pointer, + actual_length)); + } +}; + +// Test lack of initialization +TEST_F(PacketReaderTest, Uninitialized) { + WebRtc_UWord8* data_pointer = NULL; + EXPECT_EQ(-1, reader_->NextPacket(&data_pointer)); + EXPECT_EQ(NULL, data_pointer); +} + +TEST_F(PacketReaderTest, InitializeNullDataArgument) { + ASSERT_DEATH(reader_->InitializeReading(NULL, kPacketDataLength, + kPacketSizeInBytes), ""); +} + +TEST_F(PacketReaderTest, InitializeInvalidLengthArgument) { + ASSERT_DEATH(reader_->InitializeReading(packet_data_, -1, kPacketSizeInBytes), + ""); +} + +TEST_F(PacketReaderTest, InitializeZeroLengthArgument) { + reader_->InitializeReading(packet_data_, 0, kPacketSizeInBytes); + ASSERT_EQ(0, reader_->NextPacket(&packet_data_pointer_)); +} + +TEST_F(PacketReaderTest, InitializeInvalidPacketSizeArgument) { + ASSERT_DEATH(reader_->InitializeReading(packet_data_, kPacketDataLength, + 0), ""); +} + +// Test with something smaller than one packet +TEST_F(PacketReaderTest, NormalSmallData) { + const int kDataLengthInBytes = 1499; + WebRtc_UWord8 data[kDataLengthInBytes]; + WebRtc_UWord8* data_pointer = data; + memset(data, 1, kDataLengthInBytes); + + reader_->InitializeReading(data, kDataLengthInBytes, kPacketSizeInBytes); + int length_to_read = reader_->NextPacket(&data_pointer); + VerifyPacketData(kDataLengthInBytes, length_to_read, data, data_pointer); + EXPECT_EQ(0, data_pointer - data); // pointer hasn't moved + + // Reading another one shall result in 0 bytes: + length_to_read = reader_->NextPacket(&data_pointer); + EXPECT_EQ(0, length_to_read); + EXPECT_EQ(kDataLengthInBytes, data_pointer - data); +} + +// Test with data length that exactly matches one packet +TEST_F(PacketReaderTest, NormalOnePacketData) { + WebRtc_UWord8 data[kPacketSizeInBytes]; + WebRtc_UWord8* data_pointer = data; + memset(data, 1, kPacketSizeInBytes); + + reader_->InitializeReading(data, kPacketSizeInBytes, kPacketSizeInBytes); + int length_to_read = reader_->NextPacket(&data_pointer); + VerifyPacketData(kPacketSizeInBytes, length_to_read, data, data_pointer); + EXPECT_EQ(0, data_pointer - data); // pointer hasn't moved + + // Reading another one shall result in 0 bytes: + length_to_read = reader_->NextPacket(&data_pointer); + EXPECT_EQ(0, length_to_read); + EXPECT_EQ(kPacketSizeInBytes, data_pointer - data); +} + +// Test with data length that will result in 3 packets +TEST_F(PacketReaderTest, NormalLargeData) { + reader_->InitializeReading(packet_data_, kPacketDataLength, + kPacketSizeInBytes); + + int length_to_read = reader_->NextPacket(&packet_data_pointer_); + VerifyPacketData(kPacketSizeInBytes, length_to_read, + packet1_, packet_data_pointer_); + + length_to_read = reader_->NextPacket(&packet_data_pointer_); + VerifyPacketData(kPacketSizeInBytes, length_to_read, + packet2_, packet_data_pointer_); + + length_to_read = reader_->NextPacket(&packet_data_pointer_); + VerifyPacketData(1u, length_to_read, + packet3_, packet_data_pointer_); + + // Reading another one shall result in 0 bytes: + length_to_read = reader_->NextPacket(&packet_data_pointer_); + EXPECT_EQ(0, length_to_read); + EXPECT_EQ(kPacketDataLength, packet_data_pointer_ - packet_data_); +} + +// Test with empty data. +TEST_F(PacketReaderTest, EmptyData) { + const int kDataLengthInBytes = 0; + WebRtc_UWord8 data[kDataLengthInBytes]; + WebRtc_UWord8* data_pointer = data; + reader_->InitializeReading(data, kDataLengthInBytes, kPacketSizeInBytes); + EXPECT_EQ(kDataLengthInBytes, reader_->NextPacket(&data_pointer)); + EXPECT_EQ(*data, *data_pointer); + // Do it again to make sure nothing changes + EXPECT_EQ(kDataLengthInBytes, reader_->NextPacket(&data_pointer)); + EXPECT_EQ(*data, *data_pointer); +} + +} // namespace test +} // namespace webrtc diff --git a/src/modules/video_coding/codecs/test/run_all_unittests.cc b/src/modules/video_coding/codecs/test/run_all_unittests.cc new file mode 100644 index 0000000000..e6a5c09576 --- /dev/null +++ b/src/modules/video_coding/codecs/test/run_all_unittests.cc @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "gmock/gmock.h" +#include "test/test_suite.h" + +int main(int argc, char** argv) { + ::testing::InitGoogleMock(&argc, argv); + webrtc::TestSuite test_suite(argc, argv); + return test_suite.Run(); +} diff --git a/src/modules/video_coding/codecs/test/stats.cc b/src/modules/video_coding/codecs/test/stats.cc new file mode 100644 index 0000000000..4ba76b5746 --- /dev/null +++ b/src/modules/video_coding/codecs/test/stats.cc @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "stats.h" + +#include // min_element, max_element +#include +#include + +#include "util.h" + +namespace webrtc { +namespace test { + +Stats::Stats() { +} + +Stats::~Stats() { +} + +bool LessForEncodeTime(const FrameStatistic& s1, const FrameStatistic& s2) { + return s1.encode_time_in_us < s2.encode_time_in_us; +} + +bool LessForDecodeTime(const FrameStatistic& s1, const FrameStatistic& s2) { + return s1.decode_time_in_us < s2.decode_time_in_us; +} + +bool LessForEncodedSize(const FrameStatistic& s1, const FrameStatistic& s2) { + return s1.encoded_frame_length_in_bytes < s2.encoded_frame_length_in_bytes; +} + +bool LessForBitRate(const FrameStatistic& s1, const FrameStatistic& s2) { + return s1.bit_rate_in_kbps < s2.bit_rate_in_kbps; +} + + +FrameStatistic& Stats::NewFrame(int frame_number) { + assert(frame_number >= 0); + FrameStatistic stat; + stat.frame_number = frame_number; + stats_.push_back(stat); + return stats_[frame_number]; +} + +void Stats::PrintSummary() { + log("Processing summary:\n"); + if (stats_.size() == 0) { + log("No frame statistics have been logged yet.\n"); + return; + } + + // Calculate min, max, average and total encoding time + int total_encoding_time_in_us = 0; + int total_decoding_time_in_us = 0; + int total_encoded_frames_lengths = 0; + int total_encoded_key_frames_lengths = 0; + int total_encoded_nonkey_frames_lengths = 0; + int nbr_keyframes = 0; + int nbr_nonkeyframes = 0; + + for (FrameStatisticsIterator it = stats_.begin(); + it != stats_.end(); ++it) { + total_encoding_time_in_us += it->encode_time_in_us; + total_decoding_time_in_us += it->decode_time_in_us; + total_encoded_frames_lengths += it->encoded_frame_length_in_bytes; + if (it->frame_type == webrtc::kKeyFrame) { + total_encoded_key_frames_lengths += it->encoded_frame_length_in_bytes; + nbr_keyframes++; + } else { + total_encoded_nonkey_frames_lengths += it->encoded_frame_length_in_bytes; + nbr_nonkeyframes++; + } + } + + FrameStatisticsIterator frame; + + // ENCODING + log("Encoding time:\n"); + frame = min_element(stats_.begin(), + stats_.end(), LessForEncodeTime); + log(" Min : %7d us (frame %d)\n", + frame->encode_time_in_us, frame->frame_number); + + frame = max_element(stats_.begin(), + stats_.end(), LessForEncodeTime); + log(" Max : %7d us (frame %d)\n", + frame->encode_time_in_us, frame->frame_number); + + log(" Average : %7d us\n", + total_encoding_time_in_us / stats_.size()); + + // DECODING + log("Decoding time:\n"); + // only consider frames that were successfully decoded (packet loss may cause + // failures) + std::vector decoded_frames; + for (std::vector::iterator it = stats_.begin(); + it != stats_.end(); ++it) { + if (it->decoding_successful) { + decoded_frames.push_back(*it); + } + } + if (decoded_frames.size() == 0) { + printf("No successfully decoded frames exist in this statistics."); + } else { + frame = min_element(decoded_frames.begin(), + decoded_frames.end(), LessForDecodeTime); + log(" Min : %7d us (frame %d)\n", + frame->decode_time_in_us, frame->frame_number); + + frame = max_element(decoded_frames.begin(), + decoded_frames.end(), LessForDecodeTime); + log(" Max : %7d us (frame %d)\n", + frame->decode_time_in_us, frame->frame_number); + + log(" Average : %7d us\n", + total_decoding_time_in_us / decoded_frames.size()); + log(" Failures: %d frames failed to decode.\n", + (stats_.size() - decoded_frames.size())); + } + + // SIZE + log("Frame sizes:\n"); + frame = min_element(stats_.begin(), + stats_.end(), LessForEncodedSize); + log(" Min : %7d bytes (frame %d)\n", + frame->encoded_frame_length_in_bytes, frame->frame_number); + + frame = max_element(stats_.begin(), + stats_.end(), LessForEncodedSize); + log(" Max : %7d bytes (frame %d)\n", + frame->encoded_frame_length_in_bytes, frame->frame_number); + + log(" Average : %7d bytes\n", + total_encoded_frames_lengths / stats_.size()); + if (nbr_keyframes > 0) { + log(" Average key frame size : %7d bytes (%d keyframes)\n", + total_encoded_key_frames_lengths / nbr_keyframes, + nbr_keyframes); + } + if (nbr_nonkeyframes > 0) { + log(" Average non-key frame size: %7d bytes (%d frames)\n", + total_encoded_nonkey_frames_lengths / nbr_nonkeyframes, + nbr_nonkeyframes); + } + + // BIT RATE + log("Bit rates:\n"); + frame = min_element(stats_.begin(), + stats_.end(), LessForBitRate); + log(" Min bit rate: %7d kbps (frame %d)\n", + frame->bit_rate_in_kbps, frame->frame_number); + + frame = max_element(stats_.begin(), + stats_.end(), LessForBitRate); + log(" Max bit rate: %7d kbps (frame %d)\n", + frame->bit_rate_in_kbps, frame->frame_number); + + log("\n"); + log("Total encoding time : %7d ms.\n", + total_encoding_time_in_us / 1000); + log("Total decoding time : %7d ms.\n", + total_decoding_time_in_us / 1000); + log("Total processing time: %7d ms.\n", + (total_encoding_time_in_us + total_decoding_time_in_us) / 1000); +} + +} // namespace test +} // namespace webrtc diff --git a/src/modules/video_coding/codecs/test/stats.h b/src/modules/video_coding/codecs/test/stats.h new file mode 100644 index 0000000000..f448314fdb --- /dev/null +++ b/src/modules/video_coding/codecs/test/stats.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef SRC_MODULES_VIDEO_CODING_CODECS_TEST_STATS_H_ +#define SRC_MODULES_VIDEO_CODING_CODECS_TEST_STATS_H_ + +#include + +#include "video_image.h" + +namespace webrtc { +namespace test { + +// Contains statistics of a single frame that has been processed. +struct FrameStatistic { + FrameStatistic() : + encoding_successful(false), decoding_successful(false), + encode_return_code(0), decode_return_code(0), + encode_time_in_us(0), decode_time_in_us(0), + frame_number(0), packets_dropped(0), total_packets(0), + bit_rate_in_kbps(0), encoded_frame_length_in_bytes(0) { + }; + bool encoding_successful; + bool decoding_successful; + int encode_return_code; + int decode_return_code; + int encode_time_in_us; + int decode_time_in_us; + int frame_number; + // How many packets were discarded of the encoded frame data (if any) + int packets_dropped; + int total_packets; + + // Current bit rate. Calculated out of the size divided with the time + // interval per frame. + int bit_rate_in_kbps; + + // Copied from EncodedImage + int encoded_frame_length_in_bytes; + webrtc::VideoFrameType frame_type; +}; + +// Handles statistics from a single video processing run. +// Contains calculation methods for interesting metrics from these stats. +class Stats { + public: + typedef std::vector::iterator FrameStatisticsIterator; + + Stats(); + virtual ~Stats(); + + // Add a new statistic data object. + // The frame number must be incrementing and start at zero in order to use + // it as an index for the frame_statistics_ vector. + // Returns the newly created statistic object. + FrameStatistic& NewFrame(int frame_number); + + // Prints a summary of all the statistics that have been gathered during the + // processing + void PrintSummary(); + + std::vector stats_; +}; + +} // namespace test +} // namespace webrtc + +#endif // SRC_MODULES_VIDEO_CODING_CODECS_TEST_STATS_H_ diff --git a/src/modules/video_coding/codecs/test/stats_unittest.cc b/src/modules/video_coding/codecs/test/stats_unittest.cc new file mode 100644 index 0000000000..e23ad9419f --- /dev/null +++ b/src/modules/video_coding/codecs/test/stats_unittest.cc @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "gtest/gtest.h" +#include "stats.h" +#include "typedefs.h" + +namespace webrtc { +namespace test { + +class StatsTest: public testing::Test { + protected: + StatsTest() { + } + + virtual ~StatsTest() { + } + + void SetUp() { + stats_ = new Stats(); + } + + void TearDown() { + delete stats_; + } + + Stats* stats_; +}; + +// Test empty object +TEST_F(StatsTest, Uninitialized) { + EXPECT_EQ(0u, stats_->stats_.size()); + stats_->PrintSummary(); // should not crash +} + +// Add single frame stats and verify +TEST_F(StatsTest, AddOne) { + stats_->NewFrame(0u); + FrameStatistic* frameStat = &stats_->stats_[0]; + EXPECT_EQ(0, frameStat->frame_number); +} + +// Add multiple frame stats and verify +TEST_F(StatsTest, AddMany) { + int nbr_of_frames = 1000; + for (int i = 0; i < nbr_of_frames; ++i) { + FrameStatistic& frameStat = stats_->NewFrame(i); + EXPECT_EQ(i, frameStat.frame_number); + } + EXPECT_EQ(nbr_of_frames, static_cast(stats_->stats_.size())); + + stats_->PrintSummary(); // should not crash +} + +} // namespace test +} // namespace webrtc diff --git a/src/modules/video_coding/codecs/test/unittest_utils.h b/src/modules/video_coding/codecs/test/unittest_utils.h new file mode 100644 index 0000000000..0db7fbd30c --- /dev/null +++ b/src/modules/video_coding/codecs/test/unittest_utils.h @@ -0,0 +1,58 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// Author: kjellander@google.com (Henrik Kjellander) + + +#ifndef SRC_MODULES_VIDEO_CODING_CODECS_TEST_UNITTEST_UTILS_H_ +#define SRC_MODULES_VIDEO_CODING_CODECS_TEST_UNITTEST_UTILS_H_ + +namespace webrtc { +namespace test { + +const int kPacketSizeInBytes = 1500; +const int kPacketDataLength = kPacketSizeInBytes * 2 + 1; +const int kPacketDataNumberOfPackets = 3; + +// A base test fixture for packet related tests. Contains +// two full prepared packets with 1s, 2s in their data and a third packet with +// a single 3 in it (size=1). +// A packet data structure is also available, that contains these three packets +// in order. +class PacketRelatedTest: public testing::Test { + protected: + // Tree packet byte arrays with data used for verification: + WebRtc_UWord8 packet1_[kPacketSizeInBytes]; + WebRtc_UWord8 packet2_[kPacketSizeInBytes]; + WebRtc_UWord8 packet3_[1]; + // Construct a data structure containing these packets + WebRtc_UWord8 packet_data_[kPacketDataLength]; + WebRtc_UWord8* packet_data_pointer_; + + PacketRelatedTest() { + packet_data_pointer_ = packet_data_; + + memset(packet1_, 1, kPacketSizeInBytes); + memset(packet2_, 2, kPacketSizeInBytes); + memset(packet3_, 3, 1); + // Fill the packet_data: + memcpy(packet_data_pointer_, packet1_, kPacketSizeInBytes); + memcpy(packet_data_pointer_ + kPacketSizeInBytes, packet2_, + kPacketSizeInBytes); + memcpy(packet_data_pointer_ + kPacketSizeInBytes * 2, packet3_, 1); + } + + virtual ~PacketRelatedTest() { + } + + void SetUp() { + // Initialize the random generator with 0 to get determenistic behaviour + srand(0); + } + + void TearDown() { + } +}; + +} // namespace test +} // namespace webrtc + +#endif // SRC_MODULES_VIDEO_CODING_CODECS_TEST_UNITTEST_UTILS_H_ diff --git a/src/modules/video_coding/codecs/test/util.cc b/src/modules/video_coding/codecs/test/util.cc new file mode 100644 index 0000000000..60a032edb8 --- /dev/null +++ b/src/modules/video_coding/codecs/test/util.cc @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "util.h" + +#include +#include + +#include "google/gflags.h" + +DEFINE_bool(verbose, true, "Verbose mode. Prints a lot of debugging info. " + "Suitable for tracking progress but not for capturing output. " + "Default: enabled"); + +int log(const char *format, ...) { + int result = 0; + if (FLAGS_verbose) { + va_list args; + va_start(args, format); + result = vprintf(format, args); + va_end(args); + } + return result; +} + diff --git a/src/modules/video_coding/codecs/test/util.h b/src/modules/video_coding/codecs/test/util.h new file mode 100644 index 0000000000..ea54ff86c1 --- /dev/null +++ b/src/modules/video_coding/codecs/test/util.h @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef SRC_MODULES_VIDEO_CODING_CODECS_TEST_UTIL_H_ +#define SRC_MODULES_VIDEO_CODING_CODECS_TEST_UTIL_H_ + +// Custom log method that only prints if the verbose flag is given +// Supports all the standard printf parameters and formatting (just forwarded) +int log(const char *format, ...); + +#endif // SRC_MODULES_VIDEO_CODING_CODECS_TEST_UTIL_H_ diff --git a/src/modules/video_coding/codecs/test/video_codecs_test_framework.gypi b/src/modules/video_coding/codecs/test/video_codecs_test_framework.gypi new file mode 100644 index 0000000000..298c75e18f --- /dev/null +++ b/src/modules/video_coding/codecs/test/video_codecs_test_framework.gypi @@ -0,0 +1,88 @@ +# Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +{ + # Exclude the test target when building with chromium. + 'conditions': [ + ['build_with_chromium==0', { + 'targets': [ + { + 'target_name': 'video_codecs_test_framework', + 'type': '<(library)', + 'dependencies': [ + '<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers', + '<(webrtc_root)/common_video/common_video.gyp:webrtc_vplib', + '<(webrtc_root)/../testing/gtest.gyp:gtest', + '<(webrtc_root)/../third_party/google-gflags/google-gflags.gyp:google-gflags', + ], + 'include_dirs': [ + '../interface', + '<(webrtc_root)/common_video/interface', + '<(webrtc_root)/../testing/gtest/include', + ], + 'direct_dependent_settings': { + 'include_dirs': [ + '../interface', + '<(webrtc_root)/../testing/gtest/include', + ], + }, + 'sources': [ + # header files + 'file_handler.h', + 'packet_manipulator.h', + 'packet_reader.h', + 'stats.h', + 'videoprocessor.h', + 'util.h', + + # source files + 'file_handler.cc', + 'packet_manipulator.cc', + 'packet_reader.cc', + 'stats.cc', + 'videoprocessor.cc', + 'util.cc', + ], + }, + { + 'target_name': 'video_codecs_test_framework_unittests', + 'type': 'executable', + 'dependencies': [ + 'video_codecs_test_framework', + '<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers', + '<(webrtc_root)/common_video/common_video.gyp:webrtc_vplib', + '<(webrtc_root)/../testing/gmock.gyp:gmock', + '<(webrtc_root)/../test/test.gyp:test_support', + ], + 'include_dirs': [ + '<(webrtc_root)/common_video/interface', + ], + 'sources': [ + # header files + 'mocks.h', + + # source files + 'file_handler_unittest.cc', + 'packet_manipulator_unittest.cc', + 'packet_reader_unittest.cc', + # cannot use the global run all file until it supports gmock: + 'run_all_unittests.cc', + 'stats_unittest.cc', + 'videoprocessor_unittest.cc', + ], + }, + ], # targets + }], # build_with_chromium + ], # conditions +} + +# Local Variables: +# tab-width:2 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=2 shiftwidth=2: diff --git a/src/modules/video_coding/codecs/test/videoprocessor.cc b/src/modules/video_coding/codecs/test/videoprocessor.cc new file mode 100644 index 0000000000..a03071e65f --- /dev/null +++ b/src/modules/video_coding/codecs/test/videoprocessor.cc @@ -0,0 +1,243 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "videoprocessor.h" + +#include +#include +#include + +#include "cpu_wrapper.h" +#include "util.h" + +namespace webrtc { +namespace test { + +VideoProcessorImpl::VideoProcessorImpl(webrtc::VideoEncoder* encoder, + webrtc::VideoDecoder* decoder, + FileHandler* file_handler, + PacketManipulator* packet_manipulator, + const TestConfig& config, + Stats* stats) + : encoder_(encoder), + decoder_(decoder), + file_handler_(file_handler), + packet_manipulator_(packet_manipulator), + config_(config), + stats_(stats), + encode_callback_(NULL), + decode_callback_(NULL), + source_buffer_(NULL), + first_key_frame_has_been_excluded_(false), + last_frame_missing_(false), + initialized_(false) { + assert(encoder); + assert(decoder); + assert(file_handler); + assert(packet_manipulator); + assert(stats); +} + +bool VideoProcessorImpl::Init() { + // Calculate a factor used for bit rate calculations: + bit_rate_factor_ = config_.codec_settings.maxFramerate * 0.001 * 8; // bits + + int frame_length_in_bytes = file_handler_->GetFrameLength(); + + // Initialize data structures used by the encoder/decoder APIs + source_buffer_ = new WebRtc_UWord8[frame_length_in_bytes]; + last_successful_frame_buffer_ = new WebRtc_UWord8[frame_length_in_bytes]; + + // Set fixed properties common for all frames: + source_frame_._width = config_.codec_settings.width; + source_frame_._height = config_.codec_settings.height; + source_frame_._length = frame_length_in_bytes; + source_frame_._size = frame_length_in_bytes; + + // Setup required callbacks for the encoder/decoder: + encode_callback_ = new VideoProcessorEncodeCompleteCallback(this); + decode_callback_ = new VideoProcessorDecodeCompleteCallback(this); + WebRtc_Word32 register_result = + encoder_->RegisterEncodeCompleteCallback(encode_callback_); + if (register_result != WEBRTC_VIDEO_CODEC_OK) { + fprintf(stderr, "Failed to register encode complete callback, return code: " + "%d\n", register_result); + return false; + } + register_result = decoder_->RegisterDecodeCompleteCallback(decode_callback_); + if (register_result != WEBRTC_VIDEO_CODEC_OK) { + fprintf(stderr, "Failed to register decode complete callback, return code: " + "%d\n", register_result); + return false; + } + // Init the encoder and decoder + WebRtc_UWord32 nbr_of_cores = 1; + if (!config_.use_single_core) { + nbr_of_cores = CpuWrapper::DetectNumberOfCores(); + } + WebRtc_Word32 init_result = + encoder_->InitEncode(&config_.codec_settings, nbr_of_cores, + config_.networking_config.max_payload_size_in_bytes); + if (init_result != WEBRTC_VIDEO_CODEC_OK) { + fprintf(stderr, "Failed to initialize VideoEncoder, return code: %d\n", + init_result); + return false; + } + init_result = decoder_->InitDecode(&config_.codec_settings, nbr_of_cores); + if (init_result != WEBRTC_VIDEO_CODEC_OK) { + fprintf(stderr, "Failed to initialize VideoDecoder, return code: %d\n", + init_result); + return false; + } + + log("Video Processor:\n"); + log(" #CPU cores used : %d\n", nbr_of_cores); + log(" Total # of frames: %d\n", file_handler_->GetNumberOfFrames()); + log(" Codec settings:\n"); + log(" Start bitrate : %d kbps\n", config_.codec_settings.startBitrate); + log(" Width : %d\n", config_.codec_settings.width); + log(" Height : %d\n", config_.codec_settings.height); + initialized_ = true; + return true; +} + +VideoProcessorImpl::~VideoProcessorImpl() { + delete[] source_buffer_; + delete[] last_successful_frame_buffer_; + encoder_->RegisterEncodeCompleteCallback(NULL); + delete encode_callback_; + decoder_->RegisterDecodeCompleteCallback(NULL); + delete decode_callback_; +} + +bool VideoProcessorImpl::ProcessFrame(int frame_number) { + assert(frame_number >=0); + if (!initialized_) { + fprintf(stderr, "Attempting to use uninitialized VideoProcessor!\n"); + return false; + } + if (file_handler_->ReadFrame(source_buffer_)) { + // point the source frame buffer to the newly read frame data: + source_frame_._buffer = source_buffer_; + + // Ensure we have a new statistics data object we can fill: + FrameStatistic& stat = stats_->NewFrame(frame_number); + + encode_start_ = TickTime::Now(); + // Use the frame number as "timestamp" to identify frames + source_frame_._timeStamp = frame_number; + WebRtc_Word32 encode_result = encoder_->Encode(source_frame_); + if (encode_result != WEBRTC_VIDEO_CODEC_OK) { + fprintf(stderr, "Failed to encode frame %d, return code: %d\n", + frame_number, encode_result); + } + stat.encode_return_code = encode_result; + return true; + } else { + return false; // we've reached the last frame + } +} + +void VideoProcessorImpl::FrameEncoded(EncodedImage* encoded_image) { + TickTime encode_stop = TickTime::Now(); + int frame_number = encoded_image->_timeStamp; + FrameStatistic& stat = stats_->stats_[frame_number]; + stat.encode_time_in_us = GetElapsedTimeMicroseconds(encode_start_, + encode_stop); + stat.encoding_successful = true; + stat.encoded_frame_length_in_bytes = encoded_image->_length; + stat.frame_number = encoded_image->_timeStamp; + stat.frame_type = encoded_image->_frameType; + stat.bit_rate_in_kbps = encoded_image->_length * bit_rate_factor_; + stat.total_packets = encoded_image->_length / + config_.networking_config.packet_size_in_bytes + 1; + + // Perform packet loss if criteria is fullfilled: + bool exclude_this_frame = false; + // Only keyframes can be excluded + if (encoded_image->_frameType == kKeyFrame) { + switch (config_.exclude_frame_types) { + case kExcludeOnlyFirstKeyFrame: + if (!first_key_frame_has_been_excluded_) { + first_key_frame_has_been_excluded_ = true; + exclude_this_frame = true; + } + break; + case kExcludeAllKeyFrames: + exclude_this_frame = true; + break; + default: + assert(false); + } + } + if (!exclude_this_frame) { + stat.packets_dropped = + packet_manipulator_->ManipulatePackets(encoded_image); + } + + // Keep track of if frames are lost due to packet loss so we can tell + // this to the encoder (this is handled by the RTP logic in the full stack) + decode_start_ = TickTime::Now(); + // TODO(kjellander): Pass fragmentation header to the decoder when + // CL 172001 has been submitted and PacketManipulator supports this. + WebRtc_Word32 decode_result = decoder_->Decode(*encoded_image, + last_frame_missing_, NULL); + stat.decode_return_code = decode_result; + if (decode_result != WEBRTC_VIDEO_CODEC_OK) { + // Write the last successful frame the output file to avoid getting it out + // of sync with the source file for SSIM and PSNR comparisons: + file_handler_->WriteFrame(last_successful_frame_buffer_); + } + // save status for losses so we can inform the decoder for the next frame: + last_frame_missing_ = encoded_image->_length == 0; +} + +void VideoProcessorImpl::FrameDecoded(const RawImage& image) { + TickTime decode_stop = TickTime::Now(); + int frame_number = image._timeStamp; + // Report stats + FrameStatistic& stat = stats_->stats_[frame_number]; + stat.decode_time_in_us = GetElapsedTimeMicroseconds(decode_start_, + decode_stop); + stat.decoding_successful = true; + // Update our copy of the last successful frame: + memcpy(last_successful_frame_buffer_, image._buffer, image._length); + + bool write_success = file_handler_->WriteFrame(image._buffer); + if (!write_success) { + fprintf(stderr, "Failed to write frame %d to disk!", frame_number); + } +} + +int VideoProcessorImpl::GetElapsedTimeMicroseconds( + const webrtc::TickTime& start, const webrtc::TickTime& stop) { + WebRtc_UWord64 encode_time = (stop - start).Microseconds(); + assert(encode_time < + static_cast(std::numeric_limits::max())); + return static_cast(encode_time); +} + +// Callbacks +WebRtc_Word32 +VideoProcessorImpl::VideoProcessorEncodeCompleteCallback::Encoded( + EncodedImage& encoded_image, + const webrtc::CodecSpecificInfo* codec_specific_info, + const webrtc::RTPFragmentationHeader* fragmentation) { + video_processor_->FrameEncoded(&encoded_image); // forward to parent class + return 0; +} +WebRtc_Word32 +VideoProcessorImpl::VideoProcessorDecodeCompleteCallback::Decoded( + RawImage& image) { + video_processor_->FrameDecoded(image); // forward to parent class + return 0; +} + +} // namespace test +} // namespace webrtc diff --git a/src/modules/video_coding/codecs/test/videoprocessor.h b/src/modules/video_coding/codecs/test/videoprocessor.h new file mode 100644 index 0000000000..71bfb754d8 --- /dev/null +++ b/src/modules/video_coding/codecs/test/videoprocessor.h @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef SRC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_ +#define SRC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_ + +#include + +#include "file_handler.h" +#include "packet_manipulator.h" +#include "stats.h" +#include "tick_util.h" +#include "video_codec_interface.h" + +namespace webrtc { +namespace test { + +// Defines which frame types shall be excluded from packet loss and when. +enum ExcludeFrameTypes { + // Will exclude the first keyframe in the video sequence from packet loss. + // Following keyframes will be targeted for packet loss. + kExcludeOnlyFirstKeyFrame, + // Exclude all keyframes from packet loss, no matter where in the video + // sequence they occur. + kExcludeAllKeyFrames +}; + +// Test configuration for a test run +struct TestConfig { + TestConfig() + : name(""), description(""), test_number(0), + input_filename(""), output_filename(""), output_dir("out"), + networking_config(), exclude_frame_types(kExcludeOnlyFirstKeyFrame), + use_single_core(false) { + }; + + // Name of the test. This is purely metadata and does not affect + // the test in any way. + std::string name; + + // More detailed description of the test. This is purely metadata and does + // not affect the test in any way. + std::string description; + + // Number of this test. Useful if multiple runs of the same test with + // different configurations shall be managed. + int test_number; + + // File to process for the test. This must be a video file in the YUV format. + std::string input_filename; + + // File to write to during processing for the test. Will be a video file + // in the YUV format. + std::string output_filename; + + // Path to the directory where encoded files will be put + // (absolute or relative to the executable). Default: "out". + std::string output_dir; + + // Configurations related to networking. + NetworkingConfig networking_config; + + // Decides how the packet loss simulations shall exclude certain frames + // from packet loss. Default: kExcludeOnlyFirstKeyFrame. + ExcludeFrameTypes exclude_frame_types; + + // Force the encoder and decoder to use a single core for processing. + // Using a single core is necessary to get a deterministic behavior for the + // encoded frames - using multiple cores will produce different encoded frames + // since multiple cores are competing to consume the byte budget for each + // frame in parallel. + // If set to false, the maximum number of available cores will be used. + // Default: false. + bool use_single_core; + + // The codec settings to use for the test (target bitrate, video size, + // framerate and so on) + webrtc::VideoCodec codec_settings; +}; + +// Handles encoding/decoding of video using the VideoEncoder/VideoDecoder +// interfaces. This is done in a sequential manner in order to be able to +// measure times properly. +// The class processes a frame at the time for the configured input file. +// It maintains state of where in the source input file the processing is at. +// +// Regarding packet loss: Note that keyframes are excluded (first or all +// depending on the ExcludeFrameTypes setting). This is because if key frames +// would be altered, all the following delta frames would be pretty much +// worthless. VP8 has an error-resilience feature that makes it able to handle +// packet loss in key non-first keyframes, which is why only the first is +// excluded by default. +// Packet loss in such important frames is handled on a higher level in the +// Video Engine, where signaling would request a retransmit of the lost packets, +// since they're so important. +// +// Note this class is not thread safe in any way and is meant for simple testing +// purposes. +class VideoProcessor { + public: + virtual ~VideoProcessor() {} + + // Performs initial calculations about frame size, sets up callbacks etc. + // Returns false if an error has occurred, in addition to printing to stderr. + virtual bool Init() = 0; + + // Processes a single frame. Returns true as long as there's more frames + // available in the source clip. + // Frame number must be an integer >=0. + virtual bool ProcessFrame(int frame_number) = 0; +}; + +class VideoProcessorImpl : public VideoProcessor { + public: + VideoProcessorImpl(webrtc::VideoEncoder* encoder, + webrtc::VideoDecoder* decoder, + FileHandler* file_handler, + PacketManipulator* packet_manipulator, + const TestConfig& config, + Stats* stats); + virtual ~VideoProcessorImpl(); + virtual bool Init(); + virtual bool ProcessFrame(int frame_number); + + private: + // Invoked by the callback when a frame has completed encoding. + void FrameEncoded(EncodedImage* encodedImage); + // Invoked by the callback when a frame has completed decoding. + void FrameDecoded(const RawImage& image); + // Used for getting a 32-bit integer representing time + // (checks the size is within signed 32-bit bounds before casting it) + int GetElapsedTimeMicroseconds(const webrtc::TickTime& start, + const webrtc::TickTime& stop); + + webrtc::VideoEncoder* encoder_; + webrtc::VideoDecoder* decoder_; + FileHandler* file_handler_; + PacketManipulator* packet_manipulator_; + const TestConfig& config_; + Stats* stats_; + + EncodedImageCallback* encode_callback_; + DecodedImageCallback* decode_callback_; + // Buffer used for reading the source video file: + WebRtc_UWord8* source_buffer_; + // Keep track of the last successful frame, since we need to write that + // when decoding fails: + WebRtc_UWord8* last_successful_frame_buffer_; + webrtc::RawImage source_frame_; + // To keep track of if we have excluded the first key frame from packet loss: + bool first_key_frame_has_been_excluded_; + // To tell the decoder previous frame have been dropped due to packet loss: + bool last_frame_missing_; + // If Init() has executed successfully. + bool initialized_; + + // Statistics + double bit_rate_factor_; // multiply frame length with this to get bit rate + webrtc::TickTime encode_start_; + webrtc::TickTime decode_start_; + + // Callback class required to implement according to the VideoEncoder API. + class VideoProcessorEncodeCompleteCallback + : public webrtc::EncodedImageCallback { + public: + explicit VideoProcessorEncodeCompleteCallback(VideoProcessorImpl* vp) + : video_processor_(vp) { + } + WebRtc_Word32 Encoded( + webrtc::EncodedImage& encoded_image, + const webrtc::CodecSpecificInfo* codec_specific_info = NULL, + const webrtc::RTPFragmentationHeader* fragmentation = NULL); + + private: + VideoProcessorImpl* video_processor_; + }; + + // Callback class required to implement according to the VideoDecoder API. + class VideoProcessorDecodeCompleteCallback + : public webrtc::DecodedImageCallback { + public: + explicit VideoProcessorDecodeCompleteCallback(VideoProcessorImpl* vp) + : video_processor_(vp) { + } + WebRtc_Word32 Decoded(webrtc::RawImage& image); + + private: + VideoProcessorImpl* video_processor_; + }; +}; + +} // namespace test +} // namespace webrtc + +#endif // SRC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_ diff --git a/src/modules/video_coding/codecs/test/videoprocessor_unittest.cc b/src/modules/video_coding/codecs/test/videoprocessor_unittest.cc new file mode 100644 index 0000000000..ee15490db0 --- /dev/null +++ b/src/modules/video_coding/codecs/test/videoprocessor_unittest.cc @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "gtest/gtest.h" +#include "gmock/gmock.h" +#include "mocks.h" +#include "packet_reader.h" +#include "packet_manipulator.h" +#include "typedefs.h" +#include "unittest_utils.h" +#include "videoprocessor.h" + +using ::testing::_; +using ::testing::AtLeast; +using ::testing::Return; + +namespace webrtc { +namespace test { + +// Very basic testing for VideoProcessor. It's mostly tested by running the +// video_quality_measurement program. +class VideoProcessorTest: public testing::Test { + protected: + MockVideoEncoder encoder_mock_; + MockVideoDecoder decoder_mock_; + MockFileHandler file_handler_mock_; + MockPacketManipulator packet_manipulator_mock_; + Stats stats_; + TestConfig config_; + + VideoProcessorTest() { + // To avoid warnings when using ASSERT_DEATH + ::testing::FLAGS_gtest_death_test_style = "threadsafe"; + } + + virtual ~VideoProcessorTest() { + } + + void SetUp() { + } + + void TearDown() { + } + + void ExpectInit() { + EXPECT_CALL(encoder_mock_, InitEncode(_, _, _)) + .Times(1); + EXPECT_CALL(encoder_mock_, RegisterEncodeCompleteCallback(_)) + .Times(AtLeast(1)); + EXPECT_CALL(decoder_mock_, InitDecode(_, _)) + .Times(1); + EXPECT_CALL(decoder_mock_, RegisterDecodeCompleteCallback(_)) + .Times(AtLeast(1)); + EXPECT_CALL(file_handler_mock_, GetNumberOfFrames()) + .WillOnce(Return(1)); + EXPECT_CALL(file_handler_mock_, GetFrameLength()) + .WillOnce(Return(150000)); + } +}; + +TEST_F(VideoProcessorTest, ConstructorNullEncoder) { + ASSERT_DEATH(VideoProcessorImpl video_processor(NULL, + &decoder_mock_, + &file_handler_mock_, + &packet_manipulator_mock_, + config_, + &stats_), ""); +} + +TEST_F(VideoProcessorTest, ConstructorNullDecoder) { + ASSERT_DEATH(VideoProcessorImpl video_processor(&encoder_mock_, + NULL, + &file_handler_mock_, + &packet_manipulator_mock_, + config_, + &stats_), ""); +} + +TEST_F(VideoProcessorTest, ConstructorNullFileHandler) { + ASSERT_DEATH(VideoProcessorImpl video_processor(&encoder_mock_, + &decoder_mock_, + NULL, + &packet_manipulator_mock_, + config_, + &stats_), ""); +} + +TEST_F(VideoProcessorTest, ConstructorNullPacketManipulator) { + ASSERT_DEATH(VideoProcessorImpl video_processor(&encoder_mock_, + &decoder_mock_, + &file_handler_mock_, + NULL, + config_, + &stats_), ""); +} + +TEST_F(VideoProcessorTest, ConstructorNullStats) { + ASSERT_DEATH(VideoProcessorImpl video_processor(&encoder_mock_, + &decoder_mock_, + &file_handler_mock_, + &packet_manipulator_mock_, + config_, + NULL), ""); +} + +TEST_F(VideoProcessorTest, Init) { + ExpectInit(); + VideoProcessorImpl video_processor(&encoder_mock_, &decoder_mock_, + &file_handler_mock_, + &packet_manipulator_mock_, config_, + &stats_); + video_processor.Init(); +} + +TEST_F(VideoProcessorTest, ProcessFrame) { + ExpectInit(); + EXPECT_CALL(encoder_mock_, Encode(_, _, _)) + .Times(1); + EXPECT_CALL(file_handler_mock_, ReadFrame(_)) + .WillOnce(Return(true)); + // Since we don't return any callback from the mock, the decoder will not + // be more than initialized... + VideoProcessorImpl video_processor(&encoder_mock_, &decoder_mock_, + &file_handler_mock_, + &packet_manipulator_mock_, config_, + &stats_); + video_processor.Init(); + video_processor.ProcessFrame(0); +} + +TEST_F(VideoProcessorTest, ProcessFrameInvalidArgument) { + ExpectInit(); + VideoProcessorImpl video_processor(&encoder_mock_, &decoder_mock_, + &file_handler_mock_, + &packet_manipulator_mock_, config_, + &stats_); + video_processor.Init(); + ASSERT_DEATH(video_processor.ProcessFrame(-1), ""); +} + + +} // namespace test +} // namespace webrtc diff --git a/src/modules/video_coding/codecs/tools/video_codecs_tools.gypi b/src/modules/video_coding/codecs/tools/video_codecs_tools.gypi new file mode 100644 index 0000000000..dc327fc0d4 --- /dev/null +++ b/src/modules/video_coding/codecs/tools/video_codecs_tools.gypi @@ -0,0 +1,46 @@ +# Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +{ + # Exclude the test target when building with chromium. + 'conditions': [ + ['build_with_chromium==0', { + 'targets': [ + { + 'target_name': 'video_quality_measurement', + 'type': 'executable', + 'dependencies': [ + 'video_codecs_test_framework', + 'video_coding_test_lib', + 'webrtc_vp8', + '<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers', + '<(webrtc_root)/common_video/common_video.gyp:webrtc_vplib', + '<(webrtc_root)/../third_party/google-gflags/google-gflags.gyp:google-gflags', + ], + 'include_dirs': [ + '../test', + '../interface', + '<(webrtc_root)/common_video/interface', + ], + 'sources': [ + # header files + + # source files + 'video_quality_measurement.cc', + ], + }, + ], # targets + }], # build_with_chromium + ], # conditions +} + +# Local Variables: +# tab-width:2 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=2 shiftwidth=2: diff --git a/src/modules/video_coding/codecs/tools/video_quality_measurement.cc b/src/modules/video_coding/codecs/tools/video_quality_measurement.cc new file mode 100644 index 0000000000..262400040e --- /dev/null +++ b/src/modules/video_coding/codecs/tools/video_quality_measurement.cc @@ -0,0 +1,337 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include // for checking directory existence + +#include +#include + +#include "google/gflags.h" +#include "packet_manipulator.h" +#include "packet_reader.h" +#include "stats.h" +#include "trace.h" +#include "util.h" +#include "video_metrics.h" +#include "videoprocessor.h" +#include "vp8.h" + +DEFINE_string(test_name, "Quality test", "The name of the test to run. " + "Default: Quality test."); +DEFINE_string(test_description, "", "A more detailed description about what " + "the current test is about."); +DEFINE_string(input_filename, "", "Input file. " + "The source video file to be encoded and decoded. Must be in " + ".yuv format"); +DEFINE_int32(width, -1, "Width in pixels of the frames in the input file."); +DEFINE_int32(height, -1, "Height in pixels of the frames in the input file."); +DEFINE_int32(framerate, 30, "Frame rate of the input file, in FPS " + "(frames-per-second). "); +DEFINE_string(output_dir, ".", "Output directory. " + "The directory where the output file will be put. Must already " + "exist."); +DEFINE_bool(use_single_core, false, "Force using a single core. If set to " + "true, only one core will be used for processing. Using a single " + "core is necessary to get a deterministic behavior for the" + "encoded frames - using multiple cores will produce different " + "encoded frames since multiple cores are competing to consume the " + "byte budget for each frame in parallel. If set to false, " + "the maximum detected number of cores will be used. "); +DEFINE_bool(disable_fixed_random_seed , false, "Set this flag to disable the" + "usage of a fixed random seed for the random generator used " + "for packet loss. Disabling this will cause consecutive runs " + "loose packets at different locations, which is bad for " + "reproducibility."); +DEFINE_string(output_filename, "", "Output file. " + "The name of the output video file resulting of the processing " + "of the source file. By default this is the same name as the " + "input file with '_out' appended before the extension."); +DEFINE_int32(bitrate, 500, "Bit rate in kilobits/second."); +DEFINE_int32(packet_size, 1500, "Simulated network packet size in bytes (MTU). " + "Used for packet loss simulation."); +DEFINE_int32(max_payload_size, 1440, "Max payload size in bytes for the " + "encoder."); +DEFINE_string(packet_loss_mode, "uniform", "Packet loss mode. Two different " + "packet loss models are supported: uniform or burst. This " + "setting has no effect unless packet_loss_rate is >0. "); +DEFINE_double(packet_loss_probability, 0.0, "Packet loss probability. A value " + "between 0.0 and 1.0 that defines the probability of a packet " + "being lost. 0.1 means 10% and so on."); +DEFINE_int32(packet_loss_burst_length, 1, "Packet loss burst length. Defines " + "how many packets will be lost in a burst when a packet has been " + "decided to be lost. Must be >=1."); +DEFINE_bool(csv, false, "CSV output. Enabling this will output all frame " + "statistics at the end of execution. Recommended to run combined " + "with --noverbose to avoid mixing output."); + +// Runs a quality measurement on the input file supplied to the program. +// The input file must be in YUV format. +int main(int argc, char* argv[]) { + std::string program_name = argv[0]; + std::string usage = "Quality test application for video comparisons.\n" + "Run " + program_name + " --helpshort for usage.\n" + "Example usage:\n" + program_name + + " --input_filename=filename.yuv --width=352 --height=288\n"; + google::SetUsageMessage(usage); + + google::ParseCommandLineFlags(&argc, &argv, true); + + if (FLAGS_input_filename == "" || FLAGS_width == -1 || FLAGS_height == -1) { + printf("%s\n", google::ProgramUsage()); + return 1; + } else { + webrtc::test::TestConfig config; + config.name = FLAGS_test_name; + config.description = FLAGS_test_description; + + // Verify the input file exists and is readable: + FILE* test_file; + test_file = fopen(FLAGS_input_filename.c_str(), "rb"); + if (test_file == NULL) { + fprintf(stderr, "Cannot read the specified input file: %s\n", + FLAGS_input_filename.c_str()); + return 2; + } + fclose(test_file); + config.input_filename = FLAGS_input_filename; + + // Verify the output dir exists: + DIR* output_dir = opendir(FLAGS_output_dir.c_str()); + if (output_dir == NULL) { + fprintf(stderr, "Cannot find output directory: %s\n", + FLAGS_output_dir.c_str()); + return 3; + } + closedir(output_dir); + config.output_dir = FLAGS_output_dir; + + // Manufacture an output filename if none was given: + if (FLAGS_output_filename == "") { + // Cut out the filename without extension from the given input file + // (which may include a path) + int startIndex = FLAGS_input_filename.find_last_of("/") + 1; + if (startIndex == 0) { + startIndex = 0; + } + FLAGS_output_filename = + FLAGS_input_filename.substr(startIndex, + FLAGS_input_filename.find_last_of(".") + - startIndex) + "_out.yuv"; + } + + // Verify output file can be written + std::string output_filename; + if (FLAGS_output_dir == ".") { + output_filename = FLAGS_output_filename; + } else { + output_filename = FLAGS_output_dir + "/"+ FLAGS_output_filename; + } + test_file = fopen(output_filename.c_str(), "wb"); + if (test_file == NULL) { + fprintf(stderr, "Cannot write output file: %s\n", + output_filename.c_str()); + return 4; + } + fclose(test_file); + config.output_filename = output_filename; + + // Check single core flag + config.use_single_core = FLAGS_use_single_core; + + // Seed our random function if that flag is enabled. This will force + // repeatable behaviour between runs + if (!FLAGS_disable_fixed_random_seed) { + srand(0); + } + + // Check the bit rate + if (FLAGS_bitrate <= 0) { + fprintf(stderr, "Bit rate must be >0 kbps, was: %d\n", FLAGS_bitrate); + return 5; + } + config.codec_settings.startBitrate = FLAGS_bitrate; + + // Check packet size and max payload size + if (FLAGS_packet_size <= 0) { + fprintf(stderr, "Packet size must be >0 bytes, was: %d\n", + FLAGS_packet_size); + return 6; + } + config.networking_config.packet_size_in_bytes = FLAGS_packet_size; + + if (FLAGS_max_payload_size <= 0) { + fprintf(stderr, "Max payload size must be >0 bytes, was: %d\n", + FLAGS_max_payload_size); + return 7; + } + config.networking_config.max_payload_size_in_bytes = FLAGS_max_payload_size; + + // Check the width and height + if (FLAGS_width <= 0 || FLAGS_height <= 0) { + fprintf(stderr, "Width and height must be >0."); + return 8; + } + config.codec_settings.width = FLAGS_width; + config.codec_settings.height = FLAGS_height; + + // Check framerate + if (FLAGS_framerate <= 0) { + fprintf(stderr, "Framerate be >0."); + return 9; + } + config.codec_settings.maxFramerate = FLAGS_framerate; + + // Check packet loss settings + if (FLAGS_packet_loss_mode != "uniform" && + FLAGS_packet_loss_mode != "burst") { + fprintf(stderr, "Unsupported packet loss mode, must be 'uniform' or " + "'burst'\n."); + return 10; + } + config.networking_config.packet_loss_mode = webrtc::test::kUniform; + if (FLAGS_packet_loss_mode == "burst") { + config.networking_config.packet_loss_mode = webrtc::test::kBurst; + } + + if (FLAGS_packet_loss_probability < 0.0 || + FLAGS_packet_loss_probability > 1.0) { + fprintf(stderr, "Invalid packet loss probability. Must be 0.0 - 1.0, " + "was: %f\n", FLAGS_packet_loss_probability); + return 11; + } + config.networking_config.packet_loss_probability = + FLAGS_packet_loss_probability; + + if (FLAGS_packet_loss_burst_length < 1) { + fprintf(stderr, "Invalid packet loss burst length, must be >=1, " + "was: %d\n", FLAGS_packet_loss_burst_length); + return 12; + } + config.networking_config.packet_loss_burst_length = + FLAGS_packet_loss_burst_length; + + // Calculate the size of each frame to read (according to YUV spec): + int frame_length_in_bytes = + 3 * config.codec_settings.width * config.codec_settings.height / 2; + + log("Quality test with parameters:\n"); + log(" Test name : %s\n", FLAGS_test_name.c_str()); + log(" Description : %s\n", FLAGS_test_description.c_str()); + log(" Input filename : %s\n", FLAGS_input_filename.c_str()); + log(" Output directory : %s\n", config.output_dir.c_str()); + log(" Output filename : %s\n", output_filename.c_str()); + log(" Frame size : %d bytes\n", frame_length_in_bytes); + log(" Packet size : %d bytes\n", FLAGS_packet_size); + log(" Max payload size : %d bytes\n", FLAGS_max_payload_size); + log(" Packet loss:\n"); + log(" Mode : %s\n", FLAGS_packet_loss_mode.c_str()); + log(" Probability : %2.1f\n", FLAGS_packet_loss_probability); + log(" Burst length : %d packets\n", FLAGS_packet_loss_burst_length); + + webrtc::VP8Encoder encoder; + webrtc::VP8Decoder decoder; + webrtc::test::Stats stats; + webrtc::test::FileHandlerImpl file_handler(config.input_filename, + config.output_filename, + frame_length_in_bytes); + file_handler.Init(); + webrtc::test::PacketReader packet_reader; + + webrtc::test::PacketManipulatorImpl packet_manipulator( + &packet_reader, config.networking_config); + webrtc::test::VideoProcessorImpl processor(&encoder, &decoder, + &file_handler, + &packet_manipulator, + config, &stats); + processor.Init(); + + int frame_number = 0; + while (processor.ProcessFrame(frame_number)) { + if (frame_number % 80 == 0) { + log("\n"); // make the output a bit nicer. + } + log("."); + frame_number++; + } + log("\n"); + log("Processed %d frames\n", frame_number); + + // Release encoder and decoder to make sure they have finished processing: + encoder.Release(); + decoder.Release(); + + // Verify statistics are correct: + assert(frame_number == static_cast(stats.stats_.size())); + + // Close the files before we start using them for SSIM/PSNR calculations. + file_handler.Close(); + + stats.PrintSummary(); + + // Calculate SSIM + QualityMetricsResult ssimResult; + log("Calculating SSIM...\n"); + SsimFromFiles(FLAGS_input_filename.c_str(), output_filename.c_str(), + config.codec_settings.width, + config.codec_settings.height, &ssimResult); + log(" Average: %3.2f\n", ssimResult.average); + log(" Min : %3.2f (frame %d)\n", ssimResult.min, + ssimResult.min_frame_number); + log(" Max : %3.2f (frame %d)\n", ssimResult.max, + ssimResult.max_frame_number); + + QualityMetricsResult psnrResult; + log("Calculating PSNR...\n"); + PsnrFromFiles(FLAGS_input_filename.c_str(), output_filename.c_str(), + config.codec_settings.width, + config.codec_settings.height, &psnrResult); + log(" Average: %3.2f\n", psnrResult.average); + log(" Min : %3.2f (frame %d)\n", psnrResult.min, + psnrResult.min_frame_number); + log(" Max : %3.2f (frame %d)\n", psnrResult.max, + psnrResult.max_frame_number); + + if (FLAGS_csv) { + log("\nCSV output (recommended to run with --noverbose to skip the " + "above output)\n"); + printf("frame_number encoding_successful decoding_successful " + "encode_return_code decode_return_code " + "encode_time_in_us decode_time_in_us " + "bit_rate_in_kbps encoded_frame_length_in_bytes frame_type " + "packets_dropped total_packets " + "ssim psnr\n"); + + for (unsigned int i = 0; i < stats.stats_.size(); ++i) { + webrtc::test::FrameStatistic& f = stats.stats_[i]; + FrameResult& ssim = ssimResult.frames[i]; + FrameResult& psnr = psnrResult.frames[i]; + printf("%4d, %d, %d, %2d, %2d, %6d, %6d, %5d, %7d, %d, %2d, %2d, " + "%5.3f, %5.2f\n", + f.frame_number, + f.encoding_successful, + f.decoding_successful, + f.encode_return_code, + f.decode_return_code, + f.encode_time_in_us, + f.decode_time_in_us, + f.bit_rate_in_kbps, + f.encoded_frame_length_in_bytes, + f.frame_type, + f.packets_dropped, + f.total_packets, + ssim.value, + psnr.value); + } + } + log("Quality test finished!"); + return 0; + } +} + +