diff --git a/webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc b/webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc index 502b4ec890..c0094dd83e 100644 --- a/webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc +++ b/webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc @@ -13,10 +13,11 @@ #include "gtest/gtest.h" #include "testsupport/fileutils.h" #include "testsupport/metrics/video_metrics.h" -#include "video_engine/test/auto_test/interface/vie_autotest.h" -#include "video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h" -#include "video_engine/test/auto_test/primitives/framedrop_primitives.h" -#include "video_engine/test/libvietest/include/vie_to_file_renderer.h" +#include "webrtc/video_engine/test/auto_test/interface/vie_autotest.h" +#include "webrtc/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h" +#include "webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.h" +#include "webrtc/video_engine/test/libvietest/include/tb_external_transport.h" +#include "webrtc/video_engine/test/libvietest/include/vie_to_file_renderer.h" namespace { @@ -115,8 +116,7 @@ class ParameterizedFullStackTest : public ViEVideoVerificationTest, public ::testing::WithParamInterface { protected: struct TestParameters { - int packet_loss_rate; - int one_way_delay; + NetworkParameters network; int bitrate; double avg_psnr_threshold; double avg_ssim_threshold; @@ -125,22 +125,32 @@ class ParameterizedFullStackTest : public ViEVideoVerificationTest, void SetUp() { int i = 0; - parameter_table_[i].packet_loss_rate = 0; - parameter_table_[i].one_way_delay = 0; + parameter_table_[i].network.packet_loss_rate = 0; + parameter_table_[i].network.mean_one_way_delay = 0; + parameter_table_[i].network.std_dev_one_way_delay = 0; parameter_table_[i].bitrate = 300; parameter_table_[i].avg_psnr_threshold = 35; parameter_table_[i].avg_ssim_threshold = 0.96; - parameter_table_[i].test_label = "net delay 0, plr 0"; + parameter_table_[i].test_label = "net delay (0, 0), plr 0"; ++i; - parameter_table_[i].packet_loss_rate = 5; - parameter_table_[i].one_way_delay = 50; + parameter_table_[i].network.packet_loss_rate = 5; + parameter_table_[i].network.mean_one_way_delay = 50; + parameter_table_[i].network.std_dev_one_way_delay = 5; parameter_table_[i].bitrate = 300; parameter_table_[i].avg_psnr_threshold = 35; parameter_table_[i].avg_ssim_threshold = 0.96; - parameter_table_[i].test_label = "net delay 50, plr 5"; + parameter_table_[i].test_label = "net delay (50, 5), plr 5"; + ++i; + parameter_table_[i].network.packet_loss_rate = 0; + parameter_table_[i].network.mean_one_way_delay = 100; + parameter_table_[i].network.std_dev_one_way_delay = 10; + parameter_table_[i].bitrate = 300; + parameter_table_[i].avg_psnr_threshold = 35; + parameter_table_[i].avg_ssim_threshold = 0.96; + parameter_table_[i].test_label = "net delay (100, 10), plr 0"; } - TestParameters parameter_table_[2]; + TestParameters parameter_table_[3]; }; TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) { @@ -195,15 +205,14 @@ TEST_P(ParameterizedFullStackTest, RunsFullStackWithoutErrors) { // Set a low bit rate so the encoder budget will be tight, causing it to drop // frames every now and then. const int kBitRateKbps = parameter_table_[GetParam()].bitrate; - const int kPacketLossPercent = parameter_table_[GetParam()].packet_loss_rate; - const int kNetworkDelayMs = parameter_table_[GetParam()].one_way_delay; + const NetworkParameters network = parameter_table_[GetParam()].network; int width = 352; int height = 288; ViETest::Log("Bit rate : %5d kbps", kBitRateKbps); - ViETest::Log("Packet loss : %5d %%", kPacketLossPercent); - ViETest::Log("Network delay: %5d ms", kNetworkDelayMs); - tests_.TestFullStack(input_file_, width, height, kBitRateKbps, - kPacketLossPercent, kNetworkDelayMs, + ViETest::Log("Packet loss : %5d %%", network.packet_loss_rate); + ViETest::Log("Network delay: mean=%dms std dev=%d ms", + network.mean_one_way_delay, network.std_dev_one_way_delay); + tests_.TestFullStack(input_file_, width, height, kBitRateKbps, network, local_file_renderer_, remote_file_renderer_, &detector); const std::string reference_file = local_file_renderer_->GetFullOutputPath(); const std::string output_file = remote_file_renderer_->GetFullOutputPath(); @@ -258,6 +267,6 @@ TEST_P(ParameterizedFullStackTest, RunsFullStackWithoutErrors) { } INSTANTIATE_TEST_CASE_P(FullStackTests, ParameterizedFullStackTest, - ::testing::Values(0, 1)); + ::testing::Values(0, 1, 2)); } // namespace diff --git a/webrtc/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h b/webrtc/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h index b73aefbbae..fc6540e5c7 100644 --- a/webrtc/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h +++ b/webrtc/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h @@ -14,6 +14,7 @@ #include class FrameDropDetector; +struct NetworkParameters; class ViEToFileRenderer; // This class contains comparison tests, which will exercise video engine @@ -48,8 +49,7 @@ class ViEFileBasedComparisonTests { int width, int height, int bit_rate_kbps, - int packet_loss_percent, - int network_delay_ms, + const NetworkParameters& network, ViEToFileRenderer* local_file_renderer, ViEToFileRenderer* remote_file_renderer, FrameDropDetector* frame_drop_detector); diff --git a/webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.cc b/webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.cc index c64201a75f..3292fe1344 100644 --- a/webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.cc +++ b/webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.cc @@ -143,8 +143,7 @@ void TestFullStack(const TbInterfaces& interfaces, int width, int height, int bit_rate_kbps, - int packet_loss_percent, - int network_delay_ms, + const NetworkParameters& network, FrameDropDetector* frame_drop_detector, ViEToFileRenderer* remote_file_renderer, ViEToFileRenderer* local_file_renderer) { @@ -173,8 +172,7 @@ void TestFullStack(const TbInterfaces& interfaces, // Configure External transport to simulate network interference: TbExternalTransport external_transport(*interfaces.network, video_channel, NULL); - external_transport.SetPacketLoss(packet_loss_percent); - external_transport.SetNetworkDelay(network_delay_ms); + external_transport.SetNetworkParameters(network); FrameSentCallback frame_sent_callback(frame_drop_detector); FrameReceivedCallback frame_received_callback(frame_drop_detector); @@ -220,11 +218,14 @@ void TestFullStack(const TbInterfaces& interfaces, // *************************************************************** EXPECT_EQ(0, capture_interface->DisconnectCaptureDevice(video_channel)); + const int one_way_delay_99_percentile = network.mean_one_way_delay + + 3 * network.std_dev_one_way_delay; + // Wait for the last packet to arrive before we tear down the receiver. - AutoTestSleep(2*network_delay_ms); + AutoTestSleep(2 * one_way_delay_99_percentile); EXPECT_EQ(0, base_interface->StopSend(video_channel)); while (!external_transport.EmptyQueue()) { - AutoTestSleep(network_delay_ms); + AutoTestSleep(one_way_delay_99_percentile); } EXPECT_EQ(0, base_interface->StopReceive(video_channel)); EXPECT_EQ(0, network_interface->DeregisterSendTransport(video_channel)); diff --git a/webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.h b/webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.h index a79032ab91..8815c8cd87 100644 --- a/webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.h +++ b/webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.h @@ -20,6 +20,7 @@ #include "video_engine/test/libvietest/include/vie_to_file_renderer.h" class FrameDropDetector; +struct NetworkParameters; class TbInterfaces; // Initializes the Video engine and its components, runs video playback using @@ -33,8 +34,7 @@ void TestFullStack(const TbInterfaces& interfaces, int width, int height, int bit_rate_kbps, - int packet_loss_percent, - int network_delay_ms, + const NetworkParameters& network, FrameDropDetector* frame_drop_detector, ViEToFileRenderer* remote_file_renderer, ViEToFileRenderer* local_file_renderer); diff --git a/webrtc/video_engine/test/auto_test/source/vie_autotest_loopback.cc b/webrtc/video_engine/test/auto_test/source/vie_autotest_loopback.cc index 82241f8c2d..c661b894a9 100644 --- a/webrtc/video_engine/test/auto_test/source/vie_autotest_loopback.cc +++ b/webrtc/video_engine/test/auto_test/source/vie_autotest_loopback.cc @@ -492,13 +492,13 @@ int VideoEngineSampleCode(void* window1, void* window2) return -1; } + NetworkParameters network = {0, 0, 0}; // Set up packet loss value std::cout << "Enter Packet Loss Percentage" << std::endl; std::string rate_str; std::getline(std::cin, rate_str); - int rate = atoi(rate_str.c_str()); - extTransport.SetPacketLoss(rate); - if (rate) { + network.packet_loss_rate = atoi(rate_str.c_str()); + if (network.packet_loss_rate > 0) { temporalToggling = false; } @@ -506,9 +506,8 @@ int VideoEngineSampleCode(void* window1, void* window2) std::cout << "Enter network delay value [mS]" << std::endl; std::string delay_str; std::getline(std::cin, delay_str); - int delayMs = atoi(delay_str.c_str()); - extTransport.SetNetworkDelay(delayMs); - + network.mean_one_way_delay = atoi(delay_str.c_str()); + extTransport.SetNetworkParameters(network); if (numTemporalLayers > 1 && temporalToggling) { extTransport.SetTemporalToggle(numTemporalLayers); } else { diff --git a/webrtc/video_engine/test/auto_test/source/vie_autotest_rtp_rtcp.cc b/webrtc/video_engine/test/auto_test/source/vie_autotest_rtp_rtcp.cc index 9ef4ced6c6..29e1e4be7a 100644 --- a/webrtc/video_engine/test/auto_test/source/vie_autotest_rtp_rtcp.cc +++ b/webrtc/video_engine/test/auto_test/source/vie_autotest_rtp_rtcp.cc @@ -164,8 +164,9 @@ void ViEAutoTest::ViERtpRtcpStandardTest() EXPECT_EQ(0, ViE.base->StopSend(tbChannel.videoChannel)); myTransport.ClearStats(); - int rate = 20; - myTransport.SetPacketLoss(rate); + const int kPacketLossRate = 20; + NetworkParameters network = {kPacketLossRate, 0, 0}; // 20% packet loss. + myTransport.SetNetworkParameters(network); // Start send to verify sending stats @@ -243,7 +244,8 @@ void ViEAutoTest::ViERtpRtcpStandardTest() // myTransport.ClearStats(); - myTransport.SetPacketLoss(rate); + network.packet_loss_rate = kPacketLossRate; + myTransport.SetNetworkParameters(network); EXPECT_EQ(0, ViE.rtp_rtcp->SetFECStatus( tbChannel.videoChannel, true, 96, 97)); @@ -287,7 +289,8 @@ void ViEAutoTest::ViERtpRtcpStandardTest() // Test to set SSRC - myTransport.SetPacketLoss(0); + network.packet_loss_rate = 0; + myTransport.SetNetworkParameters(network); myTransport.ClearStats(); unsigned int setSSRC = 0x01234567; diff --git a/webrtc/video_engine/test/auto_test/source/vie_autotest_simulcast.cc b/webrtc/video_engine/test/auto_test/source/vie_autotest_simulcast.cc index c29558e5b1..4b5d4d1f31 100644 --- a/webrtc/video_engine/test/auto_test/source/vie_autotest_simulcast.cc +++ b/webrtc/video_engine/test/auto_test/source/vie_autotest_simulcast.cc @@ -437,10 +437,9 @@ int VideoEngineSimulcastTest(void* window1, void* window2) { } } - ext_transport.SetPacketLoss(0); - - // Set network delay value. - ext_transport.SetNetworkDelay(10); + // Set network one-way delay value. + NetworkParameters network = {0, 10, 0}; // 10 ms one-way delay. + ext_transport.SetNetworkParameters(network); if (relay_mode == kRelayOneStream) { ext_transport.SetSSRCFilter(num_streams); diff --git a/webrtc/video_engine/test/auto_test/source/vie_file_based_comparison_tests.cc b/webrtc/video_engine/test/auto_test/source/vie_file_based_comparison_tests.cc index 7fcc69335d..11f7030dcf 100644 --- a/webrtc/video_engine/test/auto_test/source/vie_file_based_comparison_tests.cc +++ b/webrtc/video_engine/test/auto_test/source/vie_file_based_comparison_tests.cc @@ -14,6 +14,7 @@ #include "video_engine/test/auto_test/primitives/base_primitives.h" #include "video_engine/test/auto_test/primitives/framedrop_primitives.h" #include "video_engine/test/auto_test/primitives/general_primitives.h" +#include "video_engine/test/libvietest/include/tb_external_transport.h" #include "video_engine/test/libvietest/include/tb_interfaces.h" #include "video_engine/test/libvietest/include/vie_external_render_filter.h" #include "video_engine/test/libvietest/include/vie_fake_camera.h" @@ -89,8 +90,7 @@ void ViEFileBasedComparisonTests::TestFullStack( int width, int height, int bit_rate_kbps, - int packet_loss_percent, - int network_delay_ms, + const NetworkParameters& network, ViEToFileRenderer* local_file_renderer, ViEToFileRenderer* remote_file_renderer, FrameDropDetector* frame_drop_detector) { @@ -118,8 +118,7 @@ void ViEFileBasedComparisonTests::TestFullStack( ConfigureRtpRtcp(interfaces.rtp_rtcp, video_channel); ::TestFullStack(interfaces, capture_id, video_channel, width, height, - bit_rate_kbps, packet_loss_percent, network_delay_ms, - frame_drop_detector, remote_file_renderer, - local_file_renderer); + bit_rate_kbps, network, frame_drop_detector, + remote_file_renderer, local_file_renderer); EXPECT_TRUE(fake_camera.StopCamera()); } diff --git a/webrtc/video_engine/test/libvietest/include/tb_external_transport.h b/webrtc/video_engine/test/libvietest/include/tb_external_transport.h index 40ac133169..e27e9e37e8 100644 --- a/webrtc/video_engine/test/libvietest/include/tb_external_transport.h +++ b/webrtc/video_engine/test/libvietest/include/tb_external_transport.h @@ -28,6 +28,12 @@ class ThreadWrapper; class ViENetwork; } +struct NetworkParameters { + int packet_loss_rate; + int mean_one_way_delay; + int std_dev_one_way_delay; +}; + // Allows to subscribe for callback when a frame is started being sent. class SendFrameCallback { @@ -80,10 +86,9 @@ public: // Only one observer can be set (multiple calls will overwrite each other). virtual void RegisterReceiveFrameCallback(ReceiveFrameCallback* callback); - // The probability of a packet of being dropped. Packets belonging to the - // first packet (same RTP timestamp) will never be dropped. - WebRtc_Word32 SetPacketLoss(WebRtc_Word32 lossRate); // Rate in % - void SetNetworkDelay(WebRtc_Word64 delayMs); + // The network parameters of the link. Regarding packet losses, packets + // belonging to the first frame (same RTP timestamp) will never be dropped. + void SetNetworkParameters(const NetworkParameters& network_parameters); void SetSSRCFilter(WebRtc_UWord32 SSRC); void ClearStats(); @@ -104,6 +109,7 @@ protected: static bool ViEExternalTransportRun(void* object); bool ViEExternalTransportProcess(); private: + static int GaussianRandom(int mean_ms, int standard_deviation_ms); WebRtc_Word64 NowMs(); enum @@ -130,8 +136,7 @@ private: webrtc::CriticalSectionWrapper& _crit; webrtc::CriticalSectionWrapper& _statCrit; - WebRtc_Word32 _lossRate; - WebRtc_Word64 _networkDelayMs; + NetworkParameters network_parameters_; WebRtc_Word32 _rtpCount; WebRtc_Word32 _rtcpCount; WebRtc_Word32 _dropCount; @@ -163,6 +168,7 @@ private: // Track RTP timestamps so we invoke callbacks properly (if registered). WebRtc_UWord32 _lastSendRTPTimestamp; WebRtc_UWord32 _lastReceiveRTPTimestamp; + int64_t last_receive_time_; }; #endif // WEBRTC_VIDEO_ENGINE_TEST_AUTOTEST_INTERFACE_TB_EXTERNAL_TRANSPORT_H_ diff --git a/webrtc/video_engine/test/libvietest/testbed/tb_external_transport.cc b/webrtc/video_engine/test/libvietest/testbed/tb_external_transport.cc index 1063953048..48e11498b0 100644 --- a/webrtc/video_engine/test/libvietest/testbed/tb_external_transport.cc +++ b/webrtc/video_engine/test/libvietest/testbed/tb_external_transport.cc @@ -10,6 +10,7 @@ #include "video_engine/test/libvietest/include/tb_external_transport.h" +#include #include // printf #include // rand #include @@ -48,8 +49,7 @@ TbExternalTransport::TbExternalTransport( _event(*webrtc::EventWrapper::Create()), _crit(*webrtc::CriticalSectionWrapper::CreateCriticalSection()), _statCrit(*webrtc::CriticalSectionWrapper::CreateCriticalSection()), - _lossRate(0), - _networkDelayMs(0), + network_parameters_(), _rtpCount(0), _rtcpCount(0), _dropCount(0), @@ -72,10 +72,12 @@ TbExternalTransport::TbExternalTransport( _firstSequenceNumber(0), _firstRTPTimestamp(0), _lastSendRTPTimestamp(0), - _lastReceiveRTPTimestamp(0) + _lastReceiveRTPTimestamp(0), + last_receive_time_(-1) { srand((int) webrtc::TickTime::MicrosecondTimestamp()); unsigned int tId = 0; + memset(&network_parameters_, 0, sizeof(NetworkParameters)); _thread.Start(tId); } @@ -191,7 +193,8 @@ int TbExternalTransport::SendPacket(int channel, const void *data, int len) // Packet loss. Never drop packets from the first RTP timestamp, i.e. the // first frame being transmitted. int dropThis = rand() % 100; - if (dropThis < _lossRate && _firstRTPTimestamp != rtp_timestamp) + if (dropThis < network_parameters_.packet_loss_rate && + _firstRTPTimestamp != rtp_timestamp) { _statCrit.Enter(); _dropCount++; @@ -223,7 +226,15 @@ int TbExternalTransport::SendPacket(int channel, const void *data, int len) newPacket->channel = channel; _crit.Enter(); - newPacket->receiveTime = NowMs() + _networkDelayMs; + // Add jitter and make sure receiveTime isn't lower than receive time of + // last frame. + int network_delay_ms = GaussianRandom( + network_parameters_.mean_one_way_delay, + network_parameters_.std_dev_one_way_delay); + newPacket->receiveTime = NowMs() + network_delay_ms; + if (newPacket->receiveTime < last_receive_time_) { + newPacket->receiveTime = last_receive_time_; + } _rtpPackets.push_back(newPacket); _event.Set(); _crit.Leave(); @@ -258,24 +269,21 @@ int TbExternalTransport::SendRTCPPacket(int channel, const void *data, int len) newPacket->channel = channel; _crit.Enter(); - newPacket->receiveTime = NowMs() + _networkDelayMs; + int network_delay_ms = GaussianRandom( + network_parameters_.mean_one_way_delay, + network_parameters_.std_dev_one_way_delay); + newPacket->receiveTime = NowMs() + network_delay_ms; _rtcpPackets.push_back(newPacket); _event.Set(); _crit.Leave(); return len; } -WebRtc_Word32 TbExternalTransport::SetPacketLoss(WebRtc_Word32 lossRate) -{ - webrtc::CriticalSectionScoped cs(&_statCrit); - _lossRate = lossRate; - return 0; -} - -void TbExternalTransport::SetNetworkDelay(WebRtc_Word64 delayMs) +void TbExternalTransport::SetNetworkParameters( + const NetworkParameters& network_parameters) { webrtc::CriticalSectionScoped cs(&_crit); - _networkDelayMs = delayMs; + network_parameters_ = network_parameters; } void TbExternalTransport::SetSSRCFilter(WebRtc_UWord32 ssrc) @@ -492,3 +500,14 @@ WebRtc_Word64 TbExternalTransport::NowMs() { return webrtc::TickTime::MillisecondTimestamp(); } + +#define PI 3.14159265 +int TbExternalTransport::GaussianRandom(int mean_ms, + int standard_deviation_ms) { + // Creating a Normal distribution variable from two independent uniform + // variables based on the Box-Muller transform. + double uniform1 = (rand() + 1.0) / (RAND_MAX + 1.0); + double uniform2 = (rand() + 1.0) / (RAND_MAX + 1.0); + return static_cast(mean_ms + standard_deviation_ms * + sqrt(-2 * log(uniform1)) * cos(2 * PI * uniform2)); +}