diff --git a/webrtc/video/full_stack.cc b/webrtc/video/full_stack.cc index 3fb1db66a5..9ae0bd97a6 100644 --- a/webrtc/video/full_stack.cc +++ b/webrtc/video/full_stack.cc @@ -9,831 +9,112 @@ */ #include -#include -#include - #include "testing/gtest/include/gtest/gtest.h" - -#include "webrtc/base/format_macros.h" -#include "webrtc/base/scoped_ptr.h" -#include "webrtc/base/thread_annotations.h" -#include "webrtc/call.h" -#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" -#include "webrtc/frame_callback.h" -#include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h" -#include "webrtc/system_wrappers/interface/clock.h" -#include "webrtc/system_wrappers/interface/cpu_info.h" -#include "webrtc/system_wrappers/interface/critical_section_wrapper.h" -#include "webrtc/system_wrappers/interface/event_wrapper.h" -#include "webrtc/system_wrappers/interface/sleep.h" -#include "webrtc/test/encoder_settings.h" -#include "webrtc/test/fake_encoder.h" -#include "webrtc/test/frame_generator.h" -#include "webrtc/test/frame_generator_capturer.h" -#include "webrtc/test/statistics.h" -#include "webrtc/test/testsupport/fileutils.h" -#include "webrtc/typedefs.h" -#include "webrtc/video/full_stack.h" +#include "webrtc/video/video_quality_test.h" namespace webrtc { static const int kFullStackTestDurationSecs = 60; -static const int kSendStatsPollingIntervalMs = 1000; -class VideoAnalyzer : public PacketReceiver, - public newapi::Transport, - public VideoRenderer, - public VideoCaptureInput, - public EncodedFrameObserver { +class FullStackTest : public VideoQualityTest { public: - VideoAnalyzer(VideoCaptureInput* input, - Transport* transport, - const char* test_label, - double avg_psnr_threshold, - double avg_ssim_threshold, - int duration_frames, - const std::string& graph_data_output_filename) - : input_(input), - transport_(transport), - receiver_(nullptr), - send_stream_(nullptr), - test_label_(test_label), - graph_data_output_filename_(graph_data_output_filename), - frames_to_process_(duration_frames), - frames_recorded_(0), - frames_processed_(0), - dropped_frames_(0), - last_render_time_(0), - rtp_timestamp_delta_(0), - avg_psnr_threshold_(avg_psnr_threshold), - avg_ssim_threshold_(avg_ssim_threshold), - comparison_available_event_(EventWrapper::Create()), - done_(EventWrapper::Create()) { - // Create thread pool for CPU-expensive PSNR/SSIM calculations. - - // Try to use about as many threads as cores, but leave kMinCoresLeft alone, - // so that we don't accidentally starve "real" worker threads (codec etc). - // Also, don't allocate more than kMaxComparisonThreads, even if there are - // spare cores. - - uint32_t num_cores = CpuInfo::DetectNumberOfCores(); - RTC_DCHECK_GE(num_cores, 1u); - static const uint32_t kMinCoresLeft = 4; - static const uint32_t kMaxComparisonThreads = 8; - - if (num_cores <= kMinCoresLeft) { - num_cores = 1; - } else { - num_cores -= kMinCoresLeft; - num_cores = std::min(num_cores, kMaxComparisonThreads); - } - - for (uint32_t i = 0; i < num_cores; ++i) { - rtc::scoped_ptr thread = - ThreadWrapper::CreateThread(&FrameComparisonThread, this, "Analyzer"); - EXPECT_TRUE(thread->Start()); - comparison_thread_pool_.push_back(thread.release()); - } - - stats_polling_thread_ = - ThreadWrapper::CreateThread(&PollStatsThread, this, "StatsPoller"); - EXPECT_TRUE(stats_polling_thread_->Start()); + void RunTest(const VideoQualityTest::Params ¶ms) { + RunWithAnalyzer(params); } - - ~VideoAnalyzer() { - for (ThreadWrapper* thread : comparison_thread_pool_) { - EXPECT_TRUE(thread->Stop()); - delete thread; - } - } - - virtual void SetReceiver(PacketReceiver* receiver) { receiver_ = receiver; } - - DeliveryStatus DeliverPacket(MediaType media_type, - const uint8_t* packet, - size_t length, - const PacketTime& packet_time) override { - rtc::scoped_ptr parser(RtpHeaderParser::Create()); - RTPHeader header; - parser->Parse(packet, length, &header); - { - rtc::CritScope lock(&crit_); - recv_times_[header.timestamp - rtp_timestamp_delta_] = - Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); - } - - return receiver_->DeliverPacket(media_type, packet, length, packet_time); - } - - void IncomingCapturedFrame(const VideoFrame& video_frame) override { - VideoFrame copy = video_frame; - copy.set_timestamp(copy.ntp_time_ms() * 90); - - { - rtc::CritScope lock(&crit_); - if (first_send_frame_.IsZeroSize() && rtp_timestamp_delta_ == 0) - first_send_frame_ = copy; - - frames_.push_back(copy); - } - - input_->IncomingCapturedFrame(video_frame); - } - - bool SendRtp(const uint8_t* packet, size_t length) override { - rtc::scoped_ptr parser(RtpHeaderParser::Create()); - RTPHeader header; - parser->Parse(packet, length, &header); - - { - rtc::CritScope lock(&crit_); - if (rtp_timestamp_delta_ == 0) { - rtp_timestamp_delta_ = - header.timestamp - first_send_frame_.timestamp(); - first_send_frame_.Reset(); - } - uint32_t timestamp = header.timestamp - rtp_timestamp_delta_; - send_times_[timestamp] = - Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); - encoded_frame_sizes_[timestamp] += - length - (header.headerLength + header.paddingLength); - } - - return transport_->SendRtp(packet, length); - } - - bool SendRtcp(const uint8_t* packet, size_t length) override { - return transport_->SendRtcp(packet, length); - } - - void EncodedFrameCallback(const EncodedFrame& frame) override { - rtc::CritScope lock(&comparison_lock_); - if (frames_recorded_ < frames_to_process_) - encoded_frame_size_.AddSample(frame.length_); - } - - void RenderFrame(const VideoFrame& video_frame, - int time_to_render_ms) override { - int64_t render_time_ms = - Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); - uint32_t send_timestamp = video_frame.timestamp() - rtp_timestamp_delta_; - - rtc::CritScope lock(&crit_); - - while (frames_.front().timestamp() < send_timestamp) { - AddFrameComparison(frames_.front(), last_rendered_frame_, true, - render_time_ms); - frames_.pop_front(); - } - - VideoFrame reference_frame = frames_.front(); - frames_.pop_front(); - assert(!reference_frame.IsZeroSize()); - EXPECT_EQ(reference_frame.timestamp(), send_timestamp); - assert(reference_frame.timestamp() == send_timestamp); - - AddFrameComparison(reference_frame, video_frame, false, render_time_ms); - - last_rendered_frame_ = video_frame; - } - - bool IsTextureSupported() const override { return false; } - - void Wait() { - // Frame comparisons can be very expensive. Wait for test to be done, but - // at time-out check if frames_processed is going up. If so, give it more - // time, otherwise fail. Hopefully this will reduce test flakiness. - - int last_frames_processed = -1; - EventTypeWrapper eventType; - int iteration = 0; - while ((eventType = done_->Wait(FullStackTest::kDefaultTimeoutMs)) != - kEventSignaled) { - int frames_processed; - { - rtc::CritScope crit(&comparison_lock_); - frames_processed = frames_processed_; - } - - // Print some output so test infrastructure won't think we've crashed. - const char* kKeepAliveMessages[3] = { - "Uh, I'm-I'm not quite dead, sir.", - "Uh, I-I think uh, I could pull through, sir.", - "Actually, I think I'm all right to come with you--"}; - printf("- %s\n", kKeepAliveMessages[iteration++ % 3]); - - if (last_frames_processed == -1) { - last_frames_processed = frames_processed; - continue; - } - ASSERT_GT(frames_processed, last_frames_processed) - << "Analyzer stalled while waiting for test to finish."; - last_frames_processed = frames_processed; - } - - if (iteration > 0) - printf("- Farewell, sweet Concorde!\n"); - - // Signal stats polling thread if that is still waiting and stop it now, - // since it uses the send_stream_ reference that might be reclaimed after - // returning from this method. - done_->Set(); - EXPECT_TRUE(stats_polling_thread_->Stop()); - } - - VideoCaptureInput* input_; - Transport* transport_; - PacketReceiver* receiver_; - VideoSendStream* send_stream_; - - private: - struct FrameComparison { - FrameComparison() - : dropped(false), - send_time_ms(0), - recv_time_ms(0), - render_time_ms(0), - encoded_frame_size(0) {} - - FrameComparison(const VideoFrame& reference, - const VideoFrame& render, - bool dropped, - int64_t send_time_ms, - int64_t recv_time_ms, - int64_t render_time_ms, - size_t encoded_frame_size) - : reference(reference), - render(render), - dropped(dropped), - send_time_ms(send_time_ms), - recv_time_ms(recv_time_ms), - render_time_ms(render_time_ms), - encoded_frame_size(encoded_frame_size) {} - - VideoFrame reference; - VideoFrame render; - bool dropped; - int64_t send_time_ms; - int64_t recv_time_ms; - int64_t render_time_ms; - size_t encoded_frame_size; - }; - - struct Sample { - Sample(double dropped, - double input_time_ms, - double send_time_ms, - double recv_time_ms, - double encoded_frame_size, - double psnr, - double ssim, - double render_time_ms) - : dropped(dropped), - input_time_ms(input_time_ms), - send_time_ms(send_time_ms), - recv_time_ms(recv_time_ms), - encoded_frame_size(encoded_frame_size), - psnr(psnr), - ssim(ssim), - render_time_ms(render_time_ms) {} - - double dropped; - double input_time_ms; - double send_time_ms; - double recv_time_ms; - double encoded_frame_size; - double psnr; - double ssim; - double render_time_ms; - }; - - void AddFrameComparison(const VideoFrame& reference, - const VideoFrame& render, - bool dropped, - int64_t render_time_ms) - EXCLUSIVE_LOCKS_REQUIRED(crit_) { - int64_t send_time_ms = send_times_[reference.timestamp()]; - send_times_.erase(reference.timestamp()); - int64_t recv_time_ms = recv_times_[reference.timestamp()]; - recv_times_.erase(reference.timestamp()); - - size_t encoded_size = encoded_frame_sizes_[reference.timestamp()]; - encoded_frame_sizes_.erase(reference.timestamp()); - - VideoFrame reference_copy; - VideoFrame render_copy; - reference_copy.CopyFrame(reference); - render_copy.CopyFrame(render); - - rtc::CritScope crit(&comparison_lock_); - comparisons_.push_back(FrameComparison(reference_copy, render_copy, dropped, - send_time_ms, recv_time_ms, - render_time_ms, encoded_size)); - comparison_available_event_->Set(); - } - - static bool PollStatsThread(void* obj) { - return static_cast(obj)->PollStats(); - } - - bool PollStats() { - switch (done_->Wait(kSendStatsPollingIntervalMs)) { - case kEventSignaled: - case kEventError: - done_->Set(); // Make sure main thread is also signaled. - return false; - case kEventTimeout: - break; - default: - RTC_NOTREACHED(); - } - - VideoSendStream::Stats stats = send_stream_->GetStats(); - - rtc::CritScope crit(&comparison_lock_); - encode_frame_rate_.AddSample(stats.encode_frame_rate); - encode_time_ms.AddSample(stats.avg_encode_time_ms); - encode_usage_percent.AddSample(stats.encode_usage_percent); - media_bitrate_bps.AddSample(stats.media_bitrate_bps); - - return true; - } - - static bool FrameComparisonThread(void* obj) { - return static_cast(obj)->CompareFrames(); - } - - bool CompareFrames() { - if (AllFramesRecorded()) - return false; - - VideoFrame reference; - VideoFrame render; - FrameComparison comparison; - - if (!PopComparison(&comparison)) { - // Wait until new comparison task is available, or test is done. - // If done, wake up remaining threads waiting. - comparison_available_event_->Wait(1000); - if (AllFramesRecorded()) { - comparison_available_event_->Set(); - return false; - } - return true; // Try again. - } - - PerformFrameComparison(comparison); - - if (FrameProcessed()) { - PrintResults(); - if (!graph_data_output_filename_.empty()) - PrintSamplesToFile(); - done_->Set(); - comparison_available_event_->Set(); - return false; - } - - return true; - } - - bool PopComparison(FrameComparison* comparison) { - rtc::CritScope crit(&comparison_lock_); - // If AllFramesRecorded() is true, it means we have already popped - // frames_to_process_ frames from comparisons_, so there is no more work - // for this thread to be done. frames_processed_ might still be lower if - // all comparisons are not done, but those frames are currently being - // worked on by other threads. - if (comparisons_.empty() || AllFramesRecorded()) - return false; - - *comparison = comparisons_.front(); - comparisons_.pop_front(); - - FrameRecorded(); - return true; - } - - // Increment counter for number of frames received for comparison. - void FrameRecorded() { - rtc::CritScope crit(&comparison_lock_); - ++frames_recorded_; - } - - // Returns true if all frames to be compared have been taken from the queue. - bool AllFramesRecorded() { - rtc::CritScope crit(&comparison_lock_); - assert(frames_recorded_ <= frames_to_process_); - return frames_recorded_ == frames_to_process_; - } - - // Increase count of number of frames processed. Returns true if this was the - // last frame to be processed. - bool FrameProcessed() { - rtc::CritScope crit(&comparison_lock_); - ++frames_processed_; - assert(frames_processed_ <= frames_to_process_); - return frames_processed_ == frames_to_process_; - } - - void PrintResults() { - rtc::CritScope crit(&comparison_lock_); - PrintResult("psnr", psnr_, " dB"); - PrintResult("ssim", ssim_, ""); - PrintResult("sender_time", sender_time_, " ms"); - printf("RESULT dropped_frames: %s = %d frames\n", test_label_, - dropped_frames_); - PrintResult("receiver_time", receiver_time_, " ms"); - PrintResult("total_delay_incl_network", end_to_end_, " ms"); - PrintResult("time_between_rendered_frames", rendered_delta_, " ms"); - PrintResult("encoded_frame_size", encoded_frame_size_, " bytes"); - PrintResult("encode_frame_rate", encode_frame_rate_, " fps"); - PrintResult("encode_time", encode_time_ms, " ms"); - PrintResult("encode_usage_percent", encode_usage_percent, " percent"); - PrintResult("media_bitrate", media_bitrate_bps, " bps"); - - EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_); - EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_); - } - - void PerformFrameComparison(const FrameComparison& comparison) { - // Perform expensive psnr and ssim calculations while not holding lock. - double psnr = I420PSNR(&comparison.reference, &comparison.render); - double ssim = I420SSIM(&comparison.reference, &comparison.render); - - int64_t input_time_ms = comparison.reference.ntp_time_ms(); - - rtc::CritScope crit(&comparison_lock_); - if (!graph_data_output_filename_.empty()) { - samples_.push_back(Sample( - comparison.dropped, input_time_ms, comparison.send_time_ms, - comparison.recv_time_ms, comparison.encoded_frame_size, psnr, ssim, - comparison.render_time_ms)); - } - psnr_.AddSample(psnr); - ssim_.AddSample(ssim); - - if (comparison.dropped) { - ++dropped_frames_; - return; - } - if (last_render_time_ != 0) - rendered_delta_.AddSample(comparison.render_time_ms - last_render_time_); - last_render_time_ = comparison.render_time_ms; - - sender_time_.AddSample(comparison.send_time_ms - input_time_ms); - receiver_time_.AddSample(comparison.render_time_ms - - comparison.recv_time_ms); - end_to_end_.AddSample(comparison.render_time_ms - input_time_ms); - encoded_frame_size_.AddSample(comparison.encoded_frame_size); - } - - void PrintResult(const char* result_type, - test::Statistics stats, - const char* unit) { - printf("RESULT %s: %s = {%f, %f}%s\n", - result_type, - test_label_, - stats.Mean(), - stats.StandardDeviation(), - unit); - } - - void PrintSamplesToFile(void) { - FILE* out = fopen(graph_data_output_filename_.c_str(), "w"); - RTC_CHECK(out != nullptr) << "Couldn't open file: " - << graph_data_output_filename_; - - rtc::CritScope crit(&comparison_lock_); - std::sort(samples_.begin(), samples_.end(), - [](const Sample& A, const Sample& B) - -> bool { return A.input_time_ms < B.input_time_ms; }); - - fprintf(out, "%s\n", test_label_); - fprintf(out, "%" PRIuS "\n", samples_.size()); - fprintf(out, - "dropped " - "input_time_ms " - "send_time_ms " - "recv_time_ms " - "encoded_frame_size " - "psnr " - "ssim " - "render_time_ms\n"); - for (const Sample& sample : samples_) { - fprintf(out, "%lf %lf %lf %lf %lf %lf %lf %lf\n", sample.dropped, - sample.input_time_ms, sample.send_time_ms, sample.recv_time_ms, - sample.encoded_frame_size, sample.psnr, sample.ssim, - sample.render_time_ms); - } - fclose(out); - } - - const char* const test_label_; - std::string graph_data_output_filename_; - std::vector samples_ GUARDED_BY(comparison_lock_); - test::Statistics sender_time_ GUARDED_BY(comparison_lock_); - test::Statistics receiver_time_ GUARDED_BY(comparison_lock_); - test::Statistics psnr_ GUARDED_BY(comparison_lock_); - test::Statistics ssim_ GUARDED_BY(comparison_lock_); - test::Statistics end_to_end_ GUARDED_BY(comparison_lock_); - test::Statistics rendered_delta_ GUARDED_BY(comparison_lock_); - test::Statistics encoded_frame_size_ GUARDED_BY(comparison_lock_); - test::Statistics encode_frame_rate_ GUARDED_BY(comparison_lock_); - test::Statistics encode_time_ms GUARDED_BY(comparison_lock_); - test::Statistics encode_usage_percent GUARDED_BY(comparison_lock_); - test::Statistics media_bitrate_bps GUARDED_BY(comparison_lock_); - - const int frames_to_process_; - int frames_recorded_; - int frames_processed_; - int dropped_frames_; - int64_t last_render_time_; - uint32_t rtp_timestamp_delta_; - - rtc::CriticalSection crit_; - std::deque frames_ GUARDED_BY(crit_); - std::deque send_stats_ GUARDED_BY(crit_); - VideoFrame last_rendered_frame_ GUARDED_BY(crit_); - std::map send_times_ GUARDED_BY(crit_); - std::map recv_times_ GUARDED_BY(crit_); - std::map encoded_frame_sizes_ GUARDED_BY(crit_); - VideoFrame first_send_frame_ GUARDED_BY(crit_); - const double avg_psnr_threshold_; - const double avg_ssim_threshold_; - - rtc::CriticalSection comparison_lock_; - std::vector comparison_thread_pool_; - rtc::scoped_ptr stats_polling_thread_; - const rtc::scoped_ptr comparison_available_event_; - std::deque comparisons_ GUARDED_BY(comparison_lock_); - const rtc::scoped_ptr done_; }; -void FullStackTest::RunTest(const FullStackTestParams& params) { - // TODO(ivica): Add num_temporal_layers as a param. - unsigned char num_temporal_layers = - params.graph_data_output_filename.empty() ? 2 : 1; - - test::DirectTransport send_transport(params.link); - test::DirectTransport recv_transport(params.link); - VideoAnalyzer analyzer(nullptr, &send_transport, params.test_label, - params.avg_psnr_threshold, params.avg_ssim_threshold, - params.test_durations_secs * params.clip.fps, - params.graph_data_output_filename); - - CreateCalls(Call::Config(), Call::Config()); - - analyzer.SetReceiver(receiver_call_->Receiver()); - send_transport.SetReceiver(&analyzer); - recv_transport.SetReceiver(sender_call_->Receiver()); - - CreateSendConfig(1, &analyzer); - - rtc::scoped_ptr encoder; - if (params.codec == "VP8") { - encoder = - rtc::scoped_ptr(VideoEncoder::Create(VideoEncoder::kVp8)); - send_config_.encoder_settings.encoder = encoder.get(); - send_config_.encoder_settings.payload_name = "VP8"; - } else if (params.codec == "VP9") { - encoder = - rtc::scoped_ptr(VideoEncoder::Create(VideoEncoder::kVp9)); - send_config_.encoder_settings.encoder = encoder.get(); - send_config_.encoder_settings.payload_name = "VP9"; - } else { - RTC_NOTREACHED() << "Codec not supported!"; - return; - } - send_config_.encoder_settings.payload_type = 124; - - send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs; - send_config_.rtp.rtx.ssrcs.push_back(kSendRtxSsrcs[0]); - send_config_.rtp.rtx.payload_type = kSendRtxPayloadType; - - VideoStream* stream = &encoder_config_.streams[0]; - stream->width = params.clip.width; - stream->height = params.clip.height; - stream->min_bitrate_bps = params.min_bitrate_bps; - stream->target_bitrate_bps = params.target_bitrate_bps; - stream->max_bitrate_bps = params.max_bitrate_bps; - stream->max_framerate = params.clip.fps; - - VideoCodecVP8 vp8_settings; - VideoCodecVP9 vp9_settings; - if (params.mode == ContentMode::kScreensharingStaticImage || - params.mode == ContentMode::kScreensharingScrollingImage) { - encoder_config_.content_type = VideoEncoderConfig::ContentType::kScreen; - encoder_config_.min_transmit_bitrate_bps = 400 * 1000; - if (params.codec == "VP8") { - vp8_settings = VideoEncoder::GetDefaultVp8Settings(); - vp8_settings.denoisingOn = false; - vp8_settings.frameDroppingOn = false; - vp8_settings.numberOfTemporalLayers = num_temporal_layers; - encoder_config_.encoder_specific_settings = &vp8_settings; - } else if (params.codec == "VP9") { - vp9_settings = VideoEncoder::GetDefaultVp9Settings(); - vp9_settings.denoisingOn = false; - vp9_settings.frameDroppingOn = false; - vp9_settings.numberOfTemporalLayers = num_temporal_layers; - encoder_config_.encoder_specific_settings = &vp9_settings; - } - - stream->temporal_layer_thresholds_bps.clear(); - if (num_temporal_layers > 1) { - stream->temporal_layer_thresholds_bps.push_back( - stream->target_bitrate_bps); - } - } - - CreateMatchingReceiveConfigs(&recv_transport); - receive_configs_[0].renderer = &analyzer; - receive_configs_[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs; - receive_configs_[0].rtp.rtx[kSendRtxPayloadType].ssrc = kSendRtxSsrcs[0]; - receive_configs_[0].rtp.rtx[kSendRtxPayloadType].payload_type = - kSendRtxPayloadType; - - for (auto& config : receive_configs_) - config.pre_decode_callback = &analyzer; - CreateStreams(); - analyzer.input_ = send_stream_->Input(); - analyzer.send_stream_ = send_stream_; - - std::vector slides; - slides.push_back(test::ResourcePath("web_screenshot_1850_1110", "yuv")); - slides.push_back(test::ResourcePath("presentation_1850_1110", "yuv")); - slides.push_back(test::ResourcePath("photo_1850_1110", "yuv")); - slides.push_back(test::ResourcePath("difficult_photo_1850_1110", "yuv")); - size_t kSlidesWidth = 1850; - size_t kSlidesHeight = 1110; - - Clock* clock = Clock::GetRealTimeClock(); - rtc::scoped_ptr frame_generator; - - switch (params.mode) { - case ContentMode::kRealTimeVideo: - frame_generator.reset(test::FrameGenerator::CreateFromYuvFile( - std::vector(1, - test::ResourcePath(params.clip.name, "yuv")), - params.clip.width, params.clip.height, 1)); - break; - case ContentMode::kScreensharingScrollingImage: - frame_generator.reset( - test::FrameGenerator::CreateScrollingInputFromYuvFiles( - clock, slides, kSlidesWidth, kSlidesHeight, params.clip.width, - params.clip.height, 2000, - 8000)); // Scroll for 2 seconds, then pause for 8. - break; - case ContentMode::kScreensharingStaticImage: - frame_generator.reset(test::FrameGenerator::CreateFromYuvFile( - slides, kSlidesWidth, kSlidesHeight, - 10 * params.clip.fps)); // Cycle image every 10 seconds. - break; - } - - ASSERT_TRUE(frame_generator.get() != nullptr); - frame_generator_capturer_.reset(new test::FrameGeneratorCapturer( - clock, &analyzer, frame_generator.release(), params.clip.fps)); - ASSERT_TRUE(frame_generator_capturer_->Init()); - - Start(); - - analyzer.Wait(); - - send_transport.StopSending(); - recv_transport.StopSending(); - - Stop(); - - DestroyStreams(); -} TEST_F(FullStackTest, ParisQcifWithoutPacketLoss) { - FullStackTestParams paris_qcif = {"net_delay_0_0_plr_0", - {"paris_qcif", 176, 144, 30}, - ContentMode::kRealTimeVideo, - 300000, - 300000, - 300000, - 36.0, - 0.96, - kFullStackTestDurationSecs, - "VP8"}; + VideoQualityTest::Params paris_qcif = { + {176, 144, 30, 300000, 300000, 300000, "VP8", 1}, + {"paris_qcif"}, + {}, + {"net_delay_0_0_plr_0", 36.0, 0.96, kFullStackTestDurationSecs}}; RunTest(paris_qcif); } TEST_F(FullStackTest, ForemanCifWithoutPacketLoss) { // TODO(pbos): Decide on psnr/ssim thresholds for foreman_cif. - FullStackTestParams foreman_cif = {"foreman_cif_net_delay_0_0_plr_0", - {"foreman_cif", 352, 288, 30}, - ContentMode::kRealTimeVideo, - 700000, - 700000, - 700000, - 0.0, - 0.0, - kFullStackTestDurationSecs, - "VP8"}; + VideoQualityTest::Params foreman_cif = { + {352, 288, 30, 700000, 700000, 700000, "VP8", 1}, + {"foreman_cif"}, + {}, + {"foreman_cif_net_delay_0_0_plr_0", 0.0, 0.0, kFullStackTestDurationSecs} + }; RunTest(foreman_cif); } TEST_F(FullStackTest, ForemanCifPlr5) { - FullStackTestParams foreman_cif = {"foreman_cif_delay_50_0_plr_5", - {"foreman_cif", 352, 288, 30}, - ContentMode::kRealTimeVideo, - 30000, - 500000, - 2000000, - 0.0, - 0.0, - kFullStackTestDurationSecs, - "VP8"}; - foreman_cif.link.loss_percent = 5; - foreman_cif.link.queue_delay_ms = 50; + VideoQualityTest::Params foreman_cif = { + {352, 288, 30, 30000, 500000, 2000000, "VP8", 1}, + {"foreman_cif"}, + {}, + {"foreman_cif_delay_50_0_plr_5", 0.0, 0.0, kFullStackTestDurationSecs}}; + foreman_cif.pipe.loss_percent = 5; + foreman_cif.pipe.queue_delay_ms = 50; RunTest(foreman_cif); } TEST_F(FullStackTest, ForemanCif500kbps) { - FullStackTestParams foreman_cif = {"foreman_cif_500kbps", - {"foreman_cif", 352, 288, 30}, - ContentMode::kRealTimeVideo, - 30000, - 500000, - 2000000, - 0.0, - 0.0, - kFullStackTestDurationSecs, - "VP8"}; - foreman_cif.link.queue_length_packets = 0; - foreman_cif.link.queue_delay_ms = 0; - foreman_cif.link.link_capacity_kbps = 500; + VideoQualityTest::Params foreman_cif = { + {352, 288, 30, 30000, 500000, 2000000, "VP8", 1}, + {"foreman_cif"}, + {}, + {"foreman_cif_500kbps", 0.0, 0.0, kFullStackTestDurationSecs}}; + foreman_cif.pipe.queue_length_packets = 0; + foreman_cif.pipe.queue_delay_ms = 0; + foreman_cif.pipe.link_capacity_kbps = 500; RunTest(foreman_cif); } TEST_F(FullStackTest, ForemanCif500kbpsLimitedQueue) { - FullStackTestParams foreman_cif = {"foreman_cif_500kbps_32pkts_queue", - {"foreman_cif", 352, 288, 30}, - ContentMode::kRealTimeVideo, - 30000, - 500000, - 2000000, - 0.0, - 0.0, - kFullStackTestDurationSecs, - "VP8"}; - foreman_cif.link.queue_length_packets = 32; - foreman_cif.link.queue_delay_ms = 0; - foreman_cif.link.link_capacity_kbps = 500; + VideoQualityTest::Params foreman_cif = { + {352, 288, 30, 30000, 500000, 2000000, "VP8", 1}, + {"foreman_cif"}, + {}, + {"foreman_cif_500kbps_32pkts_queue", 0.0, 0.0, kFullStackTestDurationSecs} + }; + foreman_cif.pipe.queue_length_packets = 32; + foreman_cif.pipe.queue_delay_ms = 0; + foreman_cif.pipe.link_capacity_kbps = 500; RunTest(foreman_cif); } TEST_F(FullStackTest, ForemanCif500kbps100ms) { - FullStackTestParams foreman_cif = {"foreman_cif_500kbps_100ms", - {"foreman_cif", 352, 288, 30}, - ContentMode::kRealTimeVideo, - 30000, - 500000, - 2000000, - 0.0, - 0.0, - kFullStackTestDurationSecs, - "VP8"}; - foreman_cif.link.queue_length_packets = 0; - foreman_cif.link.queue_delay_ms = 100; - foreman_cif.link.link_capacity_kbps = 500; + VideoQualityTest::Params foreman_cif = { + {352, 288, 30, 30000, 500000, 2000000, "VP8", 1}, + {"foreman_cif"}, + {}, + {"foreman_cif_500kbps_100ms", 0.0, 0.0, kFullStackTestDurationSecs}}; + foreman_cif.pipe.queue_length_packets = 0; + foreman_cif.pipe.queue_delay_ms = 100; + foreman_cif.pipe.link_capacity_kbps = 500; RunTest(foreman_cif); } TEST_F(FullStackTest, ForemanCif500kbps100msLimitedQueue) { - FullStackTestParams foreman_cif = {"foreman_cif_500kbps_100ms_32pkts_queue", - {"foreman_cif", 352, 288, 30}, - ContentMode::kRealTimeVideo, - 30000, - 500000, - 2000000, - 0.0, - 0.0, - kFullStackTestDurationSecs, - "VP8"}; - foreman_cif.link.queue_length_packets = 32; - foreman_cif.link.queue_delay_ms = 100; - foreman_cif.link.link_capacity_kbps = 500; + VideoQualityTest::Params foreman_cif = { + {352, 288, 30, 30000, 500000, 2000000, "VP8", 1}, + {"foreman_cif"}, + {}, + {"foreman_cif_500kbps_100ms_32pkts_queue", 0.0, 0.0, + kFullStackTestDurationSecs}}; + foreman_cif.pipe.queue_length_packets = 32; + foreman_cif.pipe.queue_delay_ms = 100; + foreman_cif.pipe.link_capacity_kbps = 500; RunTest(foreman_cif); } TEST_F(FullStackTest, ForemanCif1000kbps100msLimitedQueue) { - FullStackTestParams foreman_cif = {"foreman_cif_1000kbps_100ms_32pkts_queue", - {"foreman_cif", 352, 288, 30}, - ContentMode::kRealTimeVideo, - 30000, - 2000000, - 2000000, - 0.0, - 0.0, - kFullStackTestDurationSecs, - "VP8"}; - foreman_cif.link.queue_length_packets = 32; - foreman_cif.link.queue_delay_ms = 100; - foreman_cif.link.link_capacity_kbps = 1000; + VideoQualityTest::Params foreman_cif = { + {352, 288, 30, 30000, 500000, 2000000, "VP8", 1}, + {"foreman_cif"}, + {}, + {"foreman_cif_1000kbps_100ms_32pkts_queue", 0.0, 0.0, + kFullStackTestDurationSecs}}; + foreman_cif.pipe.queue_length_packets = 32; + foreman_cif.pipe.queue_delay_ms = 100; + foreman_cif.pipe.link_capacity_kbps = 1000; RunTest(foreman_cif); } @@ -841,49 +122,30 @@ TEST_F(FullStackTest, ForemanCif1000kbps100msLimitedQueue) { // https://code.google.com/p/chromium/issues/detail?id=513170 #include "webrtc/test/testsupport/gtest_disable.h" TEST_F(FullStackTest, DISABLED_ON_ANDROID(ScreenshareSlidesVP8_2TL)) { - FullStackTestParams screenshare_params = { - "screenshare_slides", - {"screenshare_slides", 1850, 1110, 5}, - ContentMode::kScreensharingStaticImage, - 50000, - 200000, - 2000000, - 0.0, - 0.0, - kFullStackTestDurationSecs, - "VP8"}; - RunTest(screenshare_params); + VideoQualityTest::Params screenshare = { + {1850, 1110, 5, 50000, 200000, 2000000, "VP8", 2, 400000}, + {}, // Video-specific. + {true, 10}, // Screenshare-specific. + {"screenshare_slides", 0.0, 0.0, kFullStackTestDurationSecs}}; + RunTest(screenshare); } TEST_F(FullStackTest, DISABLED_ON_ANDROID(ScreenshareSlidesVP8_2TL_Scroll)) { - FullStackTestParams screenshare_params = { - "screenshare_slides_scrolling", - // Crop height by two, scrolling vertically only. - {"screenshare_slides_scrolling", 1850, 1110 / 2, 5}, - ContentMode::kScreensharingScrollingImage, - 50000, - 200000, - 2000000, - 0.0, - 0.0, - kFullStackTestDurationSecs, - "VP8"}; - RunTest(screenshare_params); + VideoQualityTest::Params config = { + {1850, 1110 / 2, 5, 50000, 200000, 2000000, "VP8", 2, 400000}, + {}, + {true, 10, 2}, + {"screenshare_slides_scrolling", 0.0, 0.0, kFullStackTestDurationSecs}}; + RunTest(config); } // Disabled on Android along with VP8 screenshare above. TEST_F(FullStackTest, DISABLED_ON_ANDROID(ScreenshareSlidesVP9_2TL)) { - FullStackTestParams screenshare_params = { - "screenshare_slides_vp9_2tl", - {"screenshare_slides", 1850, 1110, 5}, - ContentMode::kScreensharingStaticImage, - 50000, - 200000, - 2000000, - 0.0, - 0.0, - kFullStackTestDurationSecs, - "VP9"}; - RunTest(screenshare_params); + VideoQualityTest::Params screenshare = { + {1850, 1110, 5, 50000, 200000, 2000000, "VP9", 2, 400000}, + {}, + {true, 10}, + {"screenshare_slides_vp9_2tl", 0.0, 0.0, kFullStackTestDurationSecs}}; + RunTest(screenshare); } } // namespace webrtc diff --git a/webrtc/video/full_stack.h b/webrtc/video/full_stack.h deleted file mode 100644 index 56ef4b329f..0000000000 --- a/webrtc/video/full_stack.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ -#ifndef WEBRTC_VIDEO_FULL_STACK_H_ -#define WEBRTC_VIDEO_FULL_STACK_H_ - -#include - -#include "webrtc/test/call_test.h" -#include "webrtc/test/direct_transport.h" - -namespace webrtc { - -enum class ContentMode { - kRealTimeVideo, - kScreensharingStaticImage, - kScreensharingScrollingImage, -}; - -struct FullStackTestParams { - const char* test_label; - struct { - const char* name; - size_t width, height; - int fps; - } clip; - ContentMode mode; - int min_bitrate_bps; - int target_bitrate_bps; - int max_bitrate_bps; - double avg_psnr_threshold; - double avg_ssim_threshold; - int test_durations_secs; - std::string codec; - FakeNetworkPipe::Config link; - std::string graph_data_output_filename; -}; - -class FullStackTest : public test::CallTest { - protected: - void RunTest(const FullStackTestParams& params); -}; - -} // namespace webrtc - -#endif // WEBRTC_VIDEO_FULL_STACK_H_ diff --git a/webrtc/video/full_stack_plot.py b/webrtc/video/full_stack_plot.py index 92b50a44a1..0e8990157f 100755 --- a/webrtc/video/full_stack_plot.py +++ b/webrtc/video/full_stack_plot.py @@ -7,7 +7,7 @@ # in the file PATENTS. All contributing project authors may # be found in the AUTHORS file in the root of the source tree. -"""Generate graphs for data generated by full_stack_quality_sampler.cc. +"""Generate graphs for data generated by loopback tests. Usage examples: Show end to end time for a single full stack test. @@ -76,7 +76,6 @@ _fields = [ name_to_id = {field[1]: field[0] for field in _fields} id_to_title = {field[0]: field[2] for field in _fields} - def field_arg_to_id(arg): if arg == "none": return None @@ -297,7 +296,6 @@ load_files.cache = {} def get_parser(): class CustomAction(argparse.Action): - def __call__(self, parser, namespace, values, option_string=None): if "ordered_args" not in namespace: namespace.ordered_args = [] @@ -331,7 +329,7 @@ def get_parser(): "Otherwise, a window will be shown.") parser.add_argument( "files", nargs="+", action=CustomAction, - help="List of text-based files generated by full_stack.cc") + help="List of text-based files generated by loopback tests.") return parser diff --git a/webrtc/video/full_stack_quality_sampler.cc b/webrtc/video/full_stack_quality_sampler.cc deleted file mode 100644 index a17975699a..0000000000 --- a/webrtc/video/full_stack_quality_sampler.cc +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "gflags/gflags.h" -#include "webrtc/test/field_trial.h" -#include "webrtc/test/run_test.h" -#include "webrtc/video/full_stack.h" - -namespace webrtc { -namespace flags { - -DEFINE_string(title, "Full stack graph", "Graph title."); -std::string Title() { - return static_cast(FLAGS_title); -} - -DEFINE_string(filename, "graph_data.txt", "Name of a target graph data file."); -std::string Filename() { - return static_cast(FLAGS_filename); -} - -DEFINE_string(clip_name, "screenshare_slides", "Clip name, resource name."); -std::string ClipName() { - return static_cast(FLAGS_clip_name); -} - -DEFINE_int32(width, 1850, "Video width (crops source)."); -size_t Width() { - return static_cast(FLAGS_width); -} - -DEFINE_int32(height, 1110, "Video height (crops source)."); -size_t Height() { - return static_cast(FLAGS_height); -} - -DEFINE_int32(fps, 5, "Frames per second."); -int Fps() { - return static_cast(FLAGS_fps); -} - -DEFINE_int32( - content_mode, - 1, - "0 - real time video, 1 - screenshare static, 2 - screenshare scrolling."); -ContentMode ContentModeFlag() { - switch (FLAGS_content_mode) { - case 0: - return ContentMode::kRealTimeVideo; - case 1: - return ContentMode::kScreensharingStaticImage; - case 2: - return ContentMode::kScreensharingScrollingImage; - default: - RTC_NOTREACHED() << "Unknown content mode!"; - return ContentMode::kScreensharingStaticImage; - } -} - -DEFINE_int32(test_duration, 60, "Duration of the test in seconds."); -int TestDuration() { - return static_cast(FLAGS_test_duration); -} - -DEFINE_int32(min_bitrate, 50000, "Minimum video bitrate."); -int MinBitrate() { - return static_cast(FLAGS_min_bitrate); -} - -DEFINE_int32(target_bitrate, - 500000, - "Target video bitrate. (Default value here different than in full " - "stack tests!)"); -int TargetBitrate() { - return static_cast(FLAGS_target_bitrate); -} - -DEFINE_int32(max_bitrate, - 500000, - "Maximum video bitrate. (Default value here different than in " - "full stack tests!)"); -int MaxBitrate() { - return static_cast(FLAGS_max_bitrate); -} - -DEFINE_string(codec, "VP9", "Video codec to use."); -std::string Codec() { - return static_cast(FLAGS_codec); -} - -DEFINE_string( - force_fieldtrials, - "", - "Field trials control experimental feature code which can be forced. " - "E.g. running with --force_fieldtrials=WebRTC-FooFeature/Enable/" - " will assign the group Enable to field trial WebRTC-FooFeature. Multiple " - "trials are separated by \"/\""); -} // namespace flags - -class FullStackGenGraph : public FullStackTest { - public: - void TestBody() override { - std::string title = flags::Title(); - std::string clip_name = flags::ClipName(); - FullStackTestParams params = { - title.c_str(), - {clip_name.c_str(), flags::Width(), flags::Height(), flags::Fps()}, - flags::ContentModeFlag(), - flags::MinBitrate(), - flags::TargetBitrate(), - flags::MaxBitrate(), - 0.0, // avg_psnr_threshold - 0.0, // avg_ssim_threshold - flags::TestDuration(), - flags::Codec()}; - params.graph_data_output_filename = flags::Filename(); - - RunTest(params); - } -}; - -void FullStackRun(void) { - FullStackGenGraph full_stack; - full_stack.TestBody(); -} -} // namespace webrtc - -int main(int argc, char* argv[]) { - ::testing::InitGoogleTest(&argc, argv); - google::ParseCommandLineFlags(&argc, &argv, true); - webrtc::test::InitFieldTrialsFromString( - webrtc::flags::FLAGS_force_fieldtrials); - webrtc::test::RunTest(webrtc::FullStackRun); - return 0; -} diff --git a/webrtc/video/loopback.cc b/webrtc/video/loopback.cc deleted file mode 100644 index 9083500c1e..0000000000 --- a/webrtc/video/loopback.cc +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include - -#include - -#include "webrtc/video/loopback.h" - -#include "testing/gtest/include/gtest/gtest.h" - -#include "webrtc/base/checks.h" -#include "webrtc/base/scoped_ptr.h" -#include "webrtc/call.h" -#include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h" -#include "webrtc/modules/rtp_rtcp/source/rtp_format.h" -#include "webrtc/modules/rtp_rtcp/source/rtp_utility.h" -#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" -#include "webrtc/system_wrappers/interface/clock.h" -#include "webrtc/test/encoder_settings.h" -#include "webrtc/test/fake_encoder.h" -#include "webrtc/test/layer_filtering_transport.h" -#include "webrtc/test/run_loop.h" -#include "webrtc/test/testsupport/trace_to_stderr.h" -#include "webrtc/test/video_capturer.h" -#include "webrtc/test/video_renderer.h" -#include "webrtc/typedefs.h" - -namespace webrtc { -namespace test { - -static const int kAbsSendTimeExtensionId = 7; - -static const uint32_t kSendSsrc = 0x654321; -static const uint32_t kSendRtxSsrc = 0x654322; -static const uint32_t kReceiverLocalSsrc = 0x123456; - -static const uint8_t kRtxVideoPayloadType = 96; -static const uint8_t kVideoPayloadTypeVP8 = 124; -static const uint8_t kVideoPayloadTypeVP9 = 125; - -Loopback::Loopback(const Config& config) - : config_(config), clock_(Clock::GetRealTimeClock()) { -} - -Loopback::~Loopback() { -} - -void Loopback::Run() { - rtc::scoped_ptr trace_to_stderr_; - if (config_.logs) - trace_to_stderr_.reset(new test::TraceToStderr); - - rtc::scoped_ptr local_preview( - test::VideoRenderer::Create("Local Preview", config_.width, - config_.height)); - rtc::scoped_ptr loopback_video( - test::VideoRenderer::Create("Loopback Video", config_.width, - config_.height)); - - Call::Config call_config; - call_config.bitrate_config.min_bitrate_bps = - static_cast(config_.min_bitrate_kbps) * 1000; - call_config.bitrate_config.start_bitrate_bps = - static_cast(config_.start_bitrate_kbps) * 1000; - call_config.bitrate_config.max_bitrate_bps = - static_cast(config_.max_bitrate_kbps) * 1000; - rtc::scoped_ptr call(Call::Create(call_config)); - - FakeNetworkPipe::Config pipe_config; - pipe_config.loss_percent = config_.loss_percent; - pipe_config.link_capacity_kbps = config_.link_capacity_kbps; - pipe_config.queue_length_packets = config_.queue_size; - pipe_config.queue_delay_ms = config_.avg_propagation_delay_ms; - pipe_config.delay_standard_deviation_ms = config_.std_propagation_delay_ms; - LayerFilteringTransport send_transport( - pipe_config, kVideoPayloadTypeVP8, kVideoPayloadTypeVP9, - static_cast(config_.tl_discard_threshold), - static_cast(config_.sl_discard_threshold)); - - // Loopback, call sends to itself. - send_transport.SetReceiver(call->Receiver()); - - VideoSendStream::Config send_config(&send_transport); - send_config.rtp.ssrcs.push_back(kSendSsrc); - send_config.rtp.rtx.ssrcs.push_back(kSendRtxSsrc); - send_config.rtp.rtx.payload_type = kRtxVideoPayloadType; - send_config.rtp.nack.rtp_history_ms = 1000; - send_config.rtp.extensions.push_back( - RtpExtension(RtpExtension::kAbsSendTime, kAbsSendTimeExtensionId)); - - send_config.local_renderer = local_preview.get(); - rtc::scoped_ptr encoder; - if (config_.codec == "VP8") { - encoder.reset(VideoEncoder::Create(VideoEncoder::kVp8)); - } else if (config_.codec == "VP9") { - encoder.reset(VideoEncoder::Create(VideoEncoder::kVp9)); - } else { - // Codec not supported. - RTC_NOTREACHED() << "Codec not supported!"; - return; - } - const int payload_type = - config_.codec == "VP8" ? kVideoPayloadTypeVP8 : kVideoPayloadTypeVP9; - send_config.encoder_settings.encoder = encoder.get(); - send_config.encoder_settings.payload_name = config_.codec; - send_config.encoder_settings.payload_type = payload_type; - - VideoEncoderConfig encoder_config(CreateEncoderConfig()); - - VideoSendStream* send_stream = - call->CreateVideoSendStream(send_config, encoder_config); - - rtc::scoped_ptr capturer(CreateCapturer(send_stream)); - - VideoReceiveStream::Config receive_config(&send_transport); - receive_config.rtp.remote_ssrc = send_config.rtp.ssrcs[0]; - receive_config.rtp.local_ssrc = kReceiverLocalSsrc; - receive_config.rtp.nack.rtp_history_ms = 1000; - receive_config.rtp.remb = true; - receive_config.rtp.rtx[payload_type].ssrc = kSendRtxSsrc; - receive_config.rtp.rtx[payload_type].payload_type = kRtxVideoPayloadType; - receive_config.rtp.extensions.push_back( - RtpExtension(RtpExtension::kAbsSendTime, kAbsSendTimeExtensionId)); - receive_config.renderer = loopback_video.get(); - VideoReceiveStream::Decoder decoder = - test::CreateMatchingDecoder(send_config.encoder_settings); - receive_config.decoders.push_back(decoder); - - VideoReceiveStream* receive_stream = - call->CreateVideoReceiveStream(receive_config); - - receive_stream->Start(); - send_stream->Start(); - capturer->Start(); - - test::PressEnterToContinue(); - - capturer->Stop(); - send_stream->Stop(); - receive_stream->Stop(); - - call->DestroyVideoReceiveStream(receive_stream); - call->DestroyVideoSendStream(send_stream); - - delete decoder.decoder; - - send_transport.StopSending(); -} - -VideoEncoderConfig Loopback::CreateEncoderConfig() { - VideoEncoderConfig encoder_config; - encoder_config.streams = test::CreateVideoStreams(1); - VideoStream* stream = &encoder_config.streams[0]; - stream->width = config_.width; - stream->height = config_.height; - stream->min_bitrate_bps = static_cast(config_.min_bitrate_kbps) * 1000; - stream->max_bitrate_bps = static_cast(config_.max_bitrate_kbps) * 1000; - stream->target_bitrate_bps = - static_cast(config_.max_bitrate_kbps) * 1000; - stream->max_framerate = config_.fps; - stream->max_qp = 56; - if (config_.num_temporal_layers != 0) { - stream->temporal_layer_thresholds_bps.resize(config_.num_temporal_layers - - 1); - } - return encoder_config; -} - -test::VideoCapturer* Loopback::CreateCapturer(VideoSendStream* send_stream) { - return test::VideoCapturer::Create(send_stream->Input(), config_.width, - config_.height, config_.fps, clock_); -} - -} // namespace test -} // namespace webrtc diff --git a/webrtc/video/loopback.h b/webrtc/video/loopback.h deleted file mode 100644 index c671de4e98..0000000000 --- a/webrtc/video/loopback.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include - -#include "webrtc/config.h" - -namespace webrtc { - -class VideoSendStream; -class Clock; - -namespace test { - -class VideoCapturer; - -class Loopback { - public: - struct Config { - size_t width; - size_t height; - int32_t fps; - size_t min_bitrate_kbps; - size_t start_bitrate_kbps; - size_t max_bitrate_kbps; - int32_t min_transmit_bitrate_kbps; - std::string codec; - size_t num_temporal_layers; - size_t num_spatial_layers; - // Discard all temporal/spatial layers with id greater or equal the - // threshold. 0 to disable. - size_t tl_discard_threshold; - size_t sl_discard_threshold; - int32_t loss_percent; - int32_t link_capacity_kbps; - int32_t queue_size; - int32_t avg_propagation_delay_ms; - int32_t std_propagation_delay_ms; - bool logs; - }; - - explicit Loopback(const Config& config); - virtual ~Loopback(); - - void Run(); - - protected: - virtual VideoEncoderConfig CreateEncoderConfig(); - virtual VideoCapturer* CreateCapturer(VideoSendStream* send_stream); - - const Config config_; - Clock* const clock_; -}; - -} // namespace test -} // namespace webrtc diff --git a/webrtc/video/screenshare_loopback.cc b/webrtc/video/screenshare_loopback.cc index 2dfadd1d6c..34f68947f6 100644 --- a/webrtc/video/screenshare_loopback.cc +++ b/webrtc/video/screenshare_loopback.cc @@ -10,20 +10,12 @@ #include -#include - #include "gflags/gflags.h" #include "testing/gtest/include/gtest/gtest.h" -#include "webrtc/base/checks.h" #include "webrtc/test/field_trial.h" -#include "webrtc/test/frame_generator.h" -#include "webrtc/test/frame_generator_capturer.h" #include "webrtc/test/run_test.h" -#include "webrtc/test/testsupport/fileutils.h" -#include "webrtc/typedefs.h" -#include "webrtc/video/loopback.h" -#include "webrtc/video/video_send_stream.h" +#include "webrtc/video/video_quality_test.h" namespace webrtc { namespace flags { @@ -58,49 +50,41 @@ int ScrollDuration() { return static_cast(FLAGS_scroll_duration); } -DEFINE_int32(min_bitrate, 50, "Minimum video bitrate."); -size_t MinBitrate() { - return static_cast(FLAGS_min_bitrate); +DEFINE_int32(min_bitrate, 50, "Call and stream min bitrate in kbps."); +int MinBitrateKbps() { + return static_cast(FLAGS_min_bitrate); } -DEFINE_int32(tl0_bitrate, 200, "Temporal layer 0 target bitrate."); -size_t StartBitrate() { - return static_cast(FLAGS_tl0_bitrate); +DEFINE_int32(start_bitrate, 200, "Call start bitrate in kbps."); +int StartBitrateKbps() { + return static_cast(FLAGS_start_bitrate); } -DEFINE_int32(tl1_bitrate, 2000, "Temporal layer 1 target bitrate."); -size_t MaxBitrate() { - return static_cast(FLAGS_tl1_bitrate); +DEFINE_int32(target_bitrate, 2000, "Stream target bitrate in kbps."); +int TargetBitrateKbps() { + return static_cast(FLAGS_target_bitrate); +} + +DEFINE_int32(max_bitrate, 2000, "Call and stream max bitrate in kbps."); +int MaxBitrateKbps() { + return static_cast(FLAGS_max_bitrate); } DEFINE_int32(num_temporal_layers, 2, "Number of temporal layers to use."); -int NumTemporalLayers() { - return static_cast(FLAGS_num_temporal_layers); -} - -DEFINE_int32(num_spatial_layers, 1, "Number of spatial layers to use."); -int NumSpatialLayers() { - return static_cast(FLAGS_num_spatial_layers); +size_t NumTemporalLayers() { + return static_cast(FLAGS_num_temporal_layers); } DEFINE_int32( tl_discard_threshold, 0, "Discard TLs with id greater or equal the threshold. 0 to disable."); -int TLDiscardThreshold() { - return static_cast(FLAGS_tl_discard_threshold); -} - -DEFINE_int32( - sl_discard_threshold, - 0, - "Discard SLs with id greater or equal the threshold. 0 to disable."); -int SLDiscardThreshold() { - return static_cast(FLAGS_sl_discard_threshold); +size_t TLDiscardThreshold() { + return static_cast(FLAGS_tl_discard_threshold); } DEFINE_int32(min_transmit_bitrate, 400, "Min transmit bitrate incl. padding."); -int MinTransmitBitrate() { +int MinTransmitBitrateKbps() { return FLAGS_min_transmit_bitrate; } @@ -117,7 +101,7 @@ int LossPercent() { DEFINE_int32(link_capacity, 0, "Capacity (kbps) of the fake link. 0 means infinite."); -int LinkCapacity() { +int LinkCapacityKbps() { return static_cast(FLAGS_link_capacity); } @@ -142,6 +126,19 @@ int StdPropagationDelayMs() { DEFINE_bool(logs, false, "print logs to stderr"); +DEFINE_string( + output_filename, + "", + "Name of a target graph data file. If set, no preview will be shown."); +std::string OutputFilename() { + return static_cast(FLAGS_output_filename); +} + +DEFINE_int32(duration, 60, "Duration of the test in seconds."); +int DurationSecs() { + return static_cast(FLAGS_duration); +} + DEFINE_string( force_fieldtrials, "", @@ -151,115 +148,44 @@ DEFINE_string( "trials are separated by \"/\""); } // namespace flags -class ScreenshareLoopback : public test::Loopback { - public: - explicit ScreenshareLoopback(const Config& config) : Loopback(config) { - RTC_CHECK_GE(config.num_temporal_layers, 1u); - RTC_CHECK_LE(config.num_temporal_layers, 2u); - RTC_CHECK_GE(config.num_spatial_layers, 1u); - RTC_CHECK_LE(config.num_spatial_layers, 5u); - RTC_CHECK(config.num_spatial_layers == 1 || config.codec == "VP9"); - RTC_CHECK(config.num_spatial_layers == 1 || - config.num_temporal_layers == 1); - RTC_CHECK_LT(config.tl_discard_threshold, config.num_temporal_layers); - RTC_CHECK_LT(config.sl_discard_threshold, config.num_spatial_layers); - - vp8_settings_ = VideoEncoder::GetDefaultVp8Settings(); - vp8_settings_.denoisingOn = false; - vp8_settings_.frameDroppingOn = false; - vp8_settings_.numberOfTemporalLayers = - static_cast(config.num_temporal_layers); - - vp9_settings_ = VideoEncoder::GetDefaultVp9Settings(); - vp9_settings_.denoisingOn = false; - vp9_settings_.frameDroppingOn = false; - vp9_settings_.numberOfTemporalLayers = - static_cast(config.num_temporal_layers); - vp9_settings_.numberOfSpatialLayers = - static_cast(config.num_spatial_layers); - } - virtual ~ScreenshareLoopback() {} - - protected: - VideoEncoderConfig CreateEncoderConfig() override { - VideoEncoderConfig encoder_config(test::Loopback::CreateEncoderConfig()); - VideoStream* stream = &encoder_config.streams[0]; - encoder_config.content_type = VideoEncoderConfig::ContentType::kScreen; - encoder_config.min_transmit_bitrate_bps = flags::MinTransmitBitrate(); - int num_temporal_layers; - if (config_.codec == "VP8") { - encoder_config.encoder_specific_settings = &vp8_settings_; - num_temporal_layers = vp8_settings_.numberOfTemporalLayers; - } else if (config_.codec == "VP9") { - encoder_config.encoder_specific_settings = &vp9_settings_; - num_temporal_layers = vp9_settings_.numberOfTemporalLayers; - } else { - RTC_NOTREACHED() << "Codec not supported!"; - abort(); - } - stream->temporal_layer_thresholds_bps.clear(); - stream->target_bitrate_bps = - static_cast(config_.start_bitrate_kbps) * 1000; - if (num_temporal_layers == 2) { - stream->temporal_layer_thresholds_bps.push_back( - stream->target_bitrate_bps); - } - return encoder_config; - } - - test::VideoCapturer* CreateCapturer(VideoSendStream* send_stream) override { - std::vector slides; - slides.push_back(test::ResourcePath("web_screenshot_1850_1110", "yuv")); - slides.push_back(test::ResourcePath("presentation_1850_1110", "yuv")); - slides.push_back(test::ResourcePath("photo_1850_1110", "yuv")); - slides.push_back(test::ResourcePath("difficult_photo_1850_1110", "yuv")); - - // Fixed for input resolution for prerecorded screenshare content. - const size_t kWidth = 1850; - const size_t kHeight = 1110; - RTC_CHECK_LE(flags::Width(), kWidth); - RTC_CHECK_LE(flags::Height(), kHeight); - RTC_CHECK_GT(flags::SlideChangeInterval(), 0); - const int kPauseDurationMs = - (flags::SlideChangeInterval() - flags::ScrollDuration()) * 1000; - RTC_CHECK_LE(flags::ScrollDuration(), flags::SlideChangeInterval()); - - test::FrameGenerator* frame_generator = - test::FrameGenerator::CreateScrollingInputFromYuvFiles( - Clock::GetRealTimeClock(), slides, kWidth, kHeight, flags::Width(), - flags::Height(), flags::ScrollDuration() * 1000, kPauseDurationMs); - - test::FrameGeneratorCapturer* capturer(new test::FrameGeneratorCapturer( - clock_, send_stream->Input(), frame_generator, flags::Fps())); - EXPECT_TRUE(capturer->Init()); - return capturer; - } - - VideoCodecVP8 vp8_settings_; - VideoCodecVP9 vp9_settings_; -}; - void Loopback() { - test::Loopback::Config config{flags::Width(), - flags::Height(), - flags::Fps(), - flags::MinBitrate(), - flags::StartBitrate(), - flags::MaxBitrate(), - flags::MinTransmitBitrate(), - flags::Codec(), - flags::NumTemporalLayers(), - flags::NumSpatialLayers(), - flags::TLDiscardThreshold(), - flags::SLDiscardThreshold(), - flags::LossPercent(), - flags::LinkCapacity(), - flags::QueueSize(), - flags::AvgPropagationDelayMs(), - flags::StdPropagationDelayMs(), - flags::FLAGS_logs}; - ScreenshareLoopback loopback(config); - loopback.Run(); + FakeNetworkPipe::Config pipe_config; + pipe_config.loss_percent = flags::LossPercent(); + pipe_config.link_capacity_kbps = flags::LinkCapacityKbps(); + pipe_config.queue_length_packets = flags::QueueSize(); + pipe_config.queue_delay_ms = flags::AvgPropagationDelayMs(); + pipe_config.delay_standard_deviation_ms = flags::StdPropagationDelayMs(); + + Call::Config::BitrateConfig call_bitrate_config; + call_bitrate_config.min_bitrate_bps = flags::MinBitrateKbps() * 1000; + call_bitrate_config.start_bitrate_bps = flags::StartBitrateKbps() * 1000; + call_bitrate_config.max_bitrate_bps = flags::MaxBitrateKbps() * 1000; + + VideoQualityTest::Params params{ + { + flags::Width(), + flags::Height(), + flags::Fps(), + flags::MinBitrateKbps() * 1000, + flags::TargetBitrateKbps() * 1000, + flags::MaxBitrateKbps() * 1000, + flags::Codec(), + flags::NumTemporalLayers(), + flags::MinTransmitBitrateKbps() * 1000, + call_bitrate_config, + flags::TLDiscardThreshold() + }, + {}, // Video specific. + {true, flags::SlideChangeInterval(), flags::ScrollDuration()}, + {"screenshare", 0.0, 0.0, flags::DurationSecs(), flags::OutputFilename()}, + pipe_config, + flags::FLAGS_logs}; + + VideoQualityTest test; + if (flags::OutputFilename().empty()) + test.RunWithVideoRenderer(params); + else + test.RunWithAnalyzer(params); } } // namespace webrtc diff --git a/webrtc/video/video_loopback.cc b/webrtc/video/video_loopback.cc index 789b5cceac..c5e9254c08 100644 --- a/webrtc/video/video_loopback.cc +++ b/webrtc/video/video_loopback.cc @@ -18,7 +18,7 @@ #include "webrtc/test/field_trial.h" #include "webrtc/test/run_test.h" #include "webrtc/typedefs.h" -#include "webrtc/video/loopback.h" +#include "webrtc/video/video_quality_test.h" namespace webrtc { @@ -39,24 +39,25 @@ int Fps() { return static_cast(FLAGS_fps); } -DEFINE_int32(min_bitrate, 50, "Minimum video bitrate."); -size_t MinBitrate() { - return static_cast(FLAGS_min_bitrate); +DEFINE_int32(min_bitrate, 50, "Call and stream min bitrate in kbps."); +int MinBitrateKbps() { + return static_cast(FLAGS_min_bitrate); } -DEFINE_int32(start_bitrate, 300, "Video starting bitrate."); -size_t StartBitrate() { - return static_cast(FLAGS_start_bitrate); +DEFINE_int32(start_bitrate, 300, "Call start bitrate in kbps."); +int StartBitrateKbps() { + return static_cast(FLAGS_start_bitrate); } -DEFINE_int32(max_bitrate, 800, "Maximum video bitrate."); -size_t MaxBitrate() { - return static_cast(FLAGS_max_bitrate); +DEFINE_int32(target_bitrate, 800, "Stream target bitrate in kbps."); +int TargetBitrateKbps() { + return static_cast(FLAGS_target_bitrate); } -int MinTransmitBitrate() { - return 0; -} // No min padding for regular video. +DEFINE_int32(max_bitrate, 800, "Call and stream max bitrate in kbps."); +int MaxBitrateKbps() { + return static_cast(FLAGS_max_bitrate); +} DEFINE_string(codec, "VP8", "Video codec to use."); std::string Codec() { @@ -71,7 +72,7 @@ int LossPercent() { DEFINE_int32(link_capacity, 0, "Capacity (kbps) of the fake link. 0 means infinite."); -int LinkCapacity() { +int LinkCapacityKbps() { return static_cast(FLAGS_link_capacity); } @@ -105,33 +106,82 @@ DEFINE_string( "trials are separated by \"/\""); DEFINE_int32(num_temporal_layers, - 0, + 1, "Number of temporal layers. Set to 1-4 to override."); - size_t NumTemporalLayers() { return static_cast(FLAGS_num_temporal_layers); } +DEFINE_int32( + tl_discard_threshold, + 0, + "Discard TLs with id greater or equal the threshold. 0 to disable."); +size_t TLDiscardThreshold() { + return static_cast(FLAGS_tl_discard_threshold); +} + +DEFINE_string(clip, + "", + "Name of the clip to show. If empty, using chroma generator."); +std::string Clip() { + return static_cast(FLAGS_clip); +} + +DEFINE_string( + output_filename, + "", + "Name of a target graph data file. If set, no preview will be shown."); +std::string OutputFilename() { + return static_cast(FLAGS_output_filename); +} + +DEFINE_int32(duration, 60, "Duration of the test in seconds."); +int DurationSecs() { + return static_cast(FLAGS_duration); +} + } // namespace flags void Loopback() { - test::Loopback::Config config{flags::Width(), - flags::Height(), - flags::Fps(), - flags::MinBitrate(), - flags::StartBitrate(), - flags::MaxBitrate(), - 0, // No min transmit bitrate. - flags::Codec(), - flags::NumTemporalLayers(), - flags::LossPercent(), - flags::LinkCapacity(), - flags::QueueSize(), - flags::AvgPropagationDelayMs(), - flags::StdPropagationDelayMs(), - flags::FLAGS_logs}; - test::Loopback loopback(config); - loopback.Run(); + FakeNetworkPipe::Config pipe_config; + pipe_config.loss_percent = flags::LossPercent(); + pipe_config.link_capacity_kbps = flags::LinkCapacityKbps(); + pipe_config.queue_length_packets = flags::QueueSize(); + pipe_config.queue_delay_ms = flags::AvgPropagationDelayMs(); + pipe_config.delay_standard_deviation_ms = flags::StdPropagationDelayMs(); + + Call::Config::BitrateConfig call_bitrate_config; + call_bitrate_config.min_bitrate_bps = flags::MinBitrateKbps() * 1000; + call_bitrate_config.start_bitrate_bps = flags::StartBitrateKbps() * 1000; + call_bitrate_config.max_bitrate_bps = flags::MaxBitrateKbps() * 1000; + + std::string clip = flags::Clip(); + std::string graph_title = clip.empty() ? "" : "video " + clip; + VideoQualityTest::Params params{ + { + flags::Width(), + flags::Height(), + flags::Fps(), + flags::MinBitrateKbps() * 1000, + flags::TargetBitrateKbps() * 1000, + flags::MaxBitrateKbps() * 1000, + flags::Codec(), + flags::NumTemporalLayers(), + 0, // No min transmit bitrate. + call_bitrate_config, + flags::TLDiscardThreshold() + }, + {clip}, + {}, // Screenshare specific. + {graph_title, 0.0, 0.0, flags::DurationSecs(), flags::OutputFilename()}, + pipe_config, + flags::FLAGS_logs}; + + VideoQualityTest test; + if (flags::OutputFilename().empty()) + test.RunWithVideoRenderer(params); + else + test.RunWithAnalyzer(params); } } // namespace webrtc diff --git a/webrtc/video/video_quality_test.cc b/webrtc/video/video_quality_test.cc new file mode 100644 index 0000000000..d440858155 --- /dev/null +++ b/webrtc/video/video_quality_test.cc @@ -0,0 +1,816 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include + +#include +#include +#include +#include + +#include "testing/gtest/include/gtest/gtest.h" + +#include "webrtc/base/checks.h" +#include "webrtc/base/format_macros.h" +#include "webrtc/base/scoped_ptr.h" +#include "webrtc/call.h" +#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" +#include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h" +#include "webrtc/system_wrappers/interface/cpu_info.h" +#include "webrtc/test/layer_filtering_transport.h" +#include "webrtc/test/run_loop.h" +#include "webrtc/test/statistics.h" +#include "webrtc/test/testsupport/fileutils.h" +#include "webrtc/test/video_renderer.h" +#include "webrtc/video/video_quality_test.h" + +namespace webrtc { + +static const int kSendStatsPollingIntervalMs = 1000; +static const int kPayloadTypeVP8 = 123; +static const int kPayloadTypeVP9 = 124; + +class VideoAnalyzer : public PacketReceiver, + public newapi::Transport, + public VideoRenderer, + public VideoCaptureInput, + public EncodedFrameObserver { + public: + VideoAnalyzer(VideoCaptureInput* input, + Transport* transport, + const std::string& test_label, + double avg_psnr_threshold, + double avg_ssim_threshold, + int duration_frames, + FILE* graph_data_output_file) + : input_(input), + transport_(transport), + receiver_(nullptr), + send_stream_(nullptr), + test_label_(test_label), + graph_data_output_file_(graph_data_output_file), + frames_to_process_(duration_frames), + frames_recorded_(0), + frames_processed_(0), + dropped_frames_(0), + last_render_time_(0), + rtp_timestamp_delta_(0), + avg_psnr_threshold_(avg_psnr_threshold), + avg_ssim_threshold_(avg_ssim_threshold), + comparison_available_event_(EventWrapper::Create()), + done_(EventWrapper::Create()) { + // Create thread pool for CPU-expensive PSNR/SSIM calculations. + + // Try to use about as many threads as cores, but leave kMinCoresLeft alone, + // so that we don't accidentally starve "real" worker threads (codec etc). + // Also, don't allocate more than kMaxComparisonThreads, even if there are + // spare cores. + + uint32_t num_cores = CpuInfo::DetectNumberOfCores(); + RTC_DCHECK_GE(num_cores, 1u); + static const uint32_t kMinCoresLeft = 4; + static const uint32_t kMaxComparisonThreads = 8; + + if (num_cores <= kMinCoresLeft) { + num_cores = 1; + } else { + num_cores -= kMinCoresLeft; + num_cores = std::min(num_cores, kMaxComparisonThreads); + } + + for (uint32_t i = 0; i < num_cores; ++i) { + rtc::scoped_ptr thread = + ThreadWrapper::CreateThread(&FrameComparisonThread, this, "Analyzer"); + EXPECT_TRUE(thread->Start()); + comparison_thread_pool_.push_back(thread.release()); + } + + stats_polling_thread_ = + ThreadWrapper::CreateThread(&PollStatsThread, this, "StatsPoller"); + EXPECT_TRUE(stats_polling_thread_->Start()); + } + + ~VideoAnalyzer() { + for (ThreadWrapper* thread : comparison_thread_pool_) { + EXPECT_TRUE(thread->Stop()); + delete thread; + } + } + + virtual void SetReceiver(PacketReceiver* receiver) { receiver_ = receiver; } + + DeliveryStatus DeliverPacket(MediaType media_type, + const uint8_t* packet, + size_t length, + const PacketTime& packet_time) override { + rtc::scoped_ptr parser(RtpHeaderParser::Create()); + RTPHeader header; + parser->Parse(packet, length, &header); + { + rtc::CritScope lock(&crit_); + recv_times_[header.timestamp - rtp_timestamp_delta_] = + Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); + } + + return receiver_->DeliverPacket(media_type, packet, length, packet_time); + } + + void IncomingCapturedFrame(const VideoFrame& video_frame) override { + VideoFrame copy = video_frame; + copy.set_timestamp(copy.ntp_time_ms() * 90); + + { + rtc::CritScope lock(&crit_); + if (first_send_frame_.IsZeroSize() && rtp_timestamp_delta_ == 0) + first_send_frame_ = copy; + + frames_.push_back(copy); + } + + input_->IncomingCapturedFrame(video_frame); + } + + bool SendRtp(const uint8_t* packet, size_t length) override { + rtc::scoped_ptr parser(RtpHeaderParser::Create()); + RTPHeader header; + parser->Parse(packet, length, &header); + + { + rtc::CritScope lock(&crit_); + if (rtp_timestamp_delta_ == 0) { + rtp_timestamp_delta_ = header.timestamp - first_send_frame_.timestamp(); + first_send_frame_.Reset(); + } + uint32_t timestamp = header.timestamp - rtp_timestamp_delta_; + send_times_[timestamp] = + Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); + encoded_frame_sizes_[timestamp] += + length - (header.headerLength + header.paddingLength); + } + + return transport_->SendRtp(packet, length); + } + + bool SendRtcp(const uint8_t* packet, size_t length) override { + return transport_->SendRtcp(packet, length); + } + + void EncodedFrameCallback(const EncodedFrame& frame) override { + rtc::CritScope lock(&comparison_lock_); + if (frames_recorded_ < frames_to_process_) + encoded_frame_size_.AddSample(frame.length_); + } + + void RenderFrame(const VideoFrame& video_frame, + int time_to_render_ms) override { + int64_t render_time_ms = + Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); + uint32_t send_timestamp = video_frame.timestamp() - rtp_timestamp_delta_; + + rtc::CritScope lock(&crit_); + + while (frames_.front().timestamp() < send_timestamp) { + AddFrameComparison(frames_.front(), last_rendered_frame_, true, + render_time_ms); + frames_.pop_front(); + } + + VideoFrame reference_frame = frames_.front(); + frames_.pop_front(); + assert(!reference_frame.IsZeroSize()); + EXPECT_EQ(reference_frame.timestamp(), send_timestamp); + assert(reference_frame.timestamp() == send_timestamp); + + AddFrameComparison(reference_frame, video_frame, false, render_time_ms); + + last_rendered_frame_ = video_frame; + } + + bool IsTextureSupported() const override { return false; } + + void Wait() { + // Frame comparisons can be very expensive. Wait for test to be done, but + // at time-out check if frames_processed is going up. If so, give it more + // time, otherwise fail. Hopefully this will reduce test flakiness. + + int last_frames_processed = -1; + EventTypeWrapper eventType; + int iteration = 0; + while ((eventType = done_->Wait(VideoQualityTest::kDefaultTimeoutMs)) != + kEventSignaled) { + int frames_processed; + { + rtc::CritScope crit(&comparison_lock_); + frames_processed = frames_processed_; + } + + // Print some output so test infrastructure won't think we've crashed. + const char* kKeepAliveMessages[3] = { + "Uh, I'm-I'm not quite dead, sir.", + "Uh, I-I think uh, I could pull through, sir.", + "Actually, I think I'm all right to come with you--"}; + printf("- %s\n", kKeepAliveMessages[iteration++ % 3]); + + if (last_frames_processed == -1) { + last_frames_processed = frames_processed; + continue; + } + ASSERT_GT(frames_processed, last_frames_processed) + << "Analyzer stalled while waiting for test to finish."; + last_frames_processed = frames_processed; + } + + if (iteration > 0) + printf("- Farewell, sweet Concorde!\n"); + + // Signal stats polling thread if that is still waiting and stop it now, + // since it uses the send_stream_ reference that might be reclaimed after + // returning from this method. + done_->Set(); + EXPECT_TRUE(stats_polling_thread_->Stop()); + } + + VideoCaptureInput* input_; + Transport* transport_; + PacketReceiver* receiver_; + VideoSendStream* send_stream_; + + private: + struct FrameComparison { + FrameComparison() + : dropped(false), + send_time_ms(0), + recv_time_ms(0), + render_time_ms(0), + encoded_frame_size(0) {} + + FrameComparison(const VideoFrame& reference, + const VideoFrame& render, + bool dropped, + int64_t send_time_ms, + int64_t recv_time_ms, + int64_t render_time_ms, + size_t encoded_frame_size) + : reference(reference), + render(render), + dropped(dropped), + send_time_ms(send_time_ms), + recv_time_ms(recv_time_ms), + render_time_ms(render_time_ms), + encoded_frame_size(encoded_frame_size) {} + + VideoFrame reference; + VideoFrame render; + bool dropped; + int64_t send_time_ms; + int64_t recv_time_ms; + int64_t render_time_ms; + size_t encoded_frame_size; + }; + + struct Sample { + Sample(double dropped, + double input_time_ms, + double send_time_ms, + double recv_time_ms, + double encoded_frame_size, + double psnr, + double ssim, + double render_time_ms) + : dropped(dropped), + input_time_ms(input_time_ms), + send_time_ms(send_time_ms), + recv_time_ms(recv_time_ms), + encoded_frame_size(encoded_frame_size), + psnr(psnr), + ssim(ssim), + render_time_ms(render_time_ms) {} + + double dropped; + double input_time_ms; + double send_time_ms; + double recv_time_ms; + double encoded_frame_size; + double psnr; + double ssim; + double render_time_ms; + }; + + void AddFrameComparison(const VideoFrame& reference, + const VideoFrame& render, + bool dropped, + int64_t render_time_ms) + EXCLUSIVE_LOCKS_REQUIRED(crit_) { + int64_t send_time_ms = send_times_[reference.timestamp()]; + send_times_.erase(reference.timestamp()); + int64_t recv_time_ms = recv_times_[reference.timestamp()]; + recv_times_.erase(reference.timestamp()); + + size_t encoded_size = encoded_frame_sizes_[reference.timestamp()]; + encoded_frame_sizes_.erase(reference.timestamp()); + + VideoFrame reference_copy; + VideoFrame render_copy; + reference_copy.CopyFrame(reference); + render_copy.CopyFrame(render); + + rtc::CritScope crit(&comparison_lock_); + comparisons_.push_back(FrameComparison(reference_copy, render_copy, dropped, + send_time_ms, recv_time_ms, + render_time_ms, encoded_size)); + comparison_available_event_->Set(); + } + + static bool PollStatsThread(void* obj) { + return static_cast(obj)->PollStats(); + } + + bool PollStats() { + switch (done_->Wait(kSendStatsPollingIntervalMs)) { + case kEventSignaled: + case kEventError: + done_->Set(); // Make sure main thread is also signaled. + return false; + case kEventTimeout: + break; + default: + RTC_NOTREACHED(); + } + + VideoSendStream::Stats stats = send_stream_->GetStats(); + + rtc::CritScope crit(&comparison_lock_); + encode_frame_rate_.AddSample(stats.encode_frame_rate); + encode_time_ms.AddSample(stats.avg_encode_time_ms); + encode_usage_percent.AddSample(stats.encode_usage_percent); + media_bitrate_bps.AddSample(stats.media_bitrate_bps); + + return true; + } + + static bool FrameComparisonThread(void* obj) { + return static_cast(obj)->CompareFrames(); + } + + bool CompareFrames() { + if (AllFramesRecorded()) + return false; + + VideoFrame reference; + VideoFrame render; + FrameComparison comparison; + + if (!PopComparison(&comparison)) { + // Wait until new comparison task is available, or test is done. + // If done, wake up remaining threads waiting. + comparison_available_event_->Wait(1000); + if (AllFramesRecorded()) { + comparison_available_event_->Set(); + return false; + } + return true; // Try again. + } + + PerformFrameComparison(comparison); + + if (FrameProcessed()) { + PrintResults(); + if (graph_data_output_file_) + PrintSamplesToFile(); + done_->Set(); + comparison_available_event_->Set(); + return false; + } + + return true; + } + + bool PopComparison(FrameComparison* comparison) { + rtc::CritScope crit(&comparison_lock_); + // If AllFramesRecorded() is true, it means we have already popped + // frames_to_process_ frames from comparisons_, so there is no more work + // for this thread to be done. frames_processed_ might still be lower if + // all comparisons are not done, but those frames are currently being + // worked on by other threads. + if (comparisons_.empty() || AllFramesRecorded()) + return false; + + *comparison = comparisons_.front(); + comparisons_.pop_front(); + + FrameRecorded(); + return true; + } + + // Increment counter for number of frames received for comparison. + void FrameRecorded() { + rtc::CritScope crit(&comparison_lock_); + ++frames_recorded_; + } + + // Returns true if all frames to be compared have been taken from the queue. + bool AllFramesRecorded() { + rtc::CritScope crit(&comparison_lock_); + assert(frames_recorded_ <= frames_to_process_); + return frames_recorded_ == frames_to_process_; + } + + // Increase count of number of frames processed. Returns true if this was the + // last frame to be processed. + bool FrameProcessed() { + rtc::CritScope crit(&comparison_lock_); + ++frames_processed_; + assert(frames_processed_ <= frames_to_process_); + return frames_processed_ == frames_to_process_; + } + + void PrintResults() { + rtc::CritScope crit(&comparison_lock_); + PrintResult("psnr", psnr_, " dB"); + PrintResult("ssim", ssim_, ""); + PrintResult("sender_time", sender_time_, " ms"); + printf("RESULT dropped_frames: %s = %d frames\n", test_label_.c_str(), + dropped_frames_); + PrintResult("receiver_time", receiver_time_, " ms"); + PrintResult("total_delay_incl_network", end_to_end_, " ms"); + PrintResult("time_between_rendered_frames", rendered_delta_, " ms"); + PrintResult("encoded_frame_size", encoded_frame_size_, " bytes"); + PrintResult("encode_frame_rate", encode_frame_rate_, " fps"); + PrintResult("encode_time", encode_time_ms, " ms"); + PrintResult("encode_usage_percent", encode_usage_percent, " percent"); + PrintResult("media_bitrate", media_bitrate_bps, " bps"); + + EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_); + EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_); + } + + void PerformFrameComparison(const FrameComparison& comparison) { + // Perform expensive psnr and ssim calculations while not holding lock. + double psnr = I420PSNR(&comparison.reference, &comparison.render); + double ssim = I420SSIM(&comparison.reference, &comparison.render); + + int64_t input_time_ms = comparison.reference.ntp_time_ms(); + + rtc::CritScope crit(&comparison_lock_); + if (graph_data_output_file_) { + samples_.push_back( + Sample(comparison.dropped, input_time_ms, comparison.send_time_ms, + comparison.recv_time_ms, comparison.encoded_frame_size, psnr, + ssim, comparison.render_time_ms)); + } + psnr_.AddSample(psnr); + ssim_.AddSample(ssim); + + if (comparison.dropped) { + ++dropped_frames_; + return; + } + if (last_render_time_ != 0) + rendered_delta_.AddSample(comparison.render_time_ms - last_render_time_); + last_render_time_ = comparison.render_time_ms; + + sender_time_.AddSample(comparison.send_time_ms - input_time_ms); + receiver_time_.AddSample(comparison.render_time_ms - + comparison.recv_time_ms); + end_to_end_.AddSample(comparison.render_time_ms - input_time_ms); + encoded_frame_size_.AddSample(comparison.encoded_frame_size); + } + + void PrintResult(const char* result_type, + test::Statistics stats, + const char* unit) { + printf("RESULT %s: %s = {%f, %f}%s\n", + result_type, + test_label_.c_str(), + stats.Mean(), + stats.StandardDeviation(), + unit); + } + + void PrintSamplesToFile(void) { + FILE* out = graph_data_output_file_; + rtc::CritScope crit(&comparison_lock_); + std::sort(samples_.begin(), samples_.end(), + [](const Sample& A, const Sample& B) -> bool { + return A.input_time_ms < B.input_time_ms; + }); + + fprintf(out, "%s\n", test_label_.c_str()); + fprintf(out, "%" PRIuS "\n", samples_.size()); + fprintf(out, + "dropped " + "input_time_ms " + "send_time_ms " + "recv_time_ms " + "encoded_frame_size " + "psnr " + "ssim " + "render_time_ms\n"); + for (const Sample& sample : samples_) { + fprintf(out, "%lf %lf %lf %lf %lf %lf %lf %lf\n", sample.dropped, + sample.input_time_ms, sample.send_time_ms, sample.recv_time_ms, + sample.encoded_frame_size, sample.psnr, sample.ssim, + sample.render_time_ms); + } + } + + const std::string test_label_; + FILE* const graph_data_output_file_; + std::vector samples_ GUARDED_BY(comparison_lock_); + test::Statistics sender_time_ GUARDED_BY(comparison_lock_); + test::Statistics receiver_time_ GUARDED_BY(comparison_lock_); + test::Statistics psnr_ GUARDED_BY(comparison_lock_); + test::Statistics ssim_ GUARDED_BY(comparison_lock_); + test::Statistics end_to_end_ GUARDED_BY(comparison_lock_); + test::Statistics rendered_delta_ GUARDED_BY(comparison_lock_); + test::Statistics encoded_frame_size_ GUARDED_BY(comparison_lock_); + test::Statistics encode_frame_rate_ GUARDED_BY(comparison_lock_); + test::Statistics encode_time_ms GUARDED_BY(comparison_lock_); + test::Statistics encode_usage_percent GUARDED_BY(comparison_lock_); + test::Statistics media_bitrate_bps GUARDED_BY(comparison_lock_); + + const int frames_to_process_; + int frames_recorded_; + int frames_processed_; + int dropped_frames_; + int64_t last_render_time_; + uint32_t rtp_timestamp_delta_; + + rtc::CriticalSection crit_; + std::deque frames_ GUARDED_BY(crit_); + VideoFrame last_rendered_frame_ GUARDED_BY(crit_); + std::map send_times_ GUARDED_BY(crit_); + std::map recv_times_ GUARDED_BY(crit_); + std::map encoded_frame_sizes_ GUARDED_BY(crit_); + VideoFrame first_send_frame_ GUARDED_BY(crit_); + const double avg_psnr_threshold_; + const double avg_ssim_threshold_; + + rtc::CriticalSection comparison_lock_; + std::vector comparison_thread_pool_; + rtc::scoped_ptr stats_polling_thread_; + const rtc::scoped_ptr comparison_available_event_; + std::deque comparisons_ GUARDED_BY(comparison_lock_); + const rtc::scoped_ptr done_; +}; + +VideoQualityTest::VideoQualityTest() : clock_(Clock::GetRealTimeClock()) {} + +void VideoQualityTest::ValidateParams(const Params& params) { + RTC_CHECK_GE(params.common.max_bitrate_bps, params.common.target_bitrate_bps); + RTC_CHECK_GE(params.common.target_bitrate_bps, params.common.min_bitrate_bps); + RTC_CHECK_LT(params.common.tl_discard_threshold, + params.common.num_temporal_layers); +} + +void VideoQualityTest::TestBody() {} + +void VideoQualityTest::SetupFullStack(const Params& params, + newapi::Transport* send_transport, + newapi::Transport* recv_transport) { + if (params.logs) + trace_to_stderr_.reset(new test::TraceToStderr); + + CreateSendConfig(1, send_transport); + + int payload_type; + if (params.common.codec == "VP8") { + encoder_.reset(VideoEncoder::Create(VideoEncoder::kVp8)); + payload_type = kPayloadTypeVP8; + } else if (params.common.codec == "VP9") { + encoder_.reset(VideoEncoder::Create(VideoEncoder::kVp9)); + payload_type = kPayloadTypeVP9; + } else { + RTC_NOTREACHED() << "Codec not supported!"; + return; + } + send_config_.encoder_settings.encoder = encoder_.get(); + send_config_.encoder_settings.payload_name = params.common.codec; + send_config_.encoder_settings.payload_type = payload_type; + + send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs; + send_config_.rtp.rtx.ssrcs.push_back(kSendRtxSsrcs[0]); + send_config_.rtp.rtx.payload_type = kSendRtxPayloadType; + + // Automatically fill out streams[0] with params. + VideoStream* stream = &encoder_config_.streams[0]; + stream->width = params.common.width; + stream->height = params.common.height; + stream->min_bitrate_bps = params.common.min_bitrate_bps; + stream->target_bitrate_bps = params.common.target_bitrate_bps; + stream->max_bitrate_bps = params.common.max_bitrate_bps; + stream->max_framerate = static_cast(params.common.fps); + + stream->temporal_layer_thresholds_bps.clear(); + if (params.common.num_temporal_layers > 1) { + stream->temporal_layer_thresholds_bps.push_back(stream->target_bitrate_bps); + } + + CreateMatchingReceiveConfigs(recv_transport); + + receive_configs_[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs; + receive_configs_[0].rtp.rtx[kSendRtxPayloadType].ssrc = kSendRtxSsrcs[0]; + receive_configs_[0].rtp.rtx[kSendRtxPayloadType].payload_type = + kSendRtxPayloadType; + + encoder_config_.min_transmit_bitrate_bps = params.common.min_transmit_bps; +} + +void VideoQualityTest::SetupScreenshare(const Params& params) { + RTC_CHECK(params.screenshare.enabled); + + // Fill out codec settings. + encoder_config_.content_type = VideoEncoderConfig::ContentType::kScreen; + if (params.common.codec == "VP8") { + codec_settings_.VP8 = VideoEncoder::GetDefaultVp8Settings(); + codec_settings_.VP8.denoisingOn = false; + codec_settings_.VP8.frameDroppingOn = false; + codec_settings_.VP8.numberOfTemporalLayers = + static_cast(params.common.num_temporal_layers); + encoder_config_.encoder_specific_settings = &codec_settings_.VP8; + } else if (params.common.codec == "VP9") { + codec_settings_.VP9 = VideoEncoder::GetDefaultVp9Settings(); + codec_settings_.VP9.denoisingOn = false; + codec_settings_.VP9.frameDroppingOn = false; + codec_settings_.VP9.numberOfTemporalLayers = + static_cast(params.common.num_temporal_layers); + encoder_config_.encoder_specific_settings = &codec_settings_.VP9; + } + + // Setup frame generator. + const size_t kWidth = 1850; + const size_t kHeight = 1110; + std::vector slides; + slides.push_back(test::ResourcePath("web_screenshot_1850_1110", "yuv")); + slides.push_back(test::ResourcePath("presentation_1850_1110", "yuv")); + slides.push_back(test::ResourcePath("photo_1850_1110", "yuv")); + slides.push_back(test::ResourcePath("difficult_photo_1850_1110", "yuv")); + + if (params.screenshare.scroll_duration == 0) { + // Cycle image every slide_change_interval seconds. + frame_generator_.reset(test::FrameGenerator::CreateFromYuvFile( + slides, kWidth, kHeight, + params.screenshare.slide_change_interval * params.common.fps)); + } else { + RTC_CHECK_LE(params.common.width, kWidth); + RTC_CHECK_LE(params.common.height, kHeight); + RTC_CHECK_GT(params.screenshare.slide_change_interval, 0); + const int kPauseDurationMs = (params.screenshare.slide_change_interval - + params.screenshare.scroll_duration) * 1000; + RTC_CHECK_LE(params.screenshare.scroll_duration, + params.screenshare.slide_change_interval); + + frame_generator_.reset( + test::FrameGenerator::CreateScrollingInputFromYuvFiles( + clock_, slides, kWidth, kHeight, params.common.width, + params.common.height, params.screenshare.scroll_duration * 1000, + kPauseDurationMs)); + } +} + +void VideoQualityTest::CreateCapturer(const Params& params, + VideoCaptureInput* input) { + if (params.screenshare.enabled) { + frame_generator_capturer_.reset(new test::FrameGeneratorCapturer( + clock_, input, frame_generator_.release(), params.common.fps)); + EXPECT_TRUE(frame_generator_capturer_->Init()); + } else { + if (params.video.clip_name.empty()) { + frame_generator_capturer_.reset(test::FrameGeneratorCapturer::Create( + input, params.common.width, params.common.height, params.common.fps, + clock_)); + EXPECT_TRUE(frame_generator_capturer_->Init()); + } else { + frame_generator_capturer_.reset( + test::FrameGeneratorCapturer::CreateFromYuvFile( + input, test::ResourcePath(params.video.clip_name, "yuv"), + params.common.width, params.common.height, params.common.fps, + clock_)); + ASSERT_TRUE(frame_generator_capturer_.get() != nullptr) + << "Could not create capturer for " << params.video.clip_name + << ".yuv. Is this resource file present?"; + } + } +} + +void VideoQualityTest::RunWithAnalyzer(const Params& params) { + // TODO(ivica): Merge with RunWithRenderer and use a flag / argument to + // differentiate between the analyzer and the renderer case. + ValidateParams(params); + + FILE* graph_data_output_file = nullptr; + if (!params.analyzer.graph_data_output_filename.empty()) { + graph_data_output_file = + fopen(params.analyzer.graph_data_output_filename.c_str(), "w"); + RTC_CHECK(graph_data_output_file != nullptr) + << "Can't open the file " + << params.analyzer.graph_data_output_filename << "!"; + } + + test::LayerFilteringTransport send_transport( + params.pipe, kPayloadTypeVP8, kPayloadTypeVP9, + static_cast(params.common.tl_discard_threshold), 0); + test::DirectTransport recv_transport(params.pipe); + VideoAnalyzer analyzer( + nullptr, &send_transport, params.analyzer.test_label, + params.analyzer.avg_psnr_threshold, params.analyzer.avg_ssim_threshold, + params.analyzer.test_durations_secs * params.common.fps, + graph_data_output_file); + + Call::Config call_config; + call_config.bitrate_config = params.common.call_bitrate_config; + CreateCalls(call_config, call_config); + + analyzer.SetReceiver(receiver_call_->Receiver()); + send_transport.SetReceiver(&analyzer); + recv_transport.SetReceiver(sender_call_->Receiver()); + + SetupFullStack(params, &analyzer, &recv_transport); + receive_configs_[0].renderer = &analyzer; + for (auto& config : receive_configs_) + config.pre_decode_callback = &analyzer; + + if (params.screenshare.enabled) + SetupScreenshare(params); + + CreateCapturer(params, &analyzer); + + CreateStreams(); + analyzer.input_ = send_stream_->Input(); + analyzer.send_stream_ = send_stream_; + + Start(); + + analyzer.Wait(); + + send_transport.StopSending(); + recv_transport.StopSending(); + + Stop(); + + DestroyStreams(); + + if (graph_data_output_file) + fclose(graph_data_output_file); +} + +void VideoQualityTest::RunWithVideoRenderer(const Params& params) { + ValidateParams(params); + + rtc::scoped_ptr local_preview( + test::VideoRenderer::Create("Local Preview", params.common.width, + params.common.height)); + rtc::scoped_ptr loopback_video( + test::VideoRenderer::Create("Loopback Video", params.common.width, + params.common.height)); + + // TODO(ivica): Remove bitrate_config and use the default Call::Config(), to + // match the full stack tests. + Call::Config call_config; + call_config.bitrate_config = params.common.call_bitrate_config; + rtc::scoped_ptr call(Call::Create(call_config)); + + test::LayerFilteringTransport transport( + params.pipe, kPayloadTypeVP8, kPayloadTypeVP9, + static_cast(params.common.tl_discard_threshold), 0); + // TODO(ivica): Use two calls to be able to merge with RunWithAnalyzer or at + // least share as much code as possible. That way this test would also match + // the full stack tests better. + transport.SetReceiver(call->Receiver()); + + SetupFullStack(params, &transport, &transport); + send_config_.local_renderer = local_preview.get(); + receive_configs_[0].renderer = loopback_video.get(); + + if (params.screenshare.enabled) + SetupScreenshare(params); + + send_stream_ = call->CreateVideoSendStream(send_config_, encoder_config_); + CreateCapturer(params, send_stream_->Input()); + + VideoReceiveStream* receive_stream = + call->CreateVideoReceiveStream(receive_configs_[0]); + + receive_stream->Start(); + send_stream_->Start(); + frame_generator_capturer_->Start(); + + test::PressEnterToContinue(); + + frame_generator_capturer_->Stop(); + send_stream_->Stop(); + receive_stream->Stop(); + + call->DestroyVideoReceiveStream(receive_stream); + call->DestroyVideoSendStream(send_stream_); + + transport.StopSending(); +} + +} // namespace webrtc diff --git a/webrtc/video/video_quality_test.h b/webrtc/video/video_quality_test.h new file mode 100644 index 0000000000..d606de7cac --- /dev/null +++ b/webrtc/video/video_quality_test.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef WEBRTC_VIDEO_VIDEO_QUALITY_TEST_H_ +#define WEBRTC_VIDEO_VIDEO_QUALITY_TEST_H_ + +#include + +#include "webrtc/test/call_test.h" +#include "webrtc/test/frame_generator.h" +#include "webrtc/test/testsupport/trace_to_stderr.h" + +namespace webrtc { + +class VideoQualityTest : public test::CallTest { + public: + // Parameters are grouped into smaller structs to make it easier to set + // the desired elements and skip unused, using aggregate initialization. + // Unfortunately, C++11 (as opposed to C11) doesn't support unnamed structs, + // which makes the implementation of VideoQualityTest a bit uglier. + struct Params { + struct { + size_t width; + size_t height; + int32_t fps; + int min_bitrate_bps; + int target_bitrate_bps; + int max_bitrate_bps; + std::string codec; + size_t num_temporal_layers; + + int min_transmit_bps; + Call::Config::BitrateConfig call_bitrate_config; + size_t tl_discard_threshold; + } common; + struct { // Video-specific settings. + std::string clip_name; + } video; + struct { // Screenshare-specific settings. + bool enabled; + int32_t slide_change_interval; + int32_t scroll_duration; + } screenshare; + struct { // Analyzer settings. + std::string test_label; + double avg_psnr_threshold; + double avg_ssim_threshold; + int test_durations_secs; + std::string graph_data_output_filename; + } analyzer; + FakeNetworkPipe::Config pipe; + bool logs; + }; + + VideoQualityTest(); + void RunWithAnalyzer(const Params& params); + void RunWithVideoRenderer(const Params& params); + + protected: + // No-op implementation to be able to instantiate this class from non-TEST_F + // locations. + void TestBody() override; + + void CreateCapturer(const Params& params, VideoCaptureInput* input); + void ValidateParams(const Params& params); + void SetupFullStack(const Params& params, + newapi::Transport* send_transport, + newapi::Transport* recv_transport); + void SetupScreenshare(const Params& params); + + rtc::scoped_ptr trace_to_stderr_; + rtc::scoped_ptr frame_generator_; + rtc::scoped_ptr encoder_; + VideoCodecUnion codec_settings_; + Clock* const clock_; +}; + +} // namespace webrtc + +#endif // WEBRTC_VIDEO_VIDEO_QUALITY_TEST_H_ diff --git a/webrtc/webrtc_tests.gypi b/webrtc/webrtc_tests.gypi index 8186415b6c..2515ebd101 100644 --- a/webrtc/webrtc_tests.gypi +++ b/webrtc/webrtc_tests.gypi @@ -45,45 +45,26 @@ ], }, { - 'target_name': 'full_stack_quality_sampler', - 'type': 'executable', - 'sources': [ - 'test/mac/run_test.mm', - 'test/run_test.cc', - 'test/run_test.h', - 'video/full_stack.cc', - 'video/full_stack_quality_sampler.cc', - ], - 'conditions': [ - ['OS=="mac"', { - 'sources!': [ - 'test/run_test.cc', - ], - }], - ], - 'dependencies': [ - '<(DEPTH)/testing/gtest.gyp:gtest', - '<(DEPTH)/third_party/gflags/gflags.gyp:gflags', - '<(webrtc_root)/modules/modules.gyp:video_capture', - 'test/webrtc_test_common.gyp:webrtc_test_common', - 'test/test.gyp:test_main', - 'webrtc', - ], - }, - { - 'target_name': 'loopback_base', + 'target_name': 'video_quality_test', 'type': 'static_library', 'sources': [ - 'video/loopback.cc', - 'video/loopback.h', + 'video/video_quality_test.cc', + 'video/video_quality_test.h', ], 'dependencies': [ '<(DEPTH)/testing/gtest.gyp:gtest', - '<(webrtc_root)/modules/modules.gyp:video_capture_module_internal_impl', '<(webrtc_root)/modules/modules.gyp:video_render', + '<(webrtc_root)/modules/modules.gyp:video_capture_module_internal_impl', '<(webrtc_root)/system_wrappers/system_wrappers.gyp:system_wrappers', 'webrtc', ], + 'conditions': [ + ['OS=="android"', { + 'dependencies!': [ + '<(webrtc_root)/modules/modules.gyp:video_capture_module_internal_impl', + ], + }], + ], }, { 'target_name': 'video_loopback', @@ -102,7 +83,7 @@ }], ], 'dependencies': [ - 'loopback_base', + 'video_quality_test', '<(DEPTH)/testing/gtest.gyp:gtest', '<(DEPTH)/third_party/gflags/gflags.gyp:gflags', 'test/webrtc_test_common.gyp:webrtc_test_common', @@ -128,7 +109,7 @@ }], ], 'dependencies': [ - 'loopback_base', + 'video_quality_test', '<(DEPTH)/testing/gtest.gyp:gtest', '<(DEPTH)/third_party/gflags/gflags.gyp:gflags', 'test/webrtc_test_common.gyp:webrtc_test_common', @@ -236,11 +217,13 @@ '<(webrtc_root)/modules/modules.gyp:video_capture', '<(webrtc_root)/test/test.gyp:channel_transport', '<(webrtc_root)/voice_engine/voice_engine.gyp:voice_engine', + 'video_quality_test', 'modules/modules.gyp:neteq_test_support', 'modules/modules.gyp:bwe_simulator', 'modules/modules.gyp:rtp_rtcp', 'test/test.gyp:test_main', 'test/webrtc_test_common.gyp:webrtc_test_common', + 'test/webrtc_test_common.gyp:webrtc_test_renderer', 'tools/tools.gyp:agc_manager', 'webrtc', ],