From 18bc3e19c42915ebdbbd5cc3dffc749f55c07178 Mon Sep 17 00:00:00 2001 From: Sergey Silkin Date: Wed, 17 Jan 2018 13:15:57 +0000 Subject: [PATCH] Revert "Updated analysis in videoprocessor." MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 1880c7162bd3637c433f9421c798808cd6eacaf7. Reason for revert: breaks internal tests Original change's description: > Updated analysis in videoprocessor. > > - Run analysis after all frames are processed. Before part of it was > done at bitrate change points; > - Analysis is done for whole stream as well as for each rate update > interval; > - Changed units from number of frames to time units for some metrics > and thresholds. E.g. 'num frames to hit tagret bitrate' is changed to > 'time to reach target bitrate, sec'; > - Changed data type of FrameStatistic::max_nalu_length (renamed to > max_nalu_size_bytes) from rtc::Optional to size_t. There it no need to > use such advanced data type in such low level data structure. > > Bug: webrtc:8524 > Change-Id: Ic9f6eab5b15ee12a80324b1f9c101de1bf3c702f > Reviewed-on: https://webrtc-review.googlesource.com/31901 > Commit-Queue: Sergey Silkin > Reviewed-by: Stefan Holmer > Reviewed-by: Åsa Persson > Reviewed-by: Rasmus Brandt > Cr-Commit-Position: refs/heads/master@{#21653} TBR=brandtr@webrtc.org,asapersson@webrtc.org,sprang@webrtc.org,stefan@webrtc.org,ssilkin@webrtc.org Change-Id: Id0b7d387bbba02e71637b229aeed6f6cf012af46 No-Presubmit: true No-Tree-Checks: true No-Try: true Bug: webrtc:8524 Reviewed-on: https://webrtc-review.googlesource.com/40220 Reviewed-by: Sergey Silkin Commit-Queue: Sergey Silkin Cr-Commit-Position: refs/heads/master@{#21656} --- modules/video_coding/BUILD.gn | 2 - .../codecs/test/plot_webrtc_test_logs.py | 29 +- modules/video_coding/codecs/test/stats.cc | 208 +++++- modules/video_coding/codecs/test/stats.h | 32 +- .../codecs/test/stats_unittest.cc | 15 +- .../video_coding/codecs/test/test_config.cc | 100 ++- .../video_coding/codecs/test/test_config.h | 22 +- .../codecs/test/test_config_unittest.cc | 75 ++- .../codecs/test/videoprocessor.cc | 117 ++-- .../video_coding/codecs/test/videoprocessor.h | 24 +- .../test/videoprocessor_integrationtest.cc | 616 ++++++++---------- .../test/videoprocessor_integrationtest.h | 147 +++-- .../videoprocessor_integrationtest_libvpx.cc | 166 ++--- ...deoprocessor_integrationtest_mediacodec.cc | 19 +- ...videoprocessor_integrationtest_openh264.cc | 25 +- ...processor_integrationtest_parameterized.cc | 16 +- ...oprocessor_integrationtest_videotoolbox.cc | 13 +- .../codecs/test/videoprocessor_unittest.cc | 8 + test/statistics.cc | 19 +- test/statistics.h | 4 - 20 files changed, 916 insertions(+), 741 deletions(-) diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 48dac93bd9..5b59df7dac 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -517,7 +517,6 @@ if (rtc_include_tests) { "../../test:test_support", "../../test:video_test_common", "../../test:video_test_support", - "../rtp_rtcp:rtp_rtcp_format", ] } @@ -579,7 +578,6 @@ if (rtc_include_tests) { "../../rtc_base:rtc_base_tests_utils", "../../system_wrappers", "../../test:field_trial", - "../../test:test_common", "../../test:test_support", "../../test:video_test_common", "../../test:video_test_support", diff --git a/modules/video_coding/codecs/test/plot_webrtc_test_logs.py b/modules/video_coding/codecs/test/plot_webrtc_test_logs.py index e25ea3e010..ae5621f44a 100755 --- a/modules/video_coding/codecs/test/plot_webrtc_test_logs.py +++ b/modules/video_coding/codecs/test/plot_webrtc_test_logs.py @@ -24,7 +24,6 @@ EVENT_END = 'OK ] CodecSettings/VideoProcessorIntegrationTestParameterized.' # Metrics to plot, tuple: (name to parse in file, label to use when plotting). BITRATE = ('Target bitrate', 'target bitrate (kbps)') -FRAMERATE = ('Target framerate', 'fps') WIDTH = ('Width', 'width') HEIGHT = ('Height', 'height') FILENAME = ('Filename', 'clip') @@ -36,22 +35,24 @@ CORES = ('# CPU cores used', 'CPU cores used') DENOISING = ('Denoising', 'denoising') RESILIENCE = ('Resilience', 'resilience') ERROR_CONCEALMENT = ('Error concealment', 'error concealment') -QP = ('QP', 'QP avg') +QP = ('Average QP', 'avg QP') CPU_USAGE = ('CPU usage %', 'CPU usage (%)') -PSNR = ('PSNR', 'PSNR (dB)') -SSIM = ('SSIM', 'SSIM') +PSNR = ('PSNR avg', 'PSNR (dB)') +SSIM = ('SSIM avg', 'SSIM') ENC_BITRATE = ('Encoded bitrate', 'encoded bitrate (kbps)') -NUM_FRAMES = ('# input frames', 'num frames') +FRAMERATE = ('Frame rate', 'fps') +NUM_FRAMES = ('# processed frames', 'num frames') NUM_DROPPED_FRAMES = ('# dropped frames', 'num dropped frames') -TIME_TO_TARGET = ('Time to reach target bitrate', - 'time to reach target rate (sec)') +NUM_FRAMES_TO_TARGET = ('# frames to convergence', + 'frames to reach target rate') ENCODE_TIME = ('Encoding time', 'encode time (us)') -ENCODE_TIME_AVG = ('Frame encoding time', 'encode time (us) avg') +ENCODE_TIME_AVG = ('Encoding time', 'encode time (us) avg') DECODE_TIME = ('Decoding time', 'decode time (us)') -DECODE_TIME_AVG = ('Frame decoding time', 'decode time (us) avg') -FRAME_SIZE = ('Frame size', 'frame size (bytes)') -AVG_KEY_FRAME_SIZE = ('Avg key frame size', 'avg key frame size (bytes)') -AVG_NON_KEY_FRAME_SIZE = ('Avg delta frame size', +DECODE_TIME_AVG = ('Decoding time', 'decode time (us) avg') +FRAME_SIZE = ('Frame sizes', 'frame size (bytes)') +FRAME_SIZE_AVG = ('Frame sizes', 'frame size (bytes) avg') +AVG_KEY_FRAME_SIZE = ('Average key frame size', 'avg key frame size (bytes)') +AVG_NON_KEY_FRAME_SIZE = ('Average non-key frame size', 'avg non-key frame size (bytes)') # Settings. @@ -89,7 +90,7 @@ RESULTS = [ SSIM, ENC_BITRATE, NUM_DROPPED_FRAMES, - TIME_TO_TARGET, + NUM_FRAMES_TO_TARGET, ENCODE_TIME_AVG, DECODE_TIME_AVG, QP, @@ -234,7 +235,7 @@ def TryFindMetric(parsed, line, settings_file): found, maximum = GetMetric("Max", settings_file.readline()) if not found: return - found, average = GetMetric("Avg", settings_file.readline()) + found, average = GetMetric("Average", settings_file.readline()) if not found: return diff --git a/modules/video_coding/codecs/test/stats.cc b/modules/video_coding/codecs/test/stats.cc index 0aab931b52..a3199ab6fa 100644 --- a/modules/video_coding/codecs/test/stats.cc +++ b/modules/video_coding/codecs/test/stats.cc @@ -9,35 +9,59 @@ */ #include "modules/video_coding/codecs/test/stats.h" + +#include + +#include + #include "rtc_base/checks.h" +#include "rtc_base/format_macros.h" namespace webrtc { namespace test { -std::string FrameStatistic::ToString() const { - std::stringstream ss; - ss << "frame " << frame_number; - ss << " " << decoded_width << "x" << decoded_height; - ss << " sl " << simulcast_svc_idx; - ss << " tl " << temporal_layer_idx; - ss << " type " << frame_type; - ss << " length " << encoded_frame_size_bytes; - ss << " qp " << qp; - ss << " psnr " << psnr; - ss << " ssim " << ssim; - ss << " enc_time_us " << encode_time_us; - ss << " dec_time_us " << decode_time_us; - ss << " rtp_ts " << rtp_timestamp; - ss << " bitrate_kbps " << target_bitrate_kbps; - return ss.str(); +namespace { + +bool LessForEncodeTime(const FrameStatistic& s1, const FrameStatistic& s2) { + RTC_DCHECK_NE(s1.frame_number, s2.frame_number); + return s1.encode_time_us < s2.encode_time_us; } +bool LessForDecodeTime(const FrameStatistic& s1, const FrameStatistic& s2) { + RTC_DCHECK_NE(s1.frame_number, s2.frame_number); + return s1.decode_time_us < s2.decode_time_us; +} + +bool LessForEncodedSize(const FrameStatistic& s1, const FrameStatistic& s2) { + RTC_DCHECK_NE(s1.frame_number, s2.frame_number); + return s1.encoded_frame_size_bytes < s2.encoded_frame_size_bytes; +} + +bool LessForBitRate(const FrameStatistic& s1, const FrameStatistic& s2) { + RTC_DCHECK_NE(s1.frame_number, s2.frame_number); + return s1.bitrate_kbps < s2.bitrate_kbps; +} + +bool LessForPsnr(const FrameStatistic& s1, const FrameStatistic& s2) { + RTC_DCHECK_NE(s1.frame_number, s2.frame_number); + return s1.psnr < s2.psnr; +} + +bool LessForSsim(const FrameStatistic& s1, const FrameStatistic& s2) { + RTC_DCHECK_NE(s1.frame_number, s2.frame_number); + return s1.ssim < s2.ssim; +} + +} // namespace + FrameStatistic* Stats::AddFrame() { - stats_.emplace_back(stats_.size()); + // We don't expect more frames than what can be stored in an int. + stats_.emplace_back(static_cast(stats_.size())); return &stats_.back(); } -FrameStatistic* Stats::GetFrame(size_t frame_number) { +FrameStatistic* Stats::GetFrame(int frame_number) { + RTC_CHECK_GE(frame_number, 0); RTC_CHECK_LT(frame_number, stats_.size()); return &stats_[frame_number]; } @@ -46,5 +70,153 @@ size_t Stats::size() const { return stats_.size(); } +void Stats::PrintSummary() const { + if (stats_.empty()) { + printf("No frame statistics have been logged yet.\n"); + return; + } + + printf("Encode/decode statistics\n==\n"); + + // Calculate min, max, average and total encoding time. + int total_encoding_time_us = 0; + int total_decoding_time_us = 0; + size_t total_encoded_frame_size_bytes = 0; + size_t total_encoded_key_frame_size_bytes = 0; + size_t total_encoded_delta_frame_size_bytes = 0; + size_t num_key_frames = 0; + size_t num_delta_frames = 0; + int num_encode_failures = 0; + double total_psnr = 0.0; + double total_ssim = 0.0; + + for (const FrameStatistic& stat : stats_) { + total_encoding_time_us += stat.encode_time_us; + total_decoding_time_us += stat.decode_time_us; + total_encoded_frame_size_bytes += stat.encoded_frame_size_bytes; + if (stat.frame_type == webrtc::kVideoFrameKey) { + total_encoded_key_frame_size_bytes += stat.encoded_frame_size_bytes; + ++num_key_frames; + } else { + total_encoded_delta_frame_size_bytes += stat.encoded_frame_size_bytes; + ++num_delta_frames; + } + if (stat.encode_return_code != 0) { + ++num_encode_failures; + } + if (stat.decoding_successful) { + total_psnr += stat.psnr; + total_ssim += stat.ssim; + } + } + + // Encoding stats. + printf("# Encoded frame failures: %d\n", num_encode_failures); + printf("Encoding time:\n"); + auto frame_it = + std::min_element(stats_.begin(), stats_.end(), LessForEncodeTime); + printf(" Min : %7d us (frame %d)\n", frame_it->encode_time_us, + frame_it->frame_number); + frame_it = std::max_element(stats_.begin(), stats_.end(), LessForEncodeTime); + printf(" Max : %7d us (frame %d)\n", frame_it->encode_time_us, + frame_it->frame_number); + printf(" Average : %7d us\n", + static_cast(total_encoding_time_us / stats_.size())); + + // Decoding stats. + printf("Decoding time:\n"); + // Only consider successfully decoded frames (packet loss may cause failures). + std::vector decoded_frames; + for (const FrameStatistic& stat : stats_) { + if (stat.decoding_successful) { + decoded_frames.push_back(stat); + } + } + if (decoded_frames.empty()) { + printf("No successfully decoded frames exist in this statistics.\n"); + } else { + frame_it = std::min_element(decoded_frames.begin(), decoded_frames.end(), + LessForDecodeTime); + printf(" Min : %7d us (frame %d)\n", frame_it->decode_time_us, + frame_it->frame_number); + frame_it = std::max_element(decoded_frames.begin(), decoded_frames.end(), + LessForDecodeTime); + printf(" Max : %7d us (frame %d)\n", frame_it->decode_time_us, + frame_it->frame_number); + printf(" Average : %7d us\n", + static_cast(total_decoding_time_us / decoded_frames.size())); + printf(" Failures: %d frames failed to decode.\n", + static_cast(stats_.size() - decoded_frames.size())); + } + + // Frame size stats. + printf("Frame sizes:\n"); + frame_it = std::min_element(stats_.begin(), stats_.end(), LessForEncodedSize); + printf(" Min : %7" PRIuS " bytes (frame %d)\n", + frame_it->encoded_frame_size_bytes, frame_it->frame_number); + frame_it = std::max_element(stats_.begin(), stats_.end(), LessForEncodedSize); + printf(" Max : %7" PRIuS " bytes (frame %d)\n", + frame_it->encoded_frame_size_bytes, frame_it->frame_number); + printf(" Average : %7" PRIuS " bytes\n", + total_encoded_frame_size_bytes / stats_.size()); + if (num_key_frames > 0) { + printf(" Average key frame size : %7" PRIuS " bytes (%" PRIuS + " keyframes)\n", + total_encoded_key_frame_size_bytes / num_key_frames, num_key_frames); + } + if (num_delta_frames > 0) { + printf(" Average non-key frame size: %7" PRIuS " bytes (%" PRIuS + " frames)\n", + total_encoded_delta_frame_size_bytes / num_delta_frames, + num_delta_frames); + } + + // Bitrate stats. + printf("Bitrates:\n"); + frame_it = std::min_element(stats_.begin(), stats_.end(), LessForBitRate); + printf(" Min bitrate: %7d kbps (frame %d)\n", frame_it->bitrate_kbps, + frame_it->frame_number); + frame_it = std::max_element(stats_.begin(), stats_.end(), LessForBitRate); + printf(" Max bitrate: %7d kbps (frame %d)\n", frame_it->bitrate_kbps, + frame_it->frame_number); + + // Quality. + printf("Quality:\n"); + if (decoded_frames.empty()) { + printf("No successfully decoded frames exist in this statistics.\n"); + } else { + frame_it = std::min_element(decoded_frames.begin(), decoded_frames.end(), + LessForPsnr); + printf(" PSNR min: %f (frame %d)\n", frame_it->psnr, + frame_it->frame_number); + printf(" PSNR avg: %f\n", total_psnr / decoded_frames.size()); + + frame_it = std::min_element(decoded_frames.begin(), decoded_frames.end(), + LessForSsim); + printf(" SSIM min: %f (frame %d)\n", frame_it->ssim, + frame_it->frame_number); + printf(" SSIM avg: %f\n", total_ssim / decoded_frames.size()); + } + + printf("\n"); + printf("Total encoding time : %7d ms.\n", total_encoding_time_us / 1000); + printf("Total decoding time : %7d ms.\n", total_decoding_time_us / 1000); + printf("Total processing time: %7d ms.\n", + (total_encoding_time_us + total_decoding_time_us) / 1000); + + // QP stats. + int total_qp = 0; + int total_qp_count = 0; + for (const FrameStatistic& stat : stats_) { + if (stat.qp >= 0) { + total_qp += stat.qp; + ++total_qp_count; + } + } + int avg_qp = (total_qp_count > 0) ? (total_qp / total_qp_count) : -1; + printf("Average QP: %d\n", avg_qp); + printf("\n"); +} + } // namespace test } // namespace webrtc diff --git a/modules/video_coding/codecs/test/stats.h b/modules/video_coding/codecs/test/stats.h index 42a399c016..02ca641041 100644 --- a/modules/video_coding/codecs/test/stats.h +++ b/modules/video_coding/codecs/test/stats.h @@ -11,7 +11,6 @@ #ifndef MODULES_VIDEO_CODING_CODECS_TEST_STATS_H_ #define MODULES_VIDEO_CODING_CODECS_TEST_STATS_H_ -#include #include #include "common_types.h" // NOLINT(build/include) @@ -21,42 +20,34 @@ namespace test { // Statistics for one processed frame. struct FrameStatistic { - explicit FrameStatistic(size_t frame_number) : frame_number(frame_number) {} - - std::string ToString() const; - - size_t frame_number = 0; - size_t rtp_timestamp = 0; + explicit FrameStatistic(int frame_number) : frame_number(frame_number) {} + const int frame_number = 0; // Encoding. int64_t encode_start_ns = 0; int encode_return_code = 0; bool encoding_successful = false; - size_t encode_time_us = 0; - size_t target_bitrate_kbps = 0; + int encode_time_us = 0; + int bitrate_kbps = 0; size_t encoded_frame_size_bytes = 0; webrtc::FrameType frame_type = kVideoFrameDelta; - // Layering. - size_t temporal_layer_idx = 0; - size_t simulcast_svc_idx = 0; - // H264 specific. - size_t max_nalu_size_bytes = 0; + rtc::Optional max_nalu_length; // Decoding. int64_t decode_start_ns = 0; int decode_return_code = 0; bool decoding_successful = false; - size_t decode_time_us = 0; - size_t decoded_width = 0; - size_t decoded_height = 0; + int decode_time_us = 0; + int decoded_width = 0; + int decoded_height = 0; // Quantization. int qp = -1; // How many packets were discarded of the encoded frame data (if any). - size_t packets_dropped = 0; + int packets_dropped = 0; size_t total_packets = 0; size_t manipulated_length = 0; @@ -75,10 +66,13 @@ class Stats { FrameStatistic* AddFrame(); // Returns the FrameStatistic corresponding to |frame_number|. - FrameStatistic* GetFrame(size_t frame_number); + FrameStatistic* GetFrame(int frame_number); size_t size() const; + // TODO(brandtr): Add output as CSV. + void PrintSummary() const; + private: std::vector stats_; }; diff --git a/modules/video_coding/codecs/test/stats_unittest.cc b/modules/video_coding/codecs/test/stats_unittest.cc index 08fe56fc6b..87727e2889 100644 --- a/modules/video_coding/codecs/test/stats_unittest.cc +++ b/modules/video_coding/codecs/test/stats_unittest.cc @@ -15,21 +15,28 @@ namespace webrtc { namespace test { +TEST(StatsTest, TestEmptyObject) { + Stats stats; + stats.PrintSummary(); // Should not crash. +} + TEST(StatsTest, AddSingleFrame) { Stats stats; FrameStatistic* frame_stat = stats.AddFrame(); - EXPECT_EQ(0ull, frame_stat->frame_number); + EXPECT_EQ(0, frame_stat->frame_number); EXPECT_EQ(1u, stats.size()); } TEST(StatsTest, AddMultipleFrames) { Stats stats; - const size_t kNumFrames = 1000; - for (size_t i = 0; i < kNumFrames; ++i) { + const int kNumFrames = 1000; + for (int i = 0; i < kNumFrames; ++i) { FrameStatistic* frame_stat = stats.AddFrame(); EXPECT_EQ(i, frame_stat->frame_number); } - EXPECT_EQ(kNumFrames, stats.size()); + EXPECT_EQ(kNumFrames, static_cast(stats.size())); + + stats.PrintSummary(); // Should not crash. } } // namespace test diff --git a/modules/video_coding/codecs/test/test_config.cc b/modules/video_coding/codecs/test/test_config.cc index 0c2e7acf8c..a289c619a5 100644 --- a/modules/video_coding/codecs/test/test_config.cc +++ b/modules/video_coding/codecs/test/test_config.cc @@ -27,36 +27,34 @@ std::string CodecSpecificToString(const webrtc::VideoCodec& codec) { std::stringstream ss; switch (codec.codecType) { case kVideoCodecVP8: - ss << "\n Complexity : " << codec.VP8().complexity; - ss << "\n Resilience : " << codec.VP8().resilience; - ss << "\n # temporal layers : " + ss << "\n Complexity : " << codec.VP8().complexity; + ss << "\n Resilience : " << codec.VP8().resilience; + ss << "\n # temporal layers : " << static_cast(codec.VP8().numberOfTemporalLayers); - ss << "\n Denoising : " << codec.VP8().denoisingOn; - ss << "\n Error concealment : " << codec.VP8().errorConcealmentOn; - ss << "\n Automatic resize : " << codec.VP8().automaticResizeOn; - ss << "\n Frame dropping : " << codec.VP8().frameDroppingOn; - ss << "\n Key frame interval : " << codec.VP8().keyFrameInterval; + ss << "\n Denoising : " << codec.VP8().denoisingOn; + ss << "\n Error concealment : " << codec.VP8().errorConcealmentOn; + ss << "\n Automatic resize : " << codec.VP8().automaticResizeOn; + ss << "\n Frame dropping : " << codec.VP8().frameDroppingOn; + ss << "\n Key frame interval: " << codec.VP8().keyFrameInterval; break; case kVideoCodecVP9: - ss << "\n Complexity : " << codec.VP9().complexity; - ss << "\n Resilience : " << codec.VP9().resilienceOn; - ss << "\n # temporal layers : " + ss << "\n Complexity : " << codec.VP9().complexity; + ss << "\n Resilience : " << codec.VP9().resilienceOn; + ss << "\n # temporal layers : " << static_cast(codec.VP9().numberOfTemporalLayers); - ss << "\n # spatial layers : " + ss << "\n Denoising : " << codec.VP9().denoisingOn; + ss << "\n Frame dropping : " << codec.VP9().frameDroppingOn; + ss << "\n Key frame interval: " << codec.VP9().keyFrameInterval; + ss << "\n Adaptive QP mode : " << codec.VP9().adaptiveQpMode; + ss << "\n Automatic resize : " << codec.VP9().automaticResizeOn; + ss << "\n # spatial layers : " << static_cast(codec.VP9().numberOfSpatialLayers); - ss << "\n Denoising : " << codec.VP9().denoisingOn; - ss << "\n Frame dropping : " << codec.VP9().frameDroppingOn; - ss << "\n Key frame interval : " << codec.VP9().keyFrameInterval; - ss << "\n Adaptive QP mode : " << codec.VP9().adaptiveQpMode; - ss << "\n Automatic resize : " << codec.VP9().automaticResizeOn; - ss << "\n # spatial layers : " - << static_cast(codec.VP9().numberOfSpatialLayers); - ss << "\n Flexible mode : " << codec.VP9().flexibleMode; + ss << "\n Flexible mode : " << codec.VP9().flexibleMode; break; case kVideoCodecH264: - ss << "\n Frame dropping : " << codec.H264().frameDroppingOn; - ss << "\n Key frame interval : " << codec.H264().keyFrameInterval; - ss << "\n Profile : " << codec.H264().profile; + ss << "\n Frame dropping : " << codec.H264().frameDroppingOn; + ss << "\n Key frame interval: " << codec.H264().keyFrameInterval; + ss << "\n Profile : " << codec.H264().profile; break; default: break; @@ -67,27 +65,26 @@ std::string CodecSpecificToString(const webrtc::VideoCodec& codec) { } // namespace void TestConfig::SetCodecSettings(VideoCodecType codec_type, - size_t num_temporal_layers, + int num_temporal_layers, bool error_concealment_on, bool denoising_on, bool frame_dropper_on, bool spatial_resize_on, bool resilience_on, - size_t width, - size_t height) { + int width, + int height) { webrtc::test::CodecSettings(codec_type, &codec_settings); // TODO(brandtr): Move the setting of |width| and |height| to the tests, and // DCHECK that they are set before initializing the codec instead. - codec_settings.width = static_cast(width); - codec_settings.height = static_cast(height); + codec_settings.width = width; + codec_settings.height = height; switch (codec_settings.codecType) { case kVideoCodecVP8: codec_settings.VP8()->resilience = resilience_on ? kResilientStream : kResilienceOff; - codec_settings.VP8()->numberOfTemporalLayers = - static_cast(num_temporal_layers); + codec_settings.VP8()->numberOfTemporalLayers = num_temporal_layers; codec_settings.VP8()->denoisingOn = denoising_on; codec_settings.VP8()->errorConcealmentOn = error_concealment_on; codec_settings.VP8()->automaticResizeOn = spatial_resize_on; @@ -96,8 +93,7 @@ void TestConfig::SetCodecSettings(VideoCodecType codec_type, break; case kVideoCodecVP9: codec_settings.VP9()->resilienceOn = resilience_on; - codec_settings.VP9()->numberOfTemporalLayers = - static_cast(num_temporal_layers); + codec_settings.VP9()->numberOfTemporalLayers = num_temporal_layers; codec_settings.VP9()->denoisingOn = denoising_on; codec_settings.VP9()->frameDroppingOn = frame_dropper_on; codec_settings.VP9()->keyFrameInterval = kBaseKeyFrameInterval; @@ -113,11 +109,11 @@ void TestConfig::SetCodecSettings(VideoCodecType codec_type, } } -size_t TestConfig::NumberOfCores() const { +int TestConfig::NumberOfCores() const { return use_single_core ? 1 : CpuInfo::DetectNumberOfCores(); } -size_t TestConfig::NumberOfTemporalLayers() const { +int TestConfig::NumberOfTemporalLayers() const { if (codec_settings.codecType == kVideoCodecVP8) { return codec_settings.VP8().numberOfTemporalLayers; } else if (codec_settings.codecType == kVideoCodecVP9) { @@ -127,16 +123,8 @@ size_t TestConfig::NumberOfTemporalLayers() const { } } -size_t TestConfig::NumberOfSpatialLayers() const { - if (codec_settings.codecType == kVideoCodecVP9) { - return codec_settings.VP9().numberOfSpatialLayers; - } else { - return 1; - } -} - -size_t TestConfig::TemporalLayerForFrame(size_t frame_idx) const { - size_t tl = 0; +int TestConfig::TemporalLayerForFrame(int frame_idx) const { + int tl = -1; switch (NumberOfTemporalLayers()) { case 1: tl = 0; @@ -165,7 +153,7 @@ size_t TestConfig::TemporalLayerForFrame(size_t frame_idx) const { return tl; } -std::vector TestConfig::FrameTypeForFrame(size_t frame_idx) const { +std::vector TestConfig::FrameTypeForFrame(int frame_idx) const { if (keyframe_interval > 0 && (frame_idx % keyframe_interval == 0)) { return {kVideoFrameKey}; } @@ -175,19 +163,17 @@ std::vector TestConfig::FrameTypeForFrame(size_t frame_idx) const { std::string TestConfig::ToString() const { std::string codec_type = CodecTypeToPayloadString(codec_settings.codecType); std::stringstream ss; - ss << "\n Filename : " << filename; - ss << "\n # CPU cores used : " << NumberOfCores(); + ss << "\n Filename : " << filename; + ss << "\n # CPU cores used : " << NumberOfCores(); ss << "\n General:"; - ss << "\n Codec type : " << codec_type; - ss << "\n Start bitrate : " << codec_settings.startBitrate << " kbps"; - ss << "\n Max bitrate : " << codec_settings.maxBitrate << " kbps"; - ss << "\n Min bitrate : " << codec_settings.minBitrate << " kbps"; - ss << "\n Width : " << codec_settings.width; - ss << "\n Height : " << codec_settings.height; - ss << "\n Max frame rate : " << codec_settings.maxFramerate; - ss << "\n QPmax : " << codec_settings.qpMax; - ss << "\n # simulcast streams : " - << static_cast(codec_settings.numberOfSimulcastStreams); + ss << "\n Codec type : " << codec_type; + ss << "\n Start bitrate : " << codec_settings.startBitrate << " kbps"; + ss << "\n Max bitrate : " << codec_settings.maxBitrate << " kbps"; + ss << "\n Min bitrate : " << codec_settings.minBitrate << " kbps"; + ss << "\n Width : " << codec_settings.width; + ss << "\n Height : " << codec_settings.height; + ss << "\n Max frame rate : " << codec_settings.maxFramerate; + ss << "\n QPmax : " << codec_settings.qpMax; ss << "\n " << codec_type << " specific: "; ss << CodecSpecificToString(codec_settings); return ss.str(); diff --git a/modules/video_coding/codecs/test/test_config.h b/modules/video_coding/codecs/test/test_config.h index 849967c35a..04a687878c 100644 --- a/modules/video_coding/codecs/test/test_config.h +++ b/modules/video_coding/codecs/test/test_config.h @@ -42,20 +42,19 @@ struct TestConfig { }; void SetCodecSettings(VideoCodecType codec_type, - size_t num_temporal_layers, + int num_temporal_layers, bool error_concealment_on, bool denoising_on, bool frame_dropper_on, bool spatial_resize_on, bool resilience_on, - size_t width, - size_t height); + int width, + int height); - size_t NumberOfCores() const; - size_t NumberOfTemporalLayers() const; - size_t NumberOfSpatialLayers() const; - size_t TemporalLayerForFrame(size_t frame_idx) const; - std::vector FrameTypeForFrame(size_t frame_idx) const; + int NumberOfCores() const; + int NumberOfTemporalLayers() const; + int TemporalLayerForFrame(int frame_idx) const; + std::vector FrameTypeForFrame(int frame_idx) const; std::string ToString() const; std::string CodecName() const; std::string FilenameWithParams() const; @@ -71,7 +70,7 @@ struct TestConfig { std::string output_filename; // Number of frames to process. - size_t num_frames = 0; + int num_frames = 0; // Configurations related to networking. NetworkingConfig networking_config; @@ -97,7 +96,7 @@ struct TestConfig { // to this setting. Forcing key frames may also affect encoder planning // optimizations in a negative way, since it will suddenly be forced to // produce an expensive key frame. - size_t keyframe_interval = 0; + int keyframe_interval = 0; // Codec settings to use. webrtc::VideoCodec codec_settings; @@ -119,9 +118,6 @@ struct TestConfig { // Custom checker that will be called for each frame. const EncodedFrameChecker* encoded_frame_checker = nullptr; - - // Print out frame level stats. - bool print_frame_level_stats = false; }; } // namespace test diff --git a/modules/video_coding/codecs/test/test_config_unittest.cc b/modules/video_coding/codecs/test/test_config_unittest.cc index 966f3e9f5a..968e1f3993 100644 --- a/modules/video_coding/codecs/test/test_config_unittest.cc +++ b/modules/video_coding/codecs/test/test_config_unittest.cc @@ -19,25 +19,25 @@ namespace webrtc { namespace test { namespace { -const size_t kNumTemporalLayers = 2; +const int kNumTemporalLayers = 2; } // namespace TEST(TestConfig, NumberOfCoresWithUseSingleCore) { TestConfig config; config.use_single_core = true; - EXPECT_EQ(1u, config.NumberOfCores()); + EXPECT_EQ(1, config.NumberOfCores()); } TEST(TestConfig, NumberOfCoresWithoutUseSingleCore) { TestConfig config; config.use_single_core = false; - EXPECT_GE(config.NumberOfCores(), 1u); + EXPECT_GE(config.NumberOfCores(), 1); } TEST(TestConfig, NumberOfTemporalLayersIsOne) { TestConfig config; webrtc::test::CodecSettings(kVideoCodecH264, &config.codec_settings); - EXPECT_EQ(1u, config.NumberOfTemporalLayers()); + EXPECT_EQ(1, config.NumberOfTemporalLayers()); } TEST(TestConfig, NumberOfTemporalLayers_Vp8) { @@ -58,33 +58,33 @@ TEST(TestConfig, TemporalLayersForFrame_OneLayer) { TestConfig config; webrtc::test::CodecSettings(kVideoCodecVP8, &config.codec_settings); config.codec_settings.VP8()->numberOfTemporalLayers = 1; - EXPECT_EQ(0u, config.TemporalLayerForFrame(0)); - EXPECT_EQ(0u, config.TemporalLayerForFrame(1)); - EXPECT_EQ(0u, config.TemporalLayerForFrame(2)); + EXPECT_EQ(0, config.TemporalLayerForFrame(0)); + EXPECT_EQ(0, config.TemporalLayerForFrame(1)); + EXPECT_EQ(0, config.TemporalLayerForFrame(2)); } TEST(TestConfig, TemporalLayersForFrame_TwoLayers) { TestConfig config; webrtc::test::CodecSettings(kVideoCodecVP8, &config.codec_settings); config.codec_settings.VP8()->numberOfTemporalLayers = 2; - EXPECT_EQ(0u, config.TemporalLayerForFrame(0)); - EXPECT_EQ(1u, config.TemporalLayerForFrame(1)); - EXPECT_EQ(0u, config.TemporalLayerForFrame(2)); - EXPECT_EQ(1u, config.TemporalLayerForFrame(3)); + EXPECT_EQ(0, config.TemporalLayerForFrame(0)); + EXPECT_EQ(1, config.TemporalLayerForFrame(1)); + EXPECT_EQ(0, config.TemporalLayerForFrame(2)); + EXPECT_EQ(1, config.TemporalLayerForFrame(3)); } TEST(TestConfig, TemporalLayersForFrame_ThreeLayers) { TestConfig config; webrtc::test::CodecSettings(kVideoCodecVP8, &config.codec_settings); config.codec_settings.VP8()->numberOfTemporalLayers = 3; - EXPECT_EQ(0u, config.TemporalLayerForFrame(0)); - EXPECT_EQ(2u, config.TemporalLayerForFrame(1)); - EXPECT_EQ(1u, config.TemporalLayerForFrame(2)); - EXPECT_EQ(2u, config.TemporalLayerForFrame(3)); - EXPECT_EQ(0u, config.TemporalLayerForFrame(4)); - EXPECT_EQ(2u, config.TemporalLayerForFrame(5)); - EXPECT_EQ(1u, config.TemporalLayerForFrame(6)); - EXPECT_EQ(2u, config.TemporalLayerForFrame(7)); + EXPECT_EQ(0, config.TemporalLayerForFrame(0)); + EXPECT_EQ(2, config.TemporalLayerForFrame(1)); + EXPECT_EQ(1, config.TemporalLayerForFrame(2)); + EXPECT_EQ(2, config.TemporalLayerForFrame(3)); + EXPECT_EQ(0, config.TemporalLayerForFrame(4)); + EXPECT_EQ(2, config.TemporalLayerForFrame(5)); + EXPECT_EQ(1, config.TemporalLayerForFrame(6)); + EXPECT_EQ(2, config.TemporalLayerForFrame(7)); } TEST(TestConfig, ForcedKeyFrameIntervalOff) { @@ -126,27 +126,26 @@ TEST(TestConfig, ToString_Vp8) { config.codec_settings.VP8()->keyFrameInterval = 999; EXPECT_EQ( - "\n Filename : yuvfile" - "\n # CPU cores used : 1" + "\n Filename : yuvfile" + "\n # CPU cores used : 1" "\n General:" - "\n Codec type : VP8" - "\n Start bitrate : 400 kbps" - "\n Max bitrate : 500 kbps" - "\n Min bitrate : 70 kbps" - "\n Width : 320" - "\n Height : 180" - "\n Max frame rate : 35" - "\n QPmax : 66" - "\n # simulcast streams : 0" + "\n Codec type : VP8" + "\n Start bitrate : 400 kbps" + "\n Max bitrate : 500 kbps" + "\n Min bitrate : 70 kbps" + "\n Width : 320" + "\n Height : 180" + "\n Max frame rate : 35" + "\n QPmax : 66" "\n VP8 specific: " - "\n Complexity : 0" - "\n Resilience : 0" - "\n # temporal layers : 2" - "\n Denoising : 0" - "\n Error concealment : 1" - "\n Automatic resize : 1" - "\n Frame dropping : 0" - "\n Key frame interval : 999\n", + "\n Complexity : 0" + "\n Resilience : 0" + "\n # temporal layers : 2" + "\n Denoising : 0" + "\n Error concealment : 1" + "\n Automatic resize : 1" + "\n Frame dropping : 0" + "\n Key frame interval: 999\n", config.ToString()); } diff --git a/modules/video_coding/codecs/test/videoprocessor.cc b/modules/video_coding/codecs/test/videoprocessor.cc index 63d352a7fa..06475e196d 100644 --- a/modules/video_coding/codecs/test/videoprocessor.cc +++ b/modules/video_coding/codecs/test/videoprocessor.cc @@ -17,7 +17,6 @@ #include "api/video/i420_buffer.h" #include "common_types.h" // NOLINT(build/include) #include "common_video/h264/h264_common.h" -#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/video_coding/codecs/vp8/simulcast_rate_allocator.h" #include "modules/video_coding/include/video_codec_initializer.h" #include "modules/video_coding/utility/default_video_bitrate_allocator.h" @@ -30,6 +29,8 @@ namespace test { namespace { +const int kRtpClockRateHz = 90000; + std::unique_ptr CreateBitrateAllocator( TestConfig* config) { std::unique_ptr tl_factory; @@ -42,10 +43,10 @@ std::unique_ptr CreateBitrateAllocator( std::move(tl_factory))); } -size_t GetMaxNaluSizeBytes(const EncodedImage& encoded_frame, - const TestConfig& config) { +rtc::Optional GetMaxNaluLength(const EncodedImage& encoded_frame, + const TestConfig& config) { if (config.codec_settings.codecType != kVideoCodecH264) - return 0; + return rtc::nullopt; std::vector nalu_indices = webrtc::H264::FindNaluIndices(encoded_frame._buffer, @@ -53,11 +54,11 @@ size_t GetMaxNaluSizeBytes(const EncodedImage& encoded_frame, RTC_CHECK(!nalu_indices.empty()); - size_t max_size = 0; + size_t max_length = 0; for (const webrtc::H264::NaluIndex& index : nalu_indices) - max_size = std::max(max_size, index.payload_size); + max_length = std::max(max_length, index.payload_size); - return max_size; + return max_length; } int GetElapsedTimeMicroseconds(int64_t start_ns, int64_t stop_ns) { @@ -112,14 +113,13 @@ VideoProcessor::VideoProcessor(webrtc::VideoEncoder* encoder, analysis_frame_reader_(analysis_frame_reader), encoded_frame_writer_(encoded_frame_writer), decoded_frame_writer_(decoded_frame_writer), - last_inputed_frame_num_(0), - last_encoded_frame_num_(0), - last_decoded_frame_num_(0), - num_encoded_frames_(0), - num_decoded_frames_(0), + last_inputed_frame_num_(-1), + last_encoded_frame_num_(-1), + last_decoded_frame_num_(-1), first_key_frame_has_been_excluded_(false), last_decoded_frame_buffer_(analysis_frame_reader->FrameLength()), - stats_(stats) { + stats_(stats), + rate_update_index_(-1) { RTC_DCHECK(encoder); RTC_DCHECK(decoder); RTC_DCHECK(packet_manipulator); @@ -134,13 +134,12 @@ VideoProcessor::VideoProcessor(webrtc::VideoEncoder* encoder, // Initialize the encoder and decoder. RTC_CHECK_EQ( - encoder_->InitEncode(&config_.codec_settings, - static_cast(config_.NumberOfCores()), + encoder_->InitEncode(&config_.codec_settings, config_.NumberOfCores(), config_.networking_config.max_payload_size_in_bytes), WEBRTC_VIDEO_CODEC_OK); - RTC_CHECK_EQ(decoder_->InitDecode(&config_.codec_settings, - static_cast(config_.NumberOfCores())), - WEBRTC_VIDEO_CODEC_OK); + RTC_CHECK_EQ( + decoder_->InitDecode(&config_.codec_settings, config_.NumberOfCores()), + WEBRTC_VIDEO_CODEC_OK); } VideoProcessor::~VideoProcessor() { @@ -155,7 +154,7 @@ VideoProcessor::~VideoProcessor() { void VideoProcessor::ProcessFrame() { RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); - const size_t frame_number = last_inputed_frame_num_++; + const int frame_number = ++last_inputed_frame_num_; // Get frame from file. rtc::scoped_refptr buffer( @@ -164,20 +163,18 @@ void VideoProcessor::ProcessFrame() { // Use the frame number as the basis for timestamp to identify frames. Let the // first timestamp be non-zero, to not make the IvfFileWriter believe that we // want to use capture timestamps in the IVF files. - const size_t rtp_timestamp = (frame_number + 1) * kVideoPayloadTypeFrequency / - config_.codec_settings.maxFramerate; + const uint32_t rtp_timestamp = (frame_number + 1) * kRtpClockRateHz / + config_.codec_settings.maxFramerate; const int64_t render_time_ms = (frame_number + 1) * rtc::kNumMillisecsPerSec / config_.codec_settings.maxFramerate; rtp_timestamp_to_frame_num_[rtp_timestamp] = frame_number; - input_frames_[frame_number] = - rtc::MakeUnique(buffer, static_cast(rtp_timestamp), - render_time_ms, webrtc::kVideoRotation_0); + input_frames_[frame_number] = rtc::MakeUnique( + buffer, rtp_timestamp, render_time_ms, webrtc::kVideoRotation_0); std::vector frame_types = config_.FrameTypeForFrame(frame_number); // Create frame statistics object used for aggregation at end of test run. FrameStatistic* frame_stat = stats_->AddFrame(); - frame_stat->rtp_timestamp = rtp_timestamp; // For the highest measurement accuracy of the encode time, the start/stop // time recordings should wrap the Encode call as tightly as possible. @@ -186,16 +183,27 @@ void VideoProcessor::ProcessFrame() { encoder_->Encode(*input_frames_[frame_number], nullptr, &frame_types); } -void VideoProcessor::SetRates(size_t bitrate_kbps, size_t framerate_fps) { +void VideoProcessor::SetRates(int bitrate_kbps, int framerate_fps) { RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); - config_.codec_settings.maxFramerate = static_cast(framerate_fps); - bitrate_allocation_ = bitrate_allocator_->GetAllocation( - static_cast(bitrate_kbps * 1000), - static_cast(framerate_fps)); - const int set_rates_result = encoder_->SetRateAllocation( - bitrate_allocation_, static_cast(framerate_fps)); + config_.codec_settings.maxFramerate = framerate_fps; + int set_rates_result = encoder_->SetRateAllocation( + bitrate_allocator_->GetAllocation(bitrate_kbps * 1000, framerate_fps), + framerate_fps); RTC_DCHECK_GE(set_rates_result, 0) << "Failed to update encoder with new rate " << bitrate_kbps << "."; + ++rate_update_index_; + num_dropped_frames_.push_back(0); + num_spatial_resizes_.push_back(0); +} + +std::vector VideoProcessor::NumberDroppedFramesPerRateUpdate() const { + RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); + return num_dropped_frames_; +} + +std::vector VideoProcessor::NumberSpatialResizesPerRateUpdate() const { + RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); + return num_spatial_resizes_; } void VideoProcessor::FrameEncoded(webrtc::VideoCodecType codec, @@ -210,18 +218,20 @@ void VideoProcessor::FrameEncoded(webrtc::VideoCodecType codec, config_.encoded_frame_checker->CheckEncodedFrame(codec, encoded_image); } - const size_t frame_number = + const int frame_number = rtp_timestamp_to_frame_num_[encoded_image._timeStamp]; // Ensure strict monotonicity. - if (num_encoded_frames_ > 0) { - RTC_CHECK_GT(frame_number, last_encoded_frame_num_); - } - ++num_encoded_frames_; + RTC_CHECK_GT(frame_number, last_encoded_frame_num_); // Check for dropped frames. bool last_frame_missing = false; if (frame_number > 0) { + int num_dropped_from_last_encode = + frame_number - last_encoded_frame_num_ - 1; + RTC_DCHECK_GE(num_dropped_from_last_encode, 0); + RTC_CHECK_GE(rate_update_index_, 0); + num_dropped_frames_[rate_update_index_] += num_dropped_from_last_encode; const FrameStatistic* last_encoded_frame_stat = stats_->GetFrame(last_encoded_frame_num_); last_frame_missing = (last_encoded_frame_stat->manipulated_length == 0); @@ -235,14 +245,13 @@ void VideoProcessor::FrameEncoded(webrtc::VideoCodecType codec, frame_stat->encoding_successful = true; frame_stat->encoded_frame_size_bytes = encoded_image._length; frame_stat->frame_type = encoded_image._frameType; - frame_stat->temporal_layer_idx = config_.TemporalLayerForFrame(frame_number); frame_stat->qp = encoded_image.qp_; - frame_stat->target_bitrate_kbps = - bitrate_allocation_.GetSpatialLayerSum(0) / 1000; + frame_stat->bitrate_kbps = static_cast( + encoded_image._length * config_.codec_settings.maxFramerate * 8 / 1000); frame_stat->total_packets = encoded_image._length / config_.networking_config.packet_size_in_bytes + 1; - frame_stat->max_nalu_size_bytes = GetMaxNaluSizeBytes(encoded_image, config_); + frame_stat->max_nalu_length = GetMaxNaluLength(encoded_image, config_); // Make a raw copy of |encoded_image| to feed to the decoder. size_t copied_buffer_size = encoded_image._length + @@ -279,7 +288,7 @@ void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame) { int64_t decode_stop_ns = rtc::TimeNanos(); // Update frame statistics. - const size_t frame_number = + const int frame_number = rtp_timestamp_to_frame_num_[decoded_frame.timestamp()]; FrameStatistic* frame_stat = stats_->GetFrame(frame_number); frame_stat->decoded_width = decoded_frame.width(); @@ -289,22 +298,26 @@ void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame) { frame_stat->decoding_successful = true; // Ensure strict monotonicity. - if (num_decoded_frames_ > 0) { - RTC_CHECK_GT(frame_number, last_decoded_frame_num_); - } - ++num_decoded_frames_; + RTC_CHECK_GT(frame_number, last_decoded_frame_num_); // Check if the codecs have resized the frame since previously decoded frame. if (frame_number > 0) { if (decoded_frame_writer_ && last_decoded_frame_num_ >= 0) { // For dropped/lost frames, write out the last decoded frame to make it // look like a freeze at playback. - const size_t num_dropped_frames = - frame_number - last_decoded_frame_num_ - 1; - for (size_t i = 0; i < num_dropped_frames; i++) { + const int num_dropped_frames = frame_number - last_decoded_frame_num_; + for (int i = 0; i < num_dropped_frames; i++) { WriteDecodedFrameToFile(&last_decoded_frame_buffer_); } } + // TODO(ssilkin): move to FrameEncoded when webm:1474 is implemented. + const FrameStatistic* last_decoded_frame_stat = + stats_->GetFrame(last_decoded_frame_num_); + if (decoded_frame.width() != last_decoded_frame_stat->decoded_width || + decoded_frame.height() != last_decoded_frame_stat->decoded_height) { + RTC_CHECK_GE(rate_update_index_, 0); + ++num_spatial_resizes_[rate_update_index_]; + } } last_decoded_frame_num_ = frame_number; @@ -318,8 +331,10 @@ void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame) { // Delay erasing of input frames by one frame. The current frame might // still be needed for other simulcast stream or spatial layer. - if (frame_number > 0) { - auto input_frame_erase_to = input_frames_.lower_bound(frame_number - 1); + const int frame_number_to_erase = frame_number - 1; + if (frame_number_to_erase >= 0) { + auto input_frame_erase_to = + input_frames_.lower_bound(frame_number_to_erase); input_frames_.erase(input_frames_.begin(), input_frame_erase_to); } diff --git a/modules/video_coding/codecs/test/videoprocessor.h b/modules/video_coding/codecs/test/videoprocessor.h index 190b6a3134..62a12ef871 100644 --- a/modules/video_coding/codecs/test/videoprocessor.h +++ b/modules/video_coding/codecs/test/videoprocessor.h @@ -76,7 +76,13 @@ class VideoProcessor { void ProcessFrame(); // Updates the encoder with target rates. Must be called at least once. - void SetRates(size_t bitrate_kbps, size_t framerate_fps); + void SetRates(int bitrate_kbps, int framerate_fps); + + // Returns the number of dropped frames. + std::vector NumberDroppedFramesPerRateUpdate() const; + + // Returns the number of spatial resizes. + std::vector NumberSpatialResizesPerRateUpdate() const; private: class VideoProcessorEncodeCompleteCallback @@ -184,7 +190,6 @@ class VideoProcessor { webrtc::VideoEncoder* const encoder_; webrtc::VideoDecoder* const decoder_; const std::unique_ptr bitrate_allocator_; - BitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(sequence_checker_); // Adapters for the codec callbacks. VideoProcessorEncodeCompleteCallback encode_callback_; @@ -197,7 +202,7 @@ class VideoProcessor { // Async codecs might queue frames. To handle that we keep input frame // and release it after corresponding coded frame is decoded and quality // measurement is done. - std::map> input_frames_ + std::map> input_frames_ RTC_GUARDED_BY(sequence_checker_); // These (mandatory) file manipulators are used for, e.g., objective PSNR and @@ -212,15 +217,13 @@ class VideoProcessor { FrameWriter* const decoded_frame_writer_; // Keep track of inputed/encoded/decoded frames, so we can detect frame drops. - size_t last_inputed_frame_num_ RTC_GUARDED_BY(sequence_checker_); - size_t last_encoded_frame_num_ RTC_GUARDED_BY(sequence_checker_); - size_t last_decoded_frame_num_ RTC_GUARDED_BY(sequence_checker_); - size_t num_encoded_frames_ RTC_GUARDED_BY(sequence_checker_); - size_t num_decoded_frames_ RTC_GUARDED_BY(sequence_checker_); + int last_inputed_frame_num_ RTC_GUARDED_BY(sequence_checker_); + int last_encoded_frame_num_ RTC_GUARDED_BY(sequence_checker_); + int last_decoded_frame_num_ RTC_GUARDED_BY(sequence_checker_); // Store an RTP timestamp -> frame number map, since the timestamps are // based off of the frame rate, which can change mid-test. - std::map rtp_timestamp_to_frame_num_ + std::map rtp_timestamp_to_frame_num_ RTC_GUARDED_BY(sequence_checker_); // Keep track of if we have excluded the first key frame from packet loss. @@ -232,6 +235,9 @@ class VideoProcessor { // Statistics. Stats* stats_; + std::vector num_dropped_frames_ RTC_GUARDED_BY(sequence_checker_); + std::vector num_spatial_resizes_ RTC_GUARDED_BY(sequence_checker_); + int rate_update_index_ RTC_GUARDED_BY(sequence_checker_); rtc::SequencedTaskChecker sequence_checker_; diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc index f9b0a72c62..dc9b146560 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc @@ -37,7 +37,6 @@ #include "rtc_base/file.h" #include "rtc_base/ptr_util.h" #include "system_wrappers/include/sleep.h" -#include "test/statistics.h" #include "test/testsupport/fileutils.h" #include "test/testsupport/metrics/video_metrics.h" @@ -46,10 +45,13 @@ namespace test { namespace { -const int kRtpClockRateHz = 90000; - const int kMaxBitrateMismatchPercent = 20; +// Parameters from VP8 wrapper, which control target size of key frames. +const float kInitialBufferSize = 0.5f; +const float kOptimalBufferSize = 0.6f; +const float kScaleKeyFrameSize = 0.5f; + bool RunEncodeInRealTime(const TestConfig& config) { if (config.measure_cpu) { return true; @@ -171,210 +173,118 @@ VideoProcessorIntegrationTest::~VideoProcessorIntegrationTest() = default; void VideoProcessorIntegrationTest::ProcessFramesAndMaybeVerify( const std::vector& rate_profiles, const std::vector* rc_thresholds, - const std::vector* quality_thresholds, + const QualityThresholds* quality_thresholds, const BitstreamThresholds* bs_thresholds, const VisualizationParams* visualization_params) { RTC_DCHECK(!rate_profiles.empty()); // The Android HW codec needs to be run on a task queue, so we simply always // run the test on a task queue. rtc::TaskQueue task_queue("VidProc TQ"); + rtc::Event sync_event(false, false); - SetUpAndInitObjects( - &task_queue, static_cast(rate_profiles[0].target_kbps), - static_cast(rate_profiles[0].input_fps), visualization_params); + SetUpAndInitObjects(&task_queue, rate_profiles[0].target_kbps, + rate_profiles[0].input_fps, visualization_params); PrintSettings(); - ProcessAllFrames(&task_queue, rate_profiles); - - ReleaseAndCloseObjects(&task_queue); - - AnalyzeAllFrames(rate_profiles, rc_thresholds, quality_thresholds, - bs_thresholds); -} - -void VideoProcessorIntegrationTest::ProcessAllFrames( - rtc::TaskQueue* task_queue, - const std::vector& rate_profiles) { - // Process all frames. - size_t rate_update_index = 0; - // Set initial rates. - task_queue->PostTask([this, &rate_profiles, rate_update_index] { + int rate_update_index = 0; + task_queue.PostTask([this, &rate_profiles, rate_update_index] { processor_->SetRates(rate_profiles[rate_update_index].target_kbps, rate_profiles[rate_update_index].input_fps); }); cpu_process_time_->Start(); - for (size_t frame_number = 0; frame_number < config_.num_frames; - ++frame_number) { + // Process all frames. + int frame_number = 0; + const int num_frames = config_.num_frames; + RTC_DCHECK_GE(num_frames, 1); + while (frame_number < num_frames) { + if (RunEncodeInRealTime(config_)) { + // Roughly pace the frames. + SleepMs(rtc::kNumMillisecsPerSec / + rate_profiles[rate_update_index].input_fps); + } + + task_queue.PostTask([this] { processor_->ProcessFrame(); }); + ++frame_number; + if (frame_number == rate_profiles[rate_update_index].frame_index_rate_update) { ++rate_update_index; RTC_DCHECK_GT(rate_profiles.size(), rate_update_index); - task_queue->PostTask([this, &rate_profiles, rate_update_index] { + task_queue.PostTask([this, &rate_profiles, rate_update_index] { processor_->SetRates(rate_profiles[rate_update_index].target_kbps, rate_profiles[rate_update_index].input_fps); }); } - - task_queue->PostTask([this] { processor_->ProcessFrame(); }); - - if (RunEncodeInRealTime(config_)) { - // Roughly pace the frames. - size_t frame_duration_ms = - rtc::kNumMillisecsPerSec / rate_profiles[rate_update_index].input_fps; - SleepMs(static_cast(frame_duration_ms)); - } } - rtc::Event sync_event(false, false); - task_queue->PostTask([&sync_event] { sync_event.Set(); }); - sync_event.Wait(rtc::Event::kForever); - // Give the VideoProcessor pipeline some time to process the last frame, // and then release the codecs. if (config_.hw_encoder || config_.hw_decoder) { SleepMs(1 * rtc::kNumMillisecsPerSec); } - cpu_process_time_->Stop(); -} -void VideoProcessorIntegrationTest::AnalyzeAllFrames( - const std::vector& rate_profiles, - const std::vector* rc_thresholds, - const std::vector* quality_thresholds, - const BitstreamThresholds* bs_thresholds) { - const bool is_svc = config_.NumberOfSpatialLayers() > 1; - const size_t number_of_simulcast_or_spatial_layers = - std::max(std::size_t{1}, - std::max(config_.NumberOfSpatialLayers(), - static_cast( - config_.codec_settings.numberOfSimulcastStreams))); - const size_t number_of_temporal_layers = config_.NumberOfTemporalLayers(); - printf("Rate control statistics\n==\n"); - for (size_t rate_update_index = 0; rate_update_index < rate_profiles.size(); - ++rate_update_index) { - const size_t first_frame_number = - (rate_update_index == 0) - ? 0 - : rate_profiles[rate_update_index - 1].frame_index_rate_update; - const size_t last_frame_number = - rate_profiles[rate_update_index].frame_index_rate_update - 1; - RTC_CHECK(last_frame_number >= first_frame_number); - const size_t number_of_frames = last_frame_number - first_frame_number + 1; - const float input_duration_sec = - 1.0 * number_of_frames / rate_profiles[rate_update_index].input_fps; + std::vector num_dropped_frames; + std::vector num_spatial_resizes; + sync_event.Reset(); + task_queue.PostTask( + [this, &num_dropped_frames, &num_spatial_resizes, &sync_event]() { + num_dropped_frames = processor_->NumberDroppedFramesPerRateUpdate(); + num_spatial_resizes = processor_->NumberSpatialResizesPerRateUpdate(); + sync_event.Set(); + }); + sync_event.Wait(rtc::Event::kForever); - std::vector overall_stats = - ExtractLayerStats(number_of_simulcast_or_spatial_layers - 1, - number_of_temporal_layers - 1, first_frame_number, - last_frame_number, true); + ReleaseAndCloseObjects(&task_queue); - printf("Rate update #%zu:\n", rate_update_index); + // Calculate and print rate control statistics. + rate_update_index = 0; + frame_number = 0; + quality_ = QualityMetrics(); + ResetRateControlMetrics(rate_update_index, rate_profiles); + while (frame_number < num_frames) { + UpdateRateControlMetrics(frame_number); - const RateControlThresholds* rc_threshold = - rc_thresholds ? &(*rc_thresholds)[rate_update_index] : nullptr; - const QualityThresholds* quality_threshold = - quality_thresholds ? &(*quality_thresholds)[rate_update_index] - : nullptr; - AnalyzeAndPrintStats( - overall_stats, rate_profiles[rate_update_index].target_kbps, - rate_profiles[rate_update_index].input_fps, input_duration_sec, - rc_threshold, quality_threshold, bs_thresholds); - - if (config_.print_frame_level_stats) { - PrintFrameLevelStats(overall_stats); + if (quality_thresholds) { + UpdateQualityMetrics(frame_number); } - for (size_t spatial_layer_number = 0; - spatial_layer_number < number_of_simulcast_or_spatial_layers; - ++spatial_layer_number) { - for (size_t temporal_layer_number = 0; - temporal_layer_number < number_of_temporal_layers; - ++temporal_layer_number) { - std::vector layer_stats = - ExtractLayerStats(spatial_layer_number, temporal_layer_number, - first_frame_number, last_frame_number, is_svc); + if (bs_thresholds) { + VerifyBitstream(frame_number, *bs_thresholds); + } - const size_t target_bitrate_kbps = layer_stats[0].target_bitrate_kbps; - const float target_framerate_fps = - 1.0 * rate_profiles[rate_update_index].input_fps / - (1 << (number_of_temporal_layers - temporal_layer_number - 1)); + ++frame_number; - printf("Spatial %zu temporal %zu:\n", spatial_layer_number, - temporal_layer_number); - AnalyzeAndPrintStats(layer_stats, target_bitrate_kbps, - target_framerate_fps, input_duration_sec, nullptr, - nullptr, nullptr); - - if (config_.print_frame_level_stats) { - PrintFrameLevelStats(layer_stats); - } - } + if (frame_number == + rate_profiles[rate_update_index].frame_index_rate_update) { + PrintRateControlMetrics(rate_update_index, num_dropped_frames, + num_spatial_resizes); + VerifyRateControlMetrics(rate_update_index, rc_thresholds, + num_dropped_frames, num_spatial_resizes); + ++rate_update_index; + ResetRateControlMetrics(rate_update_index, rate_profiles); } } + PrintRateControlMetrics(rate_update_index, num_dropped_frames, + num_spatial_resizes); + VerifyRateControlMetrics(rate_update_index, rc_thresholds, num_dropped_frames, + num_spatial_resizes); + + if (quality_thresholds) { + VerifyQualityMetrics(*quality_thresholds); + } + + // Calculate and print other statistics. + EXPECT_EQ(num_frames, static_cast(stats_.size())); + stats_.PrintSummary(); cpu_process_time_->Print(); } -std::vector VideoProcessorIntegrationTest::ExtractLayerStats( - size_t target_spatial_layer_number, - size_t target_temporal_layer_number, - size_t first_frame_number, - size_t last_frame_number, - bool combine_layers_stats) { - size_t target_bitrate_kbps = 0; - std::vector layer_stats; - - for (size_t frame_number = first_frame_number; - frame_number <= last_frame_number; ++frame_number) { - // TODO(ssilkin): Add layering support - // FrameStatistic superframe_stat = - // *stats_[target_spatial_layer_number].GetFrame(frame_number); - FrameStatistic superframe_stat = *stats_.GetFrame(frame_number); - const size_t tl_idx = superframe_stat.temporal_layer_idx; - if (tl_idx <= target_temporal_layer_number) { - if (combine_layers_stats) { - for (size_t spatial_layer_number = 0; - spatial_layer_number < target_spatial_layer_number; - ++spatial_layer_number) { - // TODO(ssilkin): Add layering support - // const FrameStatistic* frame_stat = - // stats_[spatial_layer_number].GetFrame(frame_number); - const FrameStatistic* frame_stat = stats_.GetFrame(frame_number); - superframe_stat.encoded_frame_size_bytes += - frame_stat->encoded_frame_size_bytes; - superframe_stat.encode_time_us = std::max( - superframe_stat.encode_time_us, frame_stat->encode_time_us); - superframe_stat.decode_time_us = std::max( - superframe_stat.decode_time_us, frame_stat->decode_time_us); - } - } - - target_bitrate_kbps = - std::max(target_bitrate_kbps, superframe_stat.target_bitrate_kbps); - - if (superframe_stat.encoding_successful) { - RTC_CHECK(superframe_stat.target_bitrate_kbps <= target_bitrate_kbps || - tl_idx == target_temporal_layer_number); - RTC_CHECK(superframe_stat.target_bitrate_kbps == target_bitrate_kbps || - tl_idx < target_temporal_layer_number); - } - - layer_stats.push_back(superframe_stat); - } - } - - for (auto& frame_stat : layer_stats) { - frame_stat.target_bitrate_kbps = target_bitrate_kbps; - } - - return layer_stats; -} - void VideoProcessorIntegrationTest::CreateEncoderAndDecoder() { std::unique_ptr encoder_factory; if (config_.hw_encoder) { @@ -526,10 +436,139 @@ void VideoProcessorIntegrationTest::ReleaseAndCloseObjects( } } +// For every encoded frame, update the rate control metrics. +void VideoProcessorIntegrationTest::UpdateRateControlMetrics(int frame_number) { + RTC_CHECK_GE(frame_number, 0); + + const int tl_idx = config_.TemporalLayerForFrame(frame_number); + ++actual_.num_frames_layer[tl_idx]; + ++actual_.num_frames; + + const FrameStatistic* frame_stat = stats_.GetFrame(frame_number); + FrameType frame_type = frame_stat->frame_type; + float framesize_kbits = frame_stat->encoded_frame_size_bytes * 8.0f / 1000.0f; + + // Update rate mismatch relative to per-frame bandwidth. + if (frame_type == kVideoFrameDelta) { + // TODO(marpan): Should we count dropped (zero size) frames in mismatch? + actual_.sum_delta_framesize_mismatch_layer[tl_idx] += + fabs(framesize_kbits - target_.framesize_kbits_layer[tl_idx]) / + target_.framesize_kbits_layer[tl_idx]; + } else { + float key_framesize_kbits = (frame_number == 0) + ? target_.key_framesize_kbits_initial + : target_.key_framesize_kbits; + actual_.sum_key_framesize_mismatch += + fabs(framesize_kbits - key_framesize_kbits) / key_framesize_kbits; + ++actual_.num_key_frames; + } + actual_.sum_framesize_kbits += framesize_kbits; + actual_.sum_framesize_kbits_layer[tl_idx] += framesize_kbits; + + // Encoded bitrate: from the start of the update/run to current frame. + actual_.kbps = actual_.sum_framesize_kbits * target_.fps / actual_.num_frames; + actual_.kbps_layer[tl_idx] = actual_.sum_framesize_kbits_layer[tl_idx] * + target_.fps_layer[tl_idx] / + actual_.num_frames_layer[tl_idx]; + + // Number of frames to hit target bitrate. + if (actual_.BitrateMismatchPercent(target_.kbps) < + kMaxBitrateMismatchPercent) { + actual_.num_frames_to_hit_target = + std::min(actual_.num_frames, actual_.num_frames_to_hit_target); + } +} + +// Verify expected behavior of rate control. +void VideoProcessorIntegrationTest::VerifyRateControlMetrics( + int rate_update_index, + const std::vector* rc_thresholds, + const std::vector& num_dropped_frames, + const std::vector& num_spatial_resizes) const { + if (!rc_thresholds) + return; + + const RateControlThresholds& rc_threshold = + (*rc_thresholds)[rate_update_index]; + + EXPECT_LE(num_dropped_frames[rate_update_index], + rc_threshold.max_num_dropped_frames); + EXPECT_EQ(rc_threshold.num_spatial_resizes, + num_spatial_resizes[rate_update_index]); + + EXPECT_LE(actual_.num_frames_to_hit_target, + rc_threshold.max_num_frames_to_hit_target); + EXPECT_EQ(rc_threshold.num_key_frames, actual_.num_key_frames); + EXPECT_LE(actual_.KeyFrameSizeMismatchPercent(), + rc_threshold.max_key_framesize_mismatch_percent); + EXPECT_LE(actual_.BitrateMismatchPercent(target_.kbps), + rc_threshold.max_bitrate_mismatch_percent); + + const int num_temporal_layers = config_.NumberOfTemporalLayers(); + for (int i = 0; i < num_temporal_layers; ++i) { + EXPECT_LE(actual_.DeltaFrameSizeMismatchPercent(i), + rc_threshold.max_delta_framesize_mismatch_percent); + EXPECT_LE(actual_.BitrateMismatchPercent(i, target_.kbps_layer[i]), + rc_threshold.max_bitrate_mismatch_percent); + } +} + +void VideoProcessorIntegrationTest::UpdateQualityMetrics(int frame_number) { + FrameStatistic* frame_stat = stats_.GetFrame(frame_number); + if (frame_stat->decoding_successful) { + ++quality_.num_decoded_frames; + quality_.total_psnr += frame_stat->psnr; + quality_.total_ssim += frame_stat->ssim; + if (frame_stat->psnr < quality_.min_psnr) + quality_.min_psnr = frame_stat->psnr; + if (frame_stat->ssim < quality_.min_ssim) + quality_.min_ssim = frame_stat->ssim; + } +} + +void VideoProcessorIntegrationTest::PrintRateControlMetrics( + int rate_update_index, + const std::vector& num_dropped_frames, + const std::vector& num_spatial_resizes) const { + if (rate_update_index == 0) { + printf("Rate control statistics\n==\n"); + } + + printf("Rate update #%d:\n", rate_update_index); + printf(" Target bitrate : %d\n", target_.kbps); + printf(" Encoded bitrate : %f\n", actual_.kbps); + printf(" Frame rate : %d\n", target_.fps); + printf(" # processed frames : %d\n", actual_.num_frames); + printf(" # frames to convergence : %d\n", actual_.num_frames_to_hit_target); + printf(" # dropped frames : %d\n", + num_dropped_frames[rate_update_index]); + printf(" # spatial resizes : %d\n", + num_spatial_resizes[rate_update_index]); + printf(" # key frames : %d\n", actual_.num_key_frames); + printf(" Key frame rate mismatch : %d\n", + actual_.KeyFrameSizeMismatchPercent()); + + const int num_temporal_layers = config_.NumberOfTemporalLayers(); + for (int i = 0; i < num_temporal_layers; ++i) { + printf(" Temporal layer #%d:\n", i); + printf(" TL%d target bitrate : %f\n", i, target_.kbps_layer[i]); + printf(" TL%d encoded bitrate : %f\n", i, actual_.kbps_layer[i]); + printf(" TL%d frame rate : %f\n", i, target_.fps_layer[i]); + printf(" TL%d # processed frames : %d\n", i, + actual_.num_frames_layer[i]); + printf(" TL%d frame size %% mismatch : %d\n", i, + actual_.DeltaFrameSizeMismatchPercent(i)); + printf(" TL%d bitrate %% mismatch : %d\n", i, + actual_.BitrateMismatchPercent(i, target_.kbps_layer[i])); + printf(" TL%d per-frame bitrate : %f\n", i, + target_.framesize_kbits_layer[i]); + } + printf("\n"); +} + void VideoProcessorIntegrationTest::PrintSettings() const { printf("VideoProcessor settings\n==\n"); - printf(" Total # of frames : %d", - analysis_frame_reader_->NumberOfFrames()); + printf(" Total # of frames: %d", analysis_frame_reader_->NumberOfFrames()); printf("%s\n", config_.ToString().c_str()); printf("VideoProcessorIntegrationTest settings\n==\n"); @@ -538,192 +577,87 @@ void VideoProcessorIntegrationTest::PrintSettings() const { const char* decoder_name = decoder_->ImplementationName(); printf(" Decoder implementation name: %s\n", decoder_name); if (strcmp(encoder_name, decoder_name) == 0) { - printf(" Codec implementation name : %s_%s\n", config_.CodecName().c_str(), + printf(" Codec implementation name : %s_%s\n", config_.CodecName().c_str(), encoder_name); } printf("\n"); } -void VideoProcessorIntegrationTest::AnalyzeAndPrintStats( - const std::vector& stats, - const float target_bitrate_kbps, - const float target_framerate_fps, - const float input_duration_sec, - const RateControlThresholds* rc_thresholds, - const QualityThresholds* quality_thresholds, - const BitstreamThresholds* bs_thresholds) { - const size_t num_input_frames = stats.size(); - size_t num_dropped_frames = 0; - size_t num_decoded_frames = 0; - size_t num_spatial_resizes = 0; - size_t num_key_frames = 0; - size_t max_nalu_size_bytes = 0; - - size_t encoded_bytes = 0; - float buffer_level_kbits = 0.0; - float time_to_reach_target_bitrate_sec = -1.0; - - Statistics buffer_level_sec; - Statistics key_frame_size_bytes; - Statistics delta_frame_size_bytes; - - Statistics encoding_time_us; - Statistics decoding_time_us; - Statistics psnr; - Statistics ssim; - - Statistics qp; - - FrameStatistic last_successfully_decoded_frame(0); - for (size_t frame_idx = 0; frame_idx < stats.size(); ++frame_idx) { - const FrameStatistic& frame_stat = stats[frame_idx]; - - const float time_since_first_input_sec = - frame_idx == 0 - ? 0.0 - : 1.0 * (frame_stat.rtp_timestamp - stats[0].rtp_timestamp) / - kRtpClockRateHz; - const float time_since_last_input_sec = - frame_idx == 0 ? 0.0 - : 1.0 * - (frame_stat.rtp_timestamp - - stats[frame_idx - 1].rtp_timestamp) / - kRtpClockRateHz; - - // Testing framework uses constant input framerate. This guarantees even - // sampling, which is important, of buffer level. - buffer_level_kbits -= time_since_last_input_sec * target_bitrate_kbps; - buffer_level_kbits = std::max(0.0f, buffer_level_kbits); - buffer_level_kbits += 8.0 * frame_stat.encoded_frame_size_bytes / 1000; - buffer_level_sec.AddSample(buffer_level_kbits / target_bitrate_kbps); - - encoded_bytes += frame_stat.encoded_frame_size_bytes; - if (frame_stat.encoded_frame_size_bytes == 0) { - ++num_dropped_frames; - } else { - if (frame_stat.frame_type == kVideoFrameKey) { - key_frame_size_bytes.AddSample(frame_stat.encoded_frame_size_bytes); - ++num_key_frames; - } else { - delta_frame_size_bytes.AddSample(frame_stat.encoded_frame_size_bytes); - } - - encoding_time_us.AddSample(frame_stat.encode_time_us); - qp.AddSample(frame_stat.qp); - - max_nalu_size_bytes = - std::max(max_nalu_size_bytes, frame_stat.max_nalu_size_bytes); - } - - if (frame_stat.decoding_successful) { - psnr.AddSample(frame_stat.psnr); - ssim.AddSample(frame_stat.ssim); - if (num_decoded_frames > 0) { - if (last_successfully_decoded_frame.decoded_width != - frame_stat.decoded_width || - last_successfully_decoded_frame.decoded_height != - frame_stat.decoded_height) { - ++num_spatial_resizes; - } - } - decoding_time_us.AddSample(frame_stat.decode_time_us); - last_successfully_decoded_frame = frame_stat; - ++num_decoded_frames; - } - - if (time_to_reach_target_bitrate_sec < 0 && frame_idx > 0) { - const float curr_bitrate_kbps = - (8.0 * encoded_bytes / 1000) / time_since_first_input_sec; - const float bitrate_mismatch_percent = - 100 * std::fabs(curr_bitrate_kbps - target_bitrate_kbps) / - target_bitrate_kbps; - if (bitrate_mismatch_percent < kMaxBitrateMismatchPercent) { - time_to_reach_target_bitrate_sec = time_since_first_input_sec; - } - } - } - - const float encoded_bitrate_kbps = - 8 * encoded_bytes / input_duration_sec / 1000; - const float bitrate_mismatch_percent = - 100 * std::fabs(encoded_bitrate_kbps - target_bitrate_kbps) / - target_bitrate_kbps; - const size_t num_encoded_frames = num_input_frames - num_dropped_frames; - const float encoded_framerate_fps = num_encoded_frames / input_duration_sec; - const float decoded_framerate_fps = num_decoded_frames / input_duration_sec; - const float framerate_mismatch_percent = - 100 * std::fabs(decoded_framerate_fps - target_framerate_fps) / - target_framerate_fps; - const float max_key_frame_delay_sec = - 8 * key_frame_size_bytes.Max() / 1000 / target_bitrate_kbps; - const float max_delta_frame_delay_sec = - 8 * delta_frame_size_bytes.Max() / 1000 / target_bitrate_kbps; - - printf("Target bitrate : %f kbps\n", target_bitrate_kbps); - printf("Encoded bitrate : %f kbps\n", encoded_bitrate_kbps); - printf("Bitrate mismatch : %f %%\n", bitrate_mismatch_percent); - printf("Time to reach target bitrate : %f sec\n", - time_to_reach_target_bitrate_sec); - printf("Target framerate : %f fps\n", target_framerate_fps); - printf("Encoding framerate : %f fps\n", encoded_framerate_fps); - printf("Decoding framerate : %f fps\n", decoded_framerate_fps); - printf("Frame encoding time : %f us\n", encoding_time_us.Mean()); - printf("Frame decoding time : %f us\n", decoding_time_us.Mean()); - printf("Framerate mismatch percent : %f %%\n", - framerate_mismatch_percent); - printf("Avg buffer level : %f sec\n", buffer_level_sec.Mean()); - printf("Max key frame delay : %f sec\n", max_key_frame_delay_sec); - printf("Max delta frame delay : %f sec\n", - max_delta_frame_delay_sec); - printf("Avg key frame size : %f bytes\n", - key_frame_size_bytes.Mean()); - printf("Avg delta frame size : %f bytes\n", - delta_frame_size_bytes.Mean()); - printf("Avg QP : %f\n", qp.Mean()); - printf("Avg PSNR : %f dB\n", psnr.Mean()); - printf("Min PSNR : %f dB\n", psnr.Min()); - printf("Avg SSIM : %f\n", ssim.Mean()); - printf("Min SSIM : %f\n", ssim.Min()); - printf("# input frames : %zu\n", num_input_frames); - printf("# encoded frames : %zu\n", num_encoded_frames); - printf("# decoded frames : %zu\n", num_decoded_frames); - printf("# dropped frames : %zu\n", num_dropped_frames); - printf("# key frames : %zu\n", num_key_frames); - printf("# encoded bytes : %zu\n", encoded_bytes); - printf("# spatial resizes : %zu\n", num_spatial_resizes); - - if (rc_thresholds) { - EXPECT_LE(bitrate_mismatch_percent, - rc_thresholds->max_avg_bitrate_mismatch_percent); - EXPECT_LE(time_to_reach_target_bitrate_sec, - rc_thresholds->max_time_to_reach_target_bitrate_sec); - EXPECT_LE(framerate_mismatch_percent, - rc_thresholds->max_avg_framerate_mismatch_percent); - EXPECT_LE(buffer_level_sec.Mean(), rc_thresholds->max_avg_buffer_level_sec); - EXPECT_LE(max_key_frame_delay_sec, - rc_thresholds->max_max_key_frame_delay_sec); - EXPECT_LE(max_delta_frame_delay_sec, - rc_thresholds->max_max_delta_frame_delay_sec); - EXPECT_LE(num_spatial_resizes, rc_thresholds->max_num_spatial_resizes); - EXPECT_LE(num_key_frames, rc_thresholds->max_num_key_frames); - } - - if (quality_thresholds) { - EXPECT_GT(psnr.Mean(), quality_thresholds->min_avg_psnr); - EXPECT_GT(psnr.Min(), quality_thresholds->min_min_psnr); - EXPECT_GT(ssim.Mean(), quality_thresholds->min_avg_ssim); - EXPECT_GT(ssim.Min(), quality_thresholds->min_min_ssim); - } - - if (bs_thresholds) { - EXPECT_LE(max_nalu_size_bytes, bs_thresholds->max_max_nalu_size_bytes); - } +void VideoProcessorIntegrationTest::VerifyBitstream( + int frame_number, + const BitstreamThresholds& bs_thresholds) { + RTC_CHECK_GE(frame_number, 0); + const FrameStatistic* frame_stat = stats_.GetFrame(frame_number); + EXPECT_LE(*(frame_stat->max_nalu_length), bs_thresholds.max_nalu_length); } -void VideoProcessorIntegrationTest::PrintFrameLevelStats( - const std::vector& stats) const { - for (auto& frame_stat : stats) { - printf("%s\n", frame_stat.ToString().c_str()); +void VideoProcessorIntegrationTest::VerifyQualityMetrics( + const QualityThresholds& quality_thresholds) { + EXPECT_GT(quality_.num_decoded_frames, 0); + EXPECT_GT(quality_.total_psnr / quality_.num_decoded_frames, + quality_thresholds.min_avg_psnr); + EXPECT_GT(quality_.min_psnr, quality_thresholds.min_min_psnr); + EXPECT_GT(quality_.total_ssim / quality_.num_decoded_frames, + quality_thresholds.min_avg_ssim); + EXPECT_GT(quality_.min_ssim, quality_thresholds.min_min_ssim); +} + +// Reset quantities before each encoder rate update. +void VideoProcessorIntegrationTest::ResetRateControlMetrics( + int rate_update_index, + const std::vector& rate_profiles) { + RTC_DCHECK_GT(rate_profiles.size(), rate_update_index); + // Set new rates. + target_.kbps = rate_profiles[rate_update_index].target_kbps; + target_.fps = rate_profiles[rate_update_index].input_fps; + SetRatesPerTemporalLayer(); + + // Set key frame target sizes. + if (rate_update_index == 0) { + target_.key_framesize_kbits_initial = + 0.5 * kInitialBufferSize * target_.kbps_layer[0]; + } + + // Set maximum size of key frames, following setting in the VP8 wrapper. + float max_key_size = kScaleKeyFrameSize * kOptimalBufferSize * target_.fps; + // We don't know exact target size of the key frames (except for first one), + // but the minimum in libvpx is ~|3 * per_frame_bandwidth| and maximum is + // set by |max_key_size_ * per_frame_bandwidth|. Take middle point/average + // as reference for mismatch. Note key frames always correspond to base + // layer frame in this test. + target_.key_framesize_kbits = + 0.5 * (3 + max_key_size) * target_.framesize_kbits_layer[0]; + + // Reset rate control metrics. + actual_ = TestResults(); + actual_.num_frames_to_hit_target = // Set to max number of frames. + rate_profiles[rate_update_index].frame_index_rate_update; +} + +void VideoProcessorIntegrationTest::SetRatesPerTemporalLayer() { + const int num_temporal_layers = config_.NumberOfTemporalLayers(); + RTC_DCHECK_LE(num_temporal_layers, kMaxNumTemporalLayers); + + for (int i = 0; i < num_temporal_layers; ++i) { + float bitrate_ratio; + if (i > 0) { + bitrate_ratio = kVp8LayerRateAlloction[num_temporal_layers - 1][i] - + kVp8LayerRateAlloction[num_temporal_layers - 1][i - 1]; + } else { + bitrate_ratio = kVp8LayerRateAlloction[num_temporal_layers - 1][i]; + } + target_.kbps_layer[i] = target_.kbps * bitrate_ratio; + target_.fps_layer[i] = + target_.fps / static_cast(1 << (num_temporal_layers - 1)); + } + if (num_temporal_layers == 3) { + target_.fps_layer[2] = target_.fps / 2.0f; + } + + // Update layer per-frame-bandwidth. + for (int i = 0; i < num_temporal_layers; ++i) { + target_.framesize_kbits_layer[i] = + target_.kbps_layer[i] / target_.fps_layer[i]; } } diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest.h b/modules/video_coding/codecs/test/videoprocessor_integrationtest.h index adcae54527..10677da2bc 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest.h +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest.h @@ -36,24 +36,35 @@ namespace test { // Rates for the encoder and the frame number when to change profile. struct RateProfile { - size_t target_kbps; - size_t input_fps; - size_t frame_index_rate_update; + int target_kbps; + int input_fps; + int frame_index_rate_update; }; +// Thresholds for the rate control metrics. The thresholds are defined for each +// rate update sequence. |max_num_frames_to_hit_target| is defined as number of +// frames, after a rate update is made to the encoder, for the encoder to reach +// |kMaxBitrateMismatchPercent| of new target rate. struct RateControlThresholds { - double max_avg_bitrate_mismatch_percent; - double max_time_to_reach_target_bitrate_sec; - // TODO(ssilkin): Use absolute threshold for framerate. - double max_avg_framerate_mismatch_percent; - double max_avg_buffer_level_sec; - double max_max_key_frame_delay_sec; - double max_max_delta_frame_delay_sec; - size_t max_num_spatial_resizes; - size_t max_num_key_frames; + int max_num_dropped_frames; + int max_key_framesize_mismatch_percent; + int max_delta_framesize_mismatch_percent; + int max_bitrate_mismatch_percent; + int max_num_frames_to_hit_target; + int num_spatial_resizes; + int num_key_frames; }; +// Thresholds for the quality metrics. struct QualityThresholds { + QualityThresholds(double min_avg_psnr, + double min_min_psnr, + double min_avg_ssim, + double min_min_ssim) + : min_avg_psnr(min_avg_psnr), + min_min_psnr(min_min_psnr), + min_avg_ssim(min_avg_ssim), + min_min_ssim(min_min_ssim) {} double min_avg_psnr; double min_min_psnr; double min_avg_ssim; @@ -61,7 +72,9 @@ struct QualityThresholds { }; struct BitstreamThresholds { - size_t max_max_nalu_size_bytes; + explicit BitstreamThresholds(size_t max_nalu_length) + : max_nalu_length(max_nalu_length) {} + size_t max_nalu_length; }; // Should video files be saved persistently to disk for post-run visualization? @@ -70,10 +83,15 @@ struct VisualizationParams { bool save_decoded_y4m; }; -// Integration test for video processor. It does rate control and frame quality -// analysis using frame statistics collected by video processor and logs the -// results. If thresholds are specified it checks that corresponding metrics -// are in desirable range. +// Integration test for video processor. Encodes+decodes a clip and +// writes it to the output directory. After completion, quality metrics +// (PSNR and SSIM) and rate control metrics are computed and compared to given +// thresholds, to verify that the quality and encoder response is acceptable. +// The rate control tests allow us to verify the behavior for changing bit rate, +// changing frame rate, frame dropping/spatial resize, and temporal layers. +// The thresholds for the rate control metrics are set to be fairly +// conservative, so failure should only happen when some significant regression +// or breakdown occurs. class VideoProcessorIntegrationTest : public testing::Test { protected: // Verifies that all H.264 keyframes contain SPS/PPS/IDR NALUs. @@ -89,7 +107,7 @@ class VideoProcessorIntegrationTest : public testing::Test { void ProcessFramesAndMaybeVerify( const std::vector& rate_profiles, const std::vector* rc_thresholds, - const std::vector* quality_thresholds, + const QualityThresholds* quality_thresholds, const BitstreamThresholds* bs_thresholds, const VisualizationParams* visualization_params); @@ -101,6 +119,54 @@ class VideoProcessorIntegrationTest : public testing::Test { private: class CpuProcessTime; + static const int kMaxNumTemporalLayers = 3; + + struct TestResults { + int KeyFrameSizeMismatchPercent() const { + if (num_key_frames == 0) { + return -1; + } + return 100 * sum_key_framesize_mismatch / num_key_frames; + } + int DeltaFrameSizeMismatchPercent(int i) const { + return 100 * sum_delta_framesize_mismatch_layer[i] / num_frames_layer[i]; + } + int BitrateMismatchPercent(float target_kbps) const { + return 100 * std::fabs(kbps - target_kbps) / target_kbps; + } + int BitrateMismatchPercent(int i, float target_kbps_layer) const { + return 100 * std::fabs(kbps_layer[i] - target_kbps_layer) / + target_kbps_layer; + } + int num_frames = 0; + int num_frames_layer[kMaxNumTemporalLayers] = {0}; + int num_key_frames = 0; + int num_frames_to_hit_target = 0; + float sum_framesize_kbits = 0.0f; + float sum_framesize_kbits_layer[kMaxNumTemporalLayers] = {0}; + float kbps = 0.0f; + float kbps_layer[kMaxNumTemporalLayers] = {0}; + float sum_key_framesize_mismatch = 0.0f; + float sum_delta_framesize_mismatch_layer[kMaxNumTemporalLayers] = {0}; + }; + + struct TargetRates { + int kbps; + int fps; + float kbps_layer[kMaxNumTemporalLayers]; + float fps_layer[kMaxNumTemporalLayers]; + float framesize_kbits_layer[kMaxNumTemporalLayers]; + float key_framesize_kbits_initial; + float key_framesize_kbits; + }; + + struct QualityMetrics { + int num_decoded_frames = 0; + double total_psnr = 0.0; + double total_ssim = 0.0; + double min_psnr = std::numeric_limits::max(); + double min_ssim = std::numeric_limits::max(); + }; void CreateEncoderAndDecoder(); void DestroyEncoderAndDecoder(); @@ -110,29 +176,26 @@ class VideoProcessorIntegrationTest : public testing::Test { const VisualizationParams* visualization_params); void ReleaseAndCloseObjects(rtc::TaskQueue* task_queue); - void ProcessAllFrames(rtc::TaskQueue* task_queue, - const std::vector& rate_profiles); - void AnalyzeAllFrames( - const std::vector& rate_profiles, + // Rate control metrics. + void ResetRateControlMetrics(int rate_update_index, + const std::vector& rate_profiles); + void SetRatesPerTemporalLayer(); + void UpdateRateControlMetrics(int frame_number); + void PrintRateControlMetrics( + int rate_update_index, + const std::vector& num_dropped_frames, + const std::vector& num_spatial_resizes) const; + void VerifyRateControlMetrics( + int rate_update_index, const std::vector* rc_thresholds, - const std::vector* quality_thresholds, - const BitstreamThresholds* bs_thresholds); + const std::vector& num_dropped_frames, + const std::vector& num_spatial_resizes) const; - std::vector ExtractLayerStats( - size_t target_spatial_layer_number, - size_t target_temporal_layer_number, - size_t first_frame_number, - size_t last_frame_number, - bool combine_layers); + void VerifyBitstream(int frame_number, + const BitstreamThresholds& bs_thresholds); - void AnalyzeAndPrintStats(const std::vector& stats, - float target_bitrate_kbps, - float target_framerate_fps, - float input_duration_sec, - const RateControlThresholds* rc_thresholds, - const QualityThresholds* quality_thresholds, - const BitstreamThresholds* bs_thresholds); - void PrintFrameLevelStats(const std::vector& stats) const; + void UpdateQualityMetrics(int frame_number); + void VerifyQualityMetrics(const QualityThresholds& quality_thresholds); void PrintSettings() const; @@ -150,6 +213,14 @@ class VideoProcessorIntegrationTest : public testing::Test { Stats stats_; std::unique_ptr processor_; std::unique_ptr cpu_process_time_; + + // Quantities updated for every encoded frame. + TestResults actual_; + + // Rates set for every encoder rate update. + TargetRates target_; + + QualityMetrics quality_; }; } // namespace test diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc index ce35ce5220..ee4a8bb6ca 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc @@ -75,96 +75,110 @@ class VideoProcessorIntegrationTestLibvpx #if !defined(WEBRTC_IOS) #if !defined(RTC_DISABLE_VP9) -TEST_F(VideoProcessorIntegrationTestLibvpx, HighBitrateVP9) { +// VP9: Run with no packet loss and fixed bitrate. Quality should be very high. +// One key frame (first frame only) in sequence. +TEST_F(VideoProcessorIntegrationTestLibvpx, Process0PercentPacketLossVP9) { config_.SetCodecSettings(kVideoCodecVP9, 1, false, false, true, false, kResilienceOn, kCifWidth, kCifHeight); config_.num_frames = kNumFramesShort; - std::vector rate_profiles = {{500, 30, kNumFramesShort}}; + std::vector rate_profiles = {{500, 30, kNumFramesShort + 1}}; std::vector rc_thresholds = { - {5, 1, 0, 0.1, 0.3, 0.1, 0, 1}}; + {0, 40, 20, 10, 20, 0, 1}}; - std::vector quality_thresholds = {{37, 36, 0.94, 0.92}}; + QualityThresholds quality_thresholds(37.0, 36.0, 0.93, 0.92); ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, kNoVisualizationParams); } -TEST_F(VideoProcessorIntegrationTestLibvpx, ChangeBitrateVP9) { +// VP9: Run with no packet loss, with varying bitrate (3 rate updates): +// low to high to medium. Check that quality and encoder response to the new +// target rate/per-frame bandwidth (for each rate update) is within limits. +// One key frame (first frame only) in sequence. +TEST_F(VideoProcessorIntegrationTestLibvpx, ProcessNoLossChangeBitRateVP9) { config_.SetCodecSettings(kVideoCodecVP9, 1, false, false, true, false, kResilienceOn, kCifWidth, kCifHeight); std::vector rate_profiles = { {200, 30, 100}, // target_kbps, input_fps, frame_index_rate_update {700, 30, 200}, - {500, 30, kNumFramesLong}}; + {500, 30, kNumFramesLong + 1}}; - std::vector rc_thresholds = { - {5, 1, 0, 0.15, 0.5, 0.1, 0, 1}, - {15, 2, 0, 0.2, 0.5, 0.1, 0, 0}, - {10, 1, 0, 0.3, 0.5, 0.1, 0, 0}}; + std::vector rc_thresholds = {{0, 35, 20, 20, 35, 0, 1}, + {2, 0, 20, 20, 60, 0, 0}, + {0, 0, 25, 20, 40, 0, 0}}; - std::vector quality_thresholds = { - {34, 33, 0.90, 0.88}, {38, 35, 0.95, 0.91}, {35, 34, 0.93, 0.90}}; + QualityThresholds quality_thresholds(35.5, 30.0, 0.90, 0.85); ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, kNoVisualizationParams); } -TEST_F(VideoProcessorIntegrationTestLibvpx, ChangeFramerateVP9) { +// VP9: Run with no packet loss, with an update (decrease) in frame rate. +// Lower frame rate means higher per-frame-bandwidth, so easier to encode. +// At the low bitrate in this test, this means better rate control after the +// update(s) to lower frame rate. So expect less frame drops, and max values +// for the rate control metrics can be lower. One key frame (first frame only). +// Note: quality after update should be higher but we currently compute quality +// metrics averaged over whole sequence run. +TEST_F(VideoProcessorIntegrationTestLibvpx, + ProcessNoLossChangeFrameRateFrameDropVP9) { config_.SetCodecSettings(kVideoCodecVP9, 1, false, false, true, false, kResilienceOn, kCifWidth, kCifHeight); std::vector rate_profiles = { {100, 24, 100}, // target_kbps, input_fps, frame_index_rate_update {100, 15, 200}, - {100, 10, kNumFramesLong}}; + {100, 10, kNumFramesLong + 1}}; - // Framerate mismatch should be lower for lower framerate. std::vector rc_thresholds = { - {10, 2, 40, 0.4, 0.5, 0.2, 0, 1}, - {8, 2, 5, 0.2, 0.5, 0.2, 0, 0}, - {5, 2, 0, 0.2, 0.5, 0.3, 0, 0}}; + {45, 50, 95, 15, 45, 0, 1}, + {20, 0, 50, 10, 30, 0, 0}, + {5, 0, 30, 5, 25, 0, 0}}; - // Quality should be higher for lower framerates for the same content. - std::vector quality_thresholds = { - {33, 32, 0.89, 0.87}, {33.5, 32, 0.90, 0.86}, {33.5, 31.5, 0.90, 0.85}}; + QualityThresholds quality_thresholds(31.5, 18.0, 0.80, 0.43); ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, kNoVisualizationParams); } -TEST_F(VideoProcessorIntegrationTestLibvpx, DenoiserOnVP9) { +// VP9: Run with no packet loss and denoiser on. One key frame (first frame). +TEST_F(VideoProcessorIntegrationTestLibvpx, ProcessNoLossDenoiserOnVP9) { config_.SetCodecSettings(kVideoCodecVP9, 1, false, true, true, false, kResilienceOn, kCifWidth, kCifHeight); config_.num_frames = kNumFramesShort; - std::vector rate_profiles = {{500, 30, kNumFramesShort}}; + std::vector rate_profiles = {{500, 30, kNumFramesShort + 1}}; std::vector rc_thresholds = { - {5, 1, 0, 0.1, 0.3, 0.1, 0, 1}}; + {0, 40, 20, 10, 20, 0, 1}}; - std::vector quality_thresholds = {{37.5, 36, 0.94, 0.93}}; + QualityThresholds quality_thresholds(36.8, 35.8, 0.92, 0.91); ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, kNoVisualizationParams); } -TEST_F(VideoProcessorIntegrationTestLibvpx, VeryLowBitrateVP9) { +// Run with no packet loss, at low bitrate. +// spatial_resize is on, for this low bitrate expect one resize in sequence. +// Resize happens on delta frame. Expect only one key frame (first frame). +TEST_F(VideoProcessorIntegrationTestLibvpx, + DISABLED_ProcessNoLossSpatialResizeFrameDropVP9) { config_.SetCodecSettings(kVideoCodecVP9, 1, false, false, true, true, kResilienceOn, kCifWidth, kCifHeight); - std::vector rate_profiles = {{50, 30, kNumFramesLong}}; + std::vector rate_profiles = {{50, 30, kNumFramesLong + 1}}; std::vector rc_thresholds = { - {15, 3, 75, 1.0, 0.5, 0.4, 1, 1}}; + {228, 70, 160, 15, 80, 1, 1}}; - std::vector quality_thresholds = {{28, 25, 0.80, 0.65}}; + QualityThresholds quality_thresholds(24.0, 13.0, 0.65, 0.37); ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, @@ -176,20 +190,20 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, VeryLowBitrateVP9) { #endif // !defined(RTC_DISABLE_VP9) -TEST_F(VideoProcessorIntegrationTestLibvpx, HighBitrateVP8) { +// VP8: Run with no packet loss and fixed bitrate. Quality should be very high. +// One key frame (first frame only) in sequence. Setting |key_frame_interval| +// to -1 below means no periodic key frames in test. +TEST_F(VideoProcessorIntegrationTestLibvpx, ProcessZeroPacketLoss) { config_.SetCodecSettings(kVideoCodecVP8, 1, false, true, true, false, kResilienceOn, kCifWidth, kCifHeight); config_.num_frames = kNumFramesShort; - std::vector rate_profiles = {{500, 30, kNumFramesShort}}; + std::vector rate_profiles = {{500, 30, kNumFramesShort + 1}}; std::vector rc_thresholds = { - {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; + {0, 40, 20, 10, 15, 0, 1}}; - // std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; - // TODO(webrtc:8757): AMR VP8 encoder's quality is significantly worse - // than quality of x86 version. Use lower thresholds for now. - std::vector quality_thresholds = {{35, 33, 0.91, 0.89}}; + QualityThresholds quality_thresholds(34.95, 33.0, 0.90, 0.89); ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, @@ -207,6 +221,10 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, HighBitrateVP8) { // disabled on Android. Some quality parameter in the above test has been // adjusted to also pass for |cpu_speed| <= 12. +// VP8: Run with no packet loss, with varying bitrate (3 rate updates): +// low to high to medium. Check that quality and encoder response to the new +// target rate/per-frame bandwidth (for each rate update) is within limits. +// One key frame (first frame only) in sequence. // Too slow to finish before timeout on iOS. See webrtc:4755. #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) #define MAYBE_ProcessNoLossChangeBitRateVP8 \ @@ -214,32 +232,34 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, HighBitrateVP8) { #else #define MAYBE_ProcessNoLossChangeBitRateVP8 ProcessNoLossChangeBitRateVP8 #endif -TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_ChangeBitrateVP8) { +TEST_F(VideoProcessorIntegrationTestLibvpx, + MAYBE_ProcessNoLossChangeBitRateVP8) { config_.SetCodecSettings(kVideoCodecVP8, 1, false, true, true, false, kResilienceOn, kCifWidth, kCifHeight); std::vector rate_profiles = { {200, 30, 100}, // target_kbps, input_fps, frame_index_rate_update {800, 30, 200}, - {500, 30, kNumFramesLong}}; + {500, 30, kNumFramesLong + 1}}; - std::vector rc_thresholds = { - {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}, - {15, 1, 0, 0.1, 0.2, 0.1, 0, 0}, - {15, 1, 0, 0.3, 0.2, 0.1, 0, 0}}; + std::vector rc_thresholds = {{0, 45, 20, 10, 15, 0, 1}, + {0, 0, 25, 20, 10, 0, 0}, + {0, 0, 25, 15, 10, 0, 0}}; - // std::vector quality_thresholds = { - // {33, 32, 0.89, 0.88}, {38, 36, 0.94, 0.93}, {35, 34, 0.92, 0.91}}; - // TODO(webrtc:8757): AMR VP8 encoder's quality is significantly worse - // than quality of x86 version. Use lower thresholds for now. - std::vector quality_thresholds = { - {31.8, 31, 0.86, 0.85}, {36, 34.8, 0.92, 0.90}, {33.5, 32, 0.90, 0.88}}; + QualityThresholds quality_thresholds(34.0, 32.0, 0.85, 0.80); ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, kNoVisualizationParams); } +// VP8: Run with no packet loss, with an update (decrease) in frame rate. +// Lower frame rate means higher per-frame-bandwidth, so easier to encode. +// At the bitrate in this test, this means better rate control after the +// update(s) to lower frame rate. So expect less frame drops, and max values +// for the rate control metrics can be lower. One key frame (first frame only). +// Note: quality after update should be higher but we currently compute quality +// metrics averaged over whole sequence run. // Too slow to finish before timeout on iOS. See webrtc:4755. #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) #define MAYBE_ProcessNoLossChangeFrameRateFrameDropVP8 \ @@ -248,38 +268,33 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_ChangeBitrateVP8) { #define MAYBE_ProcessNoLossChangeFrameRateFrameDropVP8 \ ProcessNoLossChangeFrameRateFrameDropVP8 #endif -TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_ChangeFramerateVP8) { +TEST_F(VideoProcessorIntegrationTestLibvpx, + MAYBE_ProcessNoLossChangeFrameRateFrameDropVP8) { config_.SetCodecSettings(kVideoCodecVP8, 1, false, true, true, false, kResilienceOn, kCifWidth, kCifHeight); std::vector rate_profiles = { {80, 24, 100}, // target_kbps, input_fps, frame_index_rate_update {80, 15, 200}, - {80, 10, kNumFramesLong}}; + {80, 10, kNumFramesLong + 1}}; - // std::vector rc_thresholds = { - // {10, 2, 20, 0.4, 0.3, 0.1, 0, 1}, - // {5, 2, 5, 0.3, 0.3, 0.1, 0, 0}, - // {4, 2, 1, 0.2, 0.3, 0.2, 0, 0}}; - // TODO(webrtc:8757): AMR VP8 drops more frames than x86 version. Use lower - // thresholds for now. std::vector rc_thresholds = { - {10, 2, 60, 0.5, 0.3, 0.3, 0, 1}, - {10, 2, 30, 0.3, 0.3, 0.3, 0, 0}, - {10, 2, 10, 0.2, 0.3, 0.2, 0, 0}}; + {40, 20, 75, 15, 60, 0, 1}, + {10, 0, 25, 10, 35, 0, 0}, + {0, 0, 20, 10, 15, 0, 0}}; - // std::vector quality_thresholds = { - // {31, 30, 0.87, 0.86}, {32, 31, 0.89, 0.86}, {32, 30, 0.87, 0.82}}; - // TODO(webrtc:8757): AMR VP8 encoder's quality is significantly worse - // than quality of x86 version. Use lower thresholds for now. - std::vector quality_thresholds = { - {31, 30, 0.85, 0.84}, {31.5, 30.5, 0.86, 0.84}, {30.5, 29, 0.83, 0.78}}; + QualityThresholds quality_thresholds(31.0, 22.0, 0.80, 0.65); ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, kNoVisualizationParams); } +// VP8: Run with no packet loss, with 3 temporal layers, with a rate update in +// the middle of the sequence. The max values for the frame size mismatch and +// encoding rate mismatch are applied to each layer. +// No dropped frames in this test, and internal spatial resizer is off. +// One key frame (first frame only) in sequence, so no spatial resizing. // Too slow to finish before timeout on iOS. See webrtc:4755. #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) #define MAYBE_ProcessNoLossTemporalLayersVP8 \ @@ -287,27 +302,18 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_ChangeFramerateVP8) { #else #define MAYBE_ProcessNoLossTemporalLayersVP8 ProcessNoLossTemporalLayersVP8 #endif -TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_TemporalLayersVP8) { +TEST_F(VideoProcessorIntegrationTestLibvpx, + MAYBE_ProcessNoLossTemporalLayersVP8) { config_.SetCodecSettings(kVideoCodecVP8, 3, false, true, true, false, kResilienceOn, kCifWidth, kCifHeight); std::vector rate_profiles = {{200, 30, 150}, - {400, 30, kNumFramesLong}}; + {400, 30, kNumFramesLong + 1}}; - // std::vector rc_thresholds = { - // {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}, {10, 2, 0, 0.1, 0.2, 0.1, 0, 1}}; - // TODO(webrtc:8757): AMR VP8 drops more frames than x86 version. Use lower - // thresholds for now. - std::vector rc_thresholds = { - {10, 1, 2, 0.3, 0.2, 0.1, 0, 1}, {12, 2, 3, 0.1, 0.2, 0.1, 0, 1}}; + std::vector rc_thresholds = {{0, 20, 30, 10, 10, 0, 1}, + {0, 0, 30, 15, 10, 0, 0}}; - // Min SSIM drops because of high motion scene with complex backgound (trees). - // std::vector quality_thresholds = {{32, 30, 0.88, 0.85}, - // {33, 30, 0.89, 0.83}}; - // TODO(webrtc:8757): AMR VP8 encoder's quality is significantly worse - // than quality of x86 version. Use lower thresholds for now. - std::vector quality_thresholds = {{31, 30, 0.85, 0.84}, - {31, 28, 0.85, 0.75}}; + QualityThresholds quality_thresholds(32.5, 30.0, 0.85, 0.80); ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc index 936049c0b7..7f6b40b0b9 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc @@ -44,15 +44,15 @@ TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsVp8) { config_.SetCodecSettings(kVideoCodecVP8, 1, false, false, false, false, false, 352, 288); - std::vector rate_profiles = {{500, 30, kForemanNumFrames}}; + std::vector rate_profiles = {{500, 30, kForemanNumFrames + 1}}; // The thresholds below may have to be tweaked to let even poor MediaCodec // implementations pass. If this test fails on the bots, disable it and // ping brandtr@. std::vector rc_thresholds = { - {10, 1, 1, 0.1, 0.2, 0.1, 0, 1}}; + {20, 95, 22, 11, 50, 0, 1}}; - std::vector quality_thresholds = {{36, 31, 0.92, 0.86}}; + QualityThresholds quality_thresholds(30.0, 14.0, 0.86, 0.39); ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, @@ -64,15 +64,15 @@ TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsH264CBP) { config_.SetCodecSettings(kVideoCodecH264, 1, false, false, false, false, false, 352, 288); - std::vector rate_profiles = {{500, 30, kForemanNumFrames}}; + std::vector rate_profiles = {{500, 30, kForemanNumFrames + 1}}; // The thresholds below may have to be tweaked to let even poor MediaCodec // implementations pass. If this test fails on the bots, disable it and // ping brandtr@. std::vector rc_thresholds = { - {10, 1, 1, 0.1, 0.2, 0.1, 0, 1}}; + {20, 95, 22, 11, 20, 0, 1}}; - std::vector quality_thresholds = {{36, 31, 0.92, 0.86}}; + QualityThresholds quality_thresholds(30.0, 14.0, 0.86, 0.39); ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, @@ -90,15 +90,14 @@ TEST_F(VideoProcessorIntegrationTestMediaCodec, config_.SetCodecSettings(kVideoCodecH264, 1, false, false, false, false, false, 352, 288); - std::vector rate_profiles = {{500, 30, kForemanNumFrames}}; + std::vector rate_profiles = {{500, 30, kForemanNumFrames + 1}}; // The thresholds below may have to be tweaked to let even poor MediaCodec // implementations pass. If this test fails on the bots, disable it and // ping brandtr@. - std::vector rc_thresholds = { - {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; + std::vector rc_thresholds = {{5, 60, 20, 5, 15, 0, 1}}; - std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; + QualityThresholds quality_thresholds(33.0, 30.0, 0.90, 0.85); ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc index 45a1fb0a91..9967fbddfa 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc @@ -49,16 +49,21 @@ class VideoProcessorIntegrationTestOpenH264 } }; -TEST_F(VideoProcessorIntegrationTestOpenH264, ConstantHighBitrate) { +// H264: Run with no packet loss and fixed bitrate. Quality should be very high. +// Note(hbos): The PacketManipulatorImpl code used to simulate packet loss in +// these unittests appears to drop "packets" in a way that is not compatible +// with H264. Therefore ProcessXPercentPacketLossH264, X != 0, unittests have +// not been added. +TEST_F(VideoProcessorIntegrationTestOpenH264, Process0PercentPacketLoss) { config_.SetCodecSettings(kVideoCodecH264, 1, false, false, true, false, kResilienceOn, kCifWidth, kCifHeight); - std::vector rate_profiles = {{500, 30, kNumFrames}}; + std::vector rate_profiles = {{500, 30, kNumFrames + 1}}; std::vector rc_thresholds = { - {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; + {2, 60, 20, 10, 20, 0, 1}}; - std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; + QualityThresholds quality_thresholds(35.0, 25.0, 0.93, 0.70); ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, @@ -67,22 +72,22 @@ TEST_F(VideoProcessorIntegrationTestOpenH264, ConstantHighBitrate) { // H264: Enable SingleNalUnit packetization mode. Encoder should split // large frames into multiple slices and limit length of NAL units. -TEST_F(VideoProcessorIntegrationTestOpenH264, SingleNalUnit) { +TEST_F(VideoProcessorIntegrationTestOpenH264, ProcessNoLossSingleNalUnit) { config_.h264_codec_settings.packetization_mode = H264PacketizationMode::SingleNalUnit; config_.networking_config.max_payload_size_in_bytes = 500; config_.SetCodecSettings(kVideoCodecH264, 1, false, false, true, false, kResilienceOn, kCifWidth, kCifHeight); - std::vector rate_profiles = {{500, 30, kNumFrames}}; + std::vector rate_profiles = {{500, 30, kNumFrames + 1}}; std::vector rc_thresholds = { - {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; + {2, 60, 30, 10, 20, 0, 1}}; - std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; + QualityThresholds quality_thresholds(35.0, 25.0, 0.93, 0.70); - BitstreamThresholds bs_thresholds = { - config_.networking_config.max_payload_size_in_bytes}; + BitstreamThresholds bs_thresholds( + config_.networking_config.max_payload_size_in_bytes); ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, &bs_thresholds, diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc index 550479e761..b058e2992a 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc @@ -18,13 +18,13 @@ namespace test { namespace { // Loop variables. -const size_t kBitrates[] = {500}; +const int kBitrates[] = {500}; const VideoCodecType kVideoCodecType[] = {kVideoCodecVP8}; const bool kHwCodec[] = {false}; // Codec settings. +const bool kResilienceOn = false; const int kNumTemporalLayers = 1; -const bool kResilienceOn = kNumTemporalLayers > 1; const bool kDenoisingOn = false; const bool kErrorConcealmentOn = false; const bool kSpatialResizeOn = false; @@ -46,7 +46,7 @@ const int kNumFrames = 30; class VideoProcessorIntegrationTestParameterized : public VideoProcessorIntegrationTest, public ::testing::WithParamInterface< - ::testing::tuple> { + ::testing::tuple> { protected: VideoProcessorIntegrationTestParameterized() : bitrate_(::testing::get<0>(GetParam())), @@ -54,9 +54,9 @@ class VideoProcessorIntegrationTestParameterized hw_codec_(::testing::get<2>(GetParam())) {} ~VideoProcessorIntegrationTestParameterized() override = default; - void RunTest(size_t width, - size_t height, - size_t framerate, + void RunTest(int width, + int height, + int framerate, const std::string& filename) { config_.filename = filename; config_.input_filename = ResourcePath(filename, "yuv"); @@ -72,13 +72,13 @@ class VideoProcessorIntegrationTestParameterized kSpatialResizeOn, kResilienceOn, width, height); std::vector rate_profiles = { - {bitrate_, framerate, kNumFrames}}; + {bitrate_, framerate, kNumFrames + 1}}; ProcessFramesAndMaybeVerify(rate_profiles, nullptr, nullptr, nullptr, &kVisualizationParams); } - const size_t bitrate_; + const int bitrate_; const VideoCodecType codec_type_; const bool hw_codec_; }; diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_videotoolbox.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_videotoolbox.cc index adcdab5f6b..2e040a4964 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_videotoolbox.cc +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest_videotoolbox.cc @@ -48,12 +48,12 @@ TEST_F(VideoProcessorIntegrationTestVideoToolbox, config_.SetCodecSettings(kVideoCodecH264, 1, false, false, false, false, false, 352, 288); - std::vector rate_profiles = {{500, 30, kForemanNumFrames}}; + std::vector rate_profiles = {{500, 30, kForemanNumFrames + 1}}; std::vector rc_thresholds = { - {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; + {20, 95, 60, 60, 10, 0, 1}}; - std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; + QualityThresholds quality_thresholds(30.0, 14.0, 0.86, 0.39); ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, @@ -68,12 +68,11 @@ TEST_F(VideoProcessorIntegrationTestVideoToolbox, config_.SetCodecSettings(kVideoCodecH264, 1, false, false, false, false, false, 352, 288); - std::vector rate_profiles = {{500, 30, kForemanNumFrames}}; + std::vector rate_profiles = {{500, 30, kForemanNumFrames + 1}}; - std::vector rc_thresholds = { - {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; + std::vector rc_thresholds = {{5, 75, 65, 60, 6, 0, 1}}; - std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; + QualityThresholds quality_thresholds(30.0, 14.0, 0.86, 0.39); ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, diff --git a/modules/video_coding/codecs/test/videoprocessor_unittest.cc b/modules/video_coding/codecs/test/videoprocessor_unittest.cc index f8caa05e19..1a51df223d 100644 --- a/modules/video_coding/codecs/test/videoprocessor_unittest.cc +++ b/modules/video_coding/codecs/test/videoprocessor_unittest.cc @@ -151,6 +151,10 @@ TEST_F(VideoProcessorTest, SetRates) { kFramerateFps)) .Times(1); video_processor_->SetRates(kBitrateKbps, kFramerateFps); + EXPECT_THAT(video_processor_->NumberDroppedFramesPerRateUpdate(), + ElementsAre(0)); + EXPECT_THAT(video_processor_->NumberSpatialResizesPerRateUpdate(), + ElementsAre(0)); const int kNewBitrateKbps = 456; const int kNewFramerateFps = 34; @@ -160,6 +164,10 @@ TEST_F(VideoProcessorTest, SetRates) { kNewFramerateFps)) .Times(1); video_processor_->SetRates(kNewBitrateKbps, kNewFramerateFps); + EXPECT_THAT(video_processor_->NumberDroppedFramesPerRateUpdate(), + ElementsAre(0, 0)); + EXPECT_THAT(video_processor_->NumberSpatialResizesPerRateUpdate(), + ElementsAre(0, 0)); ExpectRelease(); } diff --git a/test/statistics.cc b/test/statistics.cc index 192366aef8..c43dde96fa 100644 --- a/test/statistics.cc +++ b/test/statistics.cc @@ -11,40 +11,23 @@ #include -#include - namespace webrtc { namespace test { -Statistics::Statistics() - : sum_(0.0), - sum_squared_(0.0), - max_(std::numeric_limits::min()), - min_(std::numeric_limits::max()), - count_(0) {} +Statistics::Statistics() : sum_(0.0), sum_squared_(0.0), count_(0) {} void Statistics::AddSample(double sample) { sum_ += sample; sum_squared_ += sample * sample; - max_ = std::max(max_, sample); - min_ = std::min(min_, sample); ++count_; } -double Statistics::Max() const { - return max_; -} - double Statistics::Mean() const { if (count_ == 0) return 0.0; return sum_ / count_; } -double Statistics::Min() const { - return min_; -} - double Statistics::Variance() const { if (count_ == 0) return 0.0; diff --git a/test/statistics.h b/test/statistics.h index 0389fd1232..d52d92dc11 100644 --- a/test/statistics.h +++ b/test/statistics.h @@ -21,17 +21,13 @@ class Statistics { void AddSample(double sample); - double Max() const; double Mean() const; - double Min() const; double Variance() const; double StandardDeviation() const; private: double sum_; double sum_squared_; - double max_; - double min_; uint64_t count_; }; } // namespace test