diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn index fd6cbd219c..57e4d9f97e 100644 --- a/api/video/BUILD.gn +++ b/api/video/BUILD.gn @@ -142,6 +142,11 @@ rtc_source_set("resolution") { deps = [ "../../rtc_base/system:rtc_export" ] } +rtc_source_set("corruption_detection_filter_settings") { + visibility = [ "*" ] + public = [ "corruption_detection_filter_settings.h" ] +} + rtc_library("encoded_image") { visibility = [ "*" ] sources = [ @@ -149,6 +154,7 @@ rtc_library("encoded_image") { "encoded_image.h", ] deps = [ + ":corruption_detection_filter_settings", ":video_codec_constants", ":video_frame", ":video_frame_type", diff --git a/api/video/corruption_detection_filter_settings.h b/api/video/corruption_detection_filter_settings.h new file mode 100644 index 0000000000..9ae2414cc4 --- /dev/null +++ b/api/video/corruption_detection_filter_settings.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2024 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CORRUPTION_DETECTION_FILTER_SETTINGS_H_ +#define API_VIDEO_CORRUPTION_DETECTION_FILTER_SETTINGS_H_ + +#include + +// Filter settings for automatic corruption detection. See +// http://www.webrtc.org/experiments/rtp-hdrext/corruption-detection for more +// information. +struct CorruptionDetectionFilterSettings { + // Size of the blur kernel used. + double std_dev = 0.0; + // Allowed error thresholds (maps to `Y err` and `UV err` respectively). + int luma_error_threshold = 0; + int chroma_error_threshold = 0; +}; + +#endif // API_VIDEO_CORRUPTION_DETECTION_FILTER_SETTINGS_H_ diff --git a/api/video/encoded_image.h b/api/video/encoded_image.h index 53264be59c..553768eeb8 100644 --- a/api/video/encoded_image.h +++ b/api/video/encoded_image.h @@ -23,6 +23,7 @@ #include "api/scoped_refptr.h" #include "api/units/timestamp.h" #include "api/video/color_space.h" +#include "api/video/corruption_detection_filter_settings.h" #include "api/video/video_codec_constants.h" #include "api/video/video_content_type.h" #include "api/video/video_frame_type.h" @@ -228,6 +229,15 @@ class RTC_EXPORT EncodedImage { VideoContentType contentType() const { return content_type_; } VideoRotation rotation() const { return rotation_; } + std::optional + corruption_detection_filter_settings() const { + return corruption_detection_filter_settings_; + } + void set_corruption_detection_filter_settings( + const CorruptionDetectionFilterSettings& settings) { + corruption_detection_filter_settings_ = settings; + } + uint32_t _encodedWidth = 0; uint32_t _encodedHeight = 0; // NTP time of the capture time in local timebase in milliseconds. @@ -283,6 +293,12 @@ class RTC_EXPORT EncodedImage { // True if the frame that was encoded is a steady-state refresh frame intended // to improve the visual quality. bool is_steady_state_refresh_frame_ = false; + + // Filter settings for corruption detection suggested by the encoder + // implementation, if any. Otherwise generic per-codec-type settings will be + // used. + std::optional + corruption_detection_filter_settings_; }; } // namespace webrtc diff --git a/video/corruption_detection/BUILD.gn b/video/corruption_detection/BUILD.gn index c7ab70bc8d..9f9878387c 100644 --- a/video/corruption_detection/BUILD.gn +++ b/video/corruption_detection/BUILD.gn @@ -47,6 +47,7 @@ rtc_library("frame_instrumentation_generator") { ":generic_mapping_functions", ":halton_frame_sampler", "../../api:scoped_refptr", + "../../api/video:corruption_detection_filter_settings", "../../api/video:encoded_image", "../../api/video:video_frame", "../../api/video:video_frame_type", @@ -84,6 +85,7 @@ rtc_library("generic_mapping_functions") { "generic_mapping_functions.h", ] deps = [ + "../../api/video:corruption_detection_filter_settings", "../../api/video:video_frame", "../../api/video_codecs:video_codecs_api", "../../rtc_base:checks", diff --git a/video/corruption_detection/frame_instrumentation_generator.cc b/video/corruption_detection/frame_instrumentation_generator.cc index df6fe865a5..508afc6129 100644 --- a/video/corruption_detection/frame_instrumentation_generator.cc +++ b/video/corruption_detection/frame_instrumentation_generator.cc @@ -19,6 +19,7 @@ #include "absl/algorithm/container.h" #include "absl/types/variant.h" #include "api/scoped_refptr.h" +#include "api/video/corruption_detection_filter_settings.h" #include "api/video/encoded_image.h" #include "api/video/video_codec_type.h" #include "api/video/video_frame.h" @@ -40,36 +41,33 @@ namespace { // can lead to frame buffer pools draining. constexpr size_t kMaxPendingFrames = 3; -std::optional GetCorruptionFilterSettings( +std::optional GetCorruptionFilterSettings( const EncodedImage& encoded_image, VideoCodecType video_codec_type, int layer_id) { - /* TODO: bugs.webrtc.org/358039777 - Uncomment when parameters are available - in EncodedImage. - if (encoded_image.CorruptionDetectionParameters()) { - return FilterSettings{ - .std_dev = encoded_image.CorruptionDetectionParameters()->std_dev, - .luma_error_threshold = - encoded_image.CorruptionDetectionParameters()->luma_error_threshold, - .chroma_error_threshold = encoded_image.CorruptionDetectionParameters() - ->chroma_error_threshold}; - } - */ + std::optional filter_settings = + encoded_image.corruption_detection_filter_settings(); - int qp = encoded_image.qp_; - if (qp == -1) { - std::optional parsed_qp = QpParser().Parse( - video_codec_type, layer_id, encoded_image.data(), encoded_image.size()); - if (!parsed_qp.has_value()) { - RTC_LOG(LS_VERBOSE) << "Missing QP for " - << CodecTypeToPayloadString(video_codec_type) - << " layer " << layer_id << "."; - return std::nullopt; + if (!filter_settings.has_value()) { + // No implementation specific filter settings available, using a generic + // QP-based settings instead. + int qp = encoded_image.qp_; + if (qp == -1) { + std::optional parsed_qp = + QpParser().Parse(video_codec_type, layer_id, encoded_image.data(), + encoded_image.size()); + if (!parsed_qp.has_value()) { + RTC_LOG(LS_VERBOSE) + << "Missing QP for " << CodecTypeToPayloadString(video_codec_type) + << " layer " << layer_id << "."; + return std::nullopt; + } + qp = *parsed_qp; } - qp = *parsed_qp; - } - return GetCorruptionFilterSettings(qp, video_codec_type); + filter_settings = GetCorruptionFilterSettings(qp, video_codec_type); + } + return filter_settings; } } // namespace @@ -158,7 +156,7 @@ FrameInstrumentationGenerator::OnEncodedImage( .communicate_upper_bits = true}; } - std::optional filter_settings = + std::optional filter_settings = GetCorruptionFilterSettings(encoded_image, video_codec_type_, layer_id); if (!filter_settings.has_value()) { return std::nullopt; diff --git a/video/corruption_detection/frame_instrumentation_generator_unittest.cc b/video/corruption_detection/frame_instrumentation_generator_unittest.cc index 7f8992be51..5f3b011683 100644 --- a/video/corruption_detection/frame_instrumentation_generator_unittest.cc +++ b/video/corruption_detection/frame_instrumentation_generator_unittest.cc @@ -722,5 +722,37 @@ TEST(FrameInstrumentationGeneratorTest, QueuesAtMostThreeInputFrames) { EXPECT_THAT(frames_destroyed, ElementsAre(true, true, true, true)); } +TEST(FrameInstrumentationGeneratorTest, + UsesFilterSettingsFromFrameWhenAvailable) { + FrameInstrumentationGenerator generator(VideoCodecType::kVideoCodecVP8); + VideoFrame frame = VideoFrame::Builder() + .set_video_frame_buffer(MakeDefaultI420FrameBuffer()) + .set_rtp_timestamp(1) + .build(); + // No QP needed when frame provides filter settings. + EncodedImage encoded_image; + encoded_image.SetRtpTimestamp(1); + encoded_image.SetFrameType(VideoFrameType::kVideoFrameKey); + encoded_image._encodedWidth = kDefaultScaledWidth; + encoded_image._encodedHeight = kDefaultScaledHeight; + encoded_image.set_corruption_detection_filter_settings( + CorruptionDetectionFilterSettings{.std_dev = 1.0, + .luma_error_threshold = 2, + .chroma_error_threshold = 3}); + + generator.OnCapturedFrame(frame); + std::optional< + absl::variant> + data = generator.OnEncodedImage(encoded_image); + + ASSERT_TRUE(data.has_value()); + ASSERT_TRUE(absl::holds_alternative(*data)); + FrameInstrumentationData frame_instrumentation_data = + absl::get(*data); + EXPECT_EQ(frame_instrumentation_data.std_dev, 1.0); + EXPECT_EQ(frame_instrumentation_data.luma_error_threshold, 2); + EXPECT_EQ(frame_instrumentation_data.chroma_error_threshold, 3); +} + } // namespace } // namespace webrtc diff --git a/video/corruption_detection/frame_pair_corruption_score.cc b/video/corruption_detection/frame_pair_corruption_score.cc index 084005a1dd..91b789cb5e 100644 --- a/video/corruption_detection/frame_pair_corruption_score.cc +++ b/video/corruption_detection/frame_pair_corruption_score.cc @@ -77,7 +77,8 @@ double FramePairCorruptionScorer::CalculateScore( scoped_refptr test_i420_buffer = GetAsI420Buffer(test_buffer.ToI420()); - FilterSettings filter_settings = GetCorruptionFilterSettings(qp, codec_type_); + CorruptionDetectionFilterSettings filter_settings = + GetCorruptionFilterSettings(qp, codec_type_); const std::vector filtered_reference_sample_values = GetSampleValuesForFrame( diff --git a/video/corruption_detection/generic_mapping_functions.cc b/video/corruption_detection/generic_mapping_functions.cc index ff37734007..cb489b825a 100644 --- a/video/corruption_detection/generic_mapping_functions.cc +++ b/video/corruption_detection/generic_mapping_functions.cc @@ -73,10 +73,13 @@ double MapQpToOptimalStdDev(int qp, VideoCodecType codec_type) { } // namespace -FilterSettings GetCorruptionFilterSettings(int qp, VideoCodecType codec_type) { - return FilterSettings{.std_dev = MapQpToOptimalStdDev(qp, codec_type), - .luma_error_threshold = LumaThreshold(codec_type), - .chroma_error_threshold = ChromaThreshold(codec_type)}; +CorruptionDetectionFilterSettings GetCorruptionFilterSettings( + int qp, + VideoCodecType codec_type) { + return CorruptionDetectionFilterSettings{ + .std_dev = MapQpToOptimalStdDev(qp, codec_type), + .luma_error_threshold = LumaThreshold(codec_type), + .chroma_error_threshold = ChromaThreshold(codec_type)}; } } // namespace webrtc diff --git a/video/corruption_detection/generic_mapping_functions.h b/video/corruption_detection/generic_mapping_functions.h index 81100df850..191c229e6e 100644 --- a/video/corruption_detection/generic_mapping_functions.h +++ b/video/corruption_detection/generic_mapping_functions.h @@ -11,17 +11,17 @@ #ifndef VIDEO_CORRUPTION_DETECTION_GENERIC_MAPPING_FUNCTIONS_H_ #define VIDEO_CORRUPTION_DETECTION_GENERIC_MAPPING_FUNCTIONS_H_ +#include "api/video/corruption_detection_filter_settings.h" #include "api/video/video_codec_type.h" namespace webrtc { -struct FilterSettings { - double std_dev = 0.0; - int luma_error_threshold = 0; - int chroma_error_threshold = 0; -}; +// TODO: bugs.webrtc.org/358039777 - Remove when downstream usage is gone. +using FilterSettings = CorruptionDetectionFilterSettings; -FilterSettings GetCorruptionFilterSettings(int qp, VideoCodecType codec_type); +CorruptionDetectionFilterSettings GetCorruptionFilterSettings( + int qp, + VideoCodecType codec_type); } // namespace webrtc