From 55398a761222719242996810510bccb37b1d060d Mon Sep 17 00:00:00 2001 From: Fanny Linderborg Date: Fri, 27 Sep 2024 11:31:03 +0200 Subject: [PATCH] Add helper for comparing FrameInstrumentationData with a VideoFrame MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: webrtc:358039777 Change-Id: Ibe597160658dbc66aba427f4e30dade4d6fe56e2 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/363701 Auto-Submit: Fanny Linderborg Reviewed-by: Erik Språng Commit-Queue: Erik Språng Cr-Commit-Position: refs/heads/main@{#43134} --- video/corruption_detection/BUILD.gn | 30 ++++ .../frame_instrumentation_evaluation.cc | 97 ++++++++++++ .../frame_instrumentation_evaluation.h | 26 ++++ ...ame_instrumentation_evaluation_unittest.cc | 144 ++++++++++++++++++ .../frame_instrumentation_generator.cc | 2 +- 5 files changed, 298 insertions(+), 1 deletion(-) create mode 100644 video/corruption_detection/frame_instrumentation_evaluation.cc create mode 100644 video/corruption_detection/frame_instrumentation_evaluation.h create mode 100644 video/corruption_detection/frame_instrumentation_evaluation_unittest.cc diff --git a/video/corruption_detection/BUILD.gn b/video/corruption_detection/BUILD.gn index d37e35947f..55ab65189d 100644 --- a/video/corruption_detection/BUILD.gn +++ b/video/corruption_detection/BUILD.gn @@ -21,6 +21,23 @@ rtc_library("corruption_classifier") { ] } +rtc_library("frame_instrumentation_evaluation") { + sources = [ + "frame_instrumentation_evaluation.cc", + "frame_instrumentation_evaluation.h", + ] + deps = [ + ":corruption_classifier", + ":halton_frame_sampler", + "../../api:array_view", + "../../api:scoped_refptr", + "../../api/video:video_frame", + "../../common_video:frame_instrumentation_data", + "../../rtc_base:checks", + "../../rtc_base:logging", + ] +} + rtc_library("frame_instrumentation_generator") { sources = [ "frame_instrumentation_generator.cc", @@ -89,6 +106,18 @@ if (rtc_include_tests) { ] } + rtc_library("frame_instrumentation_evaluation_unittest") { + testonly = true + sources = [ "frame_instrumentation_evaluation_unittest.cc" ] + deps = [ + ":frame_instrumentation_evaluation", + "../../api:scoped_refptr", + "../../api/video:video_frame", + "../../common_video:frame_instrumentation_data", + "../../test:test_support", + ] + } + rtc_library("frame_instrumentation_generator_unittest") { testonly = true sources = [ "frame_instrumentation_generator_unittest.cc" ] @@ -139,6 +168,7 @@ if (rtc_include_tests) { sources = [] deps = [ ":corruption_classifier_unittest", + ":frame_instrumentation_evaluation_unittest", ":frame_instrumentation_generator_unittest", ":generic_mapping_functions_unittest", ":halton_frame_sampler_unittest", diff --git a/video/corruption_detection/frame_instrumentation_evaluation.cc b/video/corruption_detection/frame_instrumentation_evaluation.cc new file mode 100644 index 0000000000..f3367adf7b --- /dev/null +++ b/video/corruption_detection/frame_instrumentation_evaluation.cc @@ -0,0 +1,97 @@ +/* + * Copyright 2024 The WebRTC project authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/corruption_detection/frame_instrumentation_evaluation.h" + +#include +#include +#include + +#include "api/array_view.h" +#include "api/scoped_refptr.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_buffer.h" +#include "common_video/frame_instrumentation_data.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "video/corruption_detection/corruption_classifier.h" +#include "video/corruption_detection/halton_frame_sampler.h" + +namespace webrtc { + +namespace { + +std::vector ConvertSampleValuesToFilteredSamples( + rtc::ArrayView values, + rtc::ArrayView samples) { + RTC_CHECK_EQ(values.size(), samples.size()) + << "values and samples must have the same size"; + std::vector filtered_samples; + filtered_samples.reserve(values.size()); + for (size_t i = 0; i < values.size(); ++i) { + filtered_samples.push_back({.value = values[i], .plane = samples[i].plane}); + } + return filtered_samples; +} + +} // namespace + +std::optional GetCorruptionScore(const FrameInstrumentationData& data, + const VideoFrame& frame) { + if (data.sample_values.empty()) { + RTC_LOG(LS_WARNING) + << "Samples are needed to calculate a corruption score."; + return std::nullopt; + } + + scoped_refptr frame_buffer_as_i420 = + frame.video_frame_buffer()->ToI420(); + if (!frame_buffer_as_i420) { + RTC_LOG(LS_ERROR) << "Failed to convert " + << VideoFrameBufferTypeToString( + frame.video_frame_buffer()->type()) + << " image to I420"; + return std::nullopt; + } + + HaltonFrameSampler frame_sampler; + frame_sampler.SetCurrentIndex(data.sequence_index); + std::vector sample_coordinates = + frame_sampler.GetSampleCoordinatesForFrame(data.sample_values.size()); + if (sample_coordinates.empty()) { + RTC_LOG(LS_ERROR) << "Failed to get sample coordinates for frame."; + return std::nullopt; + } + + std::vector samples = + GetSampleValuesForFrame(frame_buffer_as_i420, sample_coordinates, + frame.width(), frame.height(), data.std_dev); + if (samples.empty()) { + RTC_LOG(LS_ERROR) << "Failed to get sample values for frame"; + return std::nullopt; + } + + std::vector data_samples = + ConvertSampleValuesToFilteredSamples(data.sample_values, samples); + if (data_samples.empty()) { + RTC_LOG(LS_ERROR) << "Failed to convert sample values to filtered samples"; + return std::nullopt; + } + + // TODO: bugs.webrtc.org/358039777 - Update before rollout. Which variant of + // classifier should we use? What input parameters should it have? + CorruptionClassifier classifier(2.5); + + return classifier.CalculateCorruptionProbablility( + data_samples, samples, data.luma_error_threshold, + data.chroma_error_threshold); +} + +} // namespace webrtc diff --git a/video/corruption_detection/frame_instrumentation_evaluation.h b/video/corruption_detection/frame_instrumentation_evaluation.h new file mode 100644 index 0000000000..8bd3e1c436 --- /dev/null +++ b/video/corruption_detection/frame_instrumentation_evaluation.h @@ -0,0 +1,26 @@ +/* + * Copyright 2024 The WebRTC project authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_EVALUATION_H_ +#define VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_EVALUATION_H_ + +#include + +#include "api/video/video_frame.h" +#include "common_video/frame_instrumentation_data.h" + +namespace webrtc { + +std::optional GetCorruptionScore(const FrameInstrumentationData& data, + const VideoFrame& frame); + +} // namespace webrtc + +#endif // VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_EVALUATION_H_ diff --git a/video/corruption_detection/frame_instrumentation_evaluation_unittest.cc b/video/corruption_detection/frame_instrumentation_evaluation_unittest.cc new file mode 100644 index 0000000000..cfc17e418b --- /dev/null +++ b/video/corruption_detection/frame_instrumentation_evaluation_unittest.cc @@ -0,0 +1,144 @@ +/* + * Copyright 2024 The WebRTC project authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/corruption_detection/frame_instrumentation_evaluation.h" + +#include +#include +#include + +#include "api/scoped_refptr.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "common_video/frame_instrumentation_data.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +scoped_refptr MakeI420FrameBufferWithDifferentPixelValues() { + // Create an I420 frame of size 4x4. + const int kDefaultLumaWidth = 4; + const int kDefaultLumaHeight = 4; + const int kDefaultChromaWidth = 2; + std::vector kDefaultYContent = {1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16}; + std::vector kDefaultUContent = {17, 18, 19, 20}; + std::vector kDefaultVContent = {21, 22, 23, 24}; + + return I420Buffer::Copy(kDefaultLumaWidth, kDefaultLumaHeight, + kDefaultYContent.data(), kDefaultLumaWidth, + kDefaultUContent.data(), kDefaultChromaWidth, + kDefaultVContent.data(), kDefaultChromaWidth); +} + +TEST(FrameInstrumentationEvaluationTest, + HaveNoCorruptionScoreWhenNoSampleValuesAreProvided) { + FrameInstrumentationData data = {.sequence_index = 0, + .communicate_upper_bits = false, + .std_dev = 0.0, + .luma_error_threshold = 0, + .chroma_error_threshold = 0, + .sample_values = {}}; + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) + .build(); + + std::optional corruption_score = GetCorruptionScore(data, frame); + + EXPECT_FALSE(corruption_score.has_value()); +} + +TEST(FrameInstrumentationEvaluationTest, + HaveACorruptionScoreWhenSampleValuesAreProvided) { + FrameInstrumentationData data = { + .sequence_index = 0, + .communicate_upper_bits = false, + .std_dev = 0.0, + .luma_error_threshold = 0, + .chroma_error_threshold = 0, + .sample_values = {12, 12, 12, 12, 12, 12, 12, 12}}; + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) + .build(); + + std::optional corruption_score = GetCorruptionScore(data, frame); + + ASSERT_TRUE(corruption_score.has_value()); + EXPECT_DOUBLE_EQ(*corruption_score, 1.0); +} + +TEST(FrameInstrumentationEvaluationTest, + ApplyThresholdsWhenNonNegativeThresholdsAreProvided) { + FrameInstrumentationData data = { + .sequence_index = 0, + .communicate_upper_bits = false, + .std_dev = 0.0, + .luma_error_threshold = 8, + .chroma_error_threshold = 8, + .sample_values = {12, 12, 12, 12, 12, 12, 12, 12}}; + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) + .build(); + + std::optional corruption_score = GetCorruptionScore(data, frame); + + ASSERT_TRUE(corruption_score.has_value()); + EXPECT_DOUBLE_EQ(*corruption_score, 0.55); +} + +TEST(FrameInstrumentationEvaluationTest, + ApplyStdDevWhenNonNegativeStdDevIsProvided) { + FrameInstrumentationData data = { + .sequence_index = 0, + .communicate_upper_bits = false, + .std_dev = 0.6, + .luma_error_threshold = 8, + .chroma_error_threshold = 8, + .sample_values = {12, 12, 12, 12, 12, 12, 12, 12}}; + + std::vector sample_values = {12, 12, 12, 12, 12, 12, 12, 12}; + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) + .build(); + + std::optional corruption_score = GetCorruptionScore(data, frame); + + ASSERT_TRUE(corruption_score.has_value()); + EXPECT_DOUBLE_EQ(*corruption_score, 0.3302493109581533); +} + +TEST(FrameInstrumentationEvaluationTest, ApplySequenceIndexWhenProvided) { + FrameInstrumentationData data = { + .sequence_index = 1, + .communicate_upper_bits = false, + .std_dev = 0.6, + .luma_error_threshold = 8, + .chroma_error_threshold = 8, + .sample_values = {12, 12, 12, 12, 12, 12, 12, 12}}; + + std::vector sample_values = {12, 12, 12, 12, 12, 12, 12, 12}; + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues()) + .build(); + + std::optional corruption_score = GetCorruptionScore(data, frame); + + ASSERT_TRUE(corruption_score.has_value()); + EXPECT_DOUBLE_EQ(*corruption_score, 0.12983429453668965); +} + +} // namespace +} // namespace webrtc diff --git a/video/corruption_detection/frame_instrumentation_generator.cc b/video/corruption_detection/frame_instrumentation_generator.cc index b4f03777ce..cc2b1122e4 100644 --- a/video/corruption_detection/frame_instrumentation_generator.cc +++ b/video/corruption_detection/frame_instrumentation_generator.cc @@ -142,7 +142,7 @@ FrameInstrumentationGenerator::OnEncodedImage( contexts_[layer_id] .frame_sampler.GetSampleCoordinatesForFrameIfFrameShouldBeSampled( is_key_frame, captured_frame.rtp_timestamp(), - /*sample_size=*/13); + /*num_samples=*/13); if (sample_coordinates.empty()) { if (!is_key_frame) { return std::nullopt;