From fc2175da73a1b1bc83b5d70f5e4510378aeef680 Mon Sep 17 00:00:00 2001 From: Artem Titov Date: Thu, 24 Jan 2019 11:44:26 +0100 Subject: [PATCH] Introduce QualityAnalyzingVideoEncoder and QualityAnalyzingVideoDecoder. This encoder will be used to inject VideoQualityAnalyzerInterface into VideoEncoder, so it will be able to measure its metrics and also trace frames from capturing on one peer side to rendering on another peer side. The decoder will be used for the same purpose but in VideoDecoder pert. Bug: webrtc:10138 Change-Id: Idf719753e3c0b3b1369ff206365bf0558705eb98 Reviewed-on: https://webrtc-review.googlesource.com/c/117363 Commit-Queue: Artem Titov Reviewed-by: Peter Slatala Reviewed-by: Ilya Nikolaevskiy Cr-Commit-Position: refs/heads/master@{#26381} --- test/pc/e2e/BUILD.gn | 53 ++++ .../default_encoded_image_id_injector.cc | 4 +- .../video/default_encoded_image_id_injector.h | 4 +- ...ault_encoded_image_id_injector_unittest.cc | 44 ++-- .../video/encoded_image_id_injector.h | 10 +- test/pc/e2e/analyzer/video/id_generator.cc | 24 ++ test/pc/e2e/analyzer/video/id_generator.h | 46 ++++ .../video/quality_analyzing_video_decoder.cc | 231 ++++++++++++++++++ .../video/quality_analyzing_video_decoder.h | 151 ++++++++++++ .../video/quality_analyzing_video_encoder.cc | 215 ++++++++++++++++ .../video/quality_analyzing_video_encoder.h | 128 ++++++++++ ...ingle_process_encoded_image_id_injector.cc | 8 +- ...single_process_encoded_image_id_injector.h | 4 +- ...cess_encoded_image_id_injector_unittest.cc | 54 ++-- 14 files changed, 914 insertions(+), 62 deletions(-) create mode 100644 test/pc/e2e/analyzer/video/id_generator.cc create mode 100644 test/pc/e2e/analyzer/video/id_generator.h create mode 100644 test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc create mode 100644 test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h create mode 100644 test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc create mode 100644 test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index af1204f2cb..82c334f94c 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -15,6 +15,9 @@ group("e2e") { ":default_encoded_image_id_injector", ":encoded_image_id_injector_api", ":example_video_quality_analyzer", + ":id_generator", + ":quality_analyzing_video_decoder", + ":quality_analyzing_video_encoder", ":single_process_encoded_image_id_injector", ] } @@ -73,6 +76,56 @@ rtc_source_set("single_process_encoded_image_id_injector") { ] } +rtc_source_set("id_generator") { + visibility = [ "*" ] + sources = [ + "analyzer/video/id_generator.cc", + "analyzer/video/id_generator.h", + ] + deps = [] +} + +rtc_source_set("quality_analyzing_video_decoder") { + visibility = [ "*" ] + sources = [ + "analyzer/video/quality_analyzing_video_decoder.cc", + "analyzer/video/quality_analyzing_video_decoder.h", + ] + deps = [ + ":encoded_image_id_injector_api", + ":id_generator", + "../../../api/video:encoded_image", + "../../../api/video:video_frame", + "../../../api/video_codecs:video_codecs_api", + "../../../modules/video_coding:video_codec_interface", + "../../../rtc_base:criticalsection", + "../../../rtc_base:logging", + "api:video_quality_analyzer_api", + "//third_party/abseil-cpp/absl/memory:memory", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_source_set("quality_analyzing_video_encoder") { + visibility = [ "*" ] + sources = [ + "analyzer/video/quality_analyzing_video_encoder.cc", + "analyzer/video/quality_analyzing_video_encoder.h", + ] + deps = [ + ":encoded_image_id_injector_api", + ":id_generator", + "../../../api/video:encoded_image", + "../../../api/video:video_frame", + "../../../api/video_codecs:video_codecs_api", + "../../../modules/video_coding:video_codec_interface", + "../../../rtc_base:criticalsection", + "../../../rtc_base:logging", + "api:video_quality_analyzer_api", + "//third_party/abseil-cpp/absl/memory:memory", + ] +} + if (rtc_include_tests) { rtc_source_set("single_process_encoded_image_id_injector_unittest") { testonly = true diff --git a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.cc b/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.cc index b0a5aacf7b..f52325ece6 100644 --- a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.cc +++ b/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.cc @@ -64,7 +64,7 @@ EncodedImage DefaultEncodedImageIdInjector::InjectId(uint16_t id, return out; } -std::pair DefaultEncodedImageIdInjector::ExtractId( +EncodedImageWithId DefaultEncodedImageIdInjector::ExtractId( const EncodedImage& source, int coding_entity_id) { ExtendIfRequired(coding_entity_id); @@ -100,7 +100,7 @@ std::pair DefaultEncodedImageIdInjector::ExtractId( } out.set_size(out_pos); - return std::pair(id.value(), out); + return EncodedImageWithId{id.value(), out}; } void DefaultEncodedImageIdInjector::ExtendIfRequired(int coding_entity_id) { diff --git a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h b/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h index 1f19dc35bf..6f272714cf 100644 --- a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h +++ b/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h @@ -76,8 +76,8 @@ class DefaultEncodedImageIdInjector : public EncodedImageIdInjector, EncodedImage InjectId(uint16_t id, const EncodedImage& source, int coding_entity_id) override; - std::pair ExtractId(const EncodedImage& source, - int coding_entity_id) override; + EncodedImageWithId ExtractId(const EncodedImage& source, + int coding_entity_id) override; private: void ExtendIfRequired(int coding_entity_id) RTC_LOCKS_EXCLUDED(lock_); diff --git a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector_unittest.cc b/test/pc/e2e/analyzer/video/default_encoded_image_id_injector_unittest.cc index 5886c8629c..e3e50e599e 100644 --- a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector_unittest.cc +++ b/test/pc/e2e/analyzer/video/default_encoded_image_id_injector_unittest.cc @@ -38,12 +38,12 @@ TEST(DefaultEncodedImageIdInjector, InjectExtract) { EncodedImage source(buffer.data(), 10, 10); source.SetTimestamp(123456789); - std::pair out = + EncodedImageWithId out = injector.ExtractId(injector.InjectId(512, source, 1), 2); - ASSERT_EQ(out.first, 512); - ASSERT_EQ(out.second.size(), 10ul); + ASSERT_EQ(out.id, 512); + ASSERT_EQ(out.image.size(), 10ul); for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out.second.data()[i], i + 1); + ASSERT_EQ(out.image.data()[i], i + 1); } } @@ -69,24 +69,24 @@ TEST(DefaultEncodedImageIdInjector, Inject3Extract3) { EncodedImage intermediate3 = injector.InjectId(520, source3, 1); // Extract ids in different order. - std::pair out3 = injector.ExtractId(intermediate3, 2); - std::pair out1 = injector.ExtractId(intermediate1, 2); - std::pair out2 = injector.ExtractId(intermediate2, 2); + EncodedImageWithId out3 = injector.ExtractId(intermediate3, 2); + EncodedImageWithId out1 = injector.ExtractId(intermediate1, 2); + EncodedImageWithId out2 = injector.ExtractId(intermediate2, 2); - ASSERT_EQ(out1.first, 510); - ASSERT_EQ(out1.second.size(), 10ul); + ASSERT_EQ(out1.id, 510); + ASSERT_EQ(out1.image.size(), 10ul); for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out1.second.data()[i], i + 1); + ASSERT_EQ(out1.image.data()[i], i + 1); } - ASSERT_EQ(out2.first, 520); - ASSERT_EQ(out2.second.size(), 10ul); + ASSERT_EQ(out2.id, 520); + ASSERT_EQ(out2.image.size(), 10ul); for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out2.second.data()[i], i + 11); + ASSERT_EQ(out2.image.data()[i], i + 11); } - ASSERT_EQ(out3.first, 520); - ASSERT_EQ(out3.second.size(), 10ul); + ASSERT_EQ(out3.id, 520); + ASSERT_EQ(out3.image.size(), 10ul); for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out3.second.data()[i], i + 21); + ASSERT_EQ(out3.image.data()[i], i + 21); } } @@ -121,14 +121,14 @@ TEST(DefaultEncodedImageIdInjector, InjectExtractFromConcatenated) { concatenated_length); // Extract frame id from concatenated image - std::pair out = injector.ExtractId(concatenated, 2); + EncodedImageWithId out = injector.ExtractId(concatenated, 2); - ASSERT_EQ(out.first, 512); - ASSERT_EQ(out.second.size(), 3 * 10ul); + ASSERT_EQ(out.id, 512); + ASSERT_EQ(out.image.size(), 3 * 10ul); for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out.second.data()[i], i + 1); - ASSERT_EQ(out.second.data()[i + 10], i + 11); - ASSERT_EQ(out.second.data()[i + 20], i + 21); + ASSERT_EQ(out.image.data()[i], i + 1); + ASSERT_EQ(out.image.data()[i + 10], i + 11); + ASSERT_EQ(out.image.data()[i + 20], i + 21); } } diff --git a/test/pc/e2e/analyzer/video/encoded_image_id_injector.h b/test/pc/e2e/analyzer/video/encoded_image_id_injector.h index 952c23a24c..7f53d4e876 100644 --- a/test/pc/e2e/analyzer/video/encoded_image_id_injector.h +++ b/test/pc/e2e/analyzer/video/encoded_image_id_injector.h @@ -31,6 +31,11 @@ class EncodedImageIdInjector { int coding_entity_id) = 0; }; +struct EncodedImageWithId { + uint16_t id; + EncodedImage image; +}; + // Extracts frame id from EncodedImage on decoder side. class EncodedImageIdExtractor { public: @@ -39,9 +44,8 @@ class EncodedImageIdExtractor { // Returns encoded image id, extracted from payload and also encoded image // with its original payload. For concatenated spatial layers it should be the // same id. |coding_entity_id| is unique id of decoder or encoder. - virtual std::pair ExtractId( - const EncodedImage& source, - int coding_entity_id) = 0; + virtual EncodedImageWithId ExtractId(const EncodedImage& source, + int coding_entity_id) = 0; }; } // namespace test diff --git a/test/pc/e2e/analyzer/video/id_generator.cc b/test/pc/e2e/analyzer/video/id_generator.cc new file mode 100644 index 0000000000..615defd757 --- /dev/null +++ b/test/pc/e2e/analyzer/video/id_generator.cc @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/id_generator.h" + +namespace webrtc { +namespace test { + +IntIdGenerator::IntIdGenerator(int start_value) : next_id_(start_value) {} +IntIdGenerator::~IntIdGenerator() = default; + +int IntIdGenerator::GetNextId() { + return next_id_++; +} + +} // namespace test +} // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/id_generator.h b/test/pc/e2e/analyzer/video/id_generator.h new file mode 100644 index 0000000000..47bdcaffb6 --- /dev/null +++ b/test/pc/e2e/analyzer/video/id_generator.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_ID_GENERATOR_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_ID_GENERATOR_H_ + +#include + +namespace webrtc { +namespace test { + +// IdGenerator generates ids. All provided ids have to be unique. There is no +// any order guarantees for provided ids. +template +class IdGenerator { + public: + virtual ~IdGenerator() = default; + + // Returns next unique id. There is no any order guarantees for provided ids. + virtual T GetNextId() = 0; +}; + +// Generates int ids. It is assumed, that no more then max int value ids will be +// requested from this generator. +class IntIdGenerator : public IdGenerator { + public: + explicit IntIdGenerator(int start_value); + ~IntIdGenerator() override; + + int GetNextId() override; + + private: + std::atomic next_id_; +}; + +} // namespace test +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_ID_GENERATOR_H_ diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc new file mode 100644 index 0000000000..d5b186a7a9 --- /dev/null +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h" + +#include +#include +#include + +#include "absl/memory/memory.h" +#include "absl/types/optional.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace test { + +QualityAnalyzingVideoDecoder::QualityAnalyzingVideoDecoder( + int id, + std::unique_ptr delegate, + EncodedImageIdExtractor* extractor, + VideoQualityAnalyzerInterface* analyzer) + : id_(id), + implementation_name_("AnalyzingDecoder-" + + std::string(delegate_->ImplementationName())), + delegate_(std::move(delegate)), + extractor_(extractor), + analyzer_(analyzer) { + analyzing_callback_ = absl::make_unique(this); +} +QualityAnalyzingVideoDecoder::~QualityAnalyzingVideoDecoder() = default; + +int32_t QualityAnalyzingVideoDecoder::InitDecode( + const VideoCodec* codec_settings, + int32_t number_of_cores) { + return delegate_->InitDecode(codec_settings, number_of_cores); +} + +int32_t QualityAnalyzingVideoDecoder::Decode( + const EncodedImage& input_image, + bool missing_frames, + const CodecSpecificInfo* codec_specific_info, + int64_t render_time_ms) { + // Image extractor extracts id from provided EncodedImage and also returns + // the image with the original buffer. Buffer can be modified in place, so + // owner of original buffer will be responsible for deleting it, or extractor + // can create a new buffer. In such case extractor will be responsible for + // deleting it. + EncodedImageWithId out = extractor_->ExtractId(input_image, id_); + + EncodedImage* origin_image; + { + rtc::CritScope crit(&lock_); + // Store id to be able to retrieve it in analyzing callback. + timestamp_to_frame_id_.insert({input_image.Timestamp(), out.id}); + // Store encoded image to prevent its destruction while it is used in + // decoder. + origin_image = &( + decoding_images_.insert({out.id, std::move(out.image)}).first->second); + } + // We can safely dereference |origin_image|, because it can be removed from + // the map only after |delegate_| Decode method will be invoked. Image will be + // removed inside DecodedImageCallback, which can be done on separate thread. + analyzer_->OnFrameReceived(out.id, *origin_image); + int32_t result = delegate_->Decode(*origin_image, missing_frames, + codec_specific_info, render_time_ms); + if (result != WEBRTC_VIDEO_CODEC_OK) { + // If delegate decoder failed, then cleanup data for this image. + { + rtc::CritScope crit(&lock_); + timestamp_to_frame_id_.erase(input_image.Timestamp()); + decoding_images_.erase(out.id); + } + analyzer_->OnDecoderError(out.id, result); + } + return result; +} + +int32_t QualityAnalyzingVideoDecoder::RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) { + analyzing_callback_->SetDelegateCallback(callback); + return delegate_->RegisterDecodeCompleteCallback(analyzing_callback_.get()); +} + +int32_t QualityAnalyzingVideoDecoder::Release() { + rtc::CritScope crit(&lock_); + analyzing_callback_->SetDelegateCallback(nullptr); + timestamp_to_frame_id_.clear(); + decoding_images_.clear(); + return delegate_->Release(); +} + +bool QualityAnalyzingVideoDecoder::PrefersLateDecoding() const { + return delegate_->PrefersLateDecoding(); +} + +const char* QualityAnalyzingVideoDecoder::ImplementationName() const { + return implementation_name_.c_str(); +} + +QualityAnalyzingVideoDecoder::DecoderCallback::DecoderCallback( + QualityAnalyzingVideoDecoder* decoder) + : decoder_(decoder), delegate_callback_(nullptr) {} +QualityAnalyzingVideoDecoder::DecoderCallback::~DecoderCallback() = default; + +void QualityAnalyzingVideoDecoder::DecoderCallback::SetDelegateCallback( + DecodedImageCallback* delegate) { + rtc::CritScope crit(&callback_lock_); + delegate_callback_ = delegate; +} + +// We have to implement all next 3 methods because we don't know which one +// exactly is implemented in |delegate_callback_|, so we need to call the same +// method on |delegate_callback_|, as was called on |this| callback. +int32_t QualityAnalyzingVideoDecoder::DecoderCallback::Decoded( + VideoFrame& decodedImage) { + decoder_->OnFrameDecoded(&decodedImage, /*decode_time_ms=*/absl::nullopt, + /*qp=*/absl::nullopt); + + rtc::CritScope crit(&callback_lock_); + RTC_DCHECK(delegate_callback_); + return delegate_callback_->Decoded(decodedImage); +} + +int32_t QualityAnalyzingVideoDecoder::DecoderCallback::Decoded( + VideoFrame& decodedImage, + int64_t decode_time_ms) { + decoder_->OnFrameDecoded(&decodedImage, decode_time_ms, /*qp=*/absl::nullopt); + + rtc::CritScope crit(&callback_lock_); + RTC_DCHECK(delegate_callback_); + return delegate_callback_->Decoded(decodedImage, decode_time_ms); +} + +void QualityAnalyzingVideoDecoder::DecoderCallback::Decoded( + VideoFrame& decodedImage, + absl::optional decode_time_ms, + absl::optional qp) { + decoder_->OnFrameDecoded(&decodedImage, decode_time_ms, qp); + + rtc::CritScope crit(&callback_lock_); + RTC_DCHECK(delegate_callback_); + delegate_callback_->Decoded(decodedImage, decode_time_ms, qp); +} + +int32_t +QualityAnalyzingVideoDecoder::DecoderCallback::ReceivedDecodedReferenceFrame( + const uint64_t pictureId) { + rtc::CritScope crit(&callback_lock_); + RTC_DCHECK(delegate_callback_); + return delegate_callback_->ReceivedDecodedReferenceFrame(pictureId); +} + +int32_t QualityAnalyzingVideoDecoder::DecoderCallback::ReceivedDecodedFrame( + const uint64_t pictureId) { + rtc::CritScope crit(&callback_lock_); + RTC_DCHECK(delegate_callback_); + return delegate_callback_->ReceivedDecodedFrame(pictureId); +} + +void QualityAnalyzingVideoDecoder::OnFrameDecoded( + VideoFrame* frame, + absl::optional decode_time_ms, + absl::optional qp) { + uint16_t frame_id; + { + rtc::CritScope crit(&lock_); + auto it = timestamp_to_frame_id_.find(frame->timestamp()); + if (it == timestamp_to_frame_id_.end()) { + // Ensure, that we have info about this frame. It can happen that for some + // reasons decoder response, that he failed to decode, when we were + // posting frame to it, but then call the callback for this frame. + RTC_LOG(LS_ERROR) << "QualityAnalyzingVideoDecoder::OnFrameDecoded: No " + "frame id for frame for frame->timestamp()=" + << frame->timestamp(); + return; + } + frame_id = it->second; + timestamp_to_frame_id_.erase(it); + decoding_images_.erase(frame_id); + } + // Set frame id to the value, that was extracted from corresponding encoded + // image. + frame->set_id(frame_id); + analyzer_->OnFrameDecoded(*frame, decode_time_ms, qp); +} + +QualityAnalyzingVideoDecoderFactory::QualityAnalyzingVideoDecoderFactory( + std::unique_ptr delegate, + IdGenerator* id_generator, + EncodedImageIdExtractor* extractor, + VideoQualityAnalyzerInterface* analyzer) + : delegate_(std::move(delegate)), + id_generator_(id_generator), + extractor_(extractor), + analyzer_(analyzer) {} +QualityAnalyzingVideoDecoderFactory::~QualityAnalyzingVideoDecoderFactory() = + default; + +std::vector +QualityAnalyzingVideoDecoderFactory::GetSupportedFormats() const { + return delegate_->GetSupportedFormats(); +} + +std::unique_ptr +QualityAnalyzingVideoDecoderFactory::CreateVideoDecoder( + const SdpVideoFormat& format) { + std::unique_ptr decoder = delegate_->CreateVideoDecoder(format); + return absl::make_unique( + id_generator_->GetNextId(), std::move(decoder), extractor_, analyzer_); +} + +std::unique_ptr +QualityAnalyzingVideoDecoderFactory::LegacyCreateVideoDecoder( + const SdpVideoFormat& format, + const std::string& receive_stream_id) { + std::unique_ptr decoder = + delegate_->LegacyCreateVideoDecoder(format, receive_stream_id); + return absl::make_unique( + id_generator_->GetNextId(), std::move(decoder), extractor_, analyzer_); +} + +} // namespace test +} // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h new file mode 100644 index 0000000000..2fc69b426d --- /dev/null +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_ + +#include +#include +#include +#include + +#include "api/video/encoded_image.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "rtc_base/critical_section.h" +#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/id_generator.h" +#include "test/pc/e2e/api/video_quality_analyzer_interface.h" + +namespace webrtc { +namespace test { + +// QualityAnalyzingVideoDecoder is used to wrap origin video decoder and inject +// VideoQualityAnalyzerInterface before and after decoder. +// +// QualityAnalyzingVideoDecoder propagates all calls to the origin decoder. +// It registers its own DecodedImageCallback in the origin decoder and will +// store user specified callback inside itself. +// +// When Decode(...) will be invoked, quality decoder first will extract frame id +// from passed EncodedImage with EncodedImageIdExtracor that was specified in +// constructor, then will call video quality analyzer, with correct +// EncodedImage and only then will pass image to origin decoder. +// +// When origin decoder decodes the image it will call quality decoder's special +// callback, where video analyzer will be called again and then decoded frame +// will be passed to origin callback, provided by user. +// +// Quality decoder registers its own callback in origin decoder at the same +// time, when user registers his callback in quality decoder. +class QualityAnalyzingVideoDecoder : public VideoDecoder { + public: + // Creates analyzing decoder. |id| is unique coding entity id, that will + // be used to distinguish all encoders and decoders inside + // EncodedImageIdInjector and EncodedImageIdExtracor. + QualityAnalyzingVideoDecoder(int id, + std::unique_ptr delegate, + EncodedImageIdExtractor* extractor, + VideoQualityAnalyzerInterface* analyzer); + ~QualityAnalyzingVideoDecoder() override; + + // Methods of VideoDecoder interface. + int32_t InitDecode(const VideoCodec* codec_settings, + int32_t number_of_cores) override; + int32_t Decode(const EncodedImage& input_image, + bool missing_frames, + const CodecSpecificInfo* codec_specific_info, + int64_t render_time_ms) override; + int32_t RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) override; + int32_t Release() override; + bool PrefersLateDecoding() const override; + const char* ImplementationName() const override; + + private: + class DecoderCallback : public DecodedImageCallback { + public: + explicit DecoderCallback(QualityAnalyzingVideoDecoder* decoder); + ~DecoderCallback() override; + + void SetDelegateCallback(DecodedImageCallback* delegate); + + // Methods of DecodedImageCallback interface. + int32_t Decoded(VideoFrame& decodedImage) override; + int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms) override; + void Decoded(VideoFrame& decodedImage, + absl::optional decode_time_ms, + absl::optional qp) override; + int32_t ReceivedDecodedReferenceFrame(uint64_t pictureId) override; + int32_t ReceivedDecodedFrame(uint64_t pictureId) override; + + private: + QualityAnalyzingVideoDecoder* const decoder_; + rtc::CriticalSection callback_lock_; + DecodedImageCallback* delegate_callback_ RTC_GUARDED_BY(callback_lock_); + }; + + void OnFrameDecoded(VideoFrame* frame, + absl::optional decode_time_ms, + absl::optional qp); + + const int id_; + const std::string implementation_name_; + std::unique_ptr delegate_; + EncodedImageIdExtractor* const extractor_; + VideoQualityAnalyzerInterface* const analyzer_; + std::unique_ptr analyzing_callback_; + + // VideoDecoder interface assumes async delivery of decoded video frames. + // This lock is used to protect shared state, that have to be propagated + // from received EncodedImage to resulted VideoFrame. + rtc::CriticalSection lock_; + + std::map timestamp_to_frame_id_ RTC_GUARDED_BY(lock_); + // Stores currently being decoded images by frame id. Because + // EncodedImageIdExtractor can create new copy on EncodedImage we need to + // ensure, that this image won't be deleted during async decoding. To do it + // all images are putted into this map and removed from here inside callback. + std::map decoding_images_ RTC_GUARDED_BY(lock_); +}; + +// Produces QualityAnalyzingVideoDecoder, which hold decoders, produced by +// specified factory as delegates. Forwards all other calls to specified +// factory. +class QualityAnalyzingVideoDecoderFactory : public VideoDecoderFactory { + public: + QualityAnalyzingVideoDecoderFactory( + std::unique_ptr delegate, + IdGenerator* id_generator, + EncodedImageIdExtractor* extractor, + VideoQualityAnalyzerInterface* analyzer); + ~QualityAnalyzingVideoDecoderFactory() override; + + // Methods of VideoDecoderFactory interface. + std::vector GetSupportedFormats() const override; + std::unique_ptr CreateVideoDecoder( + const SdpVideoFormat& format) override; + std::unique_ptr LegacyCreateVideoDecoder( + const SdpVideoFormat& format, + const std::string& receive_stream_id) override; + + private: + std::unique_ptr delegate_; + IdGenerator* const id_generator_; + EncodedImageIdExtractor* const extractor_; + VideoQualityAnalyzerInterface* const analyzer_; +}; + +} // namespace test +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_ diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc new file mode 100644 index 0000000000..55061cab86 --- /dev/null +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h" + +#include + +#include "absl/memory/memory.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace test { +namespace { + +constexpr size_t kMaxFrameInPipelineCount = 1000; + +} // namespace + +QualityAnalyzingVideoEncoder::QualityAnalyzingVideoEncoder( + int id, + std::unique_ptr delegate, + EncodedImageIdInjector* injector, + VideoQualityAnalyzerInterface* analyzer) + : id_(id), + delegate_(std::move(delegate)), + injector_(injector), + analyzer_(analyzer) {} +QualityAnalyzingVideoEncoder::~QualityAnalyzingVideoEncoder() = default; + +int32_t QualityAnalyzingVideoEncoder::InitEncode( + const VideoCodec* codec_settings, + int32_t number_of_cores, + size_t max_payload_size) { + return delegate_->InitEncode(codec_settings, number_of_cores, + max_payload_size); +} + +int32_t QualityAnalyzingVideoEncoder::RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) { + // We need to get a lock here because delegate_callback can be hypothetically + // accessed from different thread (encoder one) concurrently. + rtc::CritScope crit(&lock_); + delegate_callback_ = callback; + return delegate_->RegisterEncodeCompleteCallback(this); +} + +int32_t QualityAnalyzingVideoEncoder::Release() { + rtc::CritScope crit(&lock_); + delegate_callback_ = nullptr; + return delegate_->Release(); +} + +int32_t QualityAnalyzingVideoEncoder::Encode( + const VideoFrame& frame, + const CodecSpecificInfo* codec_specific_info, + const std::vector* frame_types) { + { + rtc::CritScope crit(&lock_); + // Store id to be able to retrieve it in analyzing callback. + timestamp_to_frame_id_list_.push_back({frame.timestamp(), frame.id()}); + // If this list is growing, it means that we are not receiving new encoded + // images from encoder. So it should be a bug in setup on in the encoder. + RTC_DCHECK_LT(timestamp_to_frame_id_list_.size(), kMaxFrameInPipelineCount); + } + analyzer_->OnFramePreEncode(frame); + int32_t result = delegate_->Encode(frame, codec_specific_info, frame_types); + if (result != WEBRTC_VIDEO_CODEC_OK) { + // If origin encoder failed, then cleanup data for this frame. + { + rtc::CritScope crit(&lock_); + // The timestamp-frame_id pair can be not the last one, so we need to + // find it first and then remove. We will search from the end, because + // usually it will be the last or close to the last one. + auto it = timestamp_to_frame_id_list_.end(); + while (it != timestamp_to_frame_id_list_.begin()) { + --it; + if (it->first == frame.timestamp()) { + timestamp_to_frame_id_list_.erase(it); + break; + } + } + } + analyzer_->OnEncoderError(frame, result); + } + return result; +} + +int32_t QualityAnalyzingVideoEncoder::SetRates(uint32_t bitrate, + uint32_t framerate) { + return delegate_->SetRates(bitrate, framerate); +} + +int32_t QualityAnalyzingVideoEncoder::SetRateAllocation( + const VideoBitrateAllocation& allocation, + uint32_t framerate) { + return delegate_->SetRateAllocation(allocation, framerate); +} + +VideoEncoder::EncoderInfo QualityAnalyzingVideoEncoder::GetEncoderInfo() const { + return delegate_->GetEncoderInfo(); +} + +// It is assumed, that encoded callback will be always invoked with encoded +// images that correspond to the frames if the same sequence, that frames +// arrived. In other words, assume we have frames F1, F2 and F3 and they have +// corresponding encoded images I1, I2 and I3. In such case if we will call +// encode first with F1, then with F2 and then with F3, then encoder callback +// will be called first with all spatial layers for F1 (I1), then F2 (I2) and +// then F3 (I3). +// +// Basing on it we will use a list of timestamp-frame_id pairs like this: +// 1. If current encoded image timestamp is equals to timestamp in the front +// pair - pick frame id from that pair +// 2. If current encoded image timestamp isn't equals to timestamp in the front +// pair - remove the front pair and got to the step 1. +EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage( + const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info, + const RTPFragmentationHeader* fragmentation) { + uint16_t frame_id; + { + rtc::CritScope crit(&lock_); + std::pair timestamp_frame_id; + while (!timestamp_to_frame_id_list_.empty()) { + timestamp_frame_id = timestamp_to_frame_id_list_.front(); + if (timestamp_frame_id.first == encoded_image.Timestamp()) { + break; + } + timestamp_to_frame_id_list_.pop_front(); + } + + // After the loop the first element should point to current |encoded_image| + // frame id. We don't remove it from the list, because there may be + // multiple spatial layers for this frame, so encoder can produce more + // encoded images with this timestamp. The first element will be removed + // when the next frame would be encoded and EncodedImageCallback would be + // called with the next timestamp. + + if (timestamp_to_frame_id_list_.empty()) { + // Ensure, that we have info about this frame. It can happen that for some + // reasons encoder response, that he failed to decode, when we were + // posting frame to it, but then call the callback for this frame. + RTC_LOG(LS_ERROR) << "QualityAnalyzingVideoEncoder::OnEncodedImage: No " + "frame id for encoded_image.Timestamp()=" + << encoded_image.Timestamp(); + return EncodedImageCallback::Result( + EncodedImageCallback::Result::Error::OK); + } + frame_id = timestamp_frame_id.second; + } + + analyzer_->OnFrameEncoded(frame_id, encoded_image); + + // Image id injector injects frame id into provided EncodedImage and returns + // the image with a) modified original buffer (in such case the current owner + // of the buffer will be responsible for deleting it) or b) a new buffer (in + // such case injector will be responsible for deleting it). + const EncodedImage& image = injector_->InjectId(frame_id, encoded_image, id_); + { + rtc::CritScope crit(&lock_); + RTC_DCHECK(delegate_callback_); + return delegate_callback_->OnEncodedImage(image, codec_specific_info, + fragmentation); + } +} + +void QualityAnalyzingVideoEncoder::OnDroppedFrame( + EncodedImageCallback::DropReason reason) { + rtc::CritScope crit(&lock_); + analyzer_->OnFrameDropped(reason); + RTC_DCHECK(delegate_callback_); + delegate_callback_->OnDroppedFrame(reason); +} + +QualityAnalyzingVideoEncoderFactory::QualityAnalyzingVideoEncoderFactory( + std::unique_ptr delegate, + IdGenerator* id_generator, + EncodedImageIdInjector* injector, + VideoQualityAnalyzerInterface* analyzer) + : delegate_(std::move(delegate)), + id_generator_(id_generator), + injector_(injector), + analyzer_(analyzer) {} +QualityAnalyzingVideoEncoderFactory::~QualityAnalyzingVideoEncoderFactory() = + default; + +std::vector +QualityAnalyzingVideoEncoderFactory::GetSupportedFormats() const { + return delegate_->GetSupportedFormats(); +} + +VideoEncoderFactory::CodecInfo +QualityAnalyzingVideoEncoderFactory::QueryVideoEncoder( + const SdpVideoFormat& format) const { + return delegate_->QueryVideoEncoder(format); +} + +std::unique_ptr +QualityAnalyzingVideoEncoderFactory::CreateVideoEncoder( + const SdpVideoFormat& format) { + return absl::make_unique( + id_generator_->GetNextId(), delegate_->CreateVideoEncoder(format), + injector_, analyzer_); +} + +} // namespace test +} // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h new file mode 100644 index 0000000000..56b3334dcf --- /dev/null +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_ + +#include +#include +#include +#include + +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "rtc_base/critical_section.h" +#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/id_generator.h" +#include "test/pc/e2e/api/video_quality_analyzer_interface.h" + +namespace webrtc { +namespace test { + +// QualityAnalyzingVideoEncoder is used to wrap origin video encoder and inject +// VideoQualityAnalyzerInterface before and after encoder. +// +// QualityAnalyzingVideoEncoder propagates all calls to the origin encoder. +// It registers its own EncodedImageCallback in the origin encoder and will +// store user specified callback inside itself. +// +// When Encode(...) will be invoked, quality encoder first calls video quality +// analyzer with original frame, then encodes frame with original encoder. +// +// When origin encoder encodes the image it will call quality encoder's special +// callback, where video analyzer will be called again and then frame id will be +// injected into EncodedImage with passed EncodedImageIdInjector. Then new +// EncodedImage will be passed to origin callback, provided by user. +// +// Quality encoder registers its own callback in origin encoder at the same +// time, when user registers his callback in quality encoder. +class QualityAnalyzingVideoEncoder : public VideoEncoder, + public EncodedImageCallback { + public: + // Creates analyzing encoder. |id| is unique coding entity id, that will + // be used to distinguish all encoders and decoders inside + // EncodedImageIdInjector and EncodedImageIdExtracor. + QualityAnalyzingVideoEncoder(int id, + std::unique_ptr delegate, + EncodedImageIdInjector* injector, + VideoQualityAnalyzerInterface* analyzer); + ~QualityAnalyzingVideoEncoder() override; + + // Methods of VideoEncoder interface. + int32_t InitEncode(const VideoCodec* codec_settings, + int32_t number_of_cores, + size_t max_payload_size) override; + int32_t RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) override; + int32_t Release() override; + int32_t Encode(const VideoFrame& frame, + const CodecSpecificInfo* codec_specific_info, + const std::vector* frame_types) override; + int32_t SetRates(uint32_t bitrate, uint32_t framerate) override; + int32_t SetRateAllocation(const VideoBitrateAllocation& allocation, + uint32_t framerate) override; + EncoderInfo GetEncoderInfo() const override; + + // Methods of EncodedImageCallback interface. + EncodedImageCallback::Result OnEncodedImage( + const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info, + const RTPFragmentationHeader* fragmentation) override; + void OnDroppedFrame(DropReason reason) override; + + private: + const int id_; + std::unique_ptr delegate_; + EncodedImageIdInjector* const injector_; + VideoQualityAnalyzerInterface* const analyzer_; + + // VideoEncoder interface assumes async delivery of encoded images. + // This lock is used to protect shared state, that have to be propagated + // from received VideoFrame to resulted EncodedImage. + rtc::CriticalSection lock_; + + EncodedImageCallback* delegate_callback_ RTC_GUARDED_BY(lock_); + std::list> timestamp_to_frame_id_list_ + RTC_GUARDED_BY(lock_); +}; + +// Produces QualityAnalyzingVideoEncoder, which hold decoders, produced by +// specified factory as delegates. Forwards all other calls to specified +// factory. +class QualityAnalyzingVideoEncoderFactory : public VideoEncoderFactory { + public: + QualityAnalyzingVideoEncoderFactory( + std::unique_ptr delegate, + IdGenerator* id_generator, + EncodedImageIdInjector* injector, + VideoQualityAnalyzerInterface* analyzer); + ~QualityAnalyzingVideoEncoderFactory() override; + + // Methods of VideoEncoderFactory interface. + std::vector GetSupportedFormats() const override; + VideoEncoderFactory::CodecInfo QueryVideoEncoder( + const SdpVideoFormat& format) const override; + std::unique_ptr CreateVideoEncoder( + const SdpVideoFormat& format) override; + + private: + std::unique_ptr delegate_; + IdGenerator* const id_generator_; + EncodedImageIdInjector* const injector_; + VideoQualityAnalyzerInterface* const analyzer_; +}; + +} // namespace test +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_ diff --git a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.cc b/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.cc index 0422797b35..0d72fd90f5 100644 --- a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.cc +++ b/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.cc @@ -55,9 +55,9 @@ EncodedImage SingleProcessEncodedImageIdInjector::InjectId( return out; } -std::pair -SingleProcessEncodedImageIdInjector::ExtractId(const EncodedImage& source, - int coding_entity_id) { +EncodedImageWithId SingleProcessEncodedImageIdInjector::ExtractId( + const EncodedImage& source, + int coding_entity_id) { EncodedImage out = source; size_t pos = 0; @@ -92,7 +92,7 @@ SingleProcessEncodedImageIdInjector::ExtractId(const EncodedImage& source, } out.set_size(pos); - return std::pair(id.value(), out); + return EncodedImageWithId{id.value(), out}; } SingleProcessEncodedImageIdInjector::ExtractionInfoVector:: diff --git a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h b/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h index 3570f03053..36bcffa981 100644 --- a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h +++ b/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h @@ -47,8 +47,8 @@ class SingleProcessEncodedImageIdInjector : public EncodedImageIdInjector, EncodedImage InjectId(uint16_t id, const EncodedImage& source, int coding_entity_id) override; - std::pair ExtractId(const EncodedImage& source, - int coding_entity_id) override; + EncodedImageWithId ExtractId(const EncodedImage& source, + int coding_entity_id) override; private: // Contains data required to extract frame id from EncodedImage and restore diff --git a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector_unittest.cc b/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector_unittest.cc index 930a9d9f1c..a554c025db 100644 --- a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector_unittest.cc +++ b/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector_unittest.cc @@ -38,13 +38,13 @@ TEST(SingleProcessEncodedImageIdInjector, InjectExtract) { EncodedImage source(buffer.data(), 10, 10); source.SetTimestamp(123456789); - std::pair out = + EncodedImageWithId out = injector.ExtractId(injector.InjectId(512, source, 1), 2); - ASSERT_EQ(out.first, 512); - ASSERT_EQ(out.second.size(), 10ul); - ASSERT_EQ(out.second.capacity(), 10ul); + ASSERT_EQ(out.id, 512); + ASSERT_EQ(out.image.size(), 10ul); + ASSERT_EQ(out.image.capacity(), 10ul); for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out.second.data()[i], i + 1); + ASSERT_EQ(out.image.data()[i], i + 1); } } @@ -70,27 +70,27 @@ TEST(SingleProcessEncodedImageIdInjector, Inject3Extract3) { EncodedImage intermediate3 = injector.InjectId(520, source3, 1); // Extract ids in different order. - std::pair out3 = injector.ExtractId(intermediate3, 2); - std::pair out1 = injector.ExtractId(intermediate1, 2); - std::pair out2 = injector.ExtractId(intermediate2, 2); + EncodedImageWithId out3 = injector.ExtractId(intermediate3, 2); + EncodedImageWithId out1 = injector.ExtractId(intermediate1, 2); + EncodedImageWithId out2 = injector.ExtractId(intermediate2, 2); - ASSERT_EQ(out1.first, 510); - ASSERT_EQ(out1.second.size(), 10ul); - ASSERT_EQ(out1.second.capacity(), 10ul); + ASSERT_EQ(out1.id, 510); + ASSERT_EQ(out1.image.size(), 10ul); + ASSERT_EQ(out1.image.capacity(), 10ul); for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out1.second.data()[i], i + 1); + ASSERT_EQ(out1.image.data()[i], i + 1); } - ASSERT_EQ(out2.first, 520); - ASSERT_EQ(out2.second.size(), 10ul); - ASSERT_EQ(out2.second.capacity(), 10ul); + ASSERT_EQ(out2.id, 520); + ASSERT_EQ(out2.image.size(), 10ul); + ASSERT_EQ(out2.image.capacity(), 10ul); for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out2.second.data()[i], i + 11); + ASSERT_EQ(out2.image.data()[i], i + 11); } - ASSERT_EQ(out3.first, 520); - ASSERT_EQ(out3.second.size(), 10ul); - ASSERT_EQ(out3.second.capacity(), 10ul); + ASSERT_EQ(out3.id, 520); + ASSERT_EQ(out3.image.size(), 10ul); + ASSERT_EQ(out3.image.capacity(), 10ul); for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out3.second.data()[i], i + 21); + ASSERT_EQ(out3.image.data()[i], i + 21); } } @@ -125,15 +125,15 @@ TEST(SingleProcessEncodedImageIdInjector, InjectExtractFromConcatenated) { concatenated_length); // Extract frame id from concatenated image - std::pair out = injector.ExtractId(concatenated, 2); + EncodedImageWithId out = injector.ExtractId(concatenated, 2); - ASSERT_EQ(out.first, 512); - ASSERT_EQ(out.second.size(), 3 * 10ul); - ASSERT_EQ(out.second.capacity(), 3 * 10ul); + ASSERT_EQ(out.id, 512); + ASSERT_EQ(out.image.size(), 3 * 10ul); + ASSERT_EQ(out.image.capacity(), 3 * 10ul); for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out.second.data()[i], i + 1); - ASSERT_EQ(out.second.data()[i + 10], i + 11); - ASSERT_EQ(out.second.data()[i + 20], i + 21); + ASSERT_EQ(out.image.data()[i], i + 1); + ASSERT_EQ(out.image.data()[i + 10], i + 11); + ASSERT_EQ(out.image.data()[i + 20], i + 21); } }