diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index af1204f2cb..82c334f94c 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -15,6 +15,9 @@ group("e2e") { ":default_encoded_image_id_injector", ":encoded_image_id_injector_api", ":example_video_quality_analyzer", + ":id_generator", + ":quality_analyzing_video_decoder", + ":quality_analyzing_video_encoder", ":single_process_encoded_image_id_injector", ] } @@ -73,6 +76,56 @@ rtc_source_set("single_process_encoded_image_id_injector") { ] } +rtc_source_set("id_generator") { + visibility = [ "*" ] + sources = [ + "analyzer/video/id_generator.cc", + "analyzer/video/id_generator.h", + ] + deps = [] +} + +rtc_source_set("quality_analyzing_video_decoder") { + visibility = [ "*" ] + sources = [ + "analyzer/video/quality_analyzing_video_decoder.cc", + "analyzer/video/quality_analyzing_video_decoder.h", + ] + deps = [ + ":encoded_image_id_injector_api", + ":id_generator", + "../../../api/video:encoded_image", + "../../../api/video:video_frame", + "../../../api/video_codecs:video_codecs_api", + "../../../modules/video_coding:video_codec_interface", + "../../../rtc_base:criticalsection", + "../../../rtc_base:logging", + "api:video_quality_analyzer_api", + "//third_party/abseil-cpp/absl/memory:memory", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_source_set("quality_analyzing_video_encoder") { + visibility = [ "*" ] + sources = [ + "analyzer/video/quality_analyzing_video_encoder.cc", + "analyzer/video/quality_analyzing_video_encoder.h", + ] + deps = [ + ":encoded_image_id_injector_api", + ":id_generator", + "../../../api/video:encoded_image", + "../../../api/video:video_frame", + "../../../api/video_codecs:video_codecs_api", + "../../../modules/video_coding:video_codec_interface", + "../../../rtc_base:criticalsection", + "../../../rtc_base:logging", + "api:video_quality_analyzer_api", + "//third_party/abseil-cpp/absl/memory:memory", + ] +} + if (rtc_include_tests) { rtc_source_set("single_process_encoded_image_id_injector_unittest") { testonly = true diff --git a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.cc b/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.cc index b0a5aacf7b..f52325ece6 100644 --- a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.cc +++ b/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.cc @@ -64,7 +64,7 @@ EncodedImage DefaultEncodedImageIdInjector::InjectId(uint16_t id, return out; } -std::pair DefaultEncodedImageIdInjector::ExtractId( +EncodedImageWithId DefaultEncodedImageIdInjector::ExtractId( const EncodedImage& source, int coding_entity_id) { ExtendIfRequired(coding_entity_id); @@ -100,7 +100,7 @@ std::pair DefaultEncodedImageIdInjector::ExtractId( } out.set_size(out_pos); - return std::pair(id.value(), out); + return EncodedImageWithId{id.value(), out}; } void DefaultEncodedImageIdInjector::ExtendIfRequired(int coding_entity_id) { diff --git a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h b/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h index 1f19dc35bf..6f272714cf 100644 --- a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h +++ b/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h @@ -76,8 +76,8 @@ class DefaultEncodedImageIdInjector : public EncodedImageIdInjector, EncodedImage InjectId(uint16_t id, const EncodedImage& source, int coding_entity_id) override; - std::pair ExtractId(const EncodedImage& source, - int coding_entity_id) override; + EncodedImageWithId ExtractId(const EncodedImage& source, + int coding_entity_id) override; private: void ExtendIfRequired(int coding_entity_id) RTC_LOCKS_EXCLUDED(lock_); diff --git a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector_unittest.cc b/test/pc/e2e/analyzer/video/default_encoded_image_id_injector_unittest.cc index 5886c8629c..e3e50e599e 100644 --- a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector_unittest.cc +++ b/test/pc/e2e/analyzer/video/default_encoded_image_id_injector_unittest.cc @@ -38,12 +38,12 @@ TEST(DefaultEncodedImageIdInjector, InjectExtract) { EncodedImage source(buffer.data(), 10, 10); source.SetTimestamp(123456789); - std::pair out = + EncodedImageWithId out = injector.ExtractId(injector.InjectId(512, source, 1), 2); - ASSERT_EQ(out.first, 512); - ASSERT_EQ(out.second.size(), 10ul); + ASSERT_EQ(out.id, 512); + ASSERT_EQ(out.image.size(), 10ul); for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out.second.data()[i], i + 1); + ASSERT_EQ(out.image.data()[i], i + 1); } } @@ -69,24 +69,24 @@ TEST(DefaultEncodedImageIdInjector, Inject3Extract3) { EncodedImage intermediate3 = injector.InjectId(520, source3, 1); // Extract ids in different order. - std::pair out3 = injector.ExtractId(intermediate3, 2); - std::pair out1 = injector.ExtractId(intermediate1, 2); - std::pair out2 = injector.ExtractId(intermediate2, 2); + EncodedImageWithId out3 = injector.ExtractId(intermediate3, 2); + EncodedImageWithId out1 = injector.ExtractId(intermediate1, 2); + EncodedImageWithId out2 = injector.ExtractId(intermediate2, 2); - ASSERT_EQ(out1.first, 510); - ASSERT_EQ(out1.second.size(), 10ul); + ASSERT_EQ(out1.id, 510); + ASSERT_EQ(out1.image.size(), 10ul); for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out1.second.data()[i], i + 1); + ASSERT_EQ(out1.image.data()[i], i + 1); } - ASSERT_EQ(out2.first, 520); - ASSERT_EQ(out2.second.size(), 10ul); + ASSERT_EQ(out2.id, 520); + ASSERT_EQ(out2.image.size(), 10ul); for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out2.second.data()[i], i + 11); + ASSERT_EQ(out2.image.data()[i], i + 11); } - ASSERT_EQ(out3.first, 520); - ASSERT_EQ(out3.second.size(), 10ul); + ASSERT_EQ(out3.id, 520); + ASSERT_EQ(out3.image.size(), 10ul); for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out3.second.data()[i], i + 21); + ASSERT_EQ(out3.image.data()[i], i + 21); } } @@ -121,14 +121,14 @@ TEST(DefaultEncodedImageIdInjector, InjectExtractFromConcatenated) { concatenated_length); // Extract frame id from concatenated image - std::pair out = injector.ExtractId(concatenated, 2); + EncodedImageWithId out = injector.ExtractId(concatenated, 2); - ASSERT_EQ(out.first, 512); - ASSERT_EQ(out.second.size(), 3 * 10ul); + ASSERT_EQ(out.id, 512); + ASSERT_EQ(out.image.size(), 3 * 10ul); for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out.second.data()[i], i + 1); - ASSERT_EQ(out.second.data()[i + 10], i + 11); - ASSERT_EQ(out.second.data()[i + 20], i + 21); + ASSERT_EQ(out.image.data()[i], i + 1); + ASSERT_EQ(out.image.data()[i + 10], i + 11); + ASSERT_EQ(out.image.data()[i + 20], i + 21); } } diff --git a/test/pc/e2e/analyzer/video/encoded_image_id_injector.h b/test/pc/e2e/analyzer/video/encoded_image_id_injector.h index 952c23a24c..7f53d4e876 100644 --- a/test/pc/e2e/analyzer/video/encoded_image_id_injector.h +++ b/test/pc/e2e/analyzer/video/encoded_image_id_injector.h @@ -31,6 +31,11 @@ class EncodedImageIdInjector { int coding_entity_id) = 0; }; +struct EncodedImageWithId { + uint16_t id; + EncodedImage image; +}; + // Extracts frame id from EncodedImage on decoder side. class EncodedImageIdExtractor { public: @@ -39,9 +44,8 @@ class EncodedImageIdExtractor { // Returns encoded image id, extracted from payload and also encoded image // with its original payload. For concatenated spatial layers it should be the // same id. |coding_entity_id| is unique id of decoder or encoder. - virtual std::pair ExtractId( - const EncodedImage& source, - int coding_entity_id) = 0; + virtual EncodedImageWithId ExtractId(const EncodedImage& source, + int coding_entity_id) = 0; }; } // namespace test diff --git a/test/pc/e2e/analyzer/video/id_generator.cc b/test/pc/e2e/analyzer/video/id_generator.cc new file mode 100644 index 0000000000..615defd757 --- /dev/null +++ b/test/pc/e2e/analyzer/video/id_generator.cc @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/id_generator.h" + +namespace webrtc { +namespace test { + +IntIdGenerator::IntIdGenerator(int start_value) : next_id_(start_value) {} +IntIdGenerator::~IntIdGenerator() = default; + +int IntIdGenerator::GetNextId() { + return next_id_++; +} + +} // namespace test +} // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/id_generator.h b/test/pc/e2e/analyzer/video/id_generator.h new file mode 100644 index 0000000000..47bdcaffb6 --- /dev/null +++ b/test/pc/e2e/analyzer/video/id_generator.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_ID_GENERATOR_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_ID_GENERATOR_H_ + +#include + +namespace webrtc { +namespace test { + +// IdGenerator generates ids. All provided ids have to be unique. There is no +// any order guarantees for provided ids. +template +class IdGenerator { + public: + virtual ~IdGenerator() = default; + + // Returns next unique id. There is no any order guarantees for provided ids. + virtual T GetNextId() = 0; +}; + +// Generates int ids. It is assumed, that no more then max int value ids will be +// requested from this generator. +class IntIdGenerator : public IdGenerator { + public: + explicit IntIdGenerator(int start_value); + ~IntIdGenerator() override; + + int GetNextId() override; + + private: + std::atomic next_id_; +}; + +} // namespace test +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_ID_GENERATOR_H_ diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc new file mode 100644 index 0000000000..d5b186a7a9 --- /dev/null +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h" + +#include +#include +#include + +#include "absl/memory/memory.h" +#include "absl/types/optional.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace test { + +QualityAnalyzingVideoDecoder::QualityAnalyzingVideoDecoder( + int id, + std::unique_ptr delegate, + EncodedImageIdExtractor* extractor, + VideoQualityAnalyzerInterface* analyzer) + : id_(id), + implementation_name_("AnalyzingDecoder-" + + std::string(delegate_->ImplementationName())), + delegate_(std::move(delegate)), + extractor_(extractor), + analyzer_(analyzer) { + analyzing_callback_ = absl::make_unique(this); +} +QualityAnalyzingVideoDecoder::~QualityAnalyzingVideoDecoder() = default; + +int32_t QualityAnalyzingVideoDecoder::InitDecode( + const VideoCodec* codec_settings, + int32_t number_of_cores) { + return delegate_->InitDecode(codec_settings, number_of_cores); +} + +int32_t QualityAnalyzingVideoDecoder::Decode( + const EncodedImage& input_image, + bool missing_frames, + const CodecSpecificInfo* codec_specific_info, + int64_t render_time_ms) { + // Image extractor extracts id from provided EncodedImage and also returns + // the image with the original buffer. Buffer can be modified in place, so + // owner of original buffer will be responsible for deleting it, or extractor + // can create a new buffer. In such case extractor will be responsible for + // deleting it. + EncodedImageWithId out = extractor_->ExtractId(input_image, id_); + + EncodedImage* origin_image; + { + rtc::CritScope crit(&lock_); + // Store id to be able to retrieve it in analyzing callback. + timestamp_to_frame_id_.insert({input_image.Timestamp(), out.id}); + // Store encoded image to prevent its destruction while it is used in + // decoder. + origin_image = &( + decoding_images_.insert({out.id, std::move(out.image)}).first->second); + } + // We can safely dereference |origin_image|, because it can be removed from + // the map only after |delegate_| Decode method will be invoked. Image will be + // removed inside DecodedImageCallback, which can be done on separate thread. + analyzer_->OnFrameReceived(out.id, *origin_image); + int32_t result = delegate_->Decode(*origin_image, missing_frames, + codec_specific_info, render_time_ms); + if (result != WEBRTC_VIDEO_CODEC_OK) { + // If delegate decoder failed, then cleanup data for this image. + { + rtc::CritScope crit(&lock_); + timestamp_to_frame_id_.erase(input_image.Timestamp()); + decoding_images_.erase(out.id); + } + analyzer_->OnDecoderError(out.id, result); + } + return result; +} + +int32_t QualityAnalyzingVideoDecoder::RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) { + analyzing_callback_->SetDelegateCallback(callback); + return delegate_->RegisterDecodeCompleteCallback(analyzing_callback_.get()); +} + +int32_t QualityAnalyzingVideoDecoder::Release() { + rtc::CritScope crit(&lock_); + analyzing_callback_->SetDelegateCallback(nullptr); + timestamp_to_frame_id_.clear(); + decoding_images_.clear(); + return delegate_->Release(); +} + +bool QualityAnalyzingVideoDecoder::PrefersLateDecoding() const { + return delegate_->PrefersLateDecoding(); +} + +const char* QualityAnalyzingVideoDecoder::ImplementationName() const { + return implementation_name_.c_str(); +} + +QualityAnalyzingVideoDecoder::DecoderCallback::DecoderCallback( + QualityAnalyzingVideoDecoder* decoder) + : decoder_(decoder), delegate_callback_(nullptr) {} +QualityAnalyzingVideoDecoder::DecoderCallback::~DecoderCallback() = default; + +void QualityAnalyzingVideoDecoder::DecoderCallback::SetDelegateCallback( + DecodedImageCallback* delegate) { + rtc::CritScope crit(&callback_lock_); + delegate_callback_ = delegate; +} + +// We have to implement all next 3 methods because we don't know which one +// exactly is implemented in |delegate_callback_|, so we need to call the same +// method on |delegate_callback_|, as was called on |this| callback. +int32_t QualityAnalyzingVideoDecoder::DecoderCallback::Decoded( + VideoFrame& decodedImage) { + decoder_->OnFrameDecoded(&decodedImage, /*decode_time_ms=*/absl::nullopt, + /*qp=*/absl::nullopt); + + rtc::CritScope crit(&callback_lock_); + RTC_DCHECK(delegate_callback_); + return delegate_callback_->Decoded(decodedImage); +} + +int32_t QualityAnalyzingVideoDecoder::DecoderCallback::Decoded( + VideoFrame& decodedImage, + int64_t decode_time_ms) { + decoder_->OnFrameDecoded(&decodedImage, decode_time_ms, /*qp=*/absl::nullopt); + + rtc::CritScope crit(&callback_lock_); + RTC_DCHECK(delegate_callback_); + return delegate_callback_->Decoded(decodedImage, decode_time_ms); +} + +void QualityAnalyzingVideoDecoder::DecoderCallback::Decoded( + VideoFrame& decodedImage, + absl::optional decode_time_ms, + absl::optional qp) { + decoder_->OnFrameDecoded(&decodedImage, decode_time_ms, qp); + + rtc::CritScope crit(&callback_lock_); + RTC_DCHECK(delegate_callback_); + delegate_callback_->Decoded(decodedImage, decode_time_ms, qp); +} + +int32_t +QualityAnalyzingVideoDecoder::DecoderCallback::ReceivedDecodedReferenceFrame( + const uint64_t pictureId) { + rtc::CritScope crit(&callback_lock_); + RTC_DCHECK(delegate_callback_); + return delegate_callback_->ReceivedDecodedReferenceFrame(pictureId); +} + +int32_t QualityAnalyzingVideoDecoder::DecoderCallback::ReceivedDecodedFrame( + const uint64_t pictureId) { + rtc::CritScope crit(&callback_lock_); + RTC_DCHECK(delegate_callback_); + return delegate_callback_->ReceivedDecodedFrame(pictureId); +} + +void QualityAnalyzingVideoDecoder::OnFrameDecoded( + VideoFrame* frame, + absl::optional decode_time_ms, + absl::optional qp) { + uint16_t frame_id; + { + rtc::CritScope crit(&lock_); + auto it = timestamp_to_frame_id_.find(frame->timestamp()); + if (it == timestamp_to_frame_id_.end()) { + // Ensure, that we have info about this frame. It can happen that for some + // reasons decoder response, that he failed to decode, when we were + // posting frame to it, but then call the callback for this frame. + RTC_LOG(LS_ERROR) << "QualityAnalyzingVideoDecoder::OnFrameDecoded: No " + "frame id for frame for frame->timestamp()=" + << frame->timestamp(); + return; + } + frame_id = it->second; + timestamp_to_frame_id_.erase(it); + decoding_images_.erase(frame_id); + } + // Set frame id to the value, that was extracted from corresponding encoded + // image. + frame->set_id(frame_id); + analyzer_->OnFrameDecoded(*frame, decode_time_ms, qp); +} + +QualityAnalyzingVideoDecoderFactory::QualityAnalyzingVideoDecoderFactory( + std::unique_ptr delegate, + IdGenerator* id_generator, + EncodedImageIdExtractor* extractor, + VideoQualityAnalyzerInterface* analyzer) + : delegate_(std::move(delegate)), + id_generator_(id_generator), + extractor_(extractor), + analyzer_(analyzer) {} +QualityAnalyzingVideoDecoderFactory::~QualityAnalyzingVideoDecoderFactory() = + default; + +std::vector +QualityAnalyzingVideoDecoderFactory::GetSupportedFormats() const { + return delegate_->GetSupportedFormats(); +} + +std::unique_ptr +QualityAnalyzingVideoDecoderFactory::CreateVideoDecoder( + const SdpVideoFormat& format) { + std::unique_ptr decoder = delegate_->CreateVideoDecoder(format); + return absl::make_unique( + id_generator_->GetNextId(), std::move(decoder), extractor_, analyzer_); +} + +std::unique_ptr +QualityAnalyzingVideoDecoderFactory::LegacyCreateVideoDecoder( + const SdpVideoFormat& format, + const std::string& receive_stream_id) { + std::unique_ptr decoder = + delegate_->LegacyCreateVideoDecoder(format, receive_stream_id); + return absl::make_unique( + id_generator_->GetNextId(), std::move(decoder), extractor_, analyzer_); +} + +} // namespace test +} // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h new file mode 100644 index 0000000000..2fc69b426d --- /dev/null +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_ + +#include +#include +#include +#include + +#include "api/video/encoded_image.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "rtc_base/critical_section.h" +#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/id_generator.h" +#include "test/pc/e2e/api/video_quality_analyzer_interface.h" + +namespace webrtc { +namespace test { + +// QualityAnalyzingVideoDecoder is used to wrap origin video decoder and inject +// VideoQualityAnalyzerInterface before and after decoder. +// +// QualityAnalyzingVideoDecoder propagates all calls to the origin decoder. +// It registers its own DecodedImageCallback in the origin decoder and will +// store user specified callback inside itself. +// +// When Decode(...) will be invoked, quality decoder first will extract frame id +// from passed EncodedImage with EncodedImageIdExtracor that was specified in +// constructor, then will call video quality analyzer, with correct +// EncodedImage and only then will pass image to origin decoder. +// +// When origin decoder decodes the image it will call quality decoder's special +// callback, where video analyzer will be called again and then decoded frame +// will be passed to origin callback, provided by user. +// +// Quality decoder registers its own callback in origin decoder at the same +// time, when user registers his callback in quality decoder. +class QualityAnalyzingVideoDecoder : public VideoDecoder { + public: + // Creates analyzing decoder. |id| is unique coding entity id, that will + // be used to distinguish all encoders and decoders inside + // EncodedImageIdInjector and EncodedImageIdExtracor. + QualityAnalyzingVideoDecoder(int id, + std::unique_ptr delegate, + EncodedImageIdExtractor* extractor, + VideoQualityAnalyzerInterface* analyzer); + ~QualityAnalyzingVideoDecoder() override; + + // Methods of VideoDecoder interface. + int32_t InitDecode(const VideoCodec* codec_settings, + int32_t number_of_cores) override; + int32_t Decode(const EncodedImage& input_image, + bool missing_frames, + const CodecSpecificInfo* codec_specific_info, + int64_t render_time_ms) override; + int32_t RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) override; + int32_t Release() override; + bool PrefersLateDecoding() const override; + const char* ImplementationName() const override; + + private: + class DecoderCallback : public DecodedImageCallback { + public: + explicit DecoderCallback(QualityAnalyzingVideoDecoder* decoder); + ~DecoderCallback() override; + + void SetDelegateCallback(DecodedImageCallback* delegate); + + // Methods of DecodedImageCallback interface. + int32_t Decoded(VideoFrame& decodedImage) override; + int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms) override; + void Decoded(VideoFrame& decodedImage, + absl::optional decode_time_ms, + absl::optional qp) override; + int32_t ReceivedDecodedReferenceFrame(uint64_t pictureId) override; + int32_t ReceivedDecodedFrame(uint64_t pictureId) override; + + private: + QualityAnalyzingVideoDecoder* const decoder_; + rtc::CriticalSection callback_lock_; + DecodedImageCallback* delegate_callback_ RTC_GUARDED_BY(callback_lock_); + }; + + void OnFrameDecoded(VideoFrame* frame, + absl::optional decode_time_ms, + absl::optional qp); + + const int id_; + const std::string implementation_name_; + std::unique_ptr delegate_; + EncodedImageIdExtractor* const extractor_; + VideoQualityAnalyzerInterface* const analyzer_; + std::unique_ptr analyzing_callback_; + + // VideoDecoder interface assumes async delivery of decoded video frames. + // This lock is used to protect shared state, that have to be propagated + // from received EncodedImage to resulted VideoFrame. + rtc::CriticalSection lock_; + + std::map timestamp_to_frame_id_ RTC_GUARDED_BY(lock_); + // Stores currently being decoded images by frame id. Because + // EncodedImageIdExtractor can create new copy on EncodedImage we need to + // ensure, that this image won't be deleted during async decoding. To do it + // all images are putted into this map and removed from here inside callback. + std::map decoding_images_ RTC_GUARDED_BY(lock_); +}; + +// Produces QualityAnalyzingVideoDecoder, which hold decoders, produced by +// specified factory as delegates. Forwards all other calls to specified +// factory. +class QualityAnalyzingVideoDecoderFactory : public VideoDecoderFactory { + public: + QualityAnalyzingVideoDecoderFactory( + std::unique_ptr delegate, + IdGenerator* id_generator, + EncodedImageIdExtractor* extractor, + VideoQualityAnalyzerInterface* analyzer); + ~QualityAnalyzingVideoDecoderFactory() override; + + // Methods of VideoDecoderFactory interface. + std::vector GetSupportedFormats() const override; + std::unique_ptr CreateVideoDecoder( + const SdpVideoFormat& format) override; + std::unique_ptr LegacyCreateVideoDecoder( + const SdpVideoFormat& format, + const std::string& receive_stream_id) override; + + private: + std::unique_ptr delegate_; + IdGenerator* const id_generator_; + EncodedImageIdExtractor* const extractor_; + VideoQualityAnalyzerInterface* const analyzer_; +}; + +} // namespace test +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_ diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc new file mode 100644 index 0000000000..55061cab86 --- /dev/null +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h" + +#include + +#include "absl/memory/memory.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace test { +namespace { + +constexpr size_t kMaxFrameInPipelineCount = 1000; + +} // namespace + +QualityAnalyzingVideoEncoder::QualityAnalyzingVideoEncoder( + int id, + std::unique_ptr delegate, + EncodedImageIdInjector* injector, + VideoQualityAnalyzerInterface* analyzer) + : id_(id), + delegate_(std::move(delegate)), + injector_(injector), + analyzer_(analyzer) {} +QualityAnalyzingVideoEncoder::~QualityAnalyzingVideoEncoder() = default; + +int32_t QualityAnalyzingVideoEncoder::InitEncode( + const VideoCodec* codec_settings, + int32_t number_of_cores, + size_t max_payload_size) { + return delegate_->InitEncode(codec_settings, number_of_cores, + max_payload_size); +} + +int32_t QualityAnalyzingVideoEncoder::RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) { + // We need to get a lock here because delegate_callback can be hypothetically + // accessed from different thread (encoder one) concurrently. + rtc::CritScope crit(&lock_); + delegate_callback_ = callback; + return delegate_->RegisterEncodeCompleteCallback(this); +} + +int32_t QualityAnalyzingVideoEncoder::Release() { + rtc::CritScope crit(&lock_); + delegate_callback_ = nullptr; + return delegate_->Release(); +} + +int32_t QualityAnalyzingVideoEncoder::Encode( + const VideoFrame& frame, + const CodecSpecificInfo* codec_specific_info, + const std::vector* frame_types) { + { + rtc::CritScope crit(&lock_); + // Store id to be able to retrieve it in analyzing callback. + timestamp_to_frame_id_list_.push_back({frame.timestamp(), frame.id()}); + // If this list is growing, it means that we are not receiving new encoded + // images from encoder. So it should be a bug in setup on in the encoder. + RTC_DCHECK_LT(timestamp_to_frame_id_list_.size(), kMaxFrameInPipelineCount); + } + analyzer_->OnFramePreEncode(frame); + int32_t result = delegate_->Encode(frame, codec_specific_info, frame_types); + if (result != WEBRTC_VIDEO_CODEC_OK) { + // If origin encoder failed, then cleanup data for this frame. + { + rtc::CritScope crit(&lock_); + // The timestamp-frame_id pair can be not the last one, so we need to + // find it first and then remove. We will search from the end, because + // usually it will be the last or close to the last one. + auto it = timestamp_to_frame_id_list_.end(); + while (it != timestamp_to_frame_id_list_.begin()) { + --it; + if (it->first == frame.timestamp()) { + timestamp_to_frame_id_list_.erase(it); + break; + } + } + } + analyzer_->OnEncoderError(frame, result); + } + return result; +} + +int32_t QualityAnalyzingVideoEncoder::SetRates(uint32_t bitrate, + uint32_t framerate) { + return delegate_->SetRates(bitrate, framerate); +} + +int32_t QualityAnalyzingVideoEncoder::SetRateAllocation( + const VideoBitrateAllocation& allocation, + uint32_t framerate) { + return delegate_->SetRateAllocation(allocation, framerate); +} + +VideoEncoder::EncoderInfo QualityAnalyzingVideoEncoder::GetEncoderInfo() const { + return delegate_->GetEncoderInfo(); +} + +// It is assumed, that encoded callback will be always invoked with encoded +// images that correspond to the frames if the same sequence, that frames +// arrived. In other words, assume we have frames F1, F2 and F3 and they have +// corresponding encoded images I1, I2 and I3. In such case if we will call +// encode first with F1, then with F2 and then with F3, then encoder callback +// will be called first with all spatial layers for F1 (I1), then F2 (I2) and +// then F3 (I3). +// +// Basing on it we will use a list of timestamp-frame_id pairs like this: +// 1. If current encoded image timestamp is equals to timestamp in the front +// pair - pick frame id from that pair +// 2. If current encoded image timestamp isn't equals to timestamp in the front +// pair - remove the front pair and got to the step 1. +EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage( + const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info, + const RTPFragmentationHeader* fragmentation) { + uint16_t frame_id; + { + rtc::CritScope crit(&lock_); + std::pair timestamp_frame_id; + while (!timestamp_to_frame_id_list_.empty()) { + timestamp_frame_id = timestamp_to_frame_id_list_.front(); + if (timestamp_frame_id.first == encoded_image.Timestamp()) { + break; + } + timestamp_to_frame_id_list_.pop_front(); + } + + // After the loop the first element should point to current |encoded_image| + // frame id. We don't remove it from the list, because there may be + // multiple spatial layers for this frame, so encoder can produce more + // encoded images with this timestamp. The first element will be removed + // when the next frame would be encoded and EncodedImageCallback would be + // called with the next timestamp. + + if (timestamp_to_frame_id_list_.empty()) { + // Ensure, that we have info about this frame. It can happen that for some + // reasons encoder response, that he failed to decode, when we were + // posting frame to it, but then call the callback for this frame. + RTC_LOG(LS_ERROR) << "QualityAnalyzingVideoEncoder::OnEncodedImage: No " + "frame id for encoded_image.Timestamp()=" + << encoded_image.Timestamp(); + return EncodedImageCallback::Result( + EncodedImageCallback::Result::Error::OK); + } + frame_id = timestamp_frame_id.second; + } + + analyzer_->OnFrameEncoded(frame_id, encoded_image); + + // Image id injector injects frame id into provided EncodedImage and returns + // the image with a) modified original buffer (in such case the current owner + // of the buffer will be responsible for deleting it) or b) a new buffer (in + // such case injector will be responsible for deleting it). + const EncodedImage& image = injector_->InjectId(frame_id, encoded_image, id_); + { + rtc::CritScope crit(&lock_); + RTC_DCHECK(delegate_callback_); + return delegate_callback_->OnEncodedImage(image, codec_specific_info, + fragmentation); + } +} + +void QualityAnalyzingVideoEncoder::OnDroppedFrame( + EncodedImageCallback::DropReason reason) { + rtc::CritScope crit(&lock_); + analyzer_->OnFrameDropped(reason); + RTC_DCHECK(delegate_callback_); + delegate_callback_->OnDroppedFrame(reason); +} + +QualityAnalyzingVideoEncoderFactory::QualityAnalyzingVideoEncoderFactory( + std::unique_ptr delegate, + IdGenerator* id_generator, + EncodedImageIdInjector* injector, + VideoQualityAnalyzerInterface* analyzer) + : delegate_(std::move(delegate)), + id_generator_(id_generator), + injector_(injector), + analyzer_(analyzer) {} +QualityAnalyzingVideoEncoderFactory::~QualityAnalyzingVideoEncoderFactory() = + default; + +std::vector +QualityAnalyzingVideoEncoderFactory::GetSupportedFormats() const { + return delegate_->GetSupportedFormats(); +} + +VideoEncoderFactory::CodecInfo +QualityAnalyzingVideoEncoderFactory::QueryVideoEncoder( + const SdpVideoFormat& format) const { + return delegate_->QueryVideoEncoder(format); +} + +std::unique_ptr +QualityAnalyzingVideoEncoderFactory::CreateVideoEncoder( + const SdpVideoFormat& format) { + return absl::make_unique( + id_generator_->GetNextId(), delegate_->CreateVideoEncoder(format), + injector_, analyzer_); +} + +} // namespace test +} // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h new file mode 100644 index 0000000000..56b3334dcf --- /dev/null +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_ + +#include +#include +#include +#include + +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "rtc_base/critical_section.h" +#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/id_generator.h" +#include "test/pc/e2e/api/video_quality_analyzer_interface.h" + +namespace webrtc { +namespace test { + +// QualityAnalyzingVideoEncoder is used to wrap origin video encoder and inject +// VideoQualityAnalyzerInterface before and after encoder. +// +// QualityAnalyzingVideoEncoder propagates all calls to the origin encoder. +// It registers its own EncodedImageCallback in the origin encoder and will +// store user specified callback inside itself. +// +// When Encode(...) will be invoked, quality encoder first calls video quality +// analyzer with original frame, then encodes frame with original encoder. +// +// When origin encoder encodes the image it will call quality encoder's special +// callback, where video analyzer will be called again and then frame id will be +// injected into EncodedImage with passed EncodedImageIdInjector. Then new +// EncodedImage will be passed to origin callback, provided by user. +// +// Quality encoder registers its own callback in origin encoder at the same +// time, when user registers his callback in quality encoder. +class QualityAnalyzingVideoEncoder : public VideoEncoder, + public EncodedImageCallback { + public: + // Creates analyzing encoder. |id| is unique coding entity id, that will + // be used to distinguish all encoders and decoders inside + // EncodedImageIdInjector and EncodedImageIdExtracor. + QualityAnalyzingVideoEncoder(int id, + std::unique_ptr delegate, + EncodedImageIdInjector* injector, + VideoQualityAnalyzerInterface* analyzer); + ~QualityAnalyzingVideoEncoder() override; + + // Methods of VideoEncoder interface. + int32_t InitEncode(const VideoCodec* codec_settings, + int32_t number_of_cores, + size_t max_payload_size) override; + int32_t RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) override; + int32_t Release() override; + int32_t Encode(const VideoFrame& frame, + const CodecSpecificInfo* codec_specific_info, + const std::vector* frame_types) override; + int32_t SetRates(uint32_t bitrate, uint32_t framerate) override; + int32_t SetRateAllocation(const VideoBitrateAllocation& allocation, + uint32_t framerate) override; + EncoderInfo GetEncoderInfo() const override; + + // Methods of EncodedImageCallback interface. + EncodedImageCallback::Result OnEncodedImage( + const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info, + const RTPFragmentationHeader* fragmentation) override; + void OnDroppedFrame(DropReason reason) override; + + private: + const int id_; + std::unique_ptr delegate_; + EncodedImageIdInjector* const injector_; + VideoQualityAnalyzerInterface* const analyzer_; + + // VideoEncoder interface assumes async delivery of encoded images. + // This lock is used to protect shared state, that have to be propagated + // from received VideoFrame to resulted EncodedImage. + rtc::CriticalSection lock_; + + EncodedImageCallback* delegate_callback_ RTC_GUARDED_BY(lock_); + std::list> timestamp_to_frame_id_list_ + RTC_GUARDED_BY(lock_); +}; + +// Produces QualityAnalyzingVideoEncoder, which hold decoders, produced by +// specified factory as delegates. Forwards all other calls to specified +// factory. +class QualityAnalyzingVideoEncoderFactory : public VideoEncoderFactory { + public: + QualityAnalyzingVideoEncoderFactory( + std::unique_ptr delegate, + IdGenerator* id_generator, + EncodedImageIdInjector* injector, + VideoQualityAnalyzerInterface* analyzer); + ~QualityAnalyzingVideoEncoderFactory() override; + + // Methods of VideoEncoderFactory interface. + std::vector GetSupportedFormats() const override; + VideoEncoderFactory::CodecInfo QueryVideoEncoder( + const SdpVideoFormat& format) const override; + std::unique_ptr CreateVideoEncoder( + const SdpVideoFormat& format) override; + + private: + std::unique_ptr delegate_; + IdGenerator* const id_generator_; + EncodedImageIdInjector* const injector_; + VideoQualityAnalyzerInterface* const analyzer_; +}; + +} // namespace test +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_ diff --git a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.cc b/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.cc index 0422797b35..0d72fd90f5 100644 --- a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.cc +++ b/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.cc @@ -55,9 +55,9 @@ EncodedImage SingleProcessEncodedImageIdInjector::InjectId( return out; } -std::pair -SingleProcessEncodedImageIdInjector::ExtractId(const EncodedImage& source, - int coding_entity_id) { +EncodedImageWithId SingleProcessEncodedImageIdInjector::ExtractId( + const EncodedImage& source, + int coding_entity_id) { EncodedImage out = source; size_t pos = 0; @@ -92,7 +92,7 @@ SingleProcessEncodedImageIdInjector::ExtractId(const EncodedImage& source, } out.set_size(pos); - return std::pair(id.value(), out); + return EncodedImageWithId{id.value(), out}; } SingleProcessEncodedImageIdInjector::ExtractionInfoVector:: diff --git a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h b/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h index 3570f03053..36bcffa981 100644 --- a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h +++ b/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h @@ -47,8 +47,8 @@ class SingleProcessEncodedImageIdInjector : public EncodedImageIdInjector, EncodedImage InjectId(uint16_t id, const EncodedImage& source, int coding_entity_id) override; - std::pair ExtractId(const EncodedImage& source, - int coding_entity_id) override; + EncodedImageWithId ExtractId(const EncodedImage& source, + int coding_entity_id) override; private: // Contains data required to extract frame id from EncodedImage and restore diff --git a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector_unittest.cc b/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector_unittest.cc index 930a9d9f1c..a554c025db 100644 --- a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector_unittest.cc +++ b/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector_unittest.cc @@ -38,13 +38,13 @@ TEST(SingleProcessEncodedImageIdInjector, InjectExtract) { EncodedImage source(buffer.data(), 10, 10); source.SetTimestamp(123456789); - std::pair out = + EncodedImageWithId out = injector.ExtractId(injector.InjectId(512, source, 1), 2); - ASSERT_EQ(out.first, 512); - ASSERT_EQ(out.second.size(), 10ul); - ASSERT_EQ(out.second.capacity(), 10ul); + ASSERT_EQ(out.id, 512); + ASSERT_EQ(out.image.size(), 10ul); + ASSERT_EQ(out.image.capacity(), 10ul); for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out.second.data()[i], i + 1); + ASSERT_EQ(out.image.data()[i], i + 1); } } @@ -70,27 +70,27 @@ TEST(SingleProcessEncodedImageIdInjector, Inject3Extract3) { EncodedImage intermediate3 = injector.InjectId(520, source3, 1); // Extract ids in different order. - std::pair out3 = injector.ExtractId(intermediate3, 2); - std::pair out1 = injector.ExtractId(intermediate1, 2); - std::pair out2 = injector.ExtractId(intermediate2, 2); + EncodedImageWithId out3 = injector.ExtractId(intermediate3, 2); + EncodedImageWithId out1 = injector.ExtractId(intermediate1, 2); + EncodedImageWithId out2 = injector.ExtractId(intermediate2, 2); - ASSERT_EQ(out1.first, 510); - ASSERT_EQ(out1.second.size(), 10ul); - ASSERT_EQ(out1.second.capacity(), 10ul); + ASSERT_EQ(out1.id, 510); + ASSERT_EQ(out1.image.size(), 10ul); + ASSERT_EQ(out1.image.capacity(), 10ul); for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out1.second.data()[i], i + 1); + ASSERT_EQ(out1.image.data()[i], i + 1); } - ASSERT_EQ(out2.first, 520); - ASSERT_EQ(out2.second.size(), 10ul); - ASSERT_EQ(out2.second.capacity(), 10ul); + ASSERT_EQ(out2.id, 520); + ASSERT_EQ(out2.image.size(), 10ul); + ASSERT_EQ(out2.image.capacity(), 10ul); for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out2.second.data()[i], i + 11); + ASSERT_EQ(out2.image.data()[i], i + 11); } - ASSERT_EQ(out3.first, 520); - ASSERT_EQ(out3.second.size(), 10ul); - ASSERT_EQ(out3.second.capacity(), 10ul); + ASSERT_EQ(out3.id, 520); + ASSERT_EQ(out3.image.size(), 10ul); + ASSERT_EQ(out3.image.capacity(), 10ul); for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out3.second.data()[i], i + 21); + ASSERT_EQ(out3.image.data()[i], i + 21); } } @@ -125,15 +125,15 @@ TEST(SingleProcessEncodedImageIdInjector, InjectExtractFromConcatenated) { concatenated_length); // Extract frame id from concatenated image - std::pair out = injector.ExtractId(concatenated, 2); + EncodedImageWithId out = injector.ExtractId(concatenated, 2); - ASSERT_EQ(out.first, 512); - ASSERT_EQ(out.second.size(), 3 * 10ul); - ASSERT_EQ(out.second.capacity(), 3 * 10ul); + ASSERT_EQ(out.id, 512); + ASSERT_EQ(out.image.size(), 3 * 10ul); + ASSERT_EQ(out.image.capacity(), 3 * 10ul); for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out.second.data()[i], i + 1); - ASSERT_EQ(out.second.data()[i + 10], i + 11); - ASSERT_EQ(out.second.data()[i + 20], i + 21); + ASSERT_EQ(out.image.data()[i], i + 1); + ASSERT_EQ(out.image.data()[i + 10], i + 11); + ASSERT_EQ(out.image.data()[i + 20], i + 21); } }