From d36c08623d9085dad821fbb1462a7fe03badcef5 Mon Sep 17 00:00:00 2001 From: Artem Titov Date: Tue, 5 Mar 2019 13:26:48 +0100 Subject: [PATCH] Add support for simulcast streams in QualityAnalyzingVideoDecoder. In QualityAnalyzingVideoEncoder all encoded images that belongs to unrelated simulcast streams will be marked as to be discarded. So to support simulcast streams QualityAnalyzingVideoDecoder have to return black frames when all encoded images in received concatenated encoded image are marked as to be discarded. Also QualityAnalyzingVideoDecoder shouldn't pass such encoded image into VideoQualityAnalyzerInterface. Bug: webrtc:10138 Change-Id: I0f793a7dc04b5d6b10949479bd074b2db86c5c6f Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/125460 Commit-Queue: Artem Titov Reviewed-by: Mirko Bonadei Reviewed-by: Peter Slatala Reviewed-by: Yves Gerey Reviewed-by: Ilya Nikolaevskiy Cr-Commit-Position: refs/heads/master@{#26973} --- test/pc/e2e/BUILD.gn | 1 + .../video/quality_analyzing_video_decoder.cc | 59 ++++++++++++++++++- .../video/quality_analyzing_video_decoder.h | 10 ++++ 3 files changed, 68 insertions(+), 2 deletions(-) diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index 92021f52d7..865b415c56 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -104,6 +104,7 @@ rtc_source_set("quality_analyzing_video_decoder") { ":id_generator", "../../../api/video:encoded_image", "../../../api/video:video_frame", + "../../../api/video:video_frame_i420", "../../../api/video_codecs:video_codecs_api", "../../../modules/video_coding:video_codec_interface", "../../../rtc_base:criticalsection", diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc index 13e44bf27d..e9161437ba 100644 --- a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc @@ -16,11 +16,18 @@ #include "absl/memory/memory.h" #include "absl/types/optional.h" +#include "api/video/i420_buffer.h" #include "modules/video_coding/include/video_error_codes.h" #include "rtc_base/logging.h" namespace webrtc { namespace test { +namespace { + +constexpr size_t kIrrelatedSimulcastStreamFrameWidth = 320; +constexpr size_t kIrrelatedSimulcastStreamFrameHeight = 480; + +} // namespace QualityAnalyzingVideoDecoder::QualityAnalyzingVideoDecoder( int id, @@ -55,8 +62,24 @@ int32_t QualityAnalyzingVideoDecoder::Decode( // deleting it. EncodedImageExtractionResult out = extractor_->ExtractData(input_image, id_); - // TODO(titovartem) add support for simulcast. - RTC_CHECK(!out.discard) << "Simulcast is not supported yet"; + if (out.discard) { + // To partly emulate behavior of Selective Forwarding Unit (SFU) in the + // test, on receiver side we will "discard" frames from irrelevant streams. + // When all encoded images were marked to discarded, black frame have to be + // returned. Because simulcast streams will be received by receiver as 3 + // different independent streams we don't want that irrelevant streams + // affect video quality metrics and also we don't want to use CPU time to + // decode them to prevent regressions on relevant streams. Also we can't + // just drop frame, because in such case, receiving part will be confused + // with all frames missing and will request a key frame, which will result + // into extra load on network and sender side. Because of it, discarded + // image will be always decoded as black frame and will be passed to + // callback directly without reaching decoder and video quality analyzer. + // + // For more details see QualityAnalyzingVideoEncoder. + return analyzing_callback_->IrrelevantSimulcastStreamDecoded( + out.id, input_image.Timestamp()); + } EncodedImage* origin_image; { @@ -168,6 +191,38 @@ int32_t QualityAnalyzingVideoDecoder::DecoderCallback::ReceivedDecodedFrame( return delegate_callback_->ReceivedDecodedFrame(pictureId); } +int32_t +QualityAnalyzingVideoDecoder::DecoderCallback::IrrelevantSimulcastStreamDecoded( + uint16_t frame_id, + int64_t timestamp_ms) { + webrtc::VideoFrame black_frame = + webrtc::VideoFrame::Builder() + .set_video_frame_buffer( + GetBlackFrameBuffer(kIrrelatedSimulcastStreamFrameWidth, + kIrrelatedSimulcastStreamFrameHeight)) + .set_timestamp_ms(timestamp_ms) + .set_id(frame_id) + .build(); + rtc::CritScope crit(&callback_lock_); + RTC_DCHECK(delegate_callback_); + return delegate_callback_->Decoded(black_frame); +} + +rtc::scoped_refptr +QualityAnalyzingVideoDecoder::DecoderCallback::GetBlackFrameBuffer(int width, + int height) { + if (!black_frame_buffer_ || black_frame_buffer_->width() != width || + black_frame_buffer_->height() != height) { + // Use i420 buffer here as default one and supported by all codecs. + rtc::scoped_refptr buffer = + webrtc::I420Buffer::Create(width, height); + webrtc::I420Buffer::SetBlack(buffer.get()); + black_frame_buffer_ = buffer; + } + + return black_frame_buffer_; +} + void QualityAnalyzingVideoDecoder::OnFrameDecoded( VideoFrame* frame, absl::optional decode_time_ms, diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h index 1f887f7620..a32393774e 100644 --- a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h @@ -88,8 +88,18 @@ class QualityAnalyzingVideoDecoder : public VideoDecoder { int32_t ReceivedDecodedReferenceFrame(uint64_t pictureId) override; int32_t ReceivedDecodedFrame(uint64_t pictureId) override; + int32_t IrrelevantSimulcastStreamDecoded(uint16_t frame_id, + int64_t timestamp_ms); + private: + rtc::scoped_refptr GetBlackFrameBuffer( + int width, + int height); + QualityAnalyzingVideoDecoder* const decoder_; + + rtc::scoped_refptr black_frame_buffer_; + rtc::CriticalSection callback_lock_; DecodedImageCallback* delegate_callback_ RTC_GUARDED_BY(callback_lock_); };