Add support for simulcast streams in QualityAnalyzingVideoDecoder.

In QualityAnalyzingVideoEncoder all encoded images that belongs to
unrelated simulcast streams will be marked as to be discarded. So
to support simulcast streams QualityAnalyzingVideoDecoder have to return
black frames when all encoded images in received concatenated encoded
image are marked as to be discarded. Also QualityAnalyzingVideoDecoder
shouldn't pass such encoded image into VideoQualityAnalyzerInterface.

Bug: webrtc:10138
Change-Id: I0f793a7dc04b5d6b10949479bd074b2db86c5c6f
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/125460
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Peter Slatala <psla@webrtc.org>
Reviewed-by: Yves Gerey <yvesg@google.com>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26973}

test/pc/e2e/BUILD.gn

@@ -104,6 +104,7 @@ rtc_source_set("quality_analyzing_video_decoder") {
     ":id_generator",
     "../../../api/video:encoded_image",
     "../../../api/video:video_frame",
+    "../../../api/video:video_frame_i420",
     "../../../api/video_codecs:video_codecs_api",
     "../../../modules/video_coding:video_codec_interface",
     "../../../rtc_base:criticalsection",

test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc

@@ -16,11 +16,18 @@
 
 #include "absl/memory/memory.h"
 #include "absl/types/optional.h"
+#include "api/video/i420_buffer.h"
 #include "modules/video_coding/include/video_error_codes.h"
 #include "rtc_base/logging.h"
 
 namespace webrtc {
 namespace test {
+namespace {
+
+constexpr size_t kIrrelatedSimulcastStreamFrameWidth = 320;
+constexpr size_t kIrrelatedSimulcastStreamFrameHeight = 480;
+
+}  // namespace
 
 QualityAnalyzingVideoDecoder::QualityAnalyzingVideoDecoder(
     int id,
@@ -55,8 +62,24 @@ int32_t QualityAnalyzingVideoDecoder::Decode(
   // deleting it.
   EncodedImageExtractionResult out = extractor_->ExtractData(input_image, id_);
 
-  // TODO(titovartem) add support for simulcast.
-  RTC_CHECK(!out.discard) << "Simulcast is not supported yet";
+  if (out.discard) {
+    // To partly emulate behavior of Selective Forwarding Unit (SFU) in the
+    // test, on receiver side we will "discard" frames from irrelevant streams.
+    // When all encoded images were marked to discarded, black frame have to be
+    // returned. Because simulcast streams will be received by receiver as 3
+    // different independent streams we don't want that irrelevant streams
+    // affect video quality metrics and also we don't want to use CPU time to
+    // decode them to prevent regressions on relevant streams. Also we can't
+    // just drop frame, because in such case, receiving part will be confused
+    // with all frames missing and will request a key frame, which will result
+    // into extra load on network and sender side. Because of it, discarded
+    // image will be always decoded as black frame and will be passed to
+    // callback directly without reaching decoder and video quality analyzer.
+    //
+    // For more details see QualityAnalyzingVideoEncoder.
+    return analyzing_callback_->IrrelevantSimulcastStreamDecoded(
+        out.id, input_image.Timestamp());
+  }
 
   EncodedImage* origin_image;
   {
@@ -168,6 +191,38 @@ int32_t QualityAnalyzingVideoDecoder::DecoderCallback::ReceivedDecodedFrame(
   return delegate_callback_->ReceivedDecodedFrame(pictureId);
 }
 
+int32_t
+QualityAnalyzingVideoDecoder::DecoderCallback::IrrelevantSimulcastStreamDecoded(
+    uint16_t frame_id,
+    int64_t timestamp_ms) {
+  webrtc::VideoFrame black_frame =
+      webrtc::VideoFrame::Builder()
+          .set_video_frame_buffer(
+              GetBlackFrameBuffer(kIrrelatedSimulcastStreamFrameWidth,
+                                  kIrrelatedSimulcastStreamFrameHeight))
+          .set_timestamp_ms(timestamp_ms)
+          .set_id(frame_id)
+          .build();
+  rtc::CritScope crit(&callback_lock_);
+  RTC_DCHECK(delegate_callback_);
+  return delegate_callback_->Decoded(black_frame);
+}
+
+rtc::scoped_refptr<webrtc::VideoFrameBuffer>
+QualityAnalyzingVideoDecoder::DecoderCallback::GetBlackFrameBuffer(int width,
+                                                                   int height) {
+  if (!black_frame_buffer_ || black_frame_buffer_->width() != width ||
+      black_frame_buffer_->height() != height) {
+    // Use i420 buffer here as default one and supported by all codecs.
+    rtc::scoped_refptr<webrtc::I420Buffer> buffer =
+        webrtc::I420Buffer::Create(width, height);
+    webrtc::I420Buffer::SetBlack(buffer.get());
+    black_frame_buffer_ = buffer;
+  }
+
+  return black_frame_buffer_;
+}
+
 void QualityAnalyzingVideoDecoder::OnFrameDecoded(
     VideoFrame* frame,
     absl::optional<int32_t> decode_time_ms,

test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h

@@ -88,8 +88,18 @@ class QualityAnalyzingVideoDecoder : public VideoDecoder {
     int32_t ReceivedDecodedReferenceFrame(uint64_t pictureId) override;
     int32_t ReceivedDecodedFrame(uint64_t pictureId) override;
 
+    int32_t IrrelevantSimulcastStreamDecoded(uint16_t frame_id,
+                                             int64_t timestamp_ms);
+
    private:
+    rtc::scoped_refptr<webrtc::VideoFrameBuffer> GetBlackFrameBuffer(
+        int width,
+        int height);
+
     QualityAnalyzingVideoDecoder* const decoder_;
+
+    rtc::scoped_refptr<webrtc::VideoFrameBuffer> black_frame_buffer_;
+
     rtc::CriticalSection callback_lock_;
     DecodedImageCallback* delegate_callback_ RTC_GUARDED_BY(callback_lock_);
   };