Stop decoding video for m-lines which are sendonly or inactive

by not starting the receive stream whenever it is creating.
Instead, this is controlled by the direction of the media content.

BUG=webrtc:11013

Change-Id: Iaaa0ac0aa9f90a4be776a1348f53a0f9c2b84d99
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/304661
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Commit-Queue: Philipp Hancke <phancke@microsoft.com>
Cr-Commit-Position: refs/heads/main@{#40064}

media/base/fake_media_engine.h

@@ -486,6 +486,7 @@ class FakeVideoMediaChannel : public RtpHelper<VideoMediaChannel> {
   bool HasSink(uint32_t ssrc) const;
 
   bool SetSend(bool send) override;
+  void SetReceive(bool receive) override {}
   bool SetVideoSend(
       uint32_t ssrc,
       const VideoOptions* options,

media/base/media_channel.h

@@ -975,6 +975,8 @@ class VideoMediaReceiveChannelInterface : public MediaReceiveChannelInterface {
   // Get the receive parameters for the incoming stream identified by `ssrc`.
   virtual webrtc::RtpParameters GetRtpReceiveParameters(
       uint32_t ssrc) const = 0;
+  // Starts or stops decoding of remote video.
+  virtual void SetReceive(bool receive) = 0;
   // Retrieve the receive parameters for the default receive
   // stream, which is used when SSRCs are not signaled.
   virtual webrtc::RtpParameters GetDefaultRtpReceiveParameters() const = 0;

media/base/media_channel_impl.h

@@ -750,7 +750,7 @@ class VideoMediaReceiveChannel : public VideoMediaReceiveChannelInterface {
       override {
     impl()->SetDepacketizerToDecoderFrameTransformer(ssrc, frame_transformer);
   }
-  // Implementation on videoMediaReceiveChannelInterface
+  // Implementation of VideoMediaReceiveChannelInterface
   bool SetRecvParameters(const VideoRecvParameters& params) override {
     return impl()->SetRecvParameters(params);
   }
@@ -796,6 +796,7 @@ class VideoMediaReceiveChannel : public VideoMediaReceiveChannelInterface {
                                           rtx_time);
   }
   MediaChannel* ImplForTesting() override { return impl_; }
+  void SetReceive(bool receive) override { impl()->SetReceive(receive); }
 
  private:
   VideoMediaReceiveChannelInterface* impl() { return impl_; }

media/engine/webrtc_video_engine.cc

@@ -676,6 +676,8 @@ WebRtcVideoChannel::WebRtcVideoChannel(
     webrtc::VideoBitrateAllocatorFactory* bitrate_allocator_factory)
     : VideoMediaChannel(role, call->network_thread(), config.enable_dscp),
       worker_thread_(call->worker_thread()),
+      sending_(false),
+      receiving_(false),
       call_(call),
       default_sink_(nullptr),
       video_config_(config.video),
@@ -691,7 +693,6 @@ WebRtcVideoChannel::WebRtcVideoChannel(
       crypto_options_(crypto_options) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   rtcp_receiver_report_ssrc_ = kDefaultRtcpReceiverReportSsrc;
-  sending_ = false;
   recv_codecs_ = MapCodecs(GetPayloadTypesAndDefaultCodecs(
       decoder_factory_, /*is_decoder_factory=*/true,
       /*include_rtx=*/true, call_->trials()));
@@ -1293,6 +1294,20 @@ bool WebRtcVideoChannel::GetSendCodec(VideoCodec* codec) {
   return true;
 }
 
+void WebRtcVideoChannel::SetReceive(bool receive) {
+  RTC_DCHECK_RUN_ON(&thread_checker_);
+  TRACE_EVENT0("webrtc", "WebRtcVideoChannel::SetReceive");
+  RTC_LOG(LS_VERBOSE) << "SetReceive: " << (receive ? "true" : "false");
+  for (const auto& kv : receive_streams_) {
+    if (receive) {
+      kv.second->StartReceiveStream();
+    } else {
+      kv.second->StopReceiveStream();
+    }
+  }
+  receiving_ = receive;
+}
+
 bool WebRtcVideoChannel::SetSend(bool send) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   TRACE_EVENT0("webrtc", "WebRtcVideoChannel::SetSend");
@@ -1503,10 +1518,13 @@ bool WebRtcVideoChannel::AddRecvStream(const StreamParams& sp,
   if (unsignaled_frame_transformer_ && !config.frame_transformer)
     config.frame_transformer = unsignaled_frame_transformer_;
 
-  receive_streams_[sp.first_ssrc()] =
+  auto receive_stream =
       new WebRtcVideoReceiveStream(call_, sp, std::move(config), default_stream,
                                    recv_codecs_, flexfec_config);
-
+  if (receiving_) {
+    receive_stream->StartReceiveStream();
+  }
+  receive_streams_[sp.first_ssrc()] = receive_stream;
   return true;
 }
 
@@ -1841,7 +1859,7 @@ bool WebRtcVideoChannel::MaybeCreateDefaultReceiveStream(
       // stream with a "random" SSRC and the RTX SSRC from the packet.  The
       // stream will be recreated on the first media packet, unless we are
       // extremely lucky and used the right media SSRC.
-      ReCreateDefaulReceiveStream(/*ssrc =*/14795, /*rtx_ssrc=*/packet.Ssrc());
+      ReCreateDefaultReceiveStream(/*ssrc =*/14795, /*rtx_ssrc=*/packet.Ssrc());
     }
     return true;
   } else {
@@ -1863,12 +1881,12 @@ bool WebRtcVideoChannel::MaybeCreateDefaultReceiveStream(
     }
   }
   // RTX SSRC not yet known.
-  ReCreateDefaulReceiveStream(packet.Ssrc(), absl::nullopt);
+  ReCreateDefaultReceiveStream(packet.Ssrc(), absl::nullopt);
   last_unsignalled_ssrc_creation_time_ms_ = rtc::TimeMillis();
   return true;
 }
 
-void WebRtcVideoChannel::ReCreateDefaulReceiveStream(
+void WebRtcVideoChannel::ReCreateDefaultReceiveStream(
     uint32_t ssrc,
     absl::optional<uint32_t> rtx_ssrc) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
@@ -2960,7 +2978,8 @@ WebRtcVideoChannel::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream(
       flexfec_stream_(nullptr),
       sink_(NULL),
       first_frame_timestamp_(-1),
-      estimated_remote_start_ntp_time_ms_(0) {
+      estimated_remote_start_ntp_time_ms_(0),
+      receiving_(false) {
   RTC_DCHECK(config_.decoder_factory);
   RTC_DCHECK(config_.decoders.empty())
       << "Decoder info is supplied via `recv_codecs`";
@@ -2988,7 +3007,6 @@ WebRtcVideoChannel::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream(
   flexfec_config_.payload_type = flexfec_config.payload_type;
 
   CreateReceiveStream();
-  StartReceiveStream();
 }
 
 WebRtcVideoChannel::WebRtcVideoReceiveStream::~WebRtcVideoReceiveStream() {
@@ -3189,6 +3207,7 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetRecvParameters(
 }
 
 void WebRtcVideoChannel::WebRtcVideoReceiveStream::RecreateReceiveStream() {
+  RTC_DCHECK_RUN_ON(&thread_checker_);
   RTC_DCHECK(stream_);
   absl::optional<int> base_minimum_playout_delay_ms;
   absl::optional<webrtc::VideoReceiveStreamInterface::RecordingState>
@@ -3217,8 +3236,9 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::RecreateReceiveStream() {
     stream_->SetAndGetRecordingState(std::move(*recording_state),
                                      /*generate_key_frame=*/false);
   }
-
+  if (receiving_) {
-  StartReceiveStream();
+    StartReceiveStream();
+  }
 }
 
 void WebRtcVideoChannel::WebRtcVideoReceiveStream::CreateReceiveStream() {
@@ -3235,9 +3255,18 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::CreateReceiveStream() {
 }
 
 void WebRtcVideoChannel::WebRtcVideoReceiveStream::StartReceiveStream() {
+  RTC_DCHECK_RUN_ON(&thread_checker_);
+  receiving_ = true;
   stream_->Start();
 }
 
+void WebRtcVideoChannel::WebRtcVideoReceiveStream::StopReceiveStream() {
+  RTC_DCHECK_RUN_ON(&thread_checker_);
+  receiving_ = false;
+  stream_->Stop();
+  RecreateReceiveStream();
+}
+
 void WebRtcVideoChannel::WebRtcVideoReceiveStream::OnFrame(
     const webrtc::VideoFrame& frame) {
   webrtc::MutexLock lock(&sink_lock_);

media/engine/webrtc_video_engine.h

@@ -128,6 +128,7 @@ class WebRtcVideoChannel : public VideoMediaChannel,
   webrtc::RtpParameters GetRtpReceiveParameters(uint32_t ssrc) const override;
   webrtc::RtpParameters GetDefaultRtpReceiveParameters() const override;
   bool GetSendCodec(VideoCodec* send_codec) override;
+  void SetReceive(bool receive) override;
   bool SetSend(bool send) override;
   bool SetVideoSend(
       uint32_t ssrc,
@@ -337,8 +338,8 @@ class WebRtcVideoChannel : public VideoMediaChannel,
   bool MaybeCreateDefaultReceiveStream(
       const webrtc::RtpPacketReceived& parsed_packet)
       RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
-  void ReCreateDefaulReceiveStream(uint32_t ssrc,
+  void ReCreateDefaultReceiveStream(uint32_t ssrc,
-                                   absl::optional<uint32_t> rtx_ssrc);
+                                    absl::optional<uint32_t> rtx_ssrc);
   void ConfigureReceiverRtp(
       webrtc::VideoReceiveStreamInterface::Config* config,
       webrtc::FlexfecReceiveStream::Config* flexfec_config,
@@ -540,6 +541,8 @@ class WebRtcVideoChannel : public VideoMediaChannel,
 
     void SetLocalSsrc(uint32_t local_ssrc);
     void UpdateRtxSsrc(uint32_t ssrc);
+    void StartReceiveStream();
+    void StopReceiveStream();
 
    private:
     // Attempts to reconfigure an already existing `flexfec_stream_`, create
@@ -549,7 +552,6 @@ class WebRtcVideoChannel : public VideoMediaChannel,
 
     void RecreateReceiveStream();
     void CreateReceiveStream();
-    void StartReceiveStream();
 
     // Applies a new receive codecs configration to `config_`. Returns true
     // if the internal stream needs to be reconstructed, or false if no changes
@@ -575,6 +577,9 @@ class WebRtcVideoChannel : public VideoMediaChannel,
     // Start NTP time is estimated as current remote NTP time (estimated from
     // RTCP) minus the elapsed time, as soon as remote NTP time is available.
     int64_t estimated_remote_start_ntp_time_ms_ RTC_GUARDED_BY(sink_lock_);
+
+    RTC_NO_UNIQUE_ADDRESS webrtc::SequenceChecker thread_checker_;
+    bool receiving_ RTC_GUARDED_BY(&thread_checker_);
   };
 
   void Construct(webrtc::Call* call, WebRtcVideoEngine* engine);
@@ -619,6 +624,7 @@ class WebRtcVideoChannel : public VideoMediaChannel,
 
   uint32_t rtcp_receiver_report_ssrc_ RTC_GUARDED_BY(thread_checker_);
   bool sending_ RTC_GUARDED_BY(thread_checker_);
+  bool receiving_ RTC_GUARDED_BY(&thread_checker_);
   webrtc::Call* const call_;
 
   rtc::VideoSinkInterface<webrtc::VideoFrame>* default_sink_

media/engine/webrtc_video_engine_unittest.cc

@@ -938,6 +938,7 @@ TEST_F(WebRtcVideoEngineTest, SendsFeedbackAfterUnsignaledRtxPacket) {
   ASSERT_TRUE(channel->SetRecvParameters(parameters));
   channel->SetInterface(&network);
   channel->AsVideoReceiveChannel()->OnReadyToSend(true);
+  channel->AsVideoReceiveChannel()->SetReceive(true);
 
   // Inject a RTX packet.
   webrtc::RtpHeaderExtensionMap extension_map(parameters.extensions);
@@ -972,6 +973,7 @@ TEST_F(WebRtcVideoEngineTest, UpdatesUnsignaledRtxSsrcAndRecoversPayload) {
   cricket::VideoRecvParameters parameters;
   parameters.codecs = supported_codecs;
   ASSERT_TRUE(channel->SetRecvParameters(parameters));
+  channel->AsVideoReceiveChannel()->SetReceive(true);
 
   // Receive a normal payload packet. It is not a complete frame since the
   // marker bit is not set.
@@ -1628,6 +1630,7 @@ class WebRtcVideoChannelEncodedFrameCallbackTest : public ::testing::Test {
     cricket::VideoRecvParameters parameters;
     parameters.codecs = engine_.recv_codecs();
     channel_->SetRecvParameters(parameters);
+    receive_channel_->SetReceive(true);
   }
 
   ~WebRtcVideoChannelEncodedFrameCallbackTest() override {
@@ -1739,6 +1742,31 @@ TEST_F(WebRtcVideoChannelEncodedFrameCallbackTest,
   receive_channel_->RemoveRecvStream(kSsrc + 1);
 }
 
+TEST_F(WebRtcVideoChannelEncodedFrameCallbackTest, DoesNotDecodeWhenDisabled) {
+  testing::MockFunction<void(const webrtc::RecordableEncodedFrame&)> callback;
+  EXPECT_CALL(callback, Call);
+  EXPECT_TRUE(channel_->AddRecvStream(
+      cricket::StreamParams::CreateLegacy(kSsrc), /*is_default_stream=*/true));
+  channel_->SetRecordableEncodedFrameCallback(/*ssrc=*/0,
+                                              callback.AsStdFunction());
+  EXPECT_TRUE(channel_->SetSink(kSsrc, &renderer_));
+  receive_channel_->SetReceive(false);
+  DeliverKeyFrame(kSsrc);
+  time_controller_.AdvanceTime(kFrameDuration);
+  EXPECT_EQ(renderer_.num_rendered_frames(), 0);
+
+  receive_channel_->SetReceive(true);
+  DeliverKeyFrame(kSsrc);
+  time_controller_.AdvanceTime(kFrameDuration);
+  EXPECT_EQ(renderer_.num_rendered_frames(), 1);
+
+  receive_channel_->SetReceive(false);
+  DeliverKeyFrame(kSsrc);
+  time_controller_.AdvanceTime(kFrameDuration);
+  EXPECT_EQ(renderer_.num_rendered_frames(), 1);
+  receive_channel_->RemoveRecvStream(kSsrc);
+}
+
 class WebRtcVideoChannelBaseTest : public ::testing::Test {
  protected:
   WebRtcVideoChannelBaseTest()
@@ -1781,6 +1809,7 @@ class WebRtcVideoChannelBaseTest : public ::testing::Test {
     receive_channel_ =
         std::make_unique<VideoMediaReceiveChannel>(channel_.get());
     send_channel_->OnReadyToSend(true);
+    receive_channel_->SetReceive(true);
     EXPECT_TRUE(channel_.get() != NULL);
     network_interface_.SetDestination(channel_.get());
     channel_->SetInterface(&network_interface_);
@@ -2630,6 +2659,7 @@ class WebRtcVideoChannelTest : public WebRtcVideoEngineTest {
     receive_channel_ =
         std::make_unique<cricket::VideoMediaReceiveChannel>(channel_.get());
     send_channel_->OnReadyToSend(true);
+    receive_channel_->SetReceive(true);
     last_ssrc_ = 123;
     send_parameters_.codecs = engine_.send_codecs();
     recv_parameters_.codecs = engine_.recv_codecs();
@@ -9449,6 +9479,7 @@ class WebRtcVideoChannelSimulcastTest : public ::testing::Test {
     receive_channel_ =
         std::make_unique<VideoMediaReceiveChannel>(channel_.get());
     send_channel_->OnReadyToSend(true);
+    receive_channel_->SetReceive(true);
     last_ssrc_ = 123;
   }
 

pc/channel.cc

@@ -911,16 +911,16 @@ VoiceChannel::~VoiceChannel() {
 void VoiceChannel::UpdateMediaSendRecvState_w() {
   // Render incoming data if we're the active call, and we have the local
   // content. We receive data on the default channel and multiplexed streams.
-  bool ready_to_receive = enabled() && webrtc::RtpTransceiverDirectionHasRecv(
+  bool receive = enabled() && webrtc::RtpTransceiverDirectionHasRecv(
-                                           local_content_direction());
+                                  local_content_direction());
-  media_receive_channel()->SetPlayout(ready_to_receive);
+  media_receive_channel()->SetPlayout(receive);
 
   // Send outgoing data if we're the active call, we have the remote content,
   // and we have had some form of connectivity.
   bool send = IsReadyToSendMedia_w();
   media_send_channel()->SetSend(send);
 
-  RTC_LOG(LS_INFO) << "Changing voice state, recv=" << ready_to_receive
+  RTC_LOG(LS_INFO) << "Changing voice state, recv=" << receive
                    << " send=" << send << " for " << ToString();
 }
 
@@ -1077,10 +1077,14 @@ VideoChannel::~VideoChannel() {
 void VideoChannel::UpdateMediaSendRecvState_w() {
   // Send outgoing data if we're the active call, we have the remote content,
   // and we have had some form of connectivity.
+  bool receive = enabled() && webrtc::RtpTransceiverDirectionHasRecv(
+                                  local_content_direction());
+  media_receive_channel()->SetReceive(receive);
+
   bool send = IsReadyToSendMedia_w();
   media_send_channel()->SetSend(send);
-  RTC_LOG(LS_INFO) << "Changing video state, send=" << send << " for "
+  RTC_LOG(LS_INFO) << "Changing video state, recv=" << receive
-                   << ToString();
+                   << " send=" << send << " for " << ToString();
 }
 
 bool VideoChannel::SetLocalContent_w(const MediaContentDescription* content,

pc/channel_unittest.cc

@@ -868,6 +868,30 @@ class ChannelTest : public ::testing::Test, public sigslot::has_slots<> {
       EXPECT_TRUE(media_receive_channel2_impl()->playout());
     }
     EXPECT_TRUE(media_send_channel2_impl()->sending());
+
+    // Update `content2` to be inactive on the receiver while sending at the
+    // sender.
+    content2.set_direction(RtpTransceiverDirection::kInactive);
+    EXPECT_TRUE(channel1_->SetLocalContent(&content1, SdpType::kOffer, err));
+    EXPECT_TRUE(channel2_->SetRemoteContent(&content1, SdpType::kOffer, err));
+    EXPECT_TRUE(channel2_->SetLocalContent(&content2, SdpType::kAnswer, err));
+    content2.set_direction(RtpTransceiverDirection::kRecvOnly);
+    EXPECT_TRUE(channel1_->SetRemoteContent(&content2, SdpType::kAnswer, err));
+    if (verify_playout_) {
+      EXPECT_FALSE(media_receive_channel2_impl()->playout());
+    }
+    EXPECT_TRUE(media_send_channel1_impl()->sending());
+
+    // Re-enable `content2`.
+    content2.set_direction(RtpTransceiverDirection::kSendRecv);
+    EXPECT_TRUE(channel1_->SetLocalContent(&content1, SdpType::kOffer, err));
+    EXPECT_TRUE(channel2_->SetRemoteContent(&content1, SdpType::kOffer, err));
+    EXPECT_TRUE(channel2_->SetLocalContent(&content2, SdpType::kAnswer, err));
+    EXPECT_TRUE(channel1_->SetRemoteContent(&content2, SdpType::kAnswer, err));
+    if (verify_playout_) {
+      EXPECT_TRUE(media_receive_channel2_impl()->playout());
+    }
+    EXPECT_TRUE(media_send_channel1_impl()->sending());
   }
 
   // Tests that when the transport channel signals a candidate pair change