From 32565f684bbd5e176d41fc1ce3d34aae191c3f68 Mon Sep 17 00:00:00 2001 From: Markus Handell Date: Wed, 4 Dec 2019 10:58:17 +0100 Subject: [PATCH] WebRtcVideoEngine: Enable encoded frame sink. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change ultimately enables wiring up VideoRtpReceiver::OnGenerateKeyFrame and OnEncodedSinkEnabled into internal::VideoReceiveStream so that encoded frames can flow to sinks installed in VideoTrackSourceInterface. Bug: chromium:1013590 Change-Id: I136132c210e5811547f2522ddc371d0acac90664 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/161093 Commit-Queue: Markus Handell Reviewed-by: Erik Språng Cr-Commit-Position: refs/heads/master@{#30001} --- media/BUILD.gn | 3 + media/base/fake_media_engine.cc | 10 ++ media/base/fake_media_engine.h | 7 + media/base/media_channel.h | 10 ++ media/engine/webrtc_video_engine.cc | 96 +++++++++++++ media/engine/webrtc_video_engine.h | 17 +++ media/engine/webrtc_video_engine_unittest.cc | 143 +++++++++++++++++++ 7 files changed, 286 insertions(+) diff --git a/media/BUILD.gn b/media/BUILD.gn index 39deaa3052..48fcdff313 100644 --- a/media/BUILD.gn +++ b/media/BUILD.gn @@ -92,6 +92,7 @@ rtc_library("rtc_media_base") { "../api/video:video_rtp_headers", "../api/video_codecs:video_codecs_api", "../call:call_interfaces", + "../call:video_stream_api", "../common_video", "../modules/audio_processing:audio_processing_statistics", "../modules/rtp_rtcp:rtp_rtcp_format", @@ -172,6 +173,7 @@ rtc_library("rtc_simulcast_encoder_adapter") { "../api/video:video_rtp_headers", "../api/video_codecs:rtc_software_fallback_wrappers", "../api/video_codecs:video_codecs_api", + "../call:video_stream_api", "../modules/video_coding:video_codec_interface", "../modules/video_coding:video_coding_utility", "../rtc_base:checks", @@ -571,6 +573,7 @@ if (rtc_include_tests) { "../rtc_base/experiments:min_video_bitrate_experiment", "../rtc_base/third_party/sigslot", "../test:audio_codec_mocks", + "../test:fake_video_codecs", "../test:field_trial", "../test:rtp_test_utils", "../test:test_main", diff --git a/media/base/fake_media_engine.cc b/media/base/fake_media_engine.cc index 86f6fd9371..d1302342f3 100644 --- a/media/base/fake_media_engine.cc +++ b/media/base/fake_media_engine.cc @@ -400,11 +400,21 @@ bool FakeVideoMediaChannel::SetOptions(const VideoOptions& options) { options_ = options; return true; } + bool FakeVideoMediaChannel::SetMaxSendBandwidth(int bps) { max_bps_ = bps; return true; } +void FakeVideoMediaChannel::SetRecordableEncodedFrameCallback( + uint32_t ssrc, + std::function callback) {} + +void FakeVideoMediaChannel::ClearRecordableEncodedFrameCallback(uint32_t ssrc) { +} + +void FakeVideoMediaChannel::GenerateKeyFrame(uint32_t ssrc) {} + FakeDataMediaChannel::FakeDataMediaChannel(void* unused, const DataOptions& options) : send_blocked_(false), max_bps_(-1) {} diff --git a/media/base/fake_media_engine.h b/media/base/fake_media_engine.h index 99a9b4970b..aa713d04f0 100644 --- a/media/base/fake_media_engine.h +++ b/media/base/fake_media_engine.h @@ -436,6 +436,13 @@ class FakeVideoMediaChannel : public RtpHelper { absl::optional GetBaseMinimumPlayoutDelayMs( uint32_t ssrc) const override; + void SetRecordableEncodedFrameCallback( + uint32_t ssrc, + std::function callback) + override; + void ClearRecordableEncodedFrameCallback(uint32_t ssrc) override; + void GenerateKeyFrame(uint32_t ssrc) override; + private: bool SetRecvCodecs(const std::vector& codecs); bool SetSendCodecs(const std::vector& codecs); diff --git a/media/base/media_channel.h b/media/base/media_channel.h index 696e5f7526..be4730e6b3 100644 --- a/media/base/media_channel.h +++ b/media/base/media_channel.h @@ -31,6 +31,7 @@ #include "api/video/video_source_interface.h" #include "api/video/video_timing.h" #include "api/video_codecs/video_encoder_config.h" +#include "call/video_receive_stream.h" #include "common_video/include/quality_limitation_reason.h" #include "media/base/codec.h" #include "media/base/delayable.h" @@ -41,6 +42,7 @@ #include "modules/rtp_rtcp/include/report_block_data.h" #include "rtc_base/async_packet_socket.h" #include "rtc_base/buffer.h" +#include "rtc_base/callback.h" #include "rtc_base/copy_on_write_buffer.h" #include "rtc_base/critical_section.h" #include "rtc_base/dscp.h" @@ -897,6 +899,14 @@ class VideoMediaChannel : public MediaChannel, public Delayable { virtual void FillBitrateInfo(BandwidthEstimationInfo* bwe_info) = 0; // Gets quality stats for the channel. virtual bool GetStats(VideoMediaInfo* info) = 0; + // Set recordable encoded frame callback for |ssrc| + virtual void SetRecordableEncodedFrameCallback( + uint32_t ssrc, + std::function callback) = 0; + // Clear recordable encoded frame callback for |ssrc| + virtual void ClearRecordableEncodedFrameCallback(uint32_t ssrc) = 0; + // Cause generation of a keyframe for |ssrc| + virtual void GenerateKeyFrame(uint32_t ssrc) = 0; virtual std::vector GetSources(uint32_t ssrc) const = 0; }; diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index 71d0c9b55c..7f241c86aa 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -2632,8 +2632,12 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetRecvParameters( void WebRtcVideoChannel::WebRtcVideoReceiveStream::RecreateWebRtcVideoStream() { absl::optional base_minimum_playout_delay_ms; + absl::optional recording_state; if (stream_) { base_minimum_playout_delay_ms = stream_->GetBaseMinimumPlayoutDelayMs(); + recording_state = stream_->SetAndGetRecordingState( + webrtc::VideoReceiveStream::RecordingState(), + /*generate_key_frame=*/false); MaybeDissociateFlexfecFromVideo(); call_->DestroyVideoReceiveStream(stream_); stream_ = nullptr; @@ -2646,6 +2650,10 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::RecreateWebRtcVideoStream() { stream_->SetBaseMinimumPlayoutDelayMs( base_minimum_playout_delay_ms.value()); } + if (recording_state) { + stream_->SetAndGetRecordingState(std::move(*recording_state), + /*generate_key_frame=*/false); + } MaybeAssociateFlexfecWithVideo(); stream_->Start(); @@ -2822,6 +2830,40 @@ WebRtcVideoChannel::WebRtcVideoReceiveStream::GetVideoReceiverInfo( return info; } +void WebRtcVideoChannel::WebRtcVideoReceiveStream:: + SetRecordableEncodedFrameCallback( + std::function callback) { + if (stream_) { + stream_->SetAndGetRecordingState( + webrtc::VideoReceiveStream::RecordingState(std::move(callback)), + /*generate_key_frame=*/true); + } else { + RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring setting encoded " + "frame sink"; + } +} + +void WebRtcVideoChannel::WebRtcVideoReceiveStream:: + ClearRecordableEncodedFrameCallback() { + if (stream_) { + stream_->SetAndGetRecordingState( + webrtc::VideoReceiveStream::RecordingState(), + /*generate_key_frame=*/false); + } else { + RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring clearing encoded " + "frame sink"; + } +} + +void WebRtcVideoChannel::WebRtcVideoReceiveStream::GenerateKeyFrame() { + if (stream_) { + stream_->GenerateKeyFrame(); + } else { + RTC_LOG(LS_ERROR) + << "Absent receive stream; ignoring key frame generation request."; + } +} + WebRtcVideoChannel::VideoCodecSettings::VideoCodecSettings() : flexfec_payload_type(-1), rtx_payload_type(-1) {} @@ -2968,6 +3010,60 @@ WebRtcVideoChannel::MapCodecs(const std::vector& codecs) { return video_codecs; } +WebRtcVideoChannel::WebRtcVideoReceiveStream* +WebRtcVideoChannel::FindReceiveStream(uint32_t ssrc) { + if (ssrc == 0) { + absl::optional default_ssrc = GetDefaultReceiveStreamSsrc(); + if (!default_ssrc) { + return nullptr; + } + ssrc = *default_ssrc; + } + auto it = receive_streams_.find(ssrc); + if (it != receive_streams_.end()) { + return it->second; + } + return nullptr; +} + +void WebRtcVideoChannel::SetRecordableEncodedFrameCallback( + uint32_t ssrc, + std::function callback) { + RTC_DCHECK_RUN_ON(&thread_checker_); + WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); + if (stream) { + stream->SetRecordableEncodedFrameCallback(std::move(callback)); + } else { + RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring setting encoded " + "frame sink for ssrc " + << ssrc; + } +} + +void WebRtcVideoChannel::ClearRecordableEncodedFrameCallback(uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&thread_checker_); + WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); + if (stream) { + stream->ClearRecordableEncodedFrameCallback(); + } else { + RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring clearing encoded " + "frame sink for ssrc " + << ssrc; + } +} + +void WebRtcVideoChannel::GenerateKeyFrame(uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&thread_checker_); + WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); + if (stream) { + stream->GenerateKeyFrame(); + } else { + RTC_LOG(LS_ERROR) + << "Absent receive stream; ignoring key frame generation for ssrc " + << ssrc; + } +} + // TODO(bugs.webrtc.org/8785): Consider removing max_qp as member of // EncoderStreamFactory and instead set this value individually for each stream // in the VideoEncoderConfig.simulcast_layers. diff --git a/media/engine/webrtc_video_engine.h b/media/engine/webrtc_video_engine.h index 20461ba86d..4b423243ac 100644 --- a/media/engine/webrtc_video_engine.h +++ b/media/engine/webrtc_video_engine.h @@ -210,9 +210,21 @@ class WebRtcVideoChannel : public VideoMediaChannel, void RequestEncoderFallback() override; void RequestEncoderSwitch( const EncoderSwitchRequestCallback::Config& conf) override; + void SetRecordableEncodedFrameCallback( + uint32_t ssrc, + std::function callback) + override; + void ClearRecordableEncodedFrameCallback(uint32_t ssrc) override; + void GenerateKeyFrame(uint32_t ssrc) override; private: class WebRtcVideoReceiveStream; + + // Finds VideoReceiveStream corresponding to ssrc. Aware of unsignalled + // ssrc handling. + WebRtcVideoReceiveStream* FindReceiveStream(uint32_t ssrc) + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + struct VideoCodecSettings { VideoCodecSettings(); @@ -430,6 +442,11 @@ class WebRtcVideoChannel : public VideoMediaChannel, VideoReceiverInfo GetVideoReceiverInfo(bool log_stats); + void SetRecordableEncodedFrameCallback( + std::function callback); + void ClearRecordableEncodedFrameCallback(); + void GenerateKeyFrame(); + private: void RecreateWebRtcVideoStream(); void MaybeRecreateWebRtcFlexfecStream(); diff --git a/media/engine/webrtc_video_engine_unittest.cc b/media/engine/webrtc_video_engine_unittest.cc index 2c6b524b40..b7f9266411 100644 --- a/media/engine/webrtc_video_engine_unittest.cc +++ b/media/engine/webrtc_video_engine_unittest.cc @@ -26,6 +26,7 @@ #include "api/test/mock_video_bitrate_allocator_factory.h" #include "api/test/mock_video_decoder_factory.h" #include "api/test/mock_video_encoder_factory.h" +#include "api/test/video/function_video_decoder_factory.h" #include "api/transport/field_trial_based_config.h" #include "api/transport/media/media_transport_config.h" #include "api/units/time_delta.h" @@ -57,6 +58,7 @@ #include "rtc_base/gunit.h" #include "rtc_base/numerics/safe_conversions.h" #include "rtc_base/time_utils.h" +#include "test/fake_decoder.h" #include "test/field_trial.h" #include "test/frame_generator.h" #include "test/gmock.h" @@ -1307,6 +1309,147 @@ TEST_F(WebRtcVideoEngineTest, DISABLED_RecreatesEncoderOnContentTypeChange) { EXPECT_EQ(0u, encoder_factory_->encoders().size()); } +class WebRtcVideoChannelEncodedFrameCallbackTest : public ::testing::Test { + protected: + webrtc::Call::Config GetCallConfig( + webrtc::RtcEventLogNull* event_log, + webrtc::TaskQueueFactory* task_queue_factory) { + webrtc::Call::Config call_config(event_log); + call_config.task_queue_factory = task_queue_factory; + call_config.trials = &field_trials_; + return call_config; + } + + WebRtcVideoChannelEncodedFrameCallbackTest() + : task_queue_factory_(webrtc::CreateDefaultTaskQueueFactory()), + call_(absl::WrapUnique(webrtc::Call::Create( + GetCallConfig(&event_log_, task_queue_factory_.get())))), + video_bitrate_allocator_factory_( + webrtc::CreateBuiltinVideoBitrateAllocatorFactory()), + engine_( + webrtc::CreateBuiltinVideoEncoderFactory(), + std::make_unique([]() { + return std::make_unique(); + })), + channel_(absl::WrapUnique(static_cast( + engine_.CreateMediaChannel( + call_.get(), + cricket::MediaConfig(), + cricket::VideoOptions(), + webrtc::CryptoOptions(), + video_bitrate_allocator_factory_.get())))) { + network_interface_.SetDestination(channel_.get()); + channel_->SetInterface(&network_interface_, webrtc::MediaTransportConfig()); + cricket::VideoRecvParameters parameters; + parameters.codecs = engine_.codecs(); + channel_->SetRecvParameters(parameters); + } + + void DeliverKeyFrame(uint32_t ssrc) { + webrtc::RtpPacket packet; + packet.SetMarker(true); + packet.SetPayloadType(96); // VP8 + packet.SetSsrc(ssrc); + + // VP8 Keyframe + 1 byte payload + uint8_t* buf_ptr = packet.AllocatePayload(11); + memset(buf_ptr, 0, 11); // Pass MSAN (don't care about bytes 1-9) + buf_ptr[0] = 0x10; // Partition ID 0 + beginning of partition. + call_->Receiver()->DeliverPacket(webrtc::MediaType::VIDEO, packet.Buffer(), + /*packet_time_us=*/0); + } + + void DeliverKeyFrameAndWait(uint32_t ssrc) { + DeliverKeyFrame(ssrc); + EXPECT_EQ_WAIT(1, renderer_.num_rendered_frames(), kTimeout); + EXPECT_EQ(0, renderer_.errors()); + } + + webrtc::FieldTrialBasedConfig field_trials_; + webrtc::RtcEventLogNull event_log_; + std::unique_ptr task_queue_factory_; + std::unique_ptr call_; + std::unique_ptr + video_bitrate_allocator_factory_; + WebRtcVideoEngine engine_; + std::unique_ptr channel_; + cricket::FakeNetworkInterface network_interface_; + cricket::FakeVideoRenderer renderer_; +}; + +TEST_F(WebRtcVideoChannelEncodedFrameCallbackTest, + SetEncodedFrameBufferFunction_DefaultStream) { + testing::MockFunction callback; + EXPECT_CALL(callback, Call); + EXPECT_TRUE(channel_->AddRecvStream( + cricket::StreamParams::CreateLegacy(kSsrc), /*is_default_stream=*/true)); + channel_->SetRecordableEncodedFrameCallback(/*ssrc=*/0, + callback.AsStdFunction()); + EXPECT_TRUE(channel_->SetSink(kSsrc, &renderer_)); + DeliverKeyFrame(kSsrc); + EXPECT_EQ_WAIT(1, renderer_.num_rendered_frames(), kTimeout); + EXPECT_EQ(0, renderer_.errors()); + channel_->RemoveRecvStream(kSsrc); +} + +TEST_F(WebRtcVideoChannelEncodedFrameCallbackTest, + SetEncodedFrameBufferFunction_MatchSsrcWithDefaultStream) { + testing::MockFunction callback; + EXPECT_CALL(callback, Call); + EXPECT_TRUE(channel_->AddRecvStream( + cricket::StreamParams::CreateLegacy(kSsrc), /*is_default_stream=*/true)); + EXPECT_TRUE(channel_->SetSink(kSsrc, &renderer_)); + channel_->SetRecordableEncodedFrameCallback(kSsrc, callback.AsStdFunction()); + DeliverKeyFrame(kSsrc); + EXPECT_EQ_WAIT(1, renderer_.num_rendered_frames(), kTimeout); + EXPECT_EQ(0, renderer_.errors()); + channel_->RemoveRecvStream(kSsrc); +} + +TEST_F(WebRtcVideoChannelEncodedFrameCallbackTest, + SetEncodedFrameBufferFunction_MatchSsrc) { + testing::MockFunction callback; + EXPECT_CALL(callback, Call); + EXPECT_TRUE(channel_->AddRecvStream( + cricket::StreamParams::CreateLegacy(kSsrc), /*is_default_stream=*/false)); + EXPECT_TRUE(channel_->SetSink(kSsrc, &renderer_)); + channel_->SetRecordableEncodedFrameCallback(kSsrc, callback.AsStdFunction()); + DeliverKeyFrame(kSsrc); + EXPECT_EQ_WAIT(1, renderer_.num_rendered_frames(), kTimeout); + EXPECT_EQ(0, renderer_.errors()); + channel_->RemoveRecvStream(kSsrc); +} + +TEST_F(WebRtcVideoChannelEncodedFrameCallbackTest, + SetEncodedFrameBufferFunction_MismatchSsrc) { + testing::StrictMock< + testing::MockFunction> + callback; + EXPECT_TRUE( + channel_->AddRecvStream(cricket::StreamParams::CreateLegacy(kSsrc + 1), + /*is_default_stream=*/false)); + EXPECT_TRUE(channel_->SetSink(kSsrc + 1, &renderer_)); + channel_->SetRecordableEncodedFrameCallback(kSsrc, callback.AsStdFunction()); + DeliverKeyFrame(kSsrc); // Expected to not cause function to fire. + DeliverKeyFrameAndWait(kSsrc + 1); + channel_->RemoveRecvStream(kSsrc + 1); +} + +TEST_F(WebRtcVideoChannelEncodedFrameCallbackTest, + SetEncodedFrameBufferFunction_MismatchSsrcWithDefaultStream) { + testing::StrictMock< + testing::MockFunction> + callback; + EXPECT_TRUE( + channel_->AddRecvStream(cricket::StreamParams::CreateLegacy(kSsrc + 1), + /*is_default_stream=*/true)); + EXPECT_TRUE(channel_->SetSink(kSsrc + 1, &renderer_)); + channel_->SetRecordableEncodedFrameCallback(kSsrc, callback.AsStdFunction()); + DeliverKeyFrame(kSsrc); // Expected to not cause function to fire. + DeliverKeyFrameAndWait(kSsrc + 1); + channel_->RemoveRecvStream(kSsrc + 1); +} + class WebRtcVideoChannelBaseTest : public ::testing::Test { protected: WebRtcVideoChannelBaseTest()