From 223334933f7b79269849dfad2cc5efba5f96352a Mon Sep 17 00:00:00 2001 From: Danil Chapovalov Date: Thu, 7 Dec 2023 12:10:46 +0100 Subject: [PATCH] Propagate Environment into VideoReceiveStream2 as a step to propagate Environment and thus field trials into Decoders Bug: webrtc:10335 Change-Id: Ib396421f0fbf34f2c2f90aa4a1b41b461e42253c Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/330421 Reviewed-by: Philip Eliasson Commit-Queue: Danil Chapovalov Cr-Commit-Position: refs/heads/main@{#41335} --- call/call.cc | 7 ++- video/BUILD.gn | 2 +- video/video_receive_stream2.cc | 47 +++++++++--------- video/video_receive_stream2.h | 12 ++--- video/video_receive_stream2_unittest.cc | 63 +++++++++++++------------ 5 files changed, 64 insertions(+), 67 deletions(-) diff --git a/call/call.cc b/call/call.cc index 7e6d57a37e..68e3b69a84 100644 --- a/call/call.cc +++ b/call/call.cc @@ -986,11 +986,10 @@ webrtc::VideoReceiveStreamInterface* Call::CreateVideoReceiveStream( // TODO(crbug.com/1381982): Re-enable decode synchronizer once the Chromium // API has adapted to the new Metronome interface. VideoReceiveStream2* receive_stream = new VideoReceiveStream2( - &env_.task_queue_factory(), this, num_cpu_cores_, - transport_send_->packet_router(), std::move(configuration), - call_stats_.get(), &env_.clock(), + env_, this, num_cpu_cores_, transport_send_->packet_router(), + std::move(configuration), call_stats_.get(), std::make_unique(&env_.clock(), trials()), - &nack_periodic_processor_, decode_sync_.get(), &env_.event_log()); + &nack_periodic_processor_, decode_sync_.get()); // TODO(bugs.webrtc.org/11993): Set this up asynchronously on the network // thread. receive_stream->RegisterWithTransport(&video_receiver_controller_); diff --git a/video/BUILD.gn b/video/BUILD.gn index 204c6b66f4..79edee5aa6 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -91,9 +91,9 @@ rtc_library("video") { "../api:transport_api", "../api/crypto:frame_decryptor_interface", "../api/crypto:options", + "../api/environment", "../api/task_queue", "../api/task_queue:pending_task_safety_flag", - "../api/transport:field_trial_based_config", "../api/units:data_rate", "../api/units:frequency", "../api/units:time_delta", diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc index af25c364de..9ea3936434 100644 --- a/video/video_receive_stream2.cc +++ b/video/video_receive_stream2.cc @@ -177,31 +177,28 @@ TimeDelta DetermineMaxWaitForFrame(TimeDelta rtp_history, bool is_keyframe) { } VideoReceiveStream2::VideoReceiveStream2( - TaskQueueFactory* task_queue_factory, + const Environment& env, Call* call, int num_cpu_cores, PacketRouter* packet_router, VideoReceiveStreamInterface::Config config, CallStats* call_stats, - Clock* clock, std::unique_ptr timing, NackPeriodicProcessor* nack_periodic_processor, - DecodeSynchronizer* decode_sync, - RtcEventLog* event_log) - : task_queue_factory_(task_queue_factory), + DecodeSynchronizer* decode_sync) + : env_(env), transport_adapter_(config.rtcp_send_transport), config_(std::move(config)), num_cpu_cores_(num_cpu_cores), call_(call), - clock_(clock), call_stats_(call_stats), - source_tracker_(clock_), - stats_proxy_(remote_ssrc(), clock_, call->worker_thread()), - rtp_receive_statistics_(ReceiveStatistics::Create(clock_)), + source_tracker_(&env_.clock()), + stats_proxy_(remote_ssrc(), &env_.clock(), call->worker_thread()), + rtp_receive_statistics_(ReceiveStatistics::Create(&env_.clock())), timing_(std::move(timing)), - video_receiver_(clock_, timing_.get(), call->trials()), + video_receiver_(&env_.clock(), timing_.get(), env_.field_trials()), rtp_video_stream_receiver_(call->worker_thread(), - clock_, + &env_.clock(), &transport_adapter_, call_stats->AsRtcpRttStats(), packet_router, @@ -213,8 +210,8 @@ VideoReceiveStream2::VideoReceiveStream2( this, // OnCompleteFrameCallback std::move(config_.frame_decryptor), std::move(config_.frame_transformer), - call->trials(), - event_log), + env_.field_trials(), + &env_.event_log()), rtp_stream_sync_(call->worker_thread(), this), max_wait_for_keyframe_(DetermineMaxWaitForFrame( TimeDelta::Millis(config_.rtp.nack.rtp_history_ms), @@ -222,7 +219,7 @@ VideoReceiveStream2::VideoReceiveStream2( max_wait_for_frame_(DetermineMaxWaitForFrame( TimeDelta::Millis(config_.rtp.nack.rtp_history_ms), false)), - decode_queue_(task_queue_factory_->CreateTaskQueue( + decode_queue_(env_.task_queue_factory().CreateTaskQueue( "DecodingQueue", TaskQueueFactory::Priority::HIGH)) { RTC_LOG(LS_INFO) << "VideoReceiveStream2: " << config_.ToString(); @@ -248,11 +245,11 @@ VideoReceiveStream2::VideoReceiveStream2( std::unique_ptr scheduler = decode_sync ? decode_sync->CreateSynchronizedFrameScheduler() : std::make_unique( - clock, call_->worker_thread()); + &env_.clock(), call_->worker_thread()); buffer_ = std::make_unique( - clock_, call_->worker_thread(), timing_.get(), &stats_proxy_, this, + &env_.clock(), call_->worker_thread(), timing_.get(), &stats_proxy_, this, max_wait_for_keyframe_, max_wait_for_frame_, std::move(scheduler), - call_->trials()); + env_.field_trials()); if (!config_.rtp.rtx_associated_payload_types.empty()) { rtx_receive_stream_ = std::make_unique( @@ -345,7 +342,7 @@ void VideoReceiveStream2::Start() { rtc::VideoSinkInterface* renderer = nullptr; if (config_.enable_prerenderer_smoothing) { incoming_video_stream_.reset(new IncomingVideoStream( - task_queue_factory_, config_.render_delay_ms, this)); + &env_.task_queue_factory(), config_.render_delay_ms, this)); renderer = incoming_video_stream_.get(); } else { renderer = this; @@ -356,7 +353,7 @@ void VideoReceiveStream2::Start() { settings.set_codec_type( PayloadStringToCodecType(decoder.video_format.name)); settings.set_max_render_resolution( - InitialDecoderResolution(call_->trials())); + InitialDecoderResolution(env_.field_trials())); settings.set_number_of_cores(num_cpu_cores_); const bool raw_payload = @@ -531,7 +528,7 @@ void VideoReceiveStream2::CreateAndRegisterExternalDecoder( } std::string decoded_output_file = - call_->trials().Lookup("WebRTC-DecoderDataDumpDirectory"); + env_.field_trials().Lookup("WebRTC-DecoderDataDumpDirectory"); // Because '/' can't be used inside a field trial parameter, we use ';' // instead. // This is only relevant to WebRTC-DecoderDataDumpDirectory @@ -630,7 +627,7 @@ void VideoReceiveStream2::OnFrame(const VideoFrame& video_frame) { // renderer. Frame may or may be not rendered by this time. This results in // inaccuracy but is still the best we can do in the absence of "frame // rendered" callback from the renderer. - VideoFrameMetaData frame_meta(video_frame, clock_->CurrentTime()); + VideoFrameMetaData frame_meta(video_frame, env_.clock().CurrentTime()); call_->worker_thread()->PostTask( SafeTask(task_safety_.flag(), [frame_meta, this]() { RTC_DCHECK_RUN_ON(&worker_sequence_checker_); @@ -749,7 +746,7 @@ bool VideoReceiveStream2::SetMinimumPlayoutDelay(int delay_ms) { void VideoReceiveStream2::OnEncodedFrame(std::unique_ptr frame) { RTC_DCHECK_RUN_ON(&packet_sequence_checker_); - Timestamp now = clock_->CurrentTime(); + Timestamp now = env_.clock().CurrentTime(); const bool keyframe_request_is_due = !last_keyframe_request_ || now >= (*last_keyframe_request_ + max_wait_for_keyframe_); @@ -798,7 +795,7 @@ void VideoReceiveStream2::OnEncodedFrame(std::unique_ptr frame) { void VideoReceiveStream2::OnDecodableFrameTimeout(TimeDelta wait) { RTC_DCHECK_RUN_ON(&packet_sequence_checker_); - Timestamp now = clock_->CurrentTime(); + Timestamp now = env_.clock().CurrentTime(); absl::optional last_packet_ms = rtp_video_stream_receiver_.LastReceivedPacketMs(); @@ -1054,7 +1051,7 @@ VideoReceiveStream2::SetAndGetRecordingState(RecordingState state, last_keyframe_request = last_keyframe_request_; last_keyframe_request_ = generate_key_frame - ? clock_->CurrentTime() + ? env_.clock().CurrentTime() : Timestamp::Millis(state.last_keyframe_request_ms.value_or(0)); } @@ -1086,7 +1083,7 @@ VideoReceiveStream2::SetAndGetRecordingState(RecordingState state, void VideoReceiveStream2::GenerateKeyFrame() { RTC_DCHECK_RUN_ON(&packet_sequence_checker_); - RequestKeyFrame(clock_->CurrentTime()); + RequestKeyFrame(env_.clock().CurrentTime()); keyframe_generation_requested_ = true; } diff --git a/video/video_receive_stream2.h b/video/video_receive_stream2.h index 31b9a7eb7c..b0bd164742 100644 --- a/video/video_receive_stream2.h +++ b/video/video_receive_stream2.h @@ -17,9 +17,9 @@ #include #include "absl/types/optional.h" +#include "api/environment/environment.h" #include "api/sequence_checker.h" #include "api/task_queue/pending_task_safety_flag.h" -#include "api/task_queue/task_queue_factory.h" #include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "api/video/recordable_encoded_frame.h" @@ -33,7 +33,6 @@ #include "rtc_base/system/no_unique_address.h" #include "rtc_base/task_queue.h" #include "rtc_base/thread_annotations.h" -#include "system_wrappers/include/clock.h" #include "video/receive_statistics_proxy.h" #include "video/rtp_streams_synchronizer2.h" #include "video/rtp_video_stream_receiver2.h" @@ -94,17 +93,15 @@ class VideoReceiveStream2 // configured. static constexpr size_t kBufferedEncodedFramesMaxSize = 60; - VideoReceiveStream2(TaskQueueFactory* task_queue_factory, + VideoReceiveStream2(const Environment& env, Call* call, int num_cpu_cores, PacketRouter* packet_router, VideoReceiveStreamInterface::Config config, CallStats* call_stats, - Clock* clock, std::unique_ptr timing, NackPeriodicProcessor* nack_periodic_processor, - DecodeSynchronizer* decode_sync, - RtcEventLog* event_log); + DecodeSynchronizer* decode_sync); // Destruction happens on the worker thread. Prior to destruction the caller // must ensure that a registration with the transport has been cleared. See // `RegisterWithTransport` for details. @@ -253,13 +250,12 @@ class VideoReceiveStream2 // on the network thread, this comment will be deleted. RTC_NO_UNIQUE_ADDRESS SequenceChecker packet_sequence_checker_; - TaskQueueFactory* const task_queue_factory_; + const Environment env_; TransportAdapter transport_adapter_; const VideoReceiveStreamInterface::Config config_; const int num_cpu_cores_; Call* const call_; - Clock* const clock_; CallStats* const call_stats_; diff --git a/video/video_receive_stream2_unittest.cc b/video/video_receive_stream2_unittest.cc index 084b128af8..50e00aa31b 100644 --- a/video/video_receive_stream2_unittest.cc +++ b/video/video_receive_stream2_unittest.cc @@ -23,6 +23,8 @@ #include "absl/memory/memory.h" #include "absl/types/optional.h" +#include "api/environment/environment.h" +#include "api/environment/environment_factory.h" #include "api/metronome/test/fake_metronome.h" #include "api/test/mock_video_decoder.h" #include "api/test/mock_video_decoder_factory.h" @@ -192,12 +194,13 @@ class VideoReceiveStream2Test : public ::testing::TestWithParam { VideoReceiveStream2Test() : time_controller_(kStartTime), - clock_(time_controller_.GetClock()), + env_(CreateEnvironment(time_controller_.CreateTaskQueueFactory(), + time_controller_.GetClock())), config_(&mock_transport_, &mock_h264_decoder_factory_), - call_stats_(clock_, time_controller_.GetMainThread()), + call_stats_(&env_.clock(), time_controller_.GetMainThread()), fake_renderer_(&time_controller_), fake_metronome_(TimeDelta::Millis(16)), - decode_sync_(clock_, + decode_sync_(&env_.clock(), &fake_metronome_, time_controller_.GetMainThread()), h264_decoder_factory_(&mock_decoder_) { @@ -255,13 +258,13 @@ class VideoReceiveStream2Test : public ::testing::TestWithParam { video_receive_stream_->UnregisterFromTransport(); video_receive_stream_ = nullptr; } - timing_ = new VCMTiming(clock_, fake_call_.trials()); + timing_ = new VCMTiming(&env_.clock(), env_.field_trials()); video_receive_stream_ = std::make_unique( - time_controller_.GetTaskQueueFactory(), &fake_call_, - kDefaultNumCpuCores, &packet_router_, config_.Copy(), &call_stats_, - clock_, absl::WrapUnique(timing_), &nack_periodic_processor_, - UseMetronome() ? &decode_sync_ : nullptr, nullptr); + env_, &fake_call_, kDefaultNumCpuCores, &packet_router_, + config_.Copy(), &call_stats_, absl::WrapUnique(timing_), + &nack_periodic_processor_, + UseMetronome() ? &decode_sync_ : nullptr); video_receive_stream_->RegisterWithTransport( &rtp_stream_receiver_controller_); if (state) @@ -270,7 +273,7 @@ class VideoReceiveStream2Test : public ::testing::TestWithParam { protected: GlobalSimulatedTimeController time_controller_; - Clock* const clock_; + Environment env_; NackPeriodicProcessor nack_periodic_processor_; testing::NiceMock mock_h264_decoder_factory_; VideoReceiveStreamInterface::Config config_; @@ -542,16 +545,16 @@ TEST_P(VideoReceiveStream2Test, RenderedFrameUpdatesGetSources) { info.set_csrcs({kCsrc}); info.set_rtp_timestamp(kRtpTimestamp); - info.set_receive_time(clock_->CurrentTime() - TimeDelta::Millis(5000)); + info.set_receive_time(env_.clock().CurrentTime() - TimeDelta::Millis(5000)); infos.push_back(info); - info.set_receive_time(clock_->CurrentTime() - TimeDelta::Millis(3000)); + info.set_receive_time(env_.clock().CurrentTime() - TimeDelta::Millis(3000)); infos.push_back(info); - info.set_receive_time(clock_->CurrentTime() - TimeDelta::Millis(2000)); + info.set_receive_time(env_.clock().CurrentTime() - TimeDelta::Millis(2000)); infos.push_back(info); - info.set_receive_time(clock_->CurrentTime() - TimeDelta::Millis(1000)); + info.set_receive_time(env_.clock().CurrentTime() - TimeDelta::Millis(1000)); infos.push_back(info); packet_infos = RtpPacketInfos(std::move(infos)); @@ -563,12 +566,12 @@ TEST_P(VideoReceiveStream2Test, RenderedFrameUpdatesGetSources) { EXPECT_THAT(video_receive_stream_->GetSources(), IsEmpty()); // Render one video frame. - Timestamp timestamp_min = clock_->CurrentTime(); + Timestamp timestamp_min = env_.clock().CurrentTime(); video_receive_stream_->OnCompleteFrame(std::move(test_frame)); // Verify that the per-packet information is passed to the renderer. EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrameWith(PacketInfos(ElementsAreArray(packet_infos)))); - Timestamp timestamp_max = clock_->CurrentTime(); + Timestamp timestamp_max = env_.clock().CurrentTime(); // Verify that the per-packet information also updates `GetSources()`. std::vector sources = video_receive_stream_->GetSources(); @@ -813,15 +816,15 @@ TEST_P(VideoReceiveStream2Test, FramesScheduledInOrder) { EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(RtpTimestampForFrame(2)), _)) .Times(1); - key_frame->SetReceivedTime(clock_->CurrentTime().ms()); + key_frame->SetReceivedTime(env_.clock().CurrentTime().ms()); video_receive_stream_->OnCompleteFrame(std::move(key_frame)); EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame()); - delta_frame2->SetReceivedTime(clock_->CurrentTime().ms()); + delta_frame2->SetReceivedTime(env_.clock().CurrentTime().ms()); video_receive_stream_->OnCompleteFrame(std::move(delta_frame2)); EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), DidNotReceiveFrame()); // `delta_frame1` arrives late. - delta_frame1->SetReceivedTime(clock_->CurrentTime().ms()); + delta_frame1->SetReceivedTime(env_.clock().CurrentTime().ms()); video_receive_stream_->OnCompleteFrame(std::move(delta_frame1)); EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame()); EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay * 2), RenderedFrame()); @@ -854,7 +857,7 @@ TEST_P(VideoReceiveStream2Test, WaitsforAllSpatialLayers) { // No decodes should be called until `sl2` is received. EXPECT_CALL(mock_decoder_, Decode(_, _)).Times(0); - sl0->SetReceivedTime(clock_->CurrentTime().ms()); + sl0->SetReceivedTime(env_.clock().CurrentTime().ms()); video_receive_stream_->OnCompleteFrame(std::move(sl0)); EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), DidNotReceiveFrame()); @@ -984,7 +987,7 @@ TEST_P(VideoReceiveStream2Test, RtpTimestampWrapAround) { .Id(0) .PayloadType(99) .Time(kBaseRtp) - .ReceivedTime(clock_->CurrentTime()) + .ReceivedTime(env_.clock().CurrentTime()) .AsLast() .Build()); EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame()); @@ -994,7 +997,7 @@ TEST_P(VideoReceiveStream2Test, RtpTimestampWrapAround) { .Id(1) .PayloadType(99) .Time(kBaseRtp + k30FpsRtpTimestampDelta) - .ReceivedTime(clock_->CurrentTime()) + .ReceivedTime(env_.clock().CurrentTime()) .AsLast() .Build()); EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame()); @@ -1014,7 +1017,7 @@ TEST_P(VideoReceiveStream2Test, RtpTimestampWrapAround) { .Id(2) .PayloadType(99) .Time(kWrapAroundRtp) - .ReceivedTime(clock_->CurrentTime()) + .ReceivedTime(env_.clock().CurrentTime()) .AsLast() .Build()); EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kWrapAroundRtp), _)) @@ -1067,10 +1070,11 @@ TEST_P(VideoReceiveStream2Test, PoorConnectionWithFpsChangeDuringLostFrame) { // 2 second of frames at 15 fps, and then a keyframe. time_controller_.AdvanceTime(k30FpsDelay); - Timestamp send_30fps_end_time = clock_->CurrentTime() + TimeDelta::Seconds(2); + Timestamp send_30fps_end_time = + env_.clock().CurrentTime() + TimeDelta::Seconds(2); int id = 3; EXPECT_CALL(mock_transport_, SendRtcp).Times(AnyNumber()); - while (clock_->CurrentTime() < send_30fps_end_time) { + while (env_.clock().CurrentTime() < send_30fps_end_time) { ++id; video_receive_stream_->OnCompleteFrame( test::FakeFrameBuilder() @@ -1085,8 +1089,9 @@ TEST_P(VideoReceiveStream2Test, PoorConnectionWithFpsChangeDuringLostFrame) { Eq(absl::nullopt)); } uint32_t current_rtp = RtpTimestampForFrame(id); - Timestamp send_15fps_end_time = clock_->CurrentTime() + TimeDelta::Seconds(2); - while (clock_->CurrentTime() < send_15fps_end_time) { + Timestamp send_15fps_end_time = + env_.clock().CurrentTime() + TimeDelta::Seconds(2); + while (env_.clock().CurrentTime() < send_15fps_end_time) { ++id; current_rtp += k15FpsRtpTimestampDelta; video_receive_stream_->OnCompleteFrame( @@ -1094,7 +1099,7 @@ TEST_P(VideoReceiveStream2Test, PoorConnectionWithFpsChangeDuringLostFrame) { .Id(id) .PayloadType(99) .Time(current_rtp) - .ReceivedTime(clock_->CurrentTime()) + .ReceivedTime(env_.clock().CurrentTime()) .Refs({id - 1}) .AsLast() .Build()); @@ -1112,7 +1117,7 @@ TEST_P(VideoReceiveStream2Test, PoorConnectionWithFpsChangeDuringLostFrame) { .Id(id) .PayloadType(99) .Time(current_rtp) - .ReceivedTime(clock_->CurrentTime() + kKeyframeDelay) + .ReceivedTime(env_.clock().CurrentTime() + kKeyframeDelay) .AsLast() .Build()); // If the framerate was not updated to be 15fps from the frames that arrived @@ -1166,7 +1171,7 @@ TEST_P(VideoReceiveStream2Test, StreamShouldNotTimeoutWhileWaitingForFrame) { .Id(121) .PayloadType(99) .Time(late_decode_rtp) - .ReceivedTime(clock_->CurrentTime()) + .ReceivedTime(env_.clock().CurrentTime()) .AsLast() .Build()); EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Millis(100),