diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc index 2499d35ccd..934be824a4 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -153,7 +153,7 @@ RTPSenderVideo::RTPSenderVideo(const Config& config) : (kRetransmitBaseLayer | kConditionallyRetransmitHigherLayers)), last_rotation_(kVideoRotation_0), transmit_color_space_next_frame_(false), - send_allocation_(false), + send_allocation_(SendVideoLayersAllocation::kDontSend), current_playout_delay_{-1, -1}, playout_delay_pending_(false), forced_playout_delay_(LoadVideoPlayoutDelayOverride(config.field_trials)), @@ -293,8 +293,13 @@ void RTPSenderVideo::SetVideoLayersAllocationAfterTransformation( void RTPSenderVideo::SetVideoLayersAllocationInternal( VideoLayersAllocation allocation) { RTC_DCHECK_RUNS_SERIALIZED(&send_checker_); + if (!allocation_ || allocation.active_spatial_layers.size() > + allocation_->active_spatial_layers.size()) { + send_allocation_ = SendVideoLayersAllocation::kSendWithResolution; + } else if (send_allocation_ == SendVideoLayersAllocation::kDontSend) { + send_allocation_ = SendVideoLayersAllocation::kSendWithoutResolution; + } allocation_ = std::move(allocation); - send_allocation_ = true; } void RTPSenderVideo::AddRtpHeaderExtensions( @@ -433,16 +438,14 @@ void RTPSenderVideo::AddRtpHeaderExtensions( } } - if (first_packet && send_allocation_) { - if (video_header.frame_type == VideoFrameType::kVideoFrameKey) { - packet->SetExtension( - allocation_.value()); - } else if (PacketWillLikelyBeRequestedForRestransmitionIfLost( - video_header)) { - VideoLayersAllocation allocation = allocation_.value(); - allocation.resolution_and_frame_rate_is_valid = false; - packet->SetExtension(allocation); - } + if (first_packet && + send_allocation_ != SendVideoLayersAllocation::kDontSend && + (video_header.frame_type == VideoFrameType::kVideoFrameKey || + PacketWillLikelyBeRequestedForRestransmitionIfLost(video_header))) { + VideoLayersAllocation allocation = allocation_.value(); + allocation.resolution_and_frame_rate_is_valid = + send_allocation_ == SendVideoLayersAllocation::kSendWithResolution; + packet->SetExtension(allocation); } } @@ -481,7 +484,7 @@ bool RTPSenderVideo::SendVideo( } if (allocation_) { // Send the bitrate allocation on every key frame. - send_allocation_ = true; + send_allocation_ = SendVideoLayersAllocation::kSendWithResolution; } } @@ -706,7 +709,7 @@ bool RTPSenderVideo::SendVideo( // This frame will likely be delivered, no need to populate playout // delay extensions until it changes again. playout_delay_pending_ = false; - send_allocation_ = false; + send_allocation_ = SendVideoLayersAllocation::kDontSend; } TRACE_EVENT_ASYNC_END1("webrtc", "Video", capture_time_ms, "timestamp", diff --git a/modules/rtp_rtcp/source/rtp_sender_video.h b/modules/rtp_rtcp/source/rtp_sender_video.h index 3f431dfec2..6e469900d6 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.h +++ b/modules/rtp_rtcp/source/rtp_sender_video.h @@ -159,6 +159,12 @@ class RTPSenderVideo { int64_t last_frame_time_ms; }; + enum class SendVideoLayersAllocation { + kSendWithResolution, + kSendWithoutResolution, + kDontSend + }; + void SetVideoStructureInternal( const FrameDependencyStructure* video_structure); void SetVideoLayersAllocationInternal(VideoLayersAllocation allocation); @@ -202,7 +208,7 @@ class RTPSenderVideo { absl::optional allocation_ RTC_GUARDED_BY(send_checker_); // Flag indicating if we should send |allocation_|. - bool send_allocation_ RTC_GUARDED_BY(send_checker_); + SendVideoLayersAllocation send_allocation_ RTC_GUARDED_BY(send_checker_); // Current target playout delay. VideoPlayoutDelay current_playout_delay_ RTC_GUARDED_BY(send_checker_); diff --git a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc index e415bad16f..55bafdc790 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc @@ -862,7 +862,7 @@ TEST_P(RtpSenderVideoTest, VideoLayersAllocationWithResolutionSentOnKeyFrames) { } TEST_P(RtpSenderVideoTest, - VideoLayersAllocationWithoutResolutionSentOnDeltaFrames) { + VideoLayersAllocationWithoutResolutionSentOnDeltaWhenUpdated) { const size_t kFrameSize = 100; uint8_t kFrame[kFrameSize]; rtp_module_->RegisterRtpHeaderExtension( @@ -876,14 +876,28 @@ TEST_P(RtpSenderVideoTest, allocation.resolution_and_frame_rate_is_valid = true; layer.target_bitrate_per_temporal_layer.push_back( DataRate::KilobitsPerSec(50)); - allocation.active_spatial_layers.push_back(layer); rtp_sender_video_->SetVideoLayersAllocation(allocation); RTPVideoHeader hdr; + hdr.frame_type = VideoFrameType::kVideoFrameKey; + rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr, + kDefaultExpectedRetransmissionTimeMs); + EXPECT_TRUE(transport_.last_sent_packet() + .HasExtension()); + + // No allocation sent on delta frame unless it has been updated. hdr.frame_type = VideoFrameType::kVideoFrameDelta; rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr, kDefaultExpectedRetransmissionTimeMs); + EXPECT_FALSE(transport_.last_sent_packet() + .HasExtension()); + + // Update the allocation. + rtp_sender_video_->SetVideoLayersAllocation(allocation); + rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr, + kDefaultExpectedRetransmissionTimeMs); + VideoLayersAllocation sent_allocation; EXPECT_TRUE( transport_.last_sent_packet() @@ -895,6 +909,52 @@ TEST_P(RtpSenderVideoTest, SizeIs(1)); } +TEST_P(RtpSenderVideoTest, + VideoLayersAllocationWithResolutionSentOnDeltaWhenSpatialLayerAdded) { + const size_t kFrameSize = 100; + uint8_t kFrame[kFrameSize]; + rtp_module_->RegisterRtpHeaderExtension( + RtpVideoLayersAllocationExtension::kUri, + kVideoLayersAllocationExtensionId); + + VideoLayersAllocation allocation; + allocation.resolution_and_frame_rate_is_valid = true; + VideoLayersAllocation::SpatialLayer layer; + layer.width = 360; + layer.height = 180; + layer.spatial_id = 0; + layer.target_bitrate_per_temporal_layer.push_back( + DataRate::KilobitsPerSec(50)); + allocation.active_spatial_layers.push_back(layer); + rtp_sender_video_->SetVideoLayersAllocation(allocation); + + RTPVideoHeader hdr; + hdr.frame_type = VideoFrameType::kVideoFrameKey; + rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr, + kDefaultExpectedRetransmissionTimeMs); + ASSERT_TRUE(transport_.last_sent_packet() + .HasExtension()); + + // Update the allocation. + layer.width = 640; + layer.height = 320; + layer.spatial_id = 1; + layer.target_bitrate_per_temporal_layer.push_back( + DataRate::KilobitsPerSec(100)); + allocation.active_spatial_layers.push_back(layer); + rtp_sender_video_->SetVideoLayersAllocation(allocation); + hdr.frame_type = VideoFrameType::kVideoFrameDelta; + rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr, + kDefaultExpectedRetransmissionTimeMs); + + VideoLayersAllocation sent_allocation; + EXPECT_TRUE( + transport_.last_sent_packet() + .GetExtension(&sent_allocation)); + EXPECT_THAT(sent_allocation.active_spatial_layers, SizeIs(2)); + EXPECT_TRUE(sent_allocation.resolution_and_frame_rate_is_valid); +} + TEST_P(RtpSenderVideoTest, VideoLayersAllocationSentOnDeltaFramesOnlyOnUpdate) { const size_t kFrameSize = 100; uint8_t kFrame[kFrameSize]; @@ -904,6 +964,8 @@ TEST_P(RtpSenderVideoTest, VideoLayersAllocationSentOnDeltaFramesOnlyOnUpdate) { VideoLayersAllocation allocation; VideoLayersAllocation::SpatialLayer layer; + layer.width = 360; + layer.height = 180; layer.target_bitrate_per_temporal_layer.push_back( DataRate::KilobitsPerSec(50)); allocation.active_spatial_layers.push_back(layer); @@ -943,7 +1005,10 @@ TEST_P(RtpSenderVideoTest, VideoLayersAllocationNotSentOnHigherTemporalLayers) { kVideoLayersAllocationExtensionId); VideoLayersAllocation allocation; + allocation.resolution_and_frame_rate_is_valid = true; VideoLayersAllocation::SpatialLayer layer; + layer.width = 360; + layer.height = 180; layer.target_bitrate_per_temporal_layer.push_back( DataRate::KilobitsPerSec(50)); allocation.active_spatial_layers.push_back(layer); @@ -957,18 +1022,15 @@ TEST_P(RtpSenderVideoTest, VideoLayersAllocationNotSentOnHigherTemporalLayers) { rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr, kDefaultExpectedRetransmissionTimeMs); - VideoLayersAllocation sent_allocation; - EXPECT_FALSE( - transport_.last_sent_packet() - .GetExtension(&sent_allocation)); + EXPECT_FALSE(transport_.last_sent_packet() + .HasExtension()); // Send a delta frame on tl0. vp8_header.temporalIdx = 0; rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr, kDefaultExpectedRetransmissionTimeMs); - EXPECT_TRUE( - transport_.last_sent_packet() - .GetExtension(&sent_allocation)); + EXPECT_TRUE(transport_.last_sent_packet() + .HasExtension()); } TEST_P(RtpSenderVideoTest, AbsoluteCaptureTime) {