admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Send VideoLayersAllocation with resolution if number of spatial layers

Browse Source 
increase.

VP9 and other codecs can in theory add spatial layers without a key
frame.

Bug: webrtc:12000
Change-Id: I27461af2e34c855203a130e400a6aa01144d3cf7
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/198781
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Commit-Queue: Per Kjellander <perkj@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32883}
This commit is contained in:
Per Kjellander 2020-12-22 16:12:15 +01:00 committed by Commit Bot
parent 84f3c56016
commit dbf95493ec
3 changed files with 95 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
modules/rtp_rtcp/source/rtp_sender_video.cc
View File

@ -153,7 +153,7 @@ RTPSenderVideo::RTPSenderVideo(const Config& config)
: (kRetransmitBaseLayer | kConditionallyRetransmitHigherLayers)),
last_rotation_(kVideoRotation_0),
transmit_color_space_next_frame_(false),
send_allocation_(false),
send_allocation_(SendVideoLayersAllocation::kDontSend),
current_playout_delay_{-1, -1},
playout_delay_pending_(false),
forced_playout_delay_(LoadVideoPlayoutDelayOverride(config.field_trials)),
@ -293,8 +293,13 @@ void RTPSenderVideo::SetVideoLayersAllocationAfterTransformation(
void RTPSenderVideo::SetVideoLayersAllocationInternal(
VideoLayersAllocation allocation) {
RTC_DCHECK_RUNS_SERIALIZED(&send_checker_);
if (!allocation_ || allocation.active_spatial_layers.size() >
allocation_->active_spatial_layers.size()) {
send_allocation_ = SendVideoLayersAllocation::kSendWithResolution;
} else if (send_allocation_ == SendVideoLayersAllocation::kDontSend) {
send_allocation_ = SendVideoLayersAllocation::kSendWithoutResolution;
}
allocation_ = std::move(allocation);
send_allocation_ = true;
}
void RTPSenderVideo::AddRtpHeaderExtensions(
@ -433,16 +438,14 @@ void RTPSenderVideo::AddRtpHeaderExtensions(
}
}
if (first_packet && send_allocation_) {
if (video_header.frame_type == VideoFrameType::kVideoFrameKey) {
packet->SetExtension<RtpVideoLayersAllocationExtension>(
allocation_.value());
} else if (PacketWillLikelyBeRequestedForRestransmitionIfLost(
video_header)) {
VideoLayersAllocation allocation = allocation_.value();
allocation.resolution_and_frame_rate_is_valid = false;
packet->SetExtension<RtpVideoLayersAllocationExtension>(allocation);
}
if (first_packet &&
send_allocation_ != SendVideoLayersAllocation::kDontSend &&
(video_header.frame_type == VideoFrameType::kVideoFrameKey ||
PacketWillLikelyBeRequestedForRestransmitionIfLost(video_header))) {
VideoLayersAllocation allocation = allocation_.value();
allocation.resolution_and_frame_rate_is_valid =
send_allocation_ == SendVideoLayersAllocation::kSendWithResolution;
packet->SetExtension<RtpVideoLayersAllocationExtension>(allocation);
}
}
@ -481,7 +484,7 @@ bool RTPSenderVideo::SendVideo(
}
if (allocation_) {
// Send the bitrate allocation on every key frame.
send_allocation_ = true;
send_allocation_ = SendVideoLayersAllocation::kSendWithResolution;
}
}
@ -706,7 +709,7 @@ bool RTPSenderVideo::SendVideo(
// This frame will likely be delivered, no need to populate playout
// delay extensions until it changes again.
playout_delay_pending_ = false;
send_allocation_ = false;
send_allocation_ = SendVideoLayersAllocation::kDontSend;
}
TRACE_EVENT_ASYNC_END1("webrtc", "Video", capture_time_ms, "timestamp",

8
modules/rtp_rtcp/source/rtp_sender_video.h
View File

@ -159,6 +159,12 @@ class RTPSenderVideo {
int64_t last_frame_time_ms;
};
enum class SendVideoLayersAllocation {
kSendWithResolution,
kSendWithoutResolution,
kDontSend
};
void SetVideoStructureInternal(
const FrameDependencyStructure* video_structure);
void SetVideoLayersAllocationInternal(VideoLayersAllocation allocation);
@ -202,7 +208,7 @@ class RTPSenderVideo {
absl::optional<VideoLayersAllocation> allocation_
RTC_GUARDED_BY(send_checker_);
// Flag indicating if we should send |allocation_|.
bool send_allocation_ RTC_GUARDED_BY(send_checker_);
SendVideoLayersAllocation send_allocation_ RTC_GUARDED_BY(send_checker_);
// Current target playout delay.
VideoPlayoutDelay current_playout_delay_ RTC_GUARDED_BY(send_checker_);

80
modules/rtp_rtcp/source/rtp_sender_video_unittest.cc
View File

@ -862,7 +862,7 @@ TEST_P(RtpSenderVideoTest, VideoLayersAllocationWithResolutionSentOnKeyFrames) {
}
TEST_P(RtpSenderVideoTest,
VideoLayersAllocationWithoutResolutionSentOnDeltaFrames) {
VideoLayersAllocationWithoutResolutionSentOnDeltaWhenUpdated) {
const size_t kFrameSize = 100;
uint8_t kFrame[kFrameSize];
rtp_module_->RegisterRtpHeaderExtension(
@ -876,14 +876,28 @@ TEST_P(RtpSenderVideoTest,
allocation.resolution_and_frame_rate_is_valid = true;
layer.target_bitrate_per_temporal_layer.push_back(
DataRate::KilobitsPerSec(50));
allocation.active_spatial_layers.push_back(layer);
rtp_sender_video_->SetVideoLayersAllocation(allocation);
RTPVideoHeader hdr;
hdr.frame_type = VideoFrameType::kVideoFrameKey;
rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
kDefaultExpectedRetransmissionTimeMs);
EXPECT_TRUE(transport_.last_sent_packet()
.HasExtension<RtpVideoLayersAllocationExtension>());
// No allocation sent on delta frame unless it has been updated.
hdr.frame_type = VideoFrameType::kVideoFrameDelta;
rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
kDefaultExpectedRetransmissionTimeMs);
EXPECT_FALSE(transport_.last_sent_packet()
.HasExtension<RtpVideoLayersAllocationExtension>());
// Update the allocation.
rtp_sender_video_->SetVideoLayersAllocation(allocation);
rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
kDefaultExpectedRetransmissionTimeMs);
VideoLayersAllocation sent_allocation;
EXPECT_TRUE(
transport_.last_sent_packet()
@ -895,6 +909,52 @@ TEST_P(RtpSenderVideoTest,
SizeIs(1));
}
TEST_P(RtpSenderVideoTest,
VideoLayersAllocationWithResolutionSentOnDeltaWhenSpatialLayerAdded) {
const size_t kFrameSize = 100;
uint8_t kFrame[kFrameSize];
rtp_module_->RegisterRtpHeaderExtension(
RtpVideoLayersAllocationExtension::kUri,
kVideoLayersAllocationExtensionId);
VideoLayersAllocation allocation;
allocation.resolution_and_frame_rate_is_valid = true;
VideoLayersAllocation::SpatialLayer layer;
layer.width = 360;
layer.height = 180;
layer.spatial_id = 0;
layer.target_bitrate_per_temporal_layer.push_back(
DataRate::KilobitsPerSec(50));
allocation.active_spatial_layers.push_back(layer);
rtp_sender_video_->SetVideoLayersAllocation(allocation);
RTPVideoHeader hdr;
hdr.frame_type = VideoFrameType::kVideoFrameKey;
rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
kDefaultExpectedRetransmissionTimeMs);
ASSERT_TRUE(transport_.last_sent_packet()
.HasExtension<RtpVideoLayersAllocationExtension>());
// Update the allocation.
layer.width = 640;
layer.height = 320;
layer.spatial_id = 1;
layer.target_bitrate_per_temporal_layer.push_back(
DataRate::KilobitsPerSec(100));
allocation.active_spatial_layers.push_back(layer);
rtp_sender_video_->SetVideoLayersAllocation(allocation);
hdr.frame_type = VideoFrameType::kVideoFrameDelta;
rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
kDefaultExpectedRetransmissionTimeMs);
VideoLayersAllocation sent_allocation;
EXPECT_TRUE(
transport_.last_sent_packet()
.GetExtension<RtpVideoLayersAllocationExtension>(&sent_allocation));
EXPECT_THAT(sent_allocation.active_spatial_layers, SizeIs(2));
EXPECT_TRUE(sent_allocation.resolution_and_frame_rate_is_valid);
}
TEST_P(RtpSenderVideoTest, VideoLayersAllocationSentOnDeltaFramesOnlyOnUpdate) {
const size_t kFrameSize = 100;
uint8_t kFrame[kFrameSize];
@ -904,6 +964,8 @@ TEST_P(RtpSenderVideoTest, VideoLayersAllocationSentOnDeltaFramesOnlyOnUpdate) {
VideoLayersAllocation allocation;
VideoLayersAllocation::SpatialLayer layer;
layer.width = 360;
layer.height = 180;
layer.target_bitrate_per_temporal_layer.push_back(
DataRate::KilobitsPerSec(50));
allocation.active_spatial_layers.push_back(layer);
@ -943,7 +1005,10 @@ TEST_P(RtpSenderVideoTest, VideoLayersAllocationNotSentOnHigherTemporalLayers) {
kVideoLayersAllocationExtensionId);
VideoLayersAllocation allocation;
allocation.resolution_and_frame_rate_is_valid = true;
VideoLayersAllocation::SpatialLayer layer;
layer.width = 360;
layer.height = 180;
layer.target_bitrate_per_temporal_layer.push_back(
DataRate::KilobitsPerSec(50));
allocation.active_spatial_layers.push_back(layer);
@ -957,18 +1022,15 @@ TEST_P(RtpSenderVideoTest, VideoLayersAllocationNotSentOnHigherTemporalLayers) {
rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
kDefaultExpectedRetransmissionTimeMs);
VideoLayersAllocation sent_allocation;
EXPECT_FALSE(
transport_.last_sent_packet()
.GetExtension<RtpVideoLayersAllocationExtension>(&sent_allocation));
EXPECT_FALSE(transport_.last_sent_packet()
.HasExtension<RtpVideoLayersAllocationExtension>());
// Send a delta frame on tl0.
vp8_header.temporalIdx = 0;
rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
kDefaultExpectedRetransmissionTimeMs);
EXPECT_TRUE(
transport_.last_sent_packet()
.GetExtension<RtpVideoLayersAllocationExtension>(&sent_allocation));
EXPECT_TRUE(transport_.last_sent_packet()
.HasExtension<RtpVideoLayersAllocationExtension>());
}
TEST_P(RtpSenderVideoTest, AbsoluteCaptureTime) {