diff --git a/call/rtp_payload_params.cc b/call/rtp_payload_params.cc index ddd44709e5..470d96acbb 100644 --- a/call/rtp_payload_params.cc +++ b/call/rtp_payload_params.cc @@ -398,6 +398,15 @@ void RtpPayloadParams::Vp8ToGeneric(const CodecSpecificInfoVP8& vp8_info, generic.spatial_index = spatial_index; generic.temporal_index = temporal_index; + // Generate decode target indications. + RTC_DCHECK_LT(temporal_index, kMaxTemporalStreams); + generic.decode_target_indications.resize(kMaxTemporalStreams); + auto it = std::fill_n(generic.decode_target_indications.begin(), + temporal_index, DecodeTargetIndication::kNotPresent); + std::fill(it, generic.decode_target_indications.end(), + DecodeTargetIndication::kSwitch); + + // Frame dependencies. if (vp8_info.useExplicitDependencies) { SetDependenciesVp8New(vp8_info, shared_frame_id, is_keyframe, vp8_header.layerSync, &generic); @@ -406,6 +415,15 @@ void RtpPayloadParams::Vp8ToGeneric(const CodecSpecificInfoVP8& vp8_info, spatial_index, temporal_index, vp8_header.layerSync, &generic); } + + // Calculate chains. + generic.chain_diffs = { + (is_keyframe || chain_last_frame_id_[0] < 0) + ? 0 + : static_cast(shared_frame_id - chain_last_frame_id_[0])}; + if (temporal_index == 0) { + chain_last_frame_id_[0] = shared_frame_id; + } } FrameDependencyStructure RtpPayloadParams::MinimalisticStructure( diff --git a/call/rtp_payload_params_unittest.cc b/call/rtp_payload_params_unittest.cc index b155c4c204..169a82d198 100644 --- a/call/rtp_payload_params_unittest.cc +++ b/call/rtp_payload_params_unittest.cc @@ -31,14 +31,20 @@ #include "test/gtest.h" #include "test/scoped_key_value_config.h" +namespace webrtc { +namespace { + +using ::testing::AllOf; using ::testing::Each; using ::testing::ElementsAre; using ::testing::Eq; +using ::testing::Field; using ::testing::IsEmpty; +using ::testing::Optional; using ::testing::SizeIs; -namespace webrtc { -namespace { +using GenericDescriptorInfo = RTPVideoHeader::GenericDescriptorInfo; + const uint32_t kSsrc1 = 12345; const uint32_t kSsrc2 = 23456; const int16_t kPictureId = 123; @@ -47,7 +53,6 @@ const uint8_t kTemporalIdx = 1; const int16_t kInitialPictureId1 = 222; const int16_t kInitialTl0PicIdx1 = 99; const int64_t kDontCare = 0; -} // namespace TEST(RtpPayloadParamsTest, InfoMappedToRtpVideoHeader_Vp8) { RtpPayloadState state2; @@ -193,6 +198,87 @@ TEST(RtpPayloadParamsTest, PictureIdWraps) { EXPECT_EQ(kInitialTl0PicIdx1, params.state().tl0_pic_idx); } +TEST(RtpPayloadParamsTest, CreatesGenericDescriptorForVp8) { + constexpr auto kSwitch = DecodeTargetIndication::kSwitch; + constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent; + + RtpPayloadState state; + RtpPayloadParams params(kSsrc1, &state, FieldTrialBasedConfig()); + + EncodedImage key_frame_image; + key_frame_image._frameType = VideoFrameType::kVideoFrameKey; + CodecSpecificInfo key_frame_info; + key_frame_info.codecType = kVideoCodecVP8; + key_frame_info.codecSpecific.VP8.temporalIdx = 0; + RTPVideoHeader key_frame_header = params.GetRtpVideoHeader( + key_frame_image, &key_frame_info, /*shared_frame_id=*/123); + + EncodedImage delta_t1_image; + delta_t1_image._frameType = VideoFrameType::kVideoFrameDelta; + CodecSpecificInfo delta_t1_info; + delta_t1_info.codecType = kVideoCodecVP8; + delta_t1_info.codecSpecific.VP8.temporalIdx = 1; + RTPVideoHeader delta_t1_header = params.GetRtpVideoHeader( + delta_t1_image, &delta_t1_info, /*shared_frame_id=*/124); + + EncodedImage delta_t0_image; + delta_t0_image._frameType = VideoFrameType::kVideoFrameDelta; + CodecSpecificInfo delta_t0_info; + delta_t0_info.codecType = kVideoCodecVP8; + delta_t0_info.codecSpecific.VP8.temporalIdx = 0; + RTPVideoHeader delta_t0_header = params.GetRtpVideoHeader( + delta_t0_image, &delta_t0_info, /*shared_frame_id=*/125); + + EXPECT_THAT( + key_frame_header, + AllOf(Field(&RTPVideoHeader::codec, kVideoCodecVP8), + Field(&RTPVideoHeader::frame_type, VideoFrameType::kVideoFrameKey), + Field(&RTPVideoHeader::generic, + Optional(AllOf( + Field(&GenericDescriptorInfo::frame_id, 123), + Field(&GenericDescriptorInfo::spatial_index, 0), + Field(&GenericDescriptorInfo::temporal_index, 0), + Field(&GenericDescriptorInfo::decode_target_indications, + ElementsAre(kSwitch, kSwitch, kSwitch, kSwitch)), + Field(&GenericDescriptorInfo::dependencies, IsEmpty()), + Field(&GenericDescriptorInfo::chain_diffs, + ElementsAre(0))))))); + + EXPECT_THAT( + delta_t1_header, + AllOf( + Field(&RTPVideoHeader::codec, kVideoCodecVP8), + Field(&RTPVideoHeader::frame_type, VideoFrameType::kVideoFrameDelta), + Field( + &RTPVideoHeader::generic, + Optional(AllOf( + Field(&GenericDescriptorInfo::frame_id, 124), + Field(&GenericDescriptorInfo::spatial_index, 0), + Field(&GenericDescriptorInfo::temporal_index, 1), + Field(&GenericDescriptorInfo::decode_target_indications, + ElementsAre(kNotPresent, kSwitch, kSwitch, kSwitch)), + Field(&GenericDescriptorInfo::dependencies, ElementsAre(123)), + Field(&GenericDescriptorInfo::chain_diffs, + ElementsAre(1))))))); + + EXPECT_THAT( + delta_t0_header, + AllOf( + Field(&RTPVideoHeader::codec, kVideoCodecVP8), + Field(&RTPVideoHeader::frame_type, VideoFrameType::kVideoFrameDelta), + Field( + &RTPVideoHeader::generic, + Optional(AllOf( + Field(&GenericDescriptorInfo::frame_id, 125), + Field(&GenericDescriptorInfo::spatial_index, 0), + Field(&GenericDescriptorInfo::temporal_index, 0), + Field(&GenericDescriptorInfo::decode_target_indications, + ElementsAre(kSwitch, kSwitch, kSwitch, kSwitch)), + Field(&GenericDescriptorInfo::dependencies, ElementsAre(123)), + Field(&GenericDescriptorInfo::chain_diffs, + ElementsAre(2))))))); +} + TEST(RtpPayloadParamsTest, Tl0PicIdxUpdatedForVp8) { RtpPayloadState state; state.picture_id = kInitialPictureId1; @@ -967,4 +1053,5 @@ TEST_F(RtpPayloadParamsH264ToGenericTest, FrameIdGaps) { ConvertAndCheck(1, 20, VideoFrameType::kVideoFrameDelta, kNoSync, {10, 15}); } +} // namespace } // namespace webrtc diff --git a/call/rtp_video_sender.cc b/call/rtp_video_sender.cc index 2a87b1b6f8..8dbb2267a5 100644 --- a/call/rtp_video_sender.cc +++ b/call/rtp_video_sender.cc @@ -585,12 +585,23 @@ EncodedImageCallback::Result RtpVideoSender::OnEncodedImage( } if (IsFirstFrameOfACodedVideoSequence(encoded_image, codec_specific_info)) { - // If encoder adapter produce FrameDependencyStructure, pass it so that - // dependency descriptor rtp header extension can be used. - // If not supported, disable using dependency descriptor by passing nullptr. + // In order to use the dependency descriptor RTP header extension: + // - Pass along any `FrameDependencyStructure` templates produced by the + // encoder adapter. + // - If none were produced the `RtpPayloadParams::*ToGeneric` for the + // particular codec have simulated a dependency structure, so provide a + // minimal set of templates. + // - Otherwise, don't pass along any templates at all which will disable + // the generation of a dependency descriptor. RTPSenderVideo& sender_video = *rtp_streams_[stream_index].sender_video; if (codec_specific_info && codec_specific_info->template_structure) { sender_video.SetVideoStructure(&*codec_specific_info->template_structure); + } else if (codec_specific_info && + codec_specific_info->codecType == kVideoCodecVP8) { + FrameDependencyStructure structure = + RtpPayloadParams::MinimalisticStructure(/*num_spatial_layers=*/1, + kMaxTemporalStreams); + sender_video.SetVideoStructure(&structure); } else if (codec_specific_info && codec_specific_info->codecType == kVideoCodecVP9) { const CodecSpecificInfoVP9& vp9 = codec_specific_info->codecSpecific.VP9; diff --git a/call/rtp_video_sender_unittest.cc b/call/rtp_video_sender_unittest.cc index 8c69d1b226..e59ea745a0 100644 --- a/call/rtp_video_sender_unittest.cc +++ b/call/rtp_video_sender_unittest.cc @@ -34,13 +34,14 @@ #include "video/send_delay_stats.h" #include "video/send_statistics_proxy.h" +namespace webrtc { +namespace { + using ::testing::_; using ::testing::NiceMock; using ::testing::SaveArg; using ::testing::SizeIs; -namespace webrtc { -namespace { const int8_t kPayloadType = 96; const uint32_t kSsrc1 = 12345; const uint32_t kSsrc2 = 23456; @@ -744,6 +745,53 @@ TEST(RtpVideoSenderTest, SupportsDependencyDescriptor) { sent_packets.back().HasExtension()); } +TEST(RtpVideoSenderTest, + SupportsDependencyDescriptorForVp8NotProvidedByEncoder) { + constexpr uint8_t kPayload[1] = {'a'}; + RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}); + RtpHeaderExtensionMap extensions; + extensions.Register( + kDependencyDescriptorExtensionId); + std::vector sent_packets; + ON_CALL(test.transport(), SendRtp) + .WillByDefault( + [&](const uint8_t* packet, size_t length, const PacketOptions&) { + EXPECT_TRUE( + sent_packets.emplace_back(&extensions).Parse(packet, length)); + return true; + }); + test.SetActive(true); + + EncodedImage key_frame_image; + key_frame_image._frameType = VideoFrameType::kVideoFrameKey; + key_frame_image.SetEncodedData( + EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); + CodecSpecificInfo key_frame_info; + key_frame_info.codecType = VideoCodecType::kVideoCodecVP8; + ASSERT_EQ( + test.router()->OnEncodedImage(key_frame_image, &key_frame_info).error, + EncodedImageCallback::Result::OK); + + EncodedImage delta_image; + delta_image._frameType = VideoFrameType::kVideoFrameDelta; + delta_image.SetEncodedData( + EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); + CodecSpecificInfo delta_info; + delta_info.codecType = VideoCodecType::kVideoCodecVP8; + ASSERT_EQ(test.router()->OnEncodedImage(delta_image, &delta_info).error, + EncodedImageCallback::Result::OK); + + test.AdvanceTime(TimeDelta::Millis(123)); + + DependencyDescriptor key_frame_dd; + DependencyDescriptor delta_dd; + ASSERT_THAT(sent_packets, SizeIs(2)); + EXPECT_TRUE(sent_packets[0].GetExtension( + /*structure=*/nullptr, &key_frame_dd)); + EXPECT_TRUE(sent_packets[1].GetExtension( + key_frame_dd.attached_structure.get(), &delta_dd)); +} + TEST(RtpVideoSenderTest, SupportsDependencyDescriptorForVp9) { RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}); test.SetActive(true); @@ -958,64 +1006,6 @@ TEST(RtpVideoSenderTest, SupportsStoppingUsingDependencyDescriptor) { sent_packets.back().HasExtension()); } -TEST(RtpVideoSenderTest, - SupportsStoppingUsingDependencyDescriptorForVp8Simulcast) { - RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {}, kPayloadType, {}); - test.SetActive(true); - - RtpHeaderExtensionMap extensions; - extensions.Register( - kDependencyDescriptorExtensionId); - std::vector sent_packets; - ON_CALL(test.transport(), SendRtp) - .WillByDefault([&](const uint8_t* packet, size_t length, - const PacketOptions& options) { - sent_packets.emplace_back(&extensions); - EXPECT_TRUE(sent_packets.back().Parse(packet, length)); - return true; - }); - - const uint8_t kPayload[1] = {'a'}; - EncodedImage encoded_image; - encoded_image.SetTimestamp(1); - encoded_image.capture_time_ms_ = 2; - encoded_image.SetEncodedData( - EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); - // VP8 simulcast uses spatial index to communicate simulcast stream. - encoded_image.SetSpatialIndex(1); - - CodecSpecificInfo codec_specific; - codec_specific.codecType = VideoCodecType::kVideoCodecVP8; - codec_specific.template_structure.emplace(); - codec_specific.template_structure->num_decode_targets = 1; - codec_specific.template_structure->templates = { - FrameDependencyTemplate().T(0).Dtis("S")}; - - // Send two tiny images, mapping to single RTP packets. - // Send in a key frame. - encoded_image._frameType = VideoFrameType::kVideoFrameKey; - codec_specific.generic_frame_info = - GenericFrameInfo::Builder().T(0).Dtis("S").Build(); - codec_specific.generic_frame_info->encoder_buffers = {{0, false, true}}; - EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, - EncodedImageCallback::Result::OK); - test.AdvanceTime(TimeDelta::Millis(33)); - ASSERT_THAT(sent_packets, SizeIs(1)); - EXPECT_TRUE( - sent_packets.back().HasExtension()); - - // Send in a new key frame without the support for the dependency descriptor. - encoded_image._frameType = VideoFrameType::kVideoFrameKey; - codec_specific.template_structure = absl::nullopt; - codec_specific.generic_frame_info = absl::nullopt; - EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, - EncodedImageCallback::Result::OK); - test.AdvanceTime(TimeDelta::Millis(33)); - ASSERT_THAT(sent_packets, SizeIs(2)); - EXPECT_FALSE( - sent_packets.back().HasExtension()); -} - TEST(RtpVideoSenderTest, CanSetZeroBitrate) { RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {}); test.router()->OnBitrateUpdated(CreateBitrateAllocationUpdate(0),