From da0898dfae3b0a013ca8ad3828e9adfdc749748d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20M=C3=B6ller?= Date: Mon, 27 Aug 2018 15:33:42 +0200 Subject: [PATCH] Add spatial index to EncodedImage. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the VP8 simulcast index and VP9 spatial index formely part of CodecSpecificInfo. Bug: webrtc:9378 Change-Id: I80eafd63fbdee0a25864338196a690628b4bd3d2 Reviewed-on: https://webrtc-review.googlesource.com/83161 Commit-Queue: Niels Moller Reviewed-by: Erik Språng Reviewed-by: Sebastian Jansson Reviewed-by: Magnus Jedvert Reviewed-by: Philip Eliasson Reviewed-by: Rasmus Brandt Cr-Commit-Position: refs/heads/master@{#24485} --- call/rtp_payload_params.cc | 17 +++-- call/rtp_payload_params_unittest.cc | 13 ++-- call/rtp_video_sender.cc | 24 +++---- call/rtp_video_sender_unittest.cc | 64 +++++++++---------- media/engine/simulcast_encoder_adapter.cc | 10 ++- .../simulcast_encoder_adapter_unittest.cc | 9 ++- .../codecs/h264/h264_encoder_impl.cc | 3 +- .../multiplex/multiplex_encoder_adapter.cc | 1 - .../test/multiplex_adapter_unittest.cc | 4 +- .../codecs/test/videoprocessor.cc | 24 +++---- .../codecs/vp8/libvpx_vp8_encoder.cc | 2 +- .../codecs/vp8/test/vp8_impl_unittest.cc | 2 +- .../codecs/vp9/test/vp9_impl_unittest.cc | 6 +- modules/video_coding/codecs/vp9/vp9_impl.cc | 11 ++-- modules/video_coding/codecs/vp9/vp9_impl.h | 1 + modules/video_coding/encoded_frame.cc | 3 - modules/video_coding/generic_encoder.cc | 17 +---- .../video_coding/generic_encoder_unittest.cc | 6 +- .../include/video_codec_interface.h | 7 +- .../utility/simulcast_test_fixture_impl.cc | 12 +--- sdk/android/src/jni/androidmediaencoder.cc | 2 - sdk/android/src/jni/videoencoderwrapper.cc | 2 - test/fake_encoder.cc | 2 +- test/fake_vp8_encoder.cc | 3 +- video/send_statistics_proxy.cc | 58 ++++++++--------- video/send_statistics_proxy.h | 15 ++--- video/send_statistics_proxy_unittest.cc | 32 +++++----- video/video_receive_stream.cc | 4 -- video/video_send_stream_impl.cc | 19 +++--- video/video_send_stream_tests.cc | 2 +- 30 files changed, 160 insertions(+), 215 deletions(-) diff --git a/call/rtp_payload_params.cc b/call/rtp_payload_params.cc index 608688b396..0dbe1837c6 100644 --- a/call/rtp_payload_params.cc +++ b/call/rtp_payload_params.cc @@ -22,6 +22,7 @@ namespace webrtc { namespace { void PopulateRtpWithCodecSpecifics(const CodecSpecificInfo& info, + absl::optional spatial_index, RTPVideoHeader* rtp) { rtp->codec = info.codecType; switch (info.codecType) { @@ -31,7 +32,7 @@ void PopulateRtpWithCodecSpecifics(const CodecSpecificInfo& info, rtp->vp8().temporalIdx = info.codecSpecific.VP8.temporalIdx; rtp->vp8().layerSync = info.codecSpecific.VP8.layerSync; rtp->vp8().keyIdx = info.codecSpecific.VP8.keyIdx; - rtp->simulcastIdx = info.codecSpecific.VP8.simulcastIdx; + rtp->simulcastIdx = spatial_index.value_or(0); return; } case kVideoCodecVP9: { @@ -44,13 +45,16 @@ void PopulateRtpWithCodecSpecifics(const CodecSpecificInfo& info, vp9_header.non_ref_for_inter_layer_pred = info.codecSpecific.VP9.non_ref_for_inter_layer_pred; vp9_header.temporal_idx = info.codecSpecific.VP9.temporal_idx; - vp9_header.spatial_idx = info.codecSpecific.VP9.spatial_idx; vp9_header.temporal_up_switch = info.codecSpecific.VP9.temporal_up_switch; vp9_header.inter_layer_predicted = info.codecSpecific.VP9.inter_layer_predicted; vp9_header.gof_idx = info.codecSpecific.VP9.gof_idx; vp9_header.num_spatial_layers = info.codecSpecific.VP9.num_spatial_layers; - + if (vp9_header.num_spatial_layers > 1) { + vp9_header.spatial_idx = spatial_index.value_or(kNoSpatialIdx); + } else { + vp9_header.spatial_idx = kNoSpatialIdx; + } if (info.codecSpecific.VP9.ss_data_available) { vp9_header.spatial_layer_resolution_present = info.codecSpecific.VP9.spatial_layer_resolution_present; @@ -75,13 +79,13 @@ void PopulateRtpWithCodecSpecifics(const CodecSpecificInfo& info, auto& h264_header = rtp->video_type_header.emplace(); h264_header.packetization_mode = info.codecSpecific.H264.packetization_mode; - rtp->simulcastIdx = info.codecSpecific.H264.simulcast_idx; + rtp->simulcastIdx = spatial_index.value_or(0); return; } case kVideoCodecMultiplex: case kVideoCodecGeneric: rtp->codec = kVideoCodecGeneric; - rtp->simulcastIdx = info.codecSpecific.generic.simulcast_idx; + rtp->simulcastIdx = spatial_index.value_or(0); return; default: return; @@ -131,7 +135,8 @@ RTPVideoHeader RtpPayloadParams::GetRtpVideoHeader( int64_t shared_frame_id) { RTPVideoHeader rtp_video_header; if (codec_specific_info) { - PopulateRtpWithCodecSpecifics(*codec_specific_info, &rtp_video_header); + PopulateRtpWithCodecSpecifics(*codec_specific_info, image.SpatialIndex(), + &rtp_video_header); } rtp_video_header.rotation = image.rotation_; rtp_video_header.content_type = image.content_type_; diff --git a/call/rtp_payload_params_unittest.cc b/call/rtp_payload_params_unittest.cc index 4d0b017fa0..b0c00277e5 100644 --- a/call/rtp_payload_params_unittest.cc +++ b/call/rtp_payload_params_unittest.cc @@ -38,11 +38,11 @@ TEST(RtpPayloadParamsTest, InfoMappedToRtpVideoHeader_Vp8) { EncodedImage encoded_image; encoded_image.rotation_ = kVideoRotation_90; encoded_image.content_type_ = VideoContentType::SCREENSHARE; + encoded_image.SetSpatialIndex(1); CodecSpecificInfo codec_info; memset(&codec_info, 0, sizeof(CodecSpecificInfo)); codec_info.codecType = kVideoCodecVP8; - codec_info.codecSpecific.VP8.simulcastIdx = 1; codec_info.codecSpecific.VP8.temporalIdx = 0; codec_info.codecSpecific.VP8.keyIdx = kNoKeyIdx; codec_info.codecSpecific.VP8.layerSync = false; @@ -52,7 +52,6 @@ TEST(RtpPayloadParamsTest, InfoMappedToRtpVideoHeader_Vp8) { params.GetRtpVideoHeader(encoded_image, &codec_info, kDontCare); codec_info.codecType = kVideoCodecVP8; - codec_info.codecSpecific.VP8.simulcastIdx = 1; codec_info.codecSpecific.VP8.temporalIdx = 1; codec_info.codecSpecific.VP8.layerSync = true; @@ -79,13 +78,12 @@ TEST(RtpPayloadParamsTest, InfoMappedToRtpVideoHeader_Vp9) { EncodedImage encoded_image; encoded_image.rotation_ = kVideoRotation_90; encoded_image.content_type_ = VideoContentType::SCREENSHARE; - + encoded_image.SetSpatialIndex(0); CodecSpecificInfo codec_info; memset(&codec_info, 0, sizeof(CodecSpecificInfo)); codec_info.codecType = kVideoCodecVP9; codec_info.codecSpecific.VP9.num_spatial_layers = 3; codec_info.codecSpecific.VP9.first_frame_in_picture = true; - codec_info.codecSpecific.VP9.spatial_idx = 0; codec_info.codecSpecific.VP9.temporal_idx = 2; codec_info.codecSpecific.VP9.end_of_picture = false; @@ -100,7 +98,7 @@ TEST(RtpPayloadParamsTest, InfoMappedToRtpVideoHeader_Vp9) { EXPECT_EQ(kPictureId + 1, vp9_header.picture_id); EXPECT_EQ(kTl0PicIdx, vp9_header.tl0_pic_idx); EXPECT_EQ(vp9_header.temporal_idx, codec_info.codecSpecific.VP9.temporal_idx); - EXPECT_EQ(vp9_header.spatial_idx, codec_info.codecSpecific.VP9.spatial_idx); + EXPECT_EQ(vp9_header.spatial_idx, encoded_image.SpatialIndex()); EXPECT_EQ(vp9_header.num_spatial_layers, codec_info.codecSpecific.VP9.num_spatial_layers); EXPECT_EQ(vp9_header.end_of_picture, @@ -108,9 +106,9 @@ TEST(RtpPayloadParamsTest, InfoMappedToRtpVideoHeader_Vp9) { // Next spatial layer. codec_info.codecSpecific.VP9.first_frame_in_picture = false; - codec_info.codecSpecific.VP9.spatial_idx += 1; codec_info.codecSpecific.VP9.end_of_picture = true; + encoded_image.SetSpatialIndex(1); header = params.GetRtpVideoHeader(encoded_image, &codec_info, kDontCare); EXPECT_EQ(kVideoRotation_90, header.rotation); @@ -119,7 +117,7 @@ TEST(RtpPayloadParamsTest, InfoMappedToRtpVideoHeader_Vp9) { EXPECT_EQ(kPictureId + 1, vp9_header.picture_id); EXPECT_EQ(kTl0PicIdx, vp9_header.tl0_pic_idx); EXPECT_EQ(vp9_header.temporal_idx, codec_info.codecSpecific.VP9.temporal_idx); - EXPECT_EQ(vp9_header.spatial_idx, codec_info.codecSpecific.VP9.spatial_idx); + EXPECT_EQ(vp9_header.spatial_idx, encoded_image.SpatialIndex()); EXPECT_EQ(vp9_header.num_spatial_layers, codec_info.codecSpecific.VP9.num_spatial_layers); EXPECT_EQ(vp9_header.end_of_picture, @@ -154,7 +152,6 @@ TEST(RtpPayloadParamsTest, PictureIdIsSetForVp8) { CodecSpecificInfo codec_info; memset(&codec_info, 0, sizeof(CodecSpecificInfo)); codec_info.codecType = kVideoCodecVP8; - codec_info.codecSpecific.VP8.simulcastIdx = 0; RtpPayloadParams params(kSsrc1, &state); RTPVideoHeader header = diff --git a/call/rtp_video_sender.cc b/call/rtp_video_sender.cc index 1fbd1026f5..a2df830174 100644 --- a/call/rtp_video_sender.cc +++ b/call/rtp_video_sender.cc @@ -95,21 +95,6 @@ std::vector> CreateRtpRtcpModules( return modules; } -absl::optional GetSimulcastIdx(const CodecSpecificInfo* info) { - if (!info) - return absl::nullopt; - switch (info->codecType) { - case kVideoCodecVP8: - return absl::optional(info->codecSpecific.VP8.simulcastIdx); - case kVideoCodecH264: - return absl::optional(info->codecSpecific.H264.simulcast_idx); - case kVideoCodecMultiplex: - case kVideoCodecGeneric: - return absl::optional(info->codecSpecific.generic.simulcast_idx); - default: - return absl::nullopt; - } -} bool PayloadTypeSupportsSkippingFecPackets(const std::string& payload_name) { const VideoCodecType codecType = PayloadStringToCodecType(payload_name); if (codecType == kVideoCodecVP8 || codecType == kVideoCodecVP9) { @@ -320,7 +305,14 @@ EncodedImageCallback::Result RtpVideoSender::OnEncodedImage( return Result(Result::ERROR_SEND_FAILED); shared_frame_id_++; - size_t stream_index = GetSimulcastIdx(codec_specific_info).value_or(0); + size_t stream_index = 0; + if (codec_specific_info && + (codec_specific_info->codecType == kVideoCodecVP8 || + codec_specific_info->codecType == kVideoCodecH264 || + codec_specific_info->codecType == kVideoCodecGeneric)) { + // Map spatial index to simulcast. + stream_index = encoded_image.SpatialIndex().value_or(0); + } RTC_DCHECK_LT(stream_index, rtp_modules_.size()); RTPVideoHeader rtp_video_header = params_[stream_index].GetRtpVideoHeader( encoded_image, codec_specific_info, shared_frame_id_); diff --git a/call/rtp_video_sender_unittest.cc b/call/rtp_video_sender_unittest.cc index 9e3290de4f..bd8a6a67c6 100644 --- a/call/rtp_video_sender_unittest.cc +++ b/call/rtp_video_sender_unittest.cc @@ -169,44 +169,41 @@ TEST(RtpVideoSenderTest, SendOnOneModule) { TEST(RtpVideoSenderTest, SendSimulcastSetActive) { uint8_t payload = 'a'; - EncodedImage encoded_image; - encoded_image.SetTimestamp(1); - encoded_image.capture_time_ms_ = 2; - encoded_image._frameType = kVideoFrameKey; - encoded_image._buffer = &payload; - encoded_image._length = 1; + EncodedImage encoded_image_1; + encoded_image_1.SetTimestamp(1); + encoded_image_1.capture_time_ms_ = 2; + encoded_image_1._frameType = kVideoFrameKey; + encoded_image_1._buffer = &payload; + encoded_image_1._length = 1; RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, kPayloadType, {}); - CodecSpecificInfo codec_info_1; - memset(&codec_info_1, 0, sizeof(CodecSpecificInfo)); - codec_info_1.codecType = kVideoCodecVP8; - codec_info_1.codecSpecific.VP8.simulcastIdx = 0; + CodecSpecificInfo codec_info; + memset(&codec_info, 0, sizeof(CodecSpecificInfo)); + codec_info.codecType = kVideoCodecVP8; test.router()->SetActive(true); EXPECT_EQ(EncodedImageCallback::Result::OK, test.router() - ->OnEncodedImage(encoded_image, &codec_info_1, nullptr) + ->OnEncodedImage(encoded_image_1, &codec_info, nullptr) .error); - CodecSpecificInfo codec_info_2; - memset(&codec_info_2, 0, sizeof(CodecSpecificInfo)); - codec_info_2.codecType = kVideoCodecVP8; - codec_info_2.codecSpecific.VP8.simulcastIdx = 1; + EncodedImage encoded_image_2(encoded_image_1); + encoded_image_2.SetSpatialIndex(1); EXPECT_EQ(EncodedImageCallback::Result::OK, test.router() - ->OnEncodedImage(encoded_image, &codec_info_2, nullptr) + ->OnEncodedImage(encoded_image_2, &codec_info, nullptr) .error); // Inactive. test.router()->SetActive(false); EXPECT_NE(EncodedImageCallback::Result::OK, test.router() - ->OnEncodedImage(encoded_image, &codec_info_1, nullptr) + ->OnEncodedImage(encoded_image_1, &codec_info, nullptr) .error); EXPECT_NE(EncodedImageCallback::Result::OK, test.router() - ->OnEncodedImage(encoded_image, &codec_info_2, nullptr) + ->OnEncodedImage(encoded_image_2, &codec_info, nullptr) .error); } @@ -216,22 +213,19 @@ TEST(RtpVideoSenderTest, SendSimulcastSetActive) { // be sent if both modules are inactive. TEST(RtpVideoSenderTest, SendSimulcastSetActiveModules) { uint8_t payload = 'a'; - EncodedImage encoded_image; - encoded_image.SetTimestamp(1); - encoded_image.capture_time_ms_ = 2; - encoded_image._frameType = kVideoFrameKey; - encoded_image._buffer = &payload; - encoded_image._length = 1; + EncodedImage encoded_image_1; + encoded_image_1.SetTimestamp(1); + encoded_image_1.capture_time_ms_ = 2; + encoded_image_1._frameType = kVideoFrameKey; + encoded_image_1._buffer = &payload; + encoded_image_1._length = 1; + EncodedImage encoded_image_2(encoded_image_1); + encoded_image_2.SetSpatialIndex(1); RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, kPayloadType, {}); - CodecSpecificInfo codec_info_1; - memset(&codec_info_1, 0, sizeof(CodecSpecificInfo)); - codec_info_1.codecType = kVideoCodecVP8; - codec_info_1.codecSpecific.VP8.simulcastIdx = 0; - CodecSpecificInfo codec_info_2; - memset(&codec_info_2, 0, sizeof(CodecSpecificInfo)); - codec_info_2.codecType = kVideoCodecVP8; - codec_info_2.codecSpecific.VP8.simulcastIdx = 1; + CodecSpecificInfo codec_info; + memset(&codec_info, 0, sizeof(CodecSpecificInfo)); + codec_info.codecType = kVideoCodecVP8; // Only setting one stream to active will still set the payload router to // active and allow sending data on the active stream. @@ -239,7 +233,7 @@ TEST(RtpVideoSenderTest, SendSimulcastSetActiveModules) { test.router()->SetActiveModules(active_modules); EXPECT_EQ(EncodedImageCallback::Result::OK, test.router() - ->OnEncodedImage(encoded_image, &codec_info_1, nullptr) + ->OnEncodedImage(encoded_image_1, &codec_info, nullptr) .error); // Setting both streams to inactive will turn the payload router to @@ -250,11 +244,11 @@ TEST(RtpVideoSenderTest, SendSimulcastSetActiveModules) { // because the payload router is inactive. EXPECT_NE(EncodedImageCallback::Result::OK, test.router() - ->OnEncodedImage(encoded_image, &codec_info_1, nullptr) + ->OnEncodedImage(encoded_image_1, &codec_info, nullptr) .error); EXPECT_NE(EncodedImageCallback::Result::OK, test.router() - ->OnEncodedImage(encoded_image, &codec_info_2, nullptr) + ->OnEncodedImage(encoded_image_1, &codec_info, nullptr) .error); } diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc index 4ab83a0940..5827fc5d6e 100644 --- a/media/engine/simulcast_encoder_adapter.cc +++ b/media/engine/simulcast_encoder_adapter.cc @@ -449,16 +449,14 @@ EncodedImageCallback::Result SimulcastEncoderAdapter::OnEncodedImage( const EncodedImage& encodedImage, const CodecSpecificInfo* codecSpecificInfo, const RTPFragmentationHeader* fragmentation) { + EncodedImage stream_image(encodedImage); CodecSpecificInfo stream_codec_specific = *codecSpecificInfo; stream_codec_specific.codec_name = implementation_name_.c_str(); - if (stream_codec_specific.codecType == webrtc::kVideoCodecVP8) { - stream_codec_specific.codecSpecific.VP8.simulcastIdx = stream_idx; - } else if (stream_codec_specific.codecType == webrtc::kVideoCodecH264) { - stream_codec_specific.codecSpecific.H264.simulcast_idx = stream_idx; - } + + stream_image.SetSpatialIndex(stream_idx); return encoded_complete_callback_->OnEncodedImage( - encodedImage, &stream_codec_specific, fragmentation); + stream_image, &stream_codec_specific, fragmentation); } void SimulcastEncoderAdapter::PopulateStreamCodec( diff --git a/media/engine/simulcast_encoder_adapter_unittest.cc b/media/engine/simulcast_encoder_adapter_unittest.cc index 410b85e58b..3999840a18 100644 --- a/media/engine/simulcast_encoder_adapter_unittest.cc +++ b/media/engine/simulcast_encoder_adapter_unittest.cc @@ -339,11 +339,10 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test, const RTPFragmentationHeader* fragmentation) override { last_encoded_image_width_ = encoded_image._encodedWidth; last_encoded_image_height_ = encoded_image._encodedHeight; - if (codec_specific_info) { - last_encoded_image_simulcast_index_ = - codec_specific_info->codecSpecific.VP8.simulcastIdx; - } - return Result(Result::OK, encoded_image.Timestamp()); + last_encoded_image_simulcast_index_ = + encoded_image.SpatialIndex().value_or(-1); + + return Result(Result::OK, encoded_image._timeStamp); } bool GetLastEncodedImageInfo(int* out_width, diff --git a/modules/video_coding/codecs/h264/h264_encoder_impl.cc b/modules/video_coding/codecs/h264/h264_encoder_impl.cc index 4fc9b4f536..4effcdbc52 100644 --- a/modules/video_coding/codecs/h264/h264_encoder_impl.cc +++ b/modules/video_coding/codecs/h264/h264_encoder_impl.cc @@ -506,6 +506,7 @@ int32_t H264EncoderImpl::Encode(const VideoFrame& input_frame, : VideoContentType::UNSPECIFIED; encoded_images_[i].timing_.flags = VideoSendTiming::kInvalid; encoded_images_[i]._frameType = ConvertToVideoFrameType(info.eFrameType); + encoded_images_[i].SetSpatialIndex(configurations_[i].simulcast_idx); // Split encoded image up into fragments. This also updates // |encoded_image_|. @@ -526,8 +527,6 @@ int32_t H264EncoderImpl::Encode(const VideoFrame& input_frame, codec_specific.codecType = kVideoCodecH264; codec_specific.codecSpecific.H264.packetization_mode = packetization_mode_; - codec_specific.codecSpecific.H264.simulcast_idx = - configurations_[i].simulcast_idx; encoded_image_callback_->OnEncodedImage(encoded_images_[i], &codec_specific, &frag_header); } diff --git a/modules/video_coding/codecs/multiplex/multiplex_encoder_adapter.cc b/modules/video_coding/codecs/multiplex/multiplex_encoder_adapter.cc index 4733b3a4b6..c3a75062b5 100644 --- a/modules/video_coding/codecs/multiplex/multiplex_encoder_adapter.cc +++ b/modules/video_coding/codecs/multiplex/multiplex_encoder_adapter.cc @@ -285,7 +285,6 @@ EncodedImageCallback::Result MultiplexEncoderAdapter::OnEncodedImage( CodecSpecificInfo codec_info = *codecSpecificInfo; codec_info.codecType = kVideoCodecMultiplex; - codec_info.codecSpecific.generic.simulcast_idx = 0; encoded_complete_callback_->OnEncodedImage(combined_image_, &codec_info, fragmentation); } diff --git a/modules/video_coding/codecs/multiplex/test/multiplex_adapter_unittest.cc b/modules/video_coding/codecs/multiplex/test/multiplex_adapter_unittest.cc index 303914d95e..56de138f08 100644 --- a/modules/video_coding/codecs/multiplex/test/multiplex_adapter_unittest.cc +++ b/modules/video_coding/codecs/multiplex/test/multiplex_adapter_unittest.cc @@ -232,7 +232,7 @@ TEST_P(TestMultiplexAdapter, CheckSingleFrameEncodedBitstream) { CodecSpecificInfo codec_specific_info; ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info)); EXPECT_EQ(kVideoCodecMultiplex, codec_specific_info.codecType); - EXPECT_EQ(0, codec_specific_info.codecSpecific.generic.simulcast_idx); + EXPECT_FALSE(encoded_frame.SpatialIndex()); const MultiplexImage& unpacked_frame = MultiplexEncodedImagePacker::Unpack(encoded_frame); @@ -252,7 +252,7 @@ TEST_P(TestMultiplexAdapter, CheckDoubleFramesEncodedBitstream) { CodecSpecificInfo codec_specific_info; ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info)); EXPECT_EQ(kVideoCodecMultiplex, codec_specific_info.codecType); - EXPECT_EQ(0, codec_specific_info.codecSpecific.generic.simulcast_idx); + EXPECT_FALSE(encoded_frame.SpatialIndex()); const MultiplexImage& unpacked_frame = MultiplexEncodedImagePacker::Unpack(encoded_frame); diff --git a/modules/video_coding/codecs/test/videoprocessor.cc b/modules/video_coding/codecs/test/videoprocessor.cc index ccf7053b41..75605aed8f 100644 --- a/modules/video_coding/codecs/test/videoprocessor.cc +++ b/modules/video_coding/codecs/test/videoprocessor.cc @@ -56,22 +56,17 @@ size_t GetMaxNaluSizeBytes(const EncodedImage& encoded_frame, return max_size; } -void GetLayerIndices(const CodecSpecificInfo& codec_specific, - size_t* spatial_idx, - size_t* temporal_idx) { +size_t GetTemporalLayerIndex(const CodecSpecificInfo& codec_specific) { + size_t temporal_idx = 0; if (codec_specific.codecType == kVideoCodecVP8) { - *spatial_idx = codec_specific.codecSpecific.VP8.simulcastIdx; - *temporal_idx = codec_specific.codecSpecific.VP8.temporalIdx; + temporal_idx = codec_specific.codecSpecific.VP8.temporalIdx; } else if (codec_specific.codecType == kVideoCodecVP9) { - *spatial_idx = codec_specific.codecSpecific.VP9.spatial_idx; - *temporal_idx = codec_specific.codecSpecific.VP9.temporal_idx; + temporal_idx = codec_specific.codecSpecific.VP9.temporal_idx; } - if (*spatial_idx == kNoSpatialIdx) { - *spatial_idx = 0; - } - if (*temporal_idx == kNoTemporalIdx) { - *temporal_idx = 0; + if (temporal_idx == kNoTemporalIdx) { + temporal_idx = 0; } + return temporal_idx; } int GetElapsedTimeMicroseconds(int64_t start_ns, int64_t stop_ns) { @@ -347,9 +342,8 @@ void VideoProcessor::FrameEncoded( } // Layer metadata. - size_t spatial_idx = 0; - size_t temporal_idx = 0; - GetLayerIndices(codec_specific, &spatial_idx, &temporal_idx); + size_t spatial_idx = encoded_image.SpatialIndex().value_or(0); + size_t temporal_idx = GetTemporalLayerIndex(codec_specific); FrameStatistics* frame_stat = stats_->GetFrameWithTimestamp(encoded_image.Timestamp(), spatial_idx); diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc index 3826e144f0..ef46fd2fec 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc @@ -817,7 +817,6 @@ void LibvpxVp8Encoder::PopulateCodecSpecific( codec_specific->codecType = kVideoCodecVP8; codec_specific->codec_name = ImplementationName(); CodecSpecificInfoVP8* vp8Info = &(codec_specific->codecSpecific.VP8); - vp8Info->simulcastIdx = stream_idx; vp8Info->keyIdx = kNoKeyIdx; // TODO(hlundin) populate this vp8Info->nonReference = (pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE) != 0; temporal_layers_[stream_idx]->PopulateCodecSpecific( @@ -876,6 +875,7 @@ int LibvpxVp8Encoder::GetEncodedPartitions( encoded_images_[encoder_idx]._frameType = kVideoFrameKey; is_keyframe = true; } + encoded_images_[encoder_idx].SetSpatialIndex(stream_idx); PopulateCodecSpecific(&codec_specific, tl_configs[stream_idx], *pkt, stream_idx, input_image.timestamp()); break; diff --git a/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc b/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc index aaa10dd753..dc55d178e0 100644 --- a/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc +++ b/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc @@ -70,7 +70,7 @@ class TestVp8Impl : public VideoCodecUnitTest { VerifyQpParser(*encoded_frame); EXPECT_STREQ("libvpx", codec_specific_info->codec_name); EXPECT_EQ(kVideoCodecVP8, codec_specific_info->codecType); - EXPECT_EQ(0u, codec_specific_info->codecSpecific.VP8.simulcastIdx); + EXPECT_EQ(0, encoded_frame->SpatialIndex()); } void EncodeAndExpectFrameWith(const VideoFrame& input_frame, diff --git a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc index f6594c1f7c..0f5855f2a9 100644 --- a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc +++ b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc @@ -356,7 +356,7 @@ TEST_F(TestVp9Impl, EndOfPicture) { encoder_->Encode(*NextInputFrame(), nullptr, nullptr)); ASSERT_TRUE(WaitForEncodedFrames(&frames, &codec_specific)); - EXPECT_EQ(codec_specific[0].codecSpecific.VP9.spatial_idx, kNoSpatialIdx); + EXPECT_FALSE(frames[0].SpatialIndex()); EXPECT_TRUE(codec_specific[0].codecSpecific.VP9.end_of_picture); } @@ -395,7 +395,7 @@ TEST_F(TestVp9Impl, InterLayerPred) { // Key frame. EXPECT_FALSE(codec_specific[0].codecSpecific.VP9.inter_pic_predicted); - EXPECT_EQ(codec_specific[0].codecSpecific.VP9.spatial_idx, 0); + EXPECT_EQ(frames[0].SpatialIndex(), 0); EXPECT_EQ(codec_specific[0].codecSpecific.VP9.non_ref_for_inter_layer_pred, inter_layer_pred == InterLayerPredMode::kOff); EXPECT_TRUE( @@ -408,7 +408,7 @@ TEST_F(TestVp9Impl, InterLayerPred) { // Delta frame. EXPECT_TRUE(codec_specific[0].codecSpecific.VP9.inter_pic_predicted); - EXPECT_EQ(codec_specific[0].codecSpecific.VP9.spatial_idx, 0); + EXPECT_EQ(frames[0].SpatialIndex(), 0); EXPECT_EQ(codec_specific[0].codecSpecific.VP9.non_ref_for_inter_layer_pred, inter_layer_pred == InterLayerPredMode::kOff || inter_layer_pred == InterLayerPredMode::kOnKeyPic); diff --git a/modules/video_coding/codecs/vp9/vp9_impl.cc b/modules/video_coding/codecs/vp9/vp9_impl.cc index b4c13f9ccf..e6ed0a8830 100644 --- a/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -754,6 +754,7 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image, } void VP9EncoderImpl::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, + absl::optional* spatial_idx, const vpx_codec_cx_pkt& pkt, uint32_t timestamp, bool first_frame_in_picture) { @@ -780,9 +781,9 @@ void VP9EncoderImpl::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, } if (num_active_spatial_layers_ == 1) { RTC_CHECK_EQ(layer_id.spatial_layer_id, 0); - vp9_info->spatial_idx = kNoSpatialIdx; + *spatial_idx = absl::nullopt; } else { - vp9_info->spatial_idx = layer_id.spatial_layer_id; + *spatial_idx = layer_id.spatial_layer_id; } if (layer_id.spatial_layer_id != 0) { vp9_info->ss_data_available = false; @@ -1021,8 +1022,10 @@ int VP9EncoderImpl::GetEncodedLayerFrame(const vpx_codec_cx_pkt* pkt) { RTC_DCHECK_LE(encoded_image_._length, encoded_image_._size); memset(&codec_specific_, 0, sizeof(codec_specific_)); - PopulateCodecSpecific(&codec_specific_, *pkt, input_image_->timestamp(), - first_frame_in_picture); + absl::optional spatial_index; + PopulateCodecSpecific(&codec_specific_, &spatial_index, *pkt, + input_image_->timestamp(), first_frame_in_picture); + encoded_image_.SetSpatialIndex(spatial_index); if (is_flexible_mode_) { UpdateReferenceBuffers(*pkt, pics_since_key_); diff --git a/modules/video_coding/codecs/vp9/vp9_impl.h b/modules/video_coding/codecs/vp9/vp9_impl.h index d4483060ee..4deade3e0b 100644 --- a/modules/video_coding/codecs/vp9/vp9_impl.h +++ b/modules/video_coding/codecs/vp9/vp9_impl.h @@ -61,6 +61,7 @@ class VP9EncoderImpl : public VP9Encoder { int InitAndSetControlSettings(const VideoCodec* inst); void PopulateCodecSpecific(CodecSpecificInfo* codec_specific, + absl::optional* spatial_idx, const vpx_codec_cx_pkt& pkt, uint32_t timestamp, bool first_frame_in_picture); diff --git a/modules/video_coding/encoded_frame.cc b/modules/video_coding/encoded_frame.cc index c53a737fdc..37914e455a 100644 --- a/modules/video_coding/encoded_frame.cc +++ b/modules/video_coding/encoded_frame.cc @@ -82,7 +82,6 @@ void VCMEncodedFrame::CopyCodecSpecific(const RTPVideoHeader* header) { if (_codecSpecificInfo.codecType != kVideoCodecVP9) { // This is the first packet for this frame. _codecSpecificInfo.codecSpecific.VP9.temporal_idx = 0; - _codecSpecificInfo.codecSpecific.VP9.spatial_idx = 0; _codecSpecificInfo.codecSpecific.VP9.gof_idx = 0; _codecSpecificInfo.codecSpecific.VP9.inter_layer_predicted = false; _codecSpecificInfo.codecType = kVideoCodecVP9; @@ -106,8 +105,6 @@ void VCMEncodedFrame::CopyCodecSpecific(const RTPVideoHeader* header) { vp9_header.temporal_up_switch; } if (vp9_header.spatial_idx != kNoSpatialIdx) { - _codecSpecificInfo.codecSpecific.VP9.spatial_idx = - vp9_header.spatial_idx; _codecSpecificInfo.codecSpecific.VP9.inter_layer_predicted = vp9_header.inter_layer_predicted; } diff --git a/modules/video_coding/generic_encoder.cc b/modules/video_coding/generic_encoder.cc index a8999fce6b..b1d7c28abd 100644 --- a/modules/video_coding/generic_encoder.cc +++ b/modules/video_coding/generic_encoder.cc @@ -391,21 +391,10 @@ EncodedImageCallback::Result VCMEncodedFrameCallback::OnEncodedImage( const RTPFragmentationHeader* fragmentation_header) { TRACE_EVENT_INSTANT1("webrtc", "VCMEncodedFrameCallback::Encoded", "timestamp", encoded_image.Timestamp()); - size_t simulcast_svc_idx = 0; - if (codec_specific->codecType == kVideoCodecVP9) { - if (codec_specific->codecSpecific.VP9.num_spatial_layers > 1) - simulcast_svc_idx = codec_specific->codecSpecific.VP9.spatial_idx; - } else if (codec_specific->codecType == kVideoCodecVP8) { - simulcast_svc_idx = codec_specific->codecSpecific.VP8.simulcastIdx; - } else if (codec_specific->codecType == kVideoCodecGeneric) { - simulcast_svc_idx = codec_specific->codecSpecific.generic.simulcast_idx; - } else if (codec_specific->codecType == kVideoCodecH264) { - // TODO(ilnik): When h264 simulcast is landed, extract simulcast idx here. - } - + const size_t spatial_idx = encoded_image.SpatialIndex().value_or(0); EncodedImage image_copy(encoded_image); - FillTimingInfo(simulcast_svc_idx, &image_copy); + FillTimingInfo(spatial_idx, &image_copy); // Piggyback ALR experiment group id and simulcast id into the content type. uint8_t experiment_id = @@ -421,7 +410,7 @@ EncodedImageCallback::Result VCMEncodedFrameCallback::OnEncodedImage( // id in content type to +1 of that is actual simulcast index. This is because // value 0 on the wire is reserved for 'no simulcast stream specified'. RTC_CHECK(videocontenttypehelpers::SetSimulcastId( - &image_copy.content_type_, static_cast(simulcast_svc_idx + 1))); + &image_copy.content_type_, static_cast(spatial_idx + 1))); Result result = post_encode_callback_->OnEncodedImage( image_copy, codec_specific, fragmentation_header); diff --git a/modules/video_coding/generic_encoder_unittest.cc b/modules/video_coding/generic_encoder_unittest.cc index c889769159..2be6856c40 100644 --- a/modules/video_coding/generic_encoder_unittest.cc +++ b/modules/video_coding/generic_encoder_unittest.cc @@ -95,8 +95,8 @@ std::vector> GetTimingFrames( image._length = FrameSize(min_frame_size, max_frame_size, s, i); image.capture_time_ms_ = current_timestamp; image.SetTimestamp(static_cast(current_timestamp * 90)); + image.SetSpatialIndex(s); codec_specific.codecType = kVideoCodecGeneric; - codec_specific.codecSpecific.generic.simulcast_idx = s; callback.OnEncodeStarted(static_cast(current_timestamp * 90), current_timestamp, s); if (dropped) { @@ -189,7 +189,6 @@ TEST(TestVCMEncodedFrameCallback, NoTimingFrameIfNoEncodeStartTime) { image.capture_time_ms_ = timestamp; image.SetTimestamp(static_cast(timestamp * 90)); codec_specific.codecType = kVideoCodecGeneric; - codec_specific.codecSpecific.generic.simulcast_idx = 0; FakeEncodedImageCallback sink; VCMEncodedFrameCallback callback(&sink, nullptr); VideoCodec::TimingFrameTriggerThresholds thresholds; @@ -221,7 +220,6 @@ TEST(TestVCMEncodedFrameCallback, AdjustsCaptureTimeForInternalSourceEncoder) { image.capture_time_ms_ = timestamp; image.SetTimestamp(static_cast(timestamp * 90)); codec_specific.codecType = kVideoCodecGeneric; - codec_specific.codecSpecific.generic.simulcast_idx = 0; FakeEncodedImageCallback sink; VCMEncodedFrameCallback callback(&sink, nullptr); callback.SetInternalSource(true); @@ -257,7 +255,6 @@ TEST(TestVCMEncodedFrameCallback, NotifiesAboutDroppedFrames) { const int64_t kTimestampMs3 = 47721860; const int64_t kTimestampMs4 = 47721870; codec_specific.codecType = kVideoCodecGeneric; - codec_specific.codecSpecific.generic.simulcast_idx = 0; FakeEncodedImageCallback sink; VCMEncodedFrameCallback callback(&sink, nullptr); // Any non-zero bitrate needed to be set before the first frame. @@ -293,7 +290,6 @@ TEST(TestVCMEncodedFrameCallback, RestoresCaptureTimestamps) { CodecSpecificInfo codec_specific; const int64_t kTimestampMs = 123456; codec_specific.codecType = kVideoCodecGeneric; - codec_specific.codecSpecific.generic.simulcast_idx = 0; FakeEncodedImageCallback sink; VCMEncodedFrameCallback callback(&sink, nullptr); // Any non-zero bitrate needed to be set before the first frame. diff --git a/modules/video_coding/include/video_codec_interface.h b/modules/video_coding/include/video_codec_interface.h index 9108625a54..94d4271694 100644 --- a/modules/video_coding/include/video_codec_interface.h +++ b/modules/video_coding/include/video_codec_interface.h @@ -28,6 +28,8 @@ class RTPFragmentationHeader; // forward declaration // with a copy-constructor. See below. struct CodecSpecificInfoVP8 { bool nonReference; + // TODO(bugs.webrtc.org/9378): Delete simulcastIdx, replaced by spatial index + // member in EncodedImage. Unused, but assigned in downstream code. uint8_t simulcastIdx; uint8_t temporalIdx; bool layerSync; @@ -43,6 +45,8 @@ struct CodecSpecificInfoVP9 { bool non_ref_for_inter_layer_pred; uint8_t temporal_idx; + // TODO(bugs.webrtc.org/9378): Delete spatial_idx, replaced by spatial index + // member in EncodedImage. Unused, but assigned in downstream code. uint8_t spatial_idx; bool temporal_up_switch; bool inter_layer_predicted; // Frame is dependent on directly lower spatial @@ -63,13 +67,14 @@ struct CodecSpecificInfoVP9 { bool end_of_picture; }; +// TODO(bugs.webrtc.org/9378): Delete this struct. Unused, except that +// simulcast_idx is assigned in downstream code. struct CodecSpecificInfoGeneric { uint8_t simulcast_idx; }; struct CodecSpecificInfoH264 { H264PacketizationMode packetization_mode; - uint8_t simulcast_idx; }; union CodecSpecificInfoUnion { diff --git a/modules/video_coding/utility/simulcast_test_fixture_impl.cc b/modules/video_coding/utility/simulcast_test_fixture_impl.cc index 03de1763b4..4af526c663 100644 --- a/modules/video_coding/utility/simulcast_test_fixture_impl.cc +++ b/modules/video_coding/utility/simulcast_test_fixture_impl.cc @@ -76,15 +76,9 @@ class SimulcastTestFixtureImpl::TestEncodedImageCallback virtual Result OnEncodedImage(const EncodedImage& encoded_image, const CodecSpecificInfo* codec_specific_info, const RTPFragmentationHeader* fragmentation) { - uint16_t simulcast_idx = 0; bool is_vp8 = (codec_specific_info->codecType == kVideoCodecVP8); - if (is_vp8) { - simulcast_idx = codec_specific_info->codecSpecific.VP8.simulcastIdx; - } else { - simulcast_idx = codec_specific_info->codecSpecific.H264.simulcast_idx; - } // Only store the base layer. - if (simulcast_idx) { + if (encoded_image.SpatialIndex().value_or(0) == 0) { if (encoded_image._frameType == kVideoFrameKey) { delete[] encoded_key_frame_._buffer; encoded_key_frame_._buffer = new uint8_t[encoded_image._size]; @@ -104,9 +98,9 @@ class SimulcastTestFixtureImpl::TestEncodedImageCallback } } if (is_vp8) { - layer_sync_[codec_specific_info->codecSpecific.VP8.simulcastIdx] = + layer_sync_[encoded_image.SpatialIndex().value_or(0)] = codec_specific_info->codecSpecific.VP8.layerSync; - temporal_layer_[codec_specific_info->codecSpecific.VP8.simulcastIdx] = + temporal_layer_[encoded_image.SpatialIndex().value_or(0)] = codec_specific_info->codecSpecific.VP8.temporalIdx; } return Result(Result::OK, encoded_image.Timestamp()); diff --git a/sdk/android/src/jni/androidmediaencoder.cc b/sdk/android/src/jni/androidmediaencoder.cc index 35608c888a..435bc713a8 100644 --- a/sdk/android/src/jni/androidmediaencoder.cc +++ b/sdk/android/src/jni/androidmediaencoder.cc @@ -1008,7 +1008,6 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { info.codecType = codec_type; if (codec_type == kVideoCodecVP8) { info.codecSpecific.VP8.nonReference = false; - info.codecSpecific.VP8.simulcastIdx = 0; info.codecSpecific.VP8.temporalIdx = kNoTemporalIdx; info.codecSpecific.VP8.layerSync = false; info.codecSpecific.VP8.keyIdx = kNoKeyIdx; @@ -1020,7 +1019,6 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { info.codecSpecific.VP9.flexible_mode = false; info.codecSpecific.VP9.ss_data_available = key_frame ? true : false; info.codecSpecific.VP9.temporal_idx = kNoTemporalIdx; - info.codecSpecific.VP9.spatial_idx = kNoSpatialIdx; info.codecSpecific.VP9.temporal_up_switch = true; info.codecSpecific.VP9.inter_layer_predicted = false; info.codecSpecific.VP9.gof_idx = diff --git a/sdk/android/src/jni/videoencoderwrapper.cc b/sdk/android/src/jni/videoencoderwrapper.cc index ff8be05274..c50e9f6ef9 100644 --- a/sdk/android/src/jni/videoencoderwrapper.cc +++ b/sdk/android/src/jni/videoencoderwrapper.cc @@ -390,7 +390,6 @@ CodecSpecificInfo VideoEncoderWrapper::ParseCodecSpecificInfo( switch (codec_settings_.codecType) { case kVideoCodecVP8: info.codecSpecific.VP8.nonReference = false; - info.codecSpecific.VP8.simulcastIdx = 0; info.codecSpecific.VP8.temporalIdx = kNoTemporalIdx; info.codecSpecific.VP8.layerSync = false; info.codecSpecific.VP8.keyIdx = kNoKeyIdx; @@ -403,7 +402,6 @@ CodecSpecificInfo VideoEncoderWrapper::ParseCodecSpecificInfo( info.codecSpecific.VP9.flexible_mode = false; info.codecSpecific.VP9.ss_data_available = key_frame ? true : false; info.codecSpecific.VP9.temporal_idx = kNoTemporalIdx; - info.codecSpecific.VP9.spatial_idx = kNoSpatialIdx; info.codecSpecific.VP9.temporal_up_switch = true; info.codecSpecific.VP9.inter_layer_predicted = false; info.codecSpecific.VP9.gof_idx = diff --git a/test/fake_encoder.cc b/test/fake_encoder.cc index b8d04e67b0..2075399dd9 100644 --- a/test/fake_encoder.cc +++ b/test/fake_encoder.cc @@ -102,7 +102,6 @@ int32_t FakeEncoder::Encode(const VideoFrame& input_image, CodecSpecificInfo specifics; memset(&specifics, 0, sizeof(specifics)); specifics.codecType = kVideoCodecGeneric; - specifics.codecSpecific.generic.simulcast_idx = i; std::unique_ptr encoded_buffer( new uint8_t[frame_info.layers[i].size]); memcpy(encoded_buffer.get(), encoded_buffer_, frame_info.layers[i].size); @@ -118,6 +117,7 @@ int32_t FakeEncoder::Encode(const VideoFrame& input_image, encoded.content_type_ = (mode == VideoCodecMode::kScreensharing) ? VideoContentType::SCREENSHARE : VideoContentType::UNSPECIFIED; + encoded.SetSpatialIndex(i); specifics.codec_name = ImplementationName(); if (callback->OnEncodedImage(encoded, &specifics, nullptr).error != EncodedImageCallback::Result::OK) { diff --git a/test/fake_vp8_encoder.cc b/test/fake_vp8_encoder.cc index 9d0529740c..04dff00c42 100644 --- a/test/fake_vp8_encoder.cc +++ b/test/fake_vp8_encoder.cc @@ -88,7 +88,6 @@ void FakeVP8Encoder::PopulateCodecSpecific( codec_specific->codecType = kVideoCodecVP8; codec_specific->codec_name = ImplementationName(); CodecSpecificInfoVP8* vp8Info = &(codec_specific->codecSpecific.VP8); - vp8Info->simulcastIdx = stream_idx; vp8Info->keyIdx = kNoKeyIdx; vp8Info->nonReference = false; temporal_layers_[stream_idx]->PopulateCodecSpecific( @@ -100,7 +99,7 @@ EncodedImageCallback::Result FakeVP8Encoder::OnEncodedImage( const CodecSpecificInfo* codec_specific_info, const RTPFragmentationHeader* fragments) { RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); - uint8_t stream_idx = codec_specific_info->codecSpecific.generic.simulcast_idx; + uint8_t stream_idx = encoded_image.SpatialIndex().value_or(0); CodecSpecificInfo overrided_specific_info; TemporalLayers::FrameConfig tl_config = temporal_layers_[stream_idx]->UpdateLayerConfig(encoded_image._timeStamp); diff --git a/video/send_statistics_proxy.cc b/video/send_statistics_proxy.cc index 2ab025a514..0057933c9f 100644 --- a/video/send_statistics_proxy.cc +++ b/video/send_statistics_proxy.cc @@ -81,9 +81,9 @@ void UpdateCodecTypeHistogram(const std::string& payload_name) { kVideoMax); } -bool IsForcedFallbackPossible(const CodecSpecificInfo* codec_info) { - return codec_info->codecType == kVideoCodecVP8 && - codec_info->codecSpecific.VP8.simulcastIdx == 0 && +bool IsForcedFallbackPossible(const CodecSpecificInfo* codec_info, + int simulcast_index) { + return codec_info->codecType == kVideoCodecVP8 && simulcast_index == 0 && (codec_info->codecSpecific.VP8.temporalIdx == 0 || codec_info->codecSpecific.VP8.temporalIdx == kNoTemporalIdx); } @@ -219,7 +219,7 @@ void SendStatisticsProxy::UmaSamplesContainer::RemoveOld( sent_height_counter_.Add(it->second.max_height); // Check number of encoded streams per timestamp. - if (num_streams_ > it->second.max_simulcast_idx) { + if (num_streams_ > static_cast(it->second.max_simulcast_idx)) { *is_limited_in_resolution = false; if (num_streams_ > 1) { int disabled_streams = @@ -241,7 +241,7 @@ void SendStatisticsProxy::UmaSamplesContainer::RemoveOld( bool SendStatisticsProxy::UmaSamplesContainer::InsertEncodedFrame( const EncodedImage& encoded_frame, - size_t simulcast_idx, + int simulcast_idx, bool* is_limited_in_resolution) { int64_t now_ms = clock_->TimeInMilliseconds(); RemoveOld(now_ms, is_limited_in_resolution); @@ -806,14 +806,15 @@ void SendStatisticsProxy::OnSetEncoderTargetRate(uint32_t bitrate_bps) { void SendStatisticsProxy::UpdateEncoderFallbackStats( const CodecSpecificInfo* codec_info, - int pixels) { - UpdateFallbackDisabledStats(codec_info, pixels); + int pixels, + int simulcast_index) { + UpdateFallbackDisabledStats(codec_info, pixels, simulcast_index); if (!fallback_max_pixels_ || !uma_container_->fallback_info_.is_possible) { return; } - if (!IsForcedFallbackPossible(codec_info)) { + if (!IsForcedFallbackPossible(codec_info, simulcast_index)) { uma_container_->fallback_info_.is_possible = false; return; } @@ -855,14 +856,15 @@ void SendStatisticsProxy::UpdateEncoderFallbackStats( void SendStatisticsProxy::UpdateFallbackDisabledStats( const CodecSpecificInfo* codec_info, - int pixels) { + int pixels, + int simulcast_index) { if (!fallback_max_pixels_disabled_ || !uma_container_->fallback_info_disabled_.is_possible || stats_.has_entered_low_resolution) { return; } - if (!IsForcedFallbackPossible(codec_info) || + if (!IsForcedFallbackPossible(codec_info, simulcast_index) || strcmp(codec_info->codec_name, kVp8SwCodecName) == 0) { uma_container_->fallback_info_disabled_.is_possible = false; return; @@ -882,26 +884,27 @@ void SendStatisticsProxy::OnMinPixelLimitReached() { void SendStatisticsProxy::OnSendEncodedImage( const EncodedImage& encoded_image, const CodecSpecificInfo* codec_info) { - size_t simulcast_idx = 0; + // Simulcast is used for VP8, H264 and Generic. + int simulcast_idx = + (codec_info && (codec_info->codecType == kVideoCodecVP8 || + codec_info->codecType == kVideoCodecH264 || + codec_info->codecType == kVideoCodecGeneric)) + ? encoded_image.SpatialIndex().value_or(0) + : 0; rtc::CritScope lock(&crit_); ++stats_.frames_encoded; if (codec_info) { - if (codec_info->codecType == kVideoCodecVP8) { - simulcast_idx = codec_info->codecSpecific.VP8.simulcastIdx; - } else if (codec_info->codecType == kVideoCodecH264) { - simulcast_idx = codec_info->codecSpecific.H264.simulcast_idx; - } else if (codec_info->codecType == kVideoCodecGeneric) { - simulcast_idx = codec_info->codecSpecific.generic.simulcast_idx; - } if (codec_info->codec_name) { - UpdateEncoderFallbackStats(codec_info, encoded_image._encodedWidth * - encoded_image._encodedHeight); + UpdateEncoderFallbackStats( + codec_info, + encoded_image._encodedWidth * encoded_image._encodedHeight, + simulcast_idx); stats_.encoder_implementation_name = codec_info->codec_name; } } - if (simulcast_idx >= rtp_config_.ssrcs.size()) { + if (static_cast(simulcast_idx) >= rtp_config_.ssrcs.size()) { RTC_LOG(LS_ERROR) << "Encoded image outside simulcast range (" << simulcast_idx << " >= " << rtp_config_.ssrcs.size() << ")."; @@ -928,20 +931,13 @@ void SendStatisticsProxy::OnSendEncodedImage( if (codec_info) { if (codec_info->codecType == kVideoCodecVP8) { - int spatial_idx = (rtp_config_.ssrcs.size() == 1) - ? -1 - : static_cast(simulcast_idx); + int spatial_idx = (rtp_config_.ssrcs.size() == 1) ? -1 : simulcast_idx; uma_container_->qp_counters_[spatial_idx].vp8.Add(encoded_image.qp_); } else if (codec_info->codecType == kVideoCodecVP9) { - int spatial_idx = - (codec_info->codecSpecific.VP9.num_spatial_layers == 1) - ? -1 - : codec_info->codecSpecific.VP9.spatial_idx; + int spatial_idx = encoded_image.SpatialIndex().value_or(-1); uma_container_->qp_counters_[spatial_idx].vp9.Add(encoded_image.qp_); } else if (codec_info->codecType == kVideoCodecH264) { - int spatial_idx = (rtp_config_.ssrcs.size() == 1) - ? -1 - : static_cast(simulcast_idx); + int spatial_idx = (rtp_config_.ssrcs.size() == 1) ? -1 : simulcast_idx; uma_container_->qp_counters_[spatial_idx].h264.Add(encoded_image.qp_); } } diff --git a/video/send_statistics_proxy.h b/video/send_statistics_proxy.h index 875348f162..b5868d0890 100644 --- a/video/send_statistics_proxy.h +++ b/video/send_statistics_proxy.h @@ -188,10 +188,7 @@ class SendStatisticsProxy : public VideoStreamEncoderObserver, } }; struct Frame { - Frame(int64_t send_ms, - uint32_t width, - uint32_t height, - size_t simulcast_idx) + Frame(int64_t send_ms, uint32_t width, uint32_t height, int simulcast_idx) : send_ms(send_ms), max_width(width), max_height(height), @@ -200,7 +197,7 @@ class SendStatisticsProxy : public VideoStreamEncoderObserver, send_ms; // Time when first frame with this timestamp is sent. uint32_t max_width; // Max width with this timestamp. uint32_t max_height; // Max height with this timestamp. - size_t max_simulcast_idx; // Max simulcast index with this timestamp. + int max_simulcast_idx; // Max simulcast index with this timestamp. }; typedef std::map EncodedFrameMap; @@ -218,10 +215,12 @@ class SendStatisticsProxy : public VideoStreamEncoderObserver, RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); void UpdateEncoderFallbackStats(const CodecSpecificInfo* codec_info, - int pixels) + int pixels, + int simulcast_index) RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); void UpdateFallbackDisabledStats(const CodecSpecificInfo* codec_info, - int pixels) + int pixels, + int simulcast_index) RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); Clock* const clock_; @@ -257,7 +256,7 @@ class SendStatisticsProxy : public VideoStreamEncoderObserver, void InitializeBitrateCounters(const VideoSendStream::Stats& stats); bool InsertEncodedFrame(const EncodedImage& encoded_frame, - size_t simulcast_idx, + int simulcast_idx, bool* is_limited_in_resolution); void RemoveOld(int64_t now_ms, bool* is_limited_in_resolution); diff --git a/video/send_statistics_proxy_unittest.cc b/video/send_statistics_proxy_unittest.cc index d4a7876843..4dbfc03a74 100644 --- a/video/send_statistics_proxy_unittest.cc +++ b/video/send_statistics_proxy_unittest.cc @@ -36,7 +36,6 @@ const int kRtpClockRateHz = 90000; const CodecSpecificInfo kDefaultCodecInfo = []() { CodecSpecificInfo codec_info; codec_info.codecType = kVideoCodecVP8; - codec_info.codecSpecific.VP8.simulcastIdx = 0; return codec_info; }(); } // namespace @@ -1313,10 +1312,10 @@ TEST_F(SendStatisticsProxyTest, VerifyQpHistogramStats_Vp8) { codec_info.codecType = kVideoCodecVP8; for (int i = 0; i < SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) { - codec_info.codecSpecific.VP8.simulcastIdx = 0; + encoded_image.SetSpatialIndex(0); encoded_image.qp_ = kQpIdx0; statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info); - codec_info.codecSpecific.VP8.simulcastIdx = 1; + encoded_image.SetSpatialIndex(1); encoded_image.qp_ = kQpIdx1; statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info); } @@ -1338,7 +1337,7 @@ TEST_F(SendStatisticsProxyTest, VerifyQpHistogramStats_Vp8OneSsrc) { codec_info.codecType = kVideoCodecVP8; for (int i = 0; i < SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) { - codec_info.codecSpecific.VP8.simulcastIdx = 0; + encoded_image.SetSpatialIndex(0); encoded_image.qp_ = kQpIdx0; statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info); } @@ -1355,10 +1354,10 @@ TEST_F(SendStatisticsProxyTest, VerifyQpHistogramStats_Vp9) { for (int i = 0; i < SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) { encoded_image.qp_ = kQpIdx0; - codec_info.codecSpecific.VP9.spatial_idx = 0; + encoded_image.SetSpatialIndex(0); statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info); encoded_image.qp_ = kQpIdx1; - codec_info.codecSpecific.VP9.spatial_idx = 1; + encoded_image.SetSpatialIndex(1); statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info); } statistics_proxy_.reset(); @@ -1381,7 +1380,6 @@ TEST_F(SendStatisticsProxyTest, VerifyQpHistogramStats_Vp9OneSpatialLayer) { for (int i = 0; i < SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) { encoded_image.qp_ = kQpIdx0; - codec_info.codecSpecific.VP9.spatial_idx = 0; statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info); } statistics_proxy_.reset(); @@ -1395,10 +1393,10 @@ TEST_F(SendStatisticsProxyTest, VerifyQpHistogramStats_H264) { codec_info.codecType = kVideoCodecH264; for (int i = 0; i < SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) { - codec_info.codecSpecific.H264.simulcast_idx = 0; + encoded_image.SetSpatialIndex(0); encoded_image.qp_ = kQpIdx0; statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info); - codec_info.codecSpecific.H264.simulcast_idx = 1; + encoded_image.SetSpatialIndex(1); encoded_image.qp_ = kQpIdx1; statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info); } @@ -1540,6 +1538,7 @@ TEST_F(SendStatisticsProxyTest, VideoStreamEncoderObserver::AdaptationReason::kNone, cpu_counts, quality_counts); EncodedImage encoded_image; + encoded_image.SetSpatialIndex(0); for (int i = 0; i < SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) statistics_proxy_->OnSendEncodedImage(encoded_image, &kDefaultCodecInfo); @@ -1560,6 +1559,7 @@ TEST_F(SendStatisticsProxyTest, VideoStreamEncoderObserver::AdaptationReason::kNone, cpu_counts, quality_counts); EncodedImage encoded_image; + encoded_image.SetSpatialIndex(0); for (int i = 0; i < SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) statistics_proxy_->OnSendEncodedImage(encoded_image, &kDefaultCodecInfo); @@ -1584,6 +1584,7 @@ TEST_F(SendStatisticsProxyTest, VideoStreamEncoderObserver::AdaptationReason::kNone, cpu_counts, quality_counts); EncodedImage encoded_image; + encoded_image.SetSpatialIndex(0); for (int i = 0; i < SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) statistics_proxy_->OnSendEncodedImage(encoded_image, &kDefaultCodecInfo); // Histograms are updated when the statistics_proxy_ is deleted. @@ -1717,13 +1718,13 @@ TEST_F(SendStatisticsProxyTest, EncodedResolutionTimesOut) { EncodedImage encoded_image; encoded_image._encodedWidth = kEncodedWidth; encoded_image._encodedHeight = kEncodedHeight; + encoded_image.SetSpatialIndex(0); CodecSpecificInfo codec_info; codec_info.codecType = kVideoCodecVP8; - codec_info.codecSpecific.VP8.simulcastIdx = 0; statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info); - codec_info.codecSpecific.VP8.simulcastIdx = 1; + encoded_image.SetSpatialIndex(1); statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info); VideoSendStream::Stats stats = statistics_proxy_->GetStats(); @@ -1746,7 +1747,6 @@ TEST_F(SendStatisticsProxyTest, EncodedResolutionTimesOut) { // Report stats for second SSRC to make sure it's not outdated along with the // first SSRC. - codec_info.codecSpecific.VP8.simulcastIdx = 1; statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info); // Forward 1 ms, reach timeout, substream 0 should have no resolution @@ -1765,13 +1765,13 @@ TEST_F(SendStatisticsProxyTest, ClearsResolutionFromInactiveSsrcs) { EncodedImage encoded_image; encoded_image._encodedWidth = kEncodedWidth; encoded_image._encodedHeight = kEncodedHeight; + encoded_image.SetSpatialIndex(0); CodecSpecificInfo codec_info; codec_info.codecType = kVideoCodecVP8; - codec_info.codecSpecific.VP8.simulcastIdx = 0; statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info); - codec_info.codecSpecific.VP8.simulcastIdx = 1; + encoded_image.SetSpatialIndex(1); statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info); statistics_proxy_->OnInactiveSsrc(config_.rtp.ssrcs[1]); @@ -2182,11 +2182,11 @@ class ForcedFallbackTest : public SendStatisticsProxyTest { explicit ForcedFallbackTest(const std::string& field_trials) : SendStatisticsProxyTest(field_trials) { codec_info_.codecType = kVideoCodecVP8; - codec_info_.codecSpecific.VP8.simulcastIdx = 0; codec_info_.codecSpecific.VP8.temporalIdx = 0; codec_info_.codec_name = "fake_codec"; encoded_image_._encodedWidth = kWidth; encoded_image_._encodedHeight = kHeight; + encoded_image_.SetSpatialIndex(0); } ~ForcedFallbackTest() override {} @@ -2260,7 +2260,7 @@ TEST_F(ForcedFallbackEnabled, StatsNotUpdatedForTemporalLayers) { } TEST_F(ForcedFallbackEnabled, StatsNotUpdatedForSimulcast) { - codec_info_.codecSpecific.VP8.simulcastIdx = 1; + encoded_image_.SetSpatialIndex(1); InsertEncodedFrames(kMinFrames, kFrameIntervalMs); statistics_proxy_.reset(); EXPECT_EQ(0, metrics::NumSamples(kPrefix + "FallbackTimeInPercent.Vp8")); diff --git a/video/video_receive_stream.cc b/video/video_receive_stream.cc index bd61881583..493f50f7f6 100644 --- a/video/video_receive_stream.cc +++ b/video/video_receive_stream.cc @@ -318,10 +318,6 @@ EncodedImageCallback::Result VideoReceiveStream::OnEncodedImage( const CodecSpecificInfo* codec_specific_info, const RTPFragmentationHeader* fragmentation) { stats_proxy_.OnPreDecode(encoded_image, codec_specific_info); - size_t simulcast_idx = 0; - if (codec_specific_info->codecType == kVideoCodecVP8) { - simulcast_idx = codec_specific_info->codecSpecific.VP8.simulcastIdx; - } { rtc::CritScope lock(&ivf_writer_lock_); if (ivf_writer_.get()) { diff --git a/video/video_send_stream_impl.cc b/video/video_send_stream_impl.cc index cf4bf870ac..822b951454 100644 --- a/video/video_send_stream_impl.cc +++ b/video/video_send_stream_impl.cc @@ -522,11 +522,13 @@ EncodedImageCallback::Result VideoSendStreamImpl::OnEncodedImage( // Encoded is called on whatever thread the real encoder implementation run // on. In the case of hardware encoders, there might be several encoders // running in parallel on different threads. - size_t simulcast_idx = 0; - if (codec_specific_info->codecType == kVideoCodecVP8) { - simulcast_idx = codec_specific_info->codecSpecific.VP8.simulcastIdx; - } + const size_t simulcast_idx = + (codec_specific_info->codecType != kVideoCodecVP9) + ? encoded_image.SpatialIndex().value_or(0) + : 0; if (config_->post_encode_callback) { + // TODO(nisse): Delete webrtc::EncodedFrame class, pass EncodedImage + // instead. config_->post_encode_callback->EncodedFrameCallback(EncodedFrame( encoded_image._buffer, encoded_image._length, encoded_image._frameType, simulcast_idx, encoded_image.Timestamp())); @@ -542,15 +544,10 @@ EncodedImageCallback::Result VideoSendStreamImpl::OnEncodedImage( EncodedImageCallback::Result result = rtp_video_sender_->OnEncodedImage( encoded_image, codec_specific_info, fragmentation); - RTC_DCHECK(codec_specific_info); - - int layer = codec_specific_info->codecType == kVideoCodecVP8 - ? codec_specific_info->codecSpecific.VP8.simulcastIdx - : 0; { rtc::CritScope lock(&ivf_writers_crit_); - if (file_writers_[layer].get()) { - bool ok = file_writers_[layer]->WriteFrame( + if (file_writers_[simulcast_idx].get()) { + bool ok = file_writers_[simulcast_idx]->WriteFrame( encoded_image, codec_specific_info->codecType); RTC_DCHECK(ok); } diff --git a/video/video_send_stream_tests.cc b/video/video_send_stream_tests.cc index cb11ebce7b..788b60b4da 100644 --- a/video/video_send_stream_tests.cc +++ b/video/video_send_stream_tests.cc @@ -3041,10 +3041,10 @@ TEST_F(VideoSendStreamTest, ReportsSentResolution) { encoded.capture_time_ms_ = input_image.render_time_ms(); for (size_t i = 0; i < kNumStreams; ++i) { - specifics.codecSpecific.generic.simulcast_idx = static_cast(i); encoded._frameType = (*frame_types)[i]; encoded._encodedWidth = kEncodedResolution[i].width; encoded._encodedHeight = kEncodedResolution[i].height; + encoded.SetSpatialIndex(i); EncodedImageCallback* callback; { rtc::CritScope cs(&crit_sect_);