From 9b235cd93b42bac70eb473bad60a6efc7a306af9 Mon Sep 17 00:00:00 2001 From: Evan Shrubsole Date: Tue, 6 Dec 2022 10:09:10 +0000 Subject: [PATCH] Add scalability mode to RTCOutboundRtpStreamStats stats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is in the webrtc-stats spec at https://www.w3.org/TR/webrtc-stats/#dom-rtcoutboundrtpstreamstats-scalabilitymode. This adds the scalability mode to CodecSpecificInfo which is used to plumb the modes for each simulcast layer. TBR=orphis@webrtc.org Tested: Compiled into Chrome and confirmed the scalability mode set for AV1, VP9, VP8 and H264 software encoders in chrome://webrtc-internals. Bug: webrtc:14730 Change-Id: I71ceba8f6485a4f4a73e0856031b8d5f16f913f2 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/285085 Reviewed-by: Harald Alvestrand Reviewed-by: Åsa Persson Commit-Queue: Evan Shrubsole Cr-Commit-Position: refs/heads/main@{#38847} --- api/stats/rtcstats_objects.h | 1 + call/BUILD.gn | 1 + call/video_send_stream.h | 2 + media/BUILD.gn | 1 + media/base/media_channel.h | 2 + media/engine/webrtc_video_engine.cc | 1 + modules/video_coding/BUILD.gn | 5 ++ modules/video_coding/codecs/av1/BUILD.gn | 1 + .../codecs/av1/libaom_av1_encoder.cc | 14 +++--- .../codecs/av1/libaom_av1_unittest.cc | 3 ++ .../codecs/h264/h264_encoder_impl.cc | 46 ++++++++++-------- .../codecs/h264/h264_encoder_impl.h | 6 +++ .../codecs/vp8/libvpx_vp8_encoder.cc | 12 +++++ .../codecs/vp9/libvpx_vp9_encoder.cc | 48 ++++++++++++------- .../codecs/vp9/libvpx_vp9_encoder.h | 2 + .../include/video_codec_interface.h | 2 + pc/BUILD.gn | 3 ++ pc/rtc_stats_collector.cc | 5 ++ pc/rtc_stats_collector_unittest.cc | 3 ++ pc/rtc_stats_integrationtest.cc | 2 + pc/test/svc_e2e_tests.cc | 20 ++++++++ stats/rtcstats_objects.cc | 6 ++- video/BUILD.gn | 1 + video/send_statistics_proxy.cc | 2 + video/send_statistics_proxy_unittest.cc | 32 +++++++++++++ 25 files changed, 178 insertions(+), 43 deletions(-) diff --git a/api/stats/rtcstats_objects.h b/api/stats/rtcstats_objects.h index 6f8178407f..532af53de3 100644 --- a/api/stats/rtcstats_objects.h +++ b/api/stats/rtcstats_objects.h @@ -561,6 +561,7 @@ class RTC_EXPORT RTCOutboundRTPStreamStats final : public RTCRTPStreamStats { RTCStatsMember active; RTCRestrictedStatsMember power_efficient_encoder; + RTCStatsMember scalability_mode; }; // https://w3c.github.io/webrtc-stats/#remoteinboundrtpstats-dict* diff --git a/call/BUILD.gn b/call/BUILD.gn index 6ed9a0423a..97bfdd3c4b 100644 --- a/call/BUILD.gn +++ b/call/BUILD.gn @@ -385,6 +385,7 @@ rtc_library("video_stream_api") { "../api/video:video_frame", "../api/video:video_rtp_headers", "../api/video:video_stream_encoder", + "../api/video_codecs:scalability_mode", "../api/video_codecs:video_codecs_api", "../common_video", "../common_video:frame_counts", diff --git a/call/video_send_stream.h b/call/video_send_stream.h index 9608281f2f..431c267e1e 100644 --- a/call/video_send_stream.h +++ b/call/video_send_stream.h @@ -30,6 +30,7 @@ #include "api/video/video_sink_interface.h" #include "api/video/video_source_interface.h" #include "api/video/video_stream_encoder_settings.h" +#include "api/video_codecs/scalability_mode.h" #include "call/rtp_config.h" #include "common_video/frame_counts.h" #include "common_video/include/quality_limitation_reason.h" @@ -93,6 +94,7 @@ class VideoSendStream { uint64_t total_encode_time_ms = 0; uint64_t total_encoded_bytes_target = 0; uint32_t huge_frames_sent = 0; + absl::optional scalability_mode; }; struct Stats { diff --git a/media/BUILD.gn b/media/BUILD.gn index 179246709b..7999be8ccb 100644 --- a/media/BUILD.gn +++ b/media/BUILD.gn @@ -74,6 +74,7 @@ rtc_library("rtc_media_base") { "../api/video:video_bitrate_allocator_factory", "../api/video:video_frame", "../api/video:video_rtp_headers", + "../api/video_codecs:scalability_mode", "../api/video_codecs:video_codecs_api", "../call:call_interfaces", "../call:video_stream_api", diff --git a/media/base/media_channel.h b/media/base/media_channel.h index 14d82e3edd..e181154f97 100644 --- a/media/base/media_channel.h +++ b/media/base/media_channel.h @@ -35,6 +35,7 @@ #include "api/video/video_sink_interface.h" #include "api/video/video_source_interface.h" #include "api/video/video_timing.h" +#include "api/video_codecs/scalability_mode.h" #include "api/video_codecs/video_encoder_factory.h" #include "call/video_receive_stream.h" #include "common_video/include/quality_limitation_reason.h" @@ -651,6 +652,7 @@ struct VideoSenderInfo : public MediaSenderInfo { uint32_t aggregated_huge_frames_sent = 0; absl::optional rid; absl::optional power_efficient_encoder; + absl::optional scalability_mode; }; struct VideoReceiverInfo : public MediaReceiverInfo { diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index d4bdeff992..a78162e149 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -2731,6 +2731,7 @@ WebRtcVideoChannel::WebRtcVideoSendStream::GetPerLayerVideoSenderInfos( info.total_encode_time_ms = stream_stats.total_encode_time_ms; info.total_encoded_bytes_target = stream_stats.total_encoded_bytes_target; info.huge_frames_sent = stream_stats.huge_frames_sent; + info.scalability_mode = stream_stats.scalability_mode; infos.push_back(info); } return infos; diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index d6b85a2557..2686047bfd 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -331,6 +331,7 @@ rtc_library("video_codec_interface") { ":codec_globals_headers", "../../api/video:video_frame", "../../api/video:video_rtp_headers", + "../../api/video_codecs:scalability_mode", "../../api/video_codecs:video_codecs_api", "../../common_video", "../../common_video/generic_frame_descriptor", @@ -516,6 +517,8 @@ rtc_library("webrtc_h264") { deps = [ ":video_codec_interface", ":video_coding_utility", + "../../api/transport/rtp:dependency_descriptor", + "../../api/video:video_codec_constants", "../../api/video:video_frame", "../../api/video:video_frame_i010", "../../api/video:video_rtp_headers", @@ -630,6 +633,7 @@ rtc_library("webrtc_vp8") { "../../api/video:encoded_image", "../../api/video:video_frame", "../../api/video:video_rtp_headers", + "../../api/video_codecs:scalability_mode", "../../api/video_codecs:video_codecs_api", "../../api/video_codecs:vp8_temporal_layers_factory", "../../common_video", @@ -777,6 +781,7 @@ rtc_library("webrtc_vp9") { "//third_party/abseil-cpp/absl/container:inlined_vector", "//third_party/abseil-cpp/absl/memory", "//third_party/abseil-cpp/absl/strings:strings", + "//third_party/abseil-cpp/absl/types:optional", ] if (rtc_build_libvpx) { deps += [ rtc_libvpx_dir ] diff --git a/modules/video_coding/codecs/av1/BUILD.gn b/modules/video_coding/codecs/av1/BUILD.gn index 1e23ccb082..610f958ad1 100644 --- a/modules/video_coding/codecs/av1/BUILD.gn +++ b/modules/video_coding/codecs/av1/BUILD.gn @@ -57,6 +57,7 @@ rtc_library("libaom_av1_encoder") { "../../../../api:scoped_refptr", "../../../../api/video:encoded_image", "../../../../api/video:video_frame", + "../../../../api/video_codecs:scalability_mode", "../../../../api/video_codecs:video_codecs_api", "../../../../common_video", "../../../../rtc_base:checks", diff --git a/modules/video_coding/codecs/av1/libaom_av1_encoder.cc b/modules/video_coding/codecs/av1/libaom_av1_encoder.cc index e5bf4ced90..4d8786c824 100644 --- a/modules/video_coding/codecs/av1/libaom_av1_encoder.cc +++ b/modules/video_coding/codecs/av1/libaom_av1_encoder.cc @@ -23,6 +23,7 @@ #include "api/video/encoded_image.h" #include "api/video/i420_buffer.h" #include "api/video/video_frame.h" +#include "api/video_codecs/scalability_mode.h" #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" #include "modules/video_coding/include/video_codec_interface.h" @@ -109,6 +110,7 @@ class LibaomAv1Encoder final : public VideoEncoder { void MaybeRewrapImgWithFormat(const aom_img_fmt_t fmt); std::unique_ptr svc_controller_; + absl::optional scalability_mode_; bool inited_; bool rates_configured_; absl::optional svc_params_; @@ -185,16 +187,15 @@ int LibaomAv1Encoder::InitEncode(const VideoCodec* codec_settings, RTC_LOG(LS_WARNING) << "Simulcast is not implemented by LibaomAv1Encoder."; return result; } - absl::optional scalability_mode = - encoder_settings_.GetScalabilityMode(); - if (!scalability_mode.has_value()) { + scalability_mode_ = encoder_settings_.GetScalabilityMode(); + if (!scalability_mode_.has_value()) { RTC_LOG(LS_WARNING) << "Scalability mode is not set, using 'L1T1'."; - scalability_mode = ScalabilityMode::kL1T1; + scalability_mode_ = ScalabilityMode::kL1T1; } - svc_controller_ = CreateScalabilityStructure(*scalability_mode); + svc_controller_ = CreateScalabilityStructure(*scalability_mode_); if (svc_controller_ == nullptr) { RTC_LOG(LS_WARNING) << "Failed to set scalability mode " - << static_cast(*scalability_mode); + << static_cast(*scalability_mode_); return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } @@ -709,6 +710,7 @@ int32_t LibaomAv1Encoder::Encode( CodecSpecificInfo codec_specific_info; codec_specific_info.codecType = kVideoCodecAV1; codec_specific_info.end_of_picture = end_of_picture; + codec_specific_info.scalability_mode = scalability_mode_; bool is_keyframe = layer_frame->IsKeyframe(); codec_specific_info.generic_frame_info = svc_controller_->OnEncodeDone(*layer_frame); diff --git a/modules/video_coding/codecs/av1/libaom_av1_unittest.cc b/modules/video_coding/codecs/av1/libaom_av1_unittest.cc index dbb62ea6dc..86e317f94b 100644 --- a/modules/video_coding/codecs/av1/libaom_av1_unittest.cc +++ b/modules/video_coding/codecs/av1/libaom_av1_unittest.cc @@ -44,6 +44,7 @@ using ::testing::Ge; using ::testing::IsEmpty; using ::testing::Not; using ::testing::NotNull; +using ::testing::Optional; using ::testing::Pointwise; using ::testing::SizeIs; using ::testing::Truly; @@ -248,6 +249,8 @@ TEST_P(LibaomAv1SvcTest, EncodeAndDecodeAllDecodeTargets) { requested_ids.push_back(frame_id); decoder.Decode(frame_id, frame.encoded_image); } + EXPECT_THAT(frame.codec_specific_info.scalability_mode, + Optional(param.GetScalabilityMode())); } ASSERT_THAT(requested_ids, SizeIs(Ge(2u))); diff --git a/modules/video_coding/codecs/h264/h264_encoder_impl.cc b/modules/video_coding/codecs/h264/h264_encoder_impl.cc index 80cbb60545..b8055ac85f 100644 --- a/modules/video_coding/codecs/h264/h264_encoder_impl.cc +++ b/modules/video_coding/codecs/h264/h264_encoder_impl.cc @@ -21,6 +21,9 @@ #include #include "absl/strings/match.h" +#include "absl/types/optional.h" +#include "api/video/video_codec_constants.h" +#include "api/video_codecs/scalability_mode.h" #include "common_video/libyuv/include/webrtc_libyuv.h" #include "modules/video_coding/svc/create_scalability_structure.h" #include "modules/video_coding/utility/simulcast_rate_allocator.h" @@ -86,6 +89,23 @@ VideoFrameType ConvertToVideoFrameType(EVideoFrameType type) { return VideoFrameType::kEmptyFrame; } +absl::optional ScalabilityModeFromTemporalLayers( + int num_temporal_layers) { + switch (num_temporal_layers) { + case 0: + break; + case 1: + return ScalabilityMode::kL1T1; + case 2: + return ScalabilityMode::kL1T2; + case 3: + return ScalabilityMode::kL1T3; + default: + RTC_DCHECK_NOTREACHED(); + } + return absl::nullopt; +} + } // namespace // Helper method used by H264EncoderImpl::Encode. @@ -199,6 +219,7 @@ int32_t H264EncoderImpl::InitEncode(const VideoCodec* inst, encoders_.resize(number_of_streams); pictures_.resize(number_of_streams); svc_controllers_.resize(number_of_streams); + scalability_modes_.resize(number_of_streams); configurations_.resize(number_of_streams); tl0sync_limit_.resize(number_of_streams); @@ -284,25 +305,10 @@ int32_t H264EncoderImpl::InitEncode(const VideoCodec* inst, encoded_images_[i].set_size(0); tl0sync_limit_[i] = configurations_[i].num_temporal_layers; - absl::optional scalability_mode; - switch (configurations_[i].num_temporal_layers) { - case 0: - break; - case 1: - scalability_mode = ScalabilityMode::kL1T1; - break; - case 2: - scalability_mode = ScalabilityMode::kL1T2; - break; - case 3: - scalability_mode = ScalabilityMode::kL1T3; - break; - default: - RTC_DCHECK_NOTREACHED(); - } - if (scalability_mode.has_value()) { - svc_controllers_[i] = - CreateScalabilityStructure(scalability_mode.value()); + scalability_modes_[i] = ScalabilityModeFromTemporalLayers( + configurations_[i].num_temporal_layers); + if (scalability_modes_[i].has_value()) { + svc_controllers_[i] = CreateScalabilityStructure(*scalability_modes_[i]); if (svc_controllers_[i] == nullptr) { RTC_LOG(LS_ERROR) << "Failed to create scalability structure"; Release(); @@ -335,6 +341,7 @@ int32_t H264EncoderImpl::Release() { pictures_.clear(); tl0sync_limit_.clear(); svc_controllers_.clear(); + scalability_modes_.clear(); return WEBRTC_VIDEO_CODEC_OK; } @@ -568,6 +575,7 @@ int32_t H264EncoderImpl::Encode( codec_specific.template_structure = svc_controllers_[i]->DependencyStructure(); } + codec_specific.scalability_mode = scalability_modes_[i]; } encoded_image_callback_->OnEncodedImage(encoded_images_[i], &codec_specific); diff --git a/modules/video_coding/codecs/h264/h264_encoder_impl.h b/modules/video_coding/codecs/h264/h264_encoder_impl.h index bf70c89211..f02521f0dc 100644 --- a/modules/video_coding/codecs/h264/h264_encoder_impl.h +++ b/modules/video_coding/codecs/h264/h264_encoder_impl.h @@ -24,7 +24,11 @@ #include #include +#include "absl/container/inlined_vector.h" +#include "api/transport/rtp/dependency_descriptor.h" #include "api/video/i420_buffer.h" +#include "api/video/video_codec_constants.h" +#include "api/video_codecs/scalability_mode.h" #include "api/video_codecs/video_encoder.h" #include "common_video/h264/h264_bitstream_parser.h" #include "modules/video_coding/codecs/h264/include/h264.h" @@ -99,6 +103,8 @@ class H264EncoderImpl : public H264Encoder { std::vector configurations_; std::vector encoded_images_; std::vector> svc_controllers_; + absl::InlinedVector, kMaxSimulcastStreams> + scalability_modes_; VideoCodec codec_; H264PacketizationMode packetization_mode_; diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc index 49ccf2dade..8e401fcc7b 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc @@ -25,6 +25,7 @@ #include "api/video/video_content_type.h" #include "api/video/video_frame_buffer.h" #include "api/video/video_timing.h" +#include "api/video_codecs/scalability_mode.h" #include "api/video_codecs/vp8_temporal_layers.h" #include "api/video_codecs/vp8_temporal_layers_factory.h" #include "modules/video_coding/codecs/interface/common_constants.h" @@ -1103,6 +1104,17 @@ void LibvpxVp8Encoder::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, codec_specific->template_structure->resolutions = { RenderResolution(pkt.data.frame.width[0], pkt.data.frame.height[0])}; } + switch (vpx_configs_[encoder_idx].ts_number_layers) { + case 1: + codec_specific->scalability_mode = ScalabilityMode::kL1T1; + break; + case 2: + codec_specific->scalability_mode = ScalabilityMode::kL1T2; + break; + case 3: + codec_specific->scalability_mode = ScalabilityMode::kL1T3; + break; + } } int LibvpxVp8Encoder::GetEncodedPartitions(const VideoFrame& input_image, diff --git a/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc b/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc index 4c27f4ce22..c2884c0395 100644 --- a/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc +++ b/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc @@ -9,23 +9,26 @@ * */ +#include #ifdef RTC_ENABLE_VP9 -#include "modules/video_coding/codecs/vp9/libvpx_vp9_encoder.h" - #include #include +#include #include #include #include "absl/algorithm/container.h" #include "absl/memory/memory.h" #include "absl/strings/match.h" +#include "absl/types/optional.h" #include "api/video/color_space.h" #include "api/video/i010_buffer.h" +#include "api/video_codecs/scalability_mode.h" #include "common_video/include/video_frame_buffer.h" #include "common_video/libyuv/include/webrtc_libyuv.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/video_coding/codecs/vp9/libvpx_vp9_encoder.h" #include "modules/video_coding/svc/create_scalability_structure.h" #include "modules/video_coding/svc/scalability_mode_util.h" #include "modules/video_coding/svc/scalable_video_controller.h" @@ -79,13 +82,17 @@ std::pair GetActiveLayers( return {0, 0}; } -std::unique_ptr CreateVp9ScalabilityStructure( +using Vp9ScalabilityStructure = + std::tuple, ScalabilityMode>; +absl::optional CreateVp9ScalabilityStructure( const VideoCodec& codec) { int num_spatial_layers = codec.VP9().numberOfSpatialLayers; int num_temporal_layers = std::max(1, int{codec.VP9().numberOfTemporalLayers}); if (num_spatial_layers == 1 && num_temporal_layers == 1) { - return std::make_unique(); + return absl::make_optional( + std::make_unique(), + ScalabilityMode::kL1T1); } char name[20]; @@ -93,7 +100,7 @@ std::unique_ptr CreateVp9ScalabilityStructure( if (codec.mode == VideoCodecMode::kScreensharing) { // TODO(bugs.webrtc.org/11999): Compose names of the structures when they // are implemented. - return nullptr; + return absl::nullopt; } else if (codec.VP9().interLayerPred == InterLayerPredMode::kOn || num_spatial_layers == 1) { ss << "L" << num_spatial_layers << "T" << num_temporal_layers; @@ -110,7 +117,7 @@ std::unique_ptr CreateVp9ScalabilityStructure( codec.height != codec.spatialLayers[num_spatial_layers - 1].height) { RTC_LOG(LS_WARNING) << "Top layer resolution expected to match overall resolution"; - return nullptr; + return absl::nullopt; } // Check if the ratio is one of the supported. int numerator; @@ -128,7 +135,7 @@ std::unique_ptr CreateVp9ScalabilityStructure( RTC_LOG(LS_WARNING) << "Unsupported scalability ratio " << codec.spatialLayers[0].width << ":" << codec.spatialLayers[1].width; - return nullptr; + return absl::nullopt; } // Validate ratio is consistent for all spatial layer transitions. for (int sid = 1; sid < num_spatial_layers; ++sid) { @@ -138,7 +145,7 @@ std::unique_ptr CreateVp9ScalabilityStructure( codec.spatialLayers[sid - 1].height * denominator) { RTC_LOG(LS_WARNING) << "Inconsistent scalability ratio " << numerator << ":" << denominator; - return nullptr; + return absl::nullopt; } } } @@ -147,7 +154,7 @@ std::unique_ptr CreateVp9ScalabilityStructure( ScalabilityModeFromString(name); if (!scalability_mode.has_value()) { RTC_LOG(LS_WARNING) << "Invalid scalability mode " << name; - return nullptr; + return absl::nullopt; } auto scalability_structure_controller = CreateScalabilityStructure(*scalability_mode); @@ -156,7 +163,8 @@ std::unique_ptr CreateVp9ScalabilityStructure( } else { RTC_LOG(LS_INFO) << "Created scalability structure " << name; } - return scalability_structure_controller; + return absl::make_optional( + std::move(scalability_structure_controller), *scalability_mode); } vpx_svc_ref_frame_config_t Vp9References( @@ -570,12 +578,12 @@ int LibvpxVp9Encoder::InitEncode(const VideoCodec* inst, force_key_frame_ = true; pics_since_key_ = 0; - absl::optional scalability_mode = inst->GetScalabilityMode(); - if (scalability_mode.has_value()) { + scalability_mode_ = inst->GetScalabilityMode(); + if (scalability_mode_.has_value()) { // Use settings from `ScalabilityMode` identifier. RTC_LOG(LS_INFO) << "Create scalability structure " - << ScalabilityModeToString(*scalability_mode); - svc_controller_ = CreateScalabilityStructure(*scalability_mode); + << ScalabilityModeToString(*scalability_mode_); + svc_controller_ = CreateScalabilityStructure(*scalability_mode_); if (!svc_controller_) { RTC_LOG(LS_WARNING) << "Failed to create scalability structure."; return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; @@ -584,7 +592,7 @@ int LibvpxVp9Encoder::InitEncode(const VideoCodec* inst, svc_controller_->StreamConfig(); num_spatial_layers_ = info.num_spatial_layers; num_temporal_layers_ = info.num_temporal_layers; - inter_layer_pred_ = ScalabilityModeToInterLayerPredMode(*scalability_mode); + inter_layer_pred_ = ScalabilityModeToInterLayerPredMode(*scalability_mode_); } else { num_spatial_layers_ = inst->VP9().numberOfSpatialLayers; RTC_DCHECK_GT(num_spatial_layers_, 0); @@ -593,7 +601,14 @@ int LibvpxVp9Encoder::InitEncode(const VideoCodec* inst, num_temporal_layers_ = 1; } inter_layer_pred_ = inst->VP9().interLayerPred; - svc_controller_ = CreateVp9ScalabilityStructure(*inst); + auto vp9_scalability = CreateVp9ScalabilityStructure(*inst); + if (vp9_scalability.has_value()) { + std::tie(svc_controller_, scalability_mode_) = + std::move(vp9_scalability.value()); + } else { + svc_controller_ = nullptr; + scalability_mode_ = absl::nullopt; + } } framerate_controller_ = std::vector( @@ -1443,6 +1458,7 @@ bool LibvpxVp9Encoder::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, } } } + codec_specific->scalability_mode = scalability_mode_; return true; } diff --git a/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.h b/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.h index fb6f234ead..6b662ae8f9 100644 --- a/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.h +++ b/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.h @@ -20,6 +20,7 @@ #include "api/fec_controller_override.h" #include "api/field_trials_view.h" +#include "api/video_codecs/scalability_mode.h" #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/vp9_profile.h" #include "common_video/include/video_frame_buffer_pool.h" @@ -151,6 +152,7 @@ class LibvpxVp9Encoder : public VP9Encoder { bool force_all_active_layers_; std::unique_ptr svc_controller_; + absl::optional scalability_mode_; std::vector framerate_controller_; // Used for flexible mode. diff --git a/modules/video_coding/include/video_codec_interface.h b/modules/video_coding/include/video_codec_interface.h index 261ffb11c1..46ae0d29e1 100644 --- a/modules/video_coding/include/video_codec_interface.h +++ b/modules/video_coding/include/video_codec_interface.h @@ -16,6 +16,7 @@ #include "absl/base/attributes.h" #include "absl/types/optional.h" #include "api/video/video_frame.h" +#include "api/video_codecs/scalability_mode.h" #include "api/video_codecs/video_decoder.h" #include "api/video_codecs/video_encoder.h" #include "common_video/generic_frame_descriptor/generic_frame_info.h" @@ -112,6 +113,7 @@ struct RTC_EXPORT CodecSpecificInfo { bool end_of_picture = true; absl::optional generic_frame_info; absl::optional template_structure; + absl::optional scalability_mode; }; } // namespace webrtc diff --git a/pc/BUILD.gn b/pc/BUILD.gn index 053f4e62b0..15c87dda25 100644 --- a/pc/BUILD.gn +++ b/pc/BUILD.gn @@ -1008,6 +1008,7 @@ rtc_source_set("rtc_stats_collector") { "../api/task_queue:task_queue", "../api/units:time_delta", "../api/video:video_rtp_headers", + "../api/video_codecs:scalability_mode", "../call:call_interfaces", "../common_video:common_video", "../media:rtc_media_base", @@ -2367,6 +2368,7 @@ if (rtc_include_tests && !build_with_chromium) { "../api/video:video_codec_constants", "../api/video:video_frame", "../api/video:video_rtp_headers", + "../api/video_codecs:scalability_mode", "../call/adaptation:resource_adaptation_test_utilities", "../common_video", "../logging:fake_rtc_event_log", @@ -2726,6 +2728,7 @@ if (rtc_include_tests && !build_with_chromium) { "../api:media_stream_interface", "../api:network_emulation_manager_api", "../api:peer_connection_quality_test_fixture_api", + "../api:rtc_stats_api", "../api:simulated_network_api", "../api:time_controller", "../api/test/metrics:global_metrics_logger_and_exporter", diff --git a/pc/rtc_stats_collector.cc b/pc/rtc_stats_collector.cc index 7b76953ea9..374780e941 100644 --- a/pc/rtc_stats_collector.cc +++ b/pc/rtc_stats_collector.cc @@ -33,6 +33,7 @@ #include "api/stats/rtcstats_objects.h" #include "api/units/time_delta.h" #include "api/video/video_content_type.h" +#include "api/video_codecs/scalability_mode.h" #include "common_video/include/quality_limitation_reason.h" #include "media/base/media_channel.h" #include "modules/audio_processing/include/audio_processing_statistics.h" @@ -792,6 +793,10 @@ void SetOutboundRTPStreamStatsFromVideoSenderInfo( outbound_video->power_efficient_encoder = video_sender_info.power_efficient_encoder.value(); } + if (video_sender_info.scalability_mode) { + outbound_video->scalability_mode = std::string( + ScalabilityModeToString(*video_sender_info.scalability_mode)); + } } std::unique_ptr diff --git a/pc/rtc_stats_collector_unittest.cc b/pc/rtc_stats_collector_unittest.cc index 1982446667..5388355eb7 100644 --- a/pc/rtc_stats_collector_unittest.cc +++ b/pc/rtc_stats_collector_unittest.cc @@ -39,6 +39,7 @@ #include "api/video/video_sink_interface.h" #include "api/video/video_source_interface.h" #include "api/video/video_timing.h" +#include "api/video_codecs/scalability_mode.h" #include "common_video/include/quality_limitation_reason.h" #include "media/base/media_channel.h" #include "modules/audio_processing/include/audio_processing_statistics.h" @@ -2835,6 +2836,7 @@ TEST_F(RTCStatsCollectorTest, CollectRTCOutboundRTPStreamStats_Video) { video_media_info.senders[0].frames_sent = 5; video_media_info.senders[0].huge_frames_sent = 2; video_media_info.senders[0].active = false; + video_media_info.senders[0].scalability_mode = ScalabilityMode::kL3T3_KEY; video_media_info.aggregated_senders.push_back(video_media_info.senders[0]); RtpCodecParameters codec_parameters; codec_parameters.payload_type = 42; @@ -2894,6 +2896,7 @@ TEST_F(RTCStatsCollectorTest, CollectRTCOutboundRTPStreamStats_Video) { expected_video.huge_frames_sent = 2; expected_video.active = false; expected_video.power_efficient_encoder = false; + expected_video.scalability_mode = "L3T3_KEY"; // `expected_video.content_type` should be undefined. // `expected_video.qp_sum` should be undefined. // `expected_video.encoder_implementation` should be undefined. diff --git a/pc/rtc_stats_integrationtest.cc b/pc/rtc_stats_integrationtest.cc index c63083dbfc..b5abad9d63 100644 --- a/pc/rtc_stats_integrationtest.cc +++ b/pc/rtc_stats_integrationtest.cc @@ -984,6 +984,7 @@ class RTCStatsReportVerifier { verifier.TestMemberIsNonNegative( outbound_stream.huge_frames_sent); verifier.MarkMemberTested(outbound_stream.rid, true); + verifier.TestMemberIsDefined(outbound_stream.scalability_mode); } else { verifier.TestMemberIsUndefined(outbound_stream.frames_encoded); verifier.TestMemberIsUndefined(outbound_stream.key_frames_encoded); @@ -1005,6 +1006,7 @@ class RTCStatsReportVerifier { verifier.TestMemberIsUndefined(outbound_stream.frame_width); verifier.TestMemberIsUndefined(outbound_stream.frames_sent); verifier.TestMemberIsUndefined(outbound_stream.huge_frames_sent); + verifier.TestMemberIsUndefined(outbound_stream.scalability_mode); } return verifier.ExpectAllMembersSuccessfullyTested(); } diff --git a/pc/test/svc_e2e_tests.cc b/pc/test/svc_e2e_tests.cc index 5d98f7313d..dea0763758 100644 --- a/pc/test/svc_e2e_tests.cc +++ b/pc/test/svc_e2e_tests.cc @@ -13,6 +13,7 @@ #include #include "api/media_stream_interface.h" +#include "api/stats/rtcstats_objects.h" #include "api/test/create_network_emulation_manager.h" #include "api/test/create_peer_connection_quality_test_frame_generator.h" #include "api/test/create_peerconnection_quality_test_fixture.h" @@ -46,6 +47,7 @@ using ::cricket::kH264CodecName; using ::cricket::kVp8CodecName; using ::cricket::kVp9CodecName; using ::testing::Combine; +using ::testing::Optional; using ::testing::UnitTest; using ::testing::Values; using ::testing::ValuesIn; @@ -203,16 +205,32 @@ class SvcVideoQualityAnalyzer : public DefaultVideoQualityAnalyzer { input_image); } + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr& report) override { + // Extract the scalability mode reported in the stats. + auto outbound_stats = report->GetStatsOfType(); + for (const auto& stat : outbound_stats) { + if (stat->scalability_mode.is_defined()) { + reported_scalability_mode_ = *stat->scalability_mode; + } + } + } + const SpatialTemporalLayerCounts& encoder_layers_seen() const { return encoder_layers_seen_; } const SpatialTemporalLayerCounts& decoder_layers_seen() const { return decoder_layers_seen_; } + const absl::optional reported_scalability_mode() const { + return reported_scalability_mode_; + } private: SpatialTemporalLayerCounts encoder_layers_seen_; SpatialTemporalLayerCounts decoder_layers_seen_; + absl::optional reported_scalability_mode_; }; MATCHER_P2(HasSpatialAndTemporalLayers, @@ -342,6 +360,8 @@ TEST_P(SvcTest, ScalabilityModeSupported) { SvcTestParameters().expected_spatial_layers, SvcTestParameters().expected_temporal_layers)); } + EXPECT_THAT(analyzer_ptr->reported_scalability_mode(), + Optional(SvcTestParameters().scalability_mode)); RTC_LOG(LS_INFO) << "Encoder layers seen: " << analyzer_ptr->encoder_layers_seen().size(); diff --git a/stats/rtcstats_objects.cc b/stats/rtcstats_objects.cc index 7c20b87562..734f16273c 100644 --- a/stats/rtcstats_objects.cc +++ b/stats/rtcstats_objects.cc @@ -678,7 +678,8 @@ WEBRTC_RTCSTATS_IMPL( &nack_count, &qp_sum, &active, - &power_efficient_encoder) + &power_efficient_encoder, + &scalability_mode) // clang-format on RTCOutboundRTPStreamStats::RTCOutboundRTPStreamStats(const std::string& id, @@ -719,7 +720,8 @@ RTCOutboundRTPStreamStats::RTCOutboundRTPStreamStats(std::string&& id, nack_count("nackCount"), qp_sum("qpSum"), active("active"), - power_efficient_encoder("powerEfficientEncoder") {} + power_efficient_encoder("powerEfficientEncoder"), + scalability_mode("scalabilityMode") {} RTCOutboundRTPStreamStats::RTCOutboundRTPStreamStats( const RTCOutboundRTPStreamStats& other) = default; diff --git a/video/BUILD.gn b/video/BUILD.gn index 8c755b07b6..27804f1dd6 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -859,6 +859,7 @@ if (rtc_include_tests) { "../api/video:video_frame_type", "../api/video:video_rtp_headers", "../api/video/test:video_frame_matchers", + "../api/video_codecs:scalability_mode", "../api/video_codecs:video_codecs_api", "../api/video_codecs:vp8_temporal_layers_factory", "../call:call_interfaces", diff --git a/video/send_statistics_proxy.cc b/video/send_statistics_proxy.cc index 523781558b..b6c2d60a73 100644 --- a/video/send_statistics_proxy.cc +++ b/video/send_statistics_proxy.cc @@ -985,6 +985,8 @@ void SendStatisticsProxy::OnSendEncodedImage( stats->frames_encoded++; stats->total_encode_time_ms += encoded_image.timing_.encode_finish_ms - encoded_image.timing_.encode_start_ms; + if (codec_info) + stats->scalability_mode = codec_info->scalability_mode; // Report resolution of the top spatial layer. bool is_top_spatial_layer = codec_info == nullptr || codec_info->end_of_picture; diff --git a/video/send_statistics_proxy_unittest.cc b/video/send_statistics_proxy_unittest.cc index d24b3c80a6..af3b0208e2 100644 --- a/video/send_statistics_proxy_unittest.cc +++ b/video/send_statistics_proxy_unittest.cc @@ -21,6 +21,7 @@ #include "api/video/video_adaptation_reason.h" #include "api/video/video_bitrate_allocation.h" #include "api/video/video_codec_type.h" +#include "api/video_codecs/scalability_mode.h" #include "api/video_codecs/video_codec.h" #include "rtc_base/fake_clock.h" #include "system_wrappers/include/metrics.h" @@ -32,6 +33,9 @@ namespace webrtc { namespace { + +using ::testing::Optional; + const uint32_t kFirstSsrc = 17; const uint32_t kSecondSsrc = 42; const uint32_t kFirstRtxSsrc = 18; @@ -397,6 +401,34 @@ TEST_F(SendStatisticsProxyTest, OnSendEncodedImageWithoutQpQpSumWontExist) { statistics_proxy_->GetStats().substreams[ssrc].qp_sum); } +TEST_F(SendStatisticsProxyTest, + OnSendEncodedImageSetsScalabilityModeOfCurrentLayer) { + EncodedImage encoded_image; + CodecSpecificInfo codec_info; + ScalabilityMode layer0_mode = ScalabilityMode::kL1T1; + ScalabilityMode layer1_mode = ScalabilityMode::kL1T3; + auto ssrc0 = config_.rtp.ssrcs[0]; + auto ssrc1 = config_.rtp.ssrcs[1]; + EXPECT_EQ(absl::nullopt, + statistics_proxy_->GetStats().substreams[ssrc0].scalability_mode); + EXPECT_EQ(absl::nullopt, + statistics_proxy_->GetStats().substreams[ssrc1].scalability_mode); + encoded_image.SetSpatialIndex(0); + codec_info.scalability_mode = layer0_mode; + statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info); + EXPECT_THAT(statistics_proxy_->GetStats().substreams[ssrc0].scalability_mode, + layer0_mode); + EXPECT_EQ(absl::nullopt, + statistics_proxy_->GetStats().substreams[ssrc1].scalability_mode); + encoded_image.SetSpatialIndex(1); + codec_info.scalability_mode = layer1_mode; + statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info); + EXPECT_THAT(statistics_proxy_->GetStats().substreams[ssrc0].scalability_mode, + layer0_mode); + EXPECT_THAT(statistics_proxy_->GetStats().substreams[ssrc1].scalability_mode, + layer1_mode); +} + TEST_F(SendStatisticsProxyTest, TotalEncodedBytesTargetFirstFrame) { const uint32_t kTargetBytesPerSecond = 100000; statistics_proxy_->OnSetEncoderTargetRate(kTargetBytesPerSecond * 8);