diff --git a/api/video_codecs/BUILD.gn b/api/video_codecs/BUILD.gn index af132717df..ad93d401e7 100644 --- a/api/video_codecs/BUILD.gn +++ b/api/video_codecs/BUILD.gn @@ -40,6 +40,7 @@ rtc_source_set("video_codecs_api") { "../video:video_bitrate_allocation", "../video:video_codec_constants", "../video:video_frame", + "//third_party/abseil-cpp/absl/container:inlined_vector", "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] diff --git a/api/video_codecs/video_encoder.cc b/api/video_codecs/video_encoder.cc index 478bbe1351..5b7d89c749 100644 --- a/api/video_codecs/video_encoder.cc +++ b/api/video_codecs/video_encoder.cc @@ -82,6 +82,8 @@ VideoEncoder::ScalingSettings::~ScalingSettings() {} // static constexpr VideoEncoder::ScalingSettings::KOff VideoEncoder::ScalingSettings::kOff; +// static +constexpr uint8_t VideoEncoder::EncoderInfo::kMaxFramerateFraction; VideoEncoder::EncoderInfo::EncoderInfo() : scaling_settings(VideoEncoder::ScalingSettings::kOff), @@ -89,7 +91,10 @@ VideoEncoder::EncoderInfo::EncoderInfo() implementation_name("unknown"), has_trusted_rate_controller(false), is_hardware_accelerated(true), - has_internal_source(false) {} + has_internal_source(false), + fps_allocation{absl::InlinedVector( + 1, + kMaxFramerateFraction)} {} VideoEncoder::EncoderInfo::EncoderInfo(const EncoderInfo&) = default; diff --git a/api/video_codecs/video_encoder.h b/api/video_codecs/video_encoder.h index 2856b1c3d4..d48498bcc1 100644 --- a/api/video_codecs/video_encoder.h +++ b/api/video_codecs/video_encoder.h @@ -11,10 +11,12 @@ #ifndef API_VIDEO_CODECS_VIDEO_ENCODER_H_ #define API_VIDEO_CODECS_VIDEO_ENCODER_H_ +#include #include #include #include +#include "absl/container/inlined_vector.h" #include "absl/types/optional.h" #include "api/video/encoded_image.h" #include "api/video/video_bitrate_allocation.h" @@ -120,6 +122,9 @@ class RTC_EXPORT VideoEncoder { // Struct containing metadata about the encoder implementing this interface. struct EncoderInfo { + static constexpr uint8_t kMaxFramerateFraction = + std::numeric_limits::max(); + EncoderInfo(); EncoderInfo(const EncoderInfo&); @@ -159,6 +164,32 @@ class RTC_EXPORT VideoEncoder { // Internal source encoders are deprecated and support for them will be // phased out. bool has_internal_source; + + // For each spatial layer (simulcast stream or SVC layer), represented as an + // element in |fps_allocation| a vector indicates how many temporal layers + // the encoder is using for that spatial layer. + // For each spatial/temporal layer pair, the frame rate fraction is given as + // an 8bit unsigned integer where 0 = 0% and 255 = 100%. + // + // If the vector is empty for a given spatial layer, it indicates that frame + // rates are not defined and we can't count on any specific frame rate to be + // generated. Likely this indicates Vp8TemporalLayersType::kBitrateDynamic. + // + // The encoder may update this on a per-frame basis in response to both + // internal and external signals. + // + // Spatial layers are treated independently, but temporal layers are + // cumulative. For instance, if: + // fps_allocation[0][0] = kFullFramerate / 2; + // fps_allocation[0][1] = kFullFramerate; + // Then half of the frames are in the base layer and half is in TL1, but + // since TL1 is assumed to depend on the base layer, the frame rate is + // indicated as the full 100% for the top layer. + // + // Defaults to a single spatial layer containing a single temporal layer + // with a 100% frame rate fraction. + absl::InlinedVector + fps_allocation[kMaxSpatialLayers]; }; static VideoCodecVP8 GetDefaultVp8Settings(); diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc index 0365ce2d8f..d73c59a718 100644 --- a/media/engine/simulcast_encoder_adapter.cc +++ b/media/engine/simulcast_encoder_adapter.cc @@ -312,6 +312,7 @@ int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst, encoder_info_.has_internal_source &= encoder_impl_info.has_internal_source; } + encoder_info_.fps_allocation[i] = encoder_impl_info.fps_allocation[0]; } } diff --git a/media/engine/simulcast_encoder_adapter_unittest.cc b/media/engine/simulcast_encoder_adapter_unittest.cc index 9fa354f5fa..4cbaf56db2 100644 --- a/media/engine/simulcast_encoder_adapter_unittest.cc +++ b/media/engine/simulcast_encoder_adapter_unittest.cc @@ -31,6 +31,9 @@ using ::testing::_; using ::testing::Return; +using EncoderInfo = webrtc::VideoEncoder::EncoderInfo; +using FramerateFractions = + absl::InlinedVector; namespace webrtc { namespace test { @@ -219,6 +222,7 @@ class MockVideoEncoder : public VideoEncoder { info.has_trusted_rate_controller = has_trusted_rate_controller_; info.is_hardware_accelerated = is_hardware_accelerated_; info.has_internal_source = has_internal_source_; + info.fps_allocation[0] = fps_allocation_; return info; } @@ -265,6 +269,10 @@ class MockVideoEncoder : public VideoEncoder { has_internal_source_ = has_internal_source; } + void set_fps_allocation(const FramerateFractions& fps_allocation) { + fps_allocation_ = fps_allocation; + } + VideoBitrateAllocation last_set_bitrate() const { return last_set_bitrate_; } private: @@ -277,6 +285,7 @@ class MockVideoEncoder : public VideoEncoder { bool has_internal_source_ = false; int32_t init_encode_return_value_ = 0; VideoBitrateAllocation last_set_bitrate_; + FramerateFractions fps_allocation_; VideoCodec codec_; EncodedImageCallback* callback_; @@ -1085,5 +1094,34 @@ TEST_F(TestSimulcastEncoderAdapterFake, ReportsInternalSource) { EXPECT_FALSE(adapter_->GetEncoderInfo().has_internal_source); } +TEST_F(TestSimulcastEncoderAdapterFake, ReportsFpsAllocation) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 3; + adapter_->RegisterEncodeCompleteCallback(this); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200)); + ASSERT_EQ(3u, helper_->factory()->encoders().size()); + + // Combination of three different supported mode: + // Simulcast stream 0 has undefined fps behavior. + // Simulcast stream 1 has three temporal layers. + // Simulcast stream 2 has 1 temporal layer. + FramerateFractions expected_fps_allocation[kMaxSpatialLayers]; + expected_fps_allocation[1].push_back(EncoderInfo::kMaxFramerateFraction / 4); + expected_fps_allocation[1].push_back(EncoderInfo::kMaxFramerateFraction / 2); + expected_fps_allocation[1].push_back(EncoderInfo::kMaxFramerateFraction); + expected_fps_allocation[2].push_back(EncoderInfo::kMaxFramerateFraction); + + // All encoders have internal source, simulcast adapter reports true. + for (size_t i = 0; i < codec_.numberOfSimulcastStreams; ++i) { + MockVideoEncoder* encoder = helper_->factory()->encoders()[i]; + encoder->set_fps_allocation(expected_fps_allocation[i]); + } + EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200)); + EXPECT_THAT(adapter_->GetEncoderInfo().fps_allocation, + ::testing::ElementsAreArray(expected_fps_allocation)); +} + } // namespace test } // namespace webrtc diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc index e0037b2e8d..1384dee5ab 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc @@ -963,6 +963,31 @@ VideoEncoder::EncoderInfo LibvpxVp8Encoder::GetEncoderInfo() const { ? VideoEncoder::ScalingSettings( kLowVp8QpThreshold, kHighVp8QpThreshold) : VideoEncoder::ScalingSettings::kOff; + // |encoder_idx| is libvpx index where 0 is highest resolution. + // |si| is simulcast index, where 0 is lowest resolution. + for (size_t si = 0, encoder_idx = encoders_.size() - 1; si < encoders_.size(); + ++si, --encoder_idx) { + info.fps_allocation[si].clear(); + if ((codec_.numberOfSimulcastStreams > si && + !codec_.simulcastStream[si].active) || + (si == 0 && SimulcastUtility::IsConferenceModeScreenshare(codec_))) { + // No defined frame rate fractions if not active or if using + // ScreenshareLayers, leave vector empty and continue; + continue; + } + if (configurations_[encoder_idx].ts_number_layers <= 1) { + info.fps_allocation[si].push_back(EncoderInfo::kMaxFramerateFraction); + } else { + for (size_t ti = 0; ti < configurations_[encoder_idx].ts_number_layers; + ++ti) { + RTC_DCHECK_GT(configurations_[encoder_idx].ts_rate_decimator[ti], 0); + info.fps_allocation[si].push_back(rtc::saturated_cast( + EncoderInfo::kMaxFramerateFraction / + configurations_[encoder_idx].ts_rate_decimator[ti] + + 0.5)); + } + } + } return info; } diff --git a/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc b/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc index 12ffcb5d91..12e13a900b 100644 --- a/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc +++ b/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc @@ -27,12 +27,18 @@ namespace webrtc { +using testing::_; +using testing::ElementsAreArray; using testing::Invoke; using testing::NiceMock; using testing::Return; -using testing::_; +using EncoderInfo = webrtc::VideoEncoder::EncoderInfo; +using FramerateFractions = + absl::InlinedVector; namespace { +constexpr uint32_t kLegacyScreenshareTl0BitrateKbps = 200; +constexpr uint32_t kLegacyScreenshareTl1BitrateKbps = 1000; constexpr uint32_t kInitialTimestampRtp = 123; constexpr int64_t kTestNtpTimeMs = 456; constexpr int64_t kInitialTimestampMs = 789; @@ -472,4 +478,101 @@ TEST_F(TestVp8Impl, KeepsTimestampOnReencode) { encoder.Encode(*NextInputFrame(), nullptr, &delta_frame); } +TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationNoLayers) { + FramerateFractions expected_fps_allocation[kMaxSpatialLayers] = { + FramerateFractions(1, EncoderInfo::kMaxFramerateFraction)}; + + EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation, + ::testing::ElementsAreArray(expected_fps_allocation)); +} + +TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationTwoTemporalLayers) { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); + codec_settings_.numberOfSimulcastStreams = 1; + codec_settings_.simulcastStream[0].active = true; + codec_settings_.simulcastStream[0].targetBitrate = 100; + codec_settings_.simulcastStream[0].maxBitrate = 100; + codec_settings_.simulcastStream[0].numberOfTemporalLayers = 2; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize)); + + FramerateFractions expected_fps_allocation[kMaxSpatialLayers]; + expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2); + expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction); + + EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation, + ::testing::ElementsAreArray(expected_fps_allocation)); +} + +TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationThreeTemporalLayers) { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); + codec_settings_.numberOfSimulcastStreams = 1; + codec_settings_.simulcastStream[0].active = true; + codec_settings_.simulcastStream[0].targetBitrate = 100; + codec_settings_.simulcastStream[0].maxBitrate = 100; + codec_settings_.simulcastStream[0].numberOfTemporalLayers = 3; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize)); + + FramerateFractions expected_fps_allocation[kMaxSpatialLayers]; + expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 4); + expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2); + expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction); + + EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation, + ::testing::ElementsAreArray(expected_fps_allocation)); +} + +TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationScreenshareLayers) { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); + codec_settings_.numberOfSimulcastStreams = 1; + codec_settings_.mode = VideoCodecMode::kScreensharing; + codec_settings_.simulcastStream[0].active = true; + codec_settings_.simulcastStream[0].minBitrate = 30; + codec_settings_.simulcastStream[0].targetBitrate = + kLegacyScreenshareTl0BitrateKbps; + codec_settings_.simulcastStream[0].maxBitrate = + kLegacyScreenshareTl1BitrateKbps; + codec_settings_.simulcastStream[0].numberOfTemporalLayers = 2; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize)); + + // Expect empty vector, since this mode doesn't have a fixed framerate. + FramerateFractions expected_fps_allocation[kMaxSpatialLayers]; + EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation, + ::testing::ElementsAreArray(expected_fps_allocation)); +} + +TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationSimulcastVideo) { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); + + // Set up three simulcast streams with three temporal layers each. + codec_settings_.numberOfSimulcastStreams = 3; + for (int i = 0; i < codec_settings_.numberOfSimulcastStreams; ++i) { + codec_settings_.simulcastStream[i].active = true; + codec_settings_.simulcastStream[i].minBitrate = 30; + codec_settings_.simulcastStream[i].targetBitrate = 30; + codec_settings_.simulcastStream[i].maxBitrate = 30; + codec_settings_.simulcastStream[i].numberOfTemporalLayers = 3; + codec_settings_.simulcastStream[i].width = + codec_settings_.width >> + (codec_settings_.numberOfSimulcastStreams - i - 1); + codec_settings_.simulcastStream[i].height = + codec_settings_.height >> + (codec_settings_.numberOfSimulcastStreams - i - 1); + } + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize)); + + FramerateFractions expected_fps_allocation[kMaxSpatialLayers]; + expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 4); + expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2); + expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction); + expected_fps_allocation[1] = expected_fps_allocation[0]; + expected_fps_allocation[2] = expected_fps_allocation[0]; + EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation, + ::testing::ElementsAreArray(expected_fps_allocation)); +} + } // namespace webrtc diff --git a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc index 98e0452940..9e0305291a 100644 --- a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc +++ b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc @@ -18,10 +18,17 @@ #include "modules/video_coding/codecs/vp9/include/vp9.h" #include "modules/video_coding/codecs/vp9/svc_config.h" #include "test/field_trial.h" +#include "test/gmock.h" +#include "test/gtest.h" #include "test/video_codec_settings.h" namespace webrtc { +using testing::ElementsAreArray; +using EncoderInfo = webrtc::VideoEncoder::EncoderInfo; +using FramerateFractions = + absl::InlinedVector; + namespace { const size_t kWidth = 1280; const size_t kHeight = 720; @@ -848,6 +855,79 @@ TEST_F(TestVp9Impl, ScalabilityStructureIsAvailableInFlexibleMode) { EXPECT_TRUE(codec_specific_info.codecSpecific.VP9.ss_data_available); } +TEST_F(TestVp9Impl, EncoderInfoFpsAllocation) { + const uint8_t kNumSpatialLayers = 3; + const uint8_t kNumTemporalLayers = 3; + + codec_settings_.maxFramerate = 30; + codec_settings_.VP9()->numberOfSpatialLayers = kNumSpatialLayers; + codec_settings_.VP9()->numberOfTemporalLayers = kNumTemporalLayers; + + for (uint8_t sl_idx = 0; sl_idx < kNumSpatialLayers; ++sl_idx) { + codec_settings_.spatialLayers[sl_idx].width = codec_settings_.width; + codec_settings_.spatialLayers[sl_idx].height = codec_settings_.height; + codec_settings_.spatialLayers[sl_idx].minBitrate = + codec_settings_.startBitrate; + codec_settings_.spatialLayers[sl_idx].maxBitrate = + codec_settings_.startBitrate; + codec_settings_.spatialLayers[sl_idx].targetBitrate = + codec_settings_.startBitrate; + codec_settings_.spatialLayers[sl_idx].active = true; + codec_settings_.spatialLayers[sl_idx].maxFramerate = + codec_settings_.maxFramerate; + } + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, 1 /* number of cores */, + 0 /* max payload size (unused) */)); + + FramerateFractions expected_fps_allocation[kMaxSpatialLayers]; + expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 4); + expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2); + expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction); + expected_fps_allocation[1] = expected_fps_allocation[0]; + expected_fps_allocation[2] = expected_fps_allocation[0]; + EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation, + ::testing::ElementsAreArray(expected_fps_allocation)); +} + +TEST_F(TestVp9Impl, EncoderInfoFpsAllocationFlexibleMode) { + const uint8_t kNumSpatialLayers = 3; + + codec_settings_.maxFramerate = 30; + codec_settings_.VP9()->numberOfSpatialLayers = kNumSpatialLayers; + codec_settings_.VP9()->numberOfTemporalLayers = 1; + codec_settings_.VP9()->flexibleMode = true; + + for (uint8_t sl_idx = 0; sl_idx < kNumSpatialLayers; ++sl_idx) { + codec_settings_.spatialLayers[sl_idx].width = codec_settings_.width; + codec_settings_.spatialLayers[sl_idx].height = codec_settings_.height; + codec_settings_.spatialLayers[sl_idx].minBitrate = + codec_settings_.startBitrate; + codec_settings_.spatialLayers[sl_idx].maxBitrate = + codec_settings_.startBitrate; + codec_settings_.spatialLayers[sl_idx].targetBitrate = + codec_settings_.startBitrate; + codec_settings_.spatialLayers[sl_idx].active = true; + // Force different frame rates for different layers, to verify that total + // fraction is correct. + codec_settings_.spatialLayers[sl_idx].maxFramerate = + codec_settings_.maxFramerate / (kNumSpatialLayers - sl_idx); + } + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, 1 /* number of cores */, + 0 /* max payload size (unused) */)); + + // No temporal layers allowed when spatial layers have different fps targets. + FramerateFractions expected_fps_allocation[kMaxSpatialLayers]; + expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 3); + expected_fps_allocation[1].push_back(EncoderInfo::kMaxFramerateFraction / 2); + expected_fps_allocation[2].push_back(EncoderInfo::kMaxFramerateFraction); + EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation, + ::testing::ElementsAreArray(expected_fps_allocation)); +} + class TestVp9ImplWithLayering : public TestVp9Impl, public ::testing::WithParamInterface<::testing::tuple> { diff --git a/modules/video_coding/codecs/vp9/vp9_impl.cc b/modules/video_coding/codecs/vp9/vp9_impl.cc index ae83b4d492..09a08a500d 100644 --- a/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -1354,6 +1354,22 @@ VideoEncoder::EncoderInfo VP9EncoderImpl::GetEncoderInfo() const { info.has_trusted_rate_controller = trusted_rate_controller_; info.is_hardware_accelerated = false; info.has_internal_source = false; + for (size_t si = 0; si < num_spatial_layers_; ++si) { + info.fps_allocation[si].clear(); + if (!codec_.spatialLayers[si].active) { + continue; + } + // This spatial layer may already use a fraction of the total frame rate. + const float sl_fps_fraction = + codec_.spatialLayers[si].maxFramerate / codec_.maxFramerate; + for (size_t ti = 0; ti < num_temporal_layers_; ++ti) { + const uint32_t decimator = + num_temporal_layers_ <= 1 ? 1 : config_->ts_rate_decimator[ti]; + RTC_DCHECK_GT(decimator, 0); + info.fps_allocation[si].push_back(rtc::saturated_cast( + EncoderInfo::kMaxFramerateFraction * (sl_fps_fraction / decimator))); + } + } return info; }