From 3b790f316c08c146df0fb89aa07a51b6895fd485 Mon Sep 17 00:00:00 2001 From: Ying Wang Date: Fri, 19 Jan 2018 17:58:57 +0100 Subject: [PATCH] Make fec controller plug-able. Bug: webrtc:8656 Change-Id: I3d42ffc92a7c95266e5d53bab03f388bd0de2592 Reviewed-on: https://webrtc-review.googlesource.com/39760 Reviewed-by: Stefan Holmer Commit-Queue: Ying Wang Cr-Commit-Position: refs/heads/master@{#21710} --- api/BUILD.gn | 12 ++ api/fec_controller.h | 80 +++++++++++++ call/BUILD.gn | 2 + call/DEPS | 1 + call/call.cc | 23 +++- call/call.h | 5 + modules/video_coding/BUILD.gn | 8 +- ...alculator.cc => fec_controller_default.cc} | 56 ++++----- modules/video_coding/fec_controller_default.h | 56 +++++++++ ...unittest.cc => fec_controller_unittest.cc} | 37 +++--- modules/video_coding/include/video_coding.h | 1 + .../include/video_coding_defines.h | 14 --- .../protection_bitrate_calculator.h | 78 ------------- video/BUILD.gn | 1 + video/video_quality_test.cc | 24 ++++ video/video_quality_test.h | 4 + video/video_send_stream.cc | 106 ++++++++++++++---- video/video_send_stream.h | 5 +- 18 files changed, 348 insertions(+), 165 deletions(-) create mode 100644 api/fec_controller.h rename modules/video_coding/{protection_bitrate_calculator.cc => fec_controller_default.cc} (85%) create mode 100644 modules/video_coding/fec_controller_default.h rename modules/video_coding/{protection_bitrate_calculator_unittest.cc => fec_controller_unittest.cc} (67%) delete mode 100644 modules/video_coding/protection_bitrate_calculator.h diff --git a/api/BUILD.gn b/api/BUILD.gn index 59efbf8e95..95891dface 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -217,6 +217,18 @@ rtc_source_set("transport_api") { ] } +rtc_source_set("fec_controller_api") { + visibility = [ "*" ] + sources = [ + "fec_controller.h", + ] + + deps = [ + "../common_video:common_video", + "../modules:module_api", + ] +} + rtc_source_set("video_frame_api") { visibility = [ "*" ] sources = [ diff --git a/api/fec_controller.h b/api/fec_controller.h new file mode 100644 index 0000000000..24c004ec95 --- /dev/null +++ b/api/fec_controller.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_FEC_CONTROLLER_H_ +#define API_FEC_CONTROLLER_H_ + +#include + +#include "common_video/include/video_frame.h" +#include "modules/include/module_common_types.h" + +namespace webrtc { +// TODO(yinwa): work in progress. API in class FecController should not be +// used by other users until this comment is removed. + +// Callback class used for telling the user about how to configure the FEC, +// and the rates sent the last second is returned to the VCM. +class VCMProtectionCallback { + public: + virtual int ProtectionRequest(const FecProtectionParams* delta_params, + const FecProtectionParams* key_params, + uint32_t* sent_video_rate_bps, + uint32_t* sent_nack_rate_bps, + uint32_t* sent_fec_rate_bps) = 0; + + protected: + virtual ~VCMProtectionCallback() {} +}; + +// FecController calculates how much of the allocated network +// capacity that can be used by an encoder and how much that +// is needed for redundant packets such as FEC and NACK. It uses an +// implementation of |VCMProtectionCallback| to set new FEC parameters and get +// the bitrate currently used for FEC and NACK. +// Usage: +// Setup by calling SetProtectionMethod and SetEncodingData. +// For each encoded image, call UpdateWithEncodedData. +// Each time the bandwidth estimate change, call UpdateFecRates. UpdateFecRates +// will return the bitrate that can be used by an encoder. +// A lock is used to protect internal states, so methods can be called on an +// arbitrary thread. +class FecController { + public: + virtual ~FecController() {} + + virtual void SetProtectionCallback( + VCMProtectionCallback* protection_callback) = 0; + virtual void SetProtectionMethod(bool enable_fec, bool enable_nack) = 0; + + // Informs loss protectoin logic of initial encoding state. + virtual void SetEncodingData(size_t width, + size_t height, + size_t num_temporal_layers, + size_t max_payload_size) = 0; + + // Returns target rate for the encoder given the channel parameters. + // Inputs: estimated_bitrate_bps - the estimated network bitrate in bits/s. + // actual_framerate - encoder frame rate. + // fraction_lost - packet loss rate in % in the network. + // loss_mask_vector - packet loss mask since last time this method + // was called. round_trip_time_ms - round trip time in milliseconds. + virtual uint32_t UpdateFecRates(uint32_t estimated_bitrate_bps, + int actual_framerate, + uint8_t fraction_lost, + std::vector loss_mask_vector, + int64_t round_trip_time_ms) = 0; + + // Informs of encoded output. + virtual void UpdateWithEncodedData(const EncodedImage& encoded_image) = 0; +}; + +} // namespace webrtc +#endif // API_FEC_CONTROLLER_H_ diff --git a/call/BUILD.gn b/call/BUILD.gn index 90b44cf562..636fe0bc6b 100644 --- a/call/BUILD.gn +++ b/call/BUILD.gn @@ -28,6 +28,7 @@ rtc_source_set("call_interfaces") { "..:webrtc_common", "../:typedefs", "../api:audio_mixer_api", + "../api:fec_controller_api", "../api:libjingle_peerconnection_api", "../api:optional", "../api:transport_api", @@ -149,6 +150,7 @@ rtc_static_library("call") { "../modules/rtp_rtcp", "../modules/rtp_rtcp:rtp_rtcp_format", "../modules/utility", + "../modules/video_coding:video_coding", "../rtc_base:checks", "../rtc_base:rtc_base_approved", "../rtc_base:rtc_task_queue", diff --git a/call/DEPS b/call/DEPS index 307a26e3b0..f823a7b9c3 100644 --- a/call/DEPS +++ b/call/DEPS @@ -7,6 +7,7 @@ include_rules = [ "+modules/audio_processing", "+modules/bitrate_controller", "+modules/congestion_controller", + "+modules/video_coding", "+modules/pacing", "+modules/rtp_rtcp", "+modules/utility", diff --git a/call/call.cc b/call/call.cc index fceea13768..fc8333c20a 100644 --- a/call/call.cc +++ b/call/call.cc @@ -42,6 +42,7 @@ #include "modules/rtp_rtcp/source/byte_io.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" #include "modules/utility/include/process_thread.h" +#include "modules/video_coding/fec_controller_default.h" #include "rtc_base/basictypes.h" #include "rtc_base/checks.h" #include "rtc_base/constructormagic.h" @@ -184,6 +185,10 @@ class Call : public webrtc::Call, webrtc::VideoSendStream* CreateVideoSendStream( webrtc::VideoSendStream::Config config, VideoEncoderConfig encoder_config) override; + webrtc::VideoSendStream* CreateVideoSendStream( + webrtc::VideoSendStream::Config config, + VideoEncoderConfig encoder_config, + std::unique_ptr fec_controller) override; void DestroyVideoSendStream(webrtc::VideoSendStream* send_stream) override; webrtc::VideoReceiveStream* CreateVideoReceiveStream( @@ -402,6 +407,13 @@ Call* Call::Create( return new internal::Call(config, std::move(transport_send)); } +VideoSendStream* Call::CreateVideoSendStream( + VideoSendStream::Config config, + VideoEncoderConfig encoder_config, + std::unique_ptr fec_controller) { + return nullptr; +} + namespace internal { Call::Call(const Call::Config& config, @@ -714,6 +726,15 @@ void Call::DestroyAudioReceiveStream( webrtc::VideoSendStream* Call::CreateVideoSendStream( webrtc::VideoSendStream::Config config, VideoEncoderConfig encoder_config) { + return CreateVideoSendStream( + std::move(config), std::move(encoder_config), + rtc::MakeUnique(Clock::GetRealTimeClock())); +} + +webrtc::VideoSendStream* Call::CreateVideoSendStream( + webrtc::VideoSendStream::Config config, + VideoEncoderConfig encoder_config, + std::unique_ptr fec_controller) { TRACE_EVENT0("webrtc", "Call::CreateVideoSendStream"); RTC_DCHECK_CALLED_SEQUENTIALLY(&configuration_sequence_checker_); @@ -733,7 +754,7 @@ webrtc::VideoSendStream* Call::CreateVideoSendStream( call_stats_.get(), transport_send_.get(), bitrate_allocator_.get(), video_send_delay_stats_.get(), event_log_, std::move(config), std::move(encoder_config), suspended_video_send_ssrcs_, - suspended_video_payload_states_); + suspended_video_payload_states_, std::move(fec_controller)); { WriteLockScoped write_lock(*send_crit_); diff --git a/call/call.h b/call/call.h index b6e0aeabbf..eb23e8b28f 100644 --- a/call/call.h +++ b/call/call.h @@ -15,6 +15,7 @@ #include #include +#include "api/fec_controller.h" #include "api/rtcerror.h" #include "call/audio_receive_stream.h" #include "call/audio_send_stream.h" @@ -144,6 +145,10 @@ class Call { virtual VideoSendStream* CreateVideoSendStream( VideoSendStream::Config config, VideoEncoderConfig encoder_config) = 0; + virtual VideoSendStream* CreateVideoSendStream( + VideoSendStream::Config config, + VideoEncoderConfig encoder_config, + std::unique_ptr fec_controller); virtual void DestroyVideoSendStream(VideoSendStream* send_stream) = 0; virtual VideoReceiveStream* CreateVideoReceiveStream( diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index d3dbcf6277..1ab974d0f5 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -19,6 +19,8 @@ rtc_static_library("video_coding") { "decoding_state.h", "encoded_frame.cc", "encoded_frame.h", + "fec_controller_default.cc", + "fec_controller_default.h", "fec_rate_table.h", "frame_buffer.cc", "frame_buffer.h", @@ -37,6 +39,7 @@ rtc_static_library("video_coding") { "histogram.cc", "histogram.h", "include/video_codec_initializer.h", + "include/video_coding.h", "inter_frame_delay.cc", "inter_frame_delay.h", "internal_defines.h", @@ -56,8 +59,6 @@ rtc_static_library("video_coding") { "packet.h", "packet_buffer.cc", "packet_buffer.h", - "protection_bitrate_calculator.cc", - "protection_bitrate_calculator.h", "qp_parser.cc", "qp_parser.h", "receiver.cc", @@ -101,6 +102,7 @@ rtc_static_library("video_coding") { "..:module_api_public", "../..:webrtc_common", "../../:typedefs", + "../../api:fec_controller_api", "../../api:optional", "../../api:video_frame_api", "../../api:video_frame_api_i420", @@ -635,6 +637,7 @@ if (rtc_include_tests) { "codecs/vp8/screenshare_layers_unittest.cc", "codecs/vp8/simulcast_unittest.cc", "decoding_state_unittest.cc", + "fec_controller_unittest.cc", "frame_buffer2_unittest.cc", "generic_encoder_unittest.cc", "h264_sprop_parameter_sets_unittest.cc", @@ -644,7 +647,6 @@ if (rtc_include_tests) { "jitter_buffer_unittest.cc", "jitter_estimator_tests.cc", "nack_module_unittest.cc", - "protection_bitrate_calculator_unittest.cc", "receiver_unittest.cc", "rtp_frame_reference_finder_unittest.cc", "session_info_unittest.cc", diff --git a/modules/video_coding/protection_bitrate_calculator.cc b/modules/video_coding/fec_controller_default.cc similarity index 85% rename from modules/video_coding/protection_bitrate_calculator.cc rename to modules/video_coding/fec_controller_default.cc index 3c1da234d9..9d26aace56 100644 --- a/modules/video_coding/protection_bitrate_calculator.cc +++ b/modules/video_coding/fec_controller_default.cc @@ -8,13 +8,11 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/video_coding/protection_bitrate_calculator.h" +#include "modules/video_coding/fec_controller_default.h" namespace webrtc { - using rtc::CritScope; - -ProtectionBitrateCalculator::ProtectionBitrateCalculator( +FecControllerDefault::FecControllerDefault( Clock* clock, VCMProtectionCallback* protection_callback) : clock_(clock), @@ -23,24 +21,36 @@ ProtectionBitrateCalculator::ProtectionBitrateCalculator( clock_->TimeInMilliseconds())), max_payload_size_(1460) {} -ProtectionBitrateCalculator::~ProtectionBitrateCalculator(void) { +FecControllerDefault::FecControllerDefault(Clock* clock) + : clock_(clock), + loss_prot_logic_(new media_optimization::VCMLossProtectionLogic( + clock_->TimeInMilliseconds())), + max_payload_size_(1460) {} + +FecControllerDefault::~FecControllerDefault(void) { loss_prot_logic_->Release(); } -void ProtectionBitrateCalculator::SetEncodingData(size_t width, - size_t height, - size_t num_temporal_layers, - size_t max_payload_size) { +void FecControllerDefault::SetProtectionCallback( + VCMProtectionCallback* protection_callback) { + protection_callback_ = protection_callback; +} + +void FecControllerDefault::SetEncodingData(size_t width, + size_t height, + size_t num_temporal_layers, + size_t max_payload_size) { CritScope lock(&crit_sect_); loss_prot_logic_->UpdateFrameSize(width, height); loss_prot_logic_->UpdateNumLayers(num_temporal_layers); max_payload_size_ = max_payload_size; } -uint32_t ProtectionBitrateCalculator::SetTargetRates( +uint32_t FecControllerDefault::UpdateFecRates( uint32_t estimated_bitrate_bps, int actual_framerate_fps, uint8_t fraction_lost, + std::vector loss_mask_vector, int64_t round_trip_time_ms) { float target_bitrate_kbps = static_cast(estimated_bitrate_bps) / 1000.0f; @@ -48,19 +58,15 @@ uint32_t ProtectionBitrateCalculator::SetTargetRates( if (actual_framerate_fps < 1.0) { actual_framerate_fps = 1.0; } - FecProtectionParams delta_fec_params; FecProtectionParams key_fec_params; { CritScope lock(&crit_sect_); - loss_prot_logic_->UpdateBitRate(target_bitrate_kbps); loss_prot_logic_->UpdateRtt(round_trip_time_ms); - // Update frame rate for the loss protection logic class: frame rate should // be the actual/sent rate. loss_prot_logic_->UpdateFrameRate(actual_framerate_fps); - // Returns the filtered packet loss, used for the protection setting. // The filtered loss may be the received loss (no filter), or some // filtered value (average or max window filter). @@ -69,31 +75,24 @@ uint32_t ProtectionBitrateCalculator::SetTargetRates( media_optimization::kMaxFilter; uint8_t packet_loss_enc = loss_prot_logic_->FilteredLoss( clock_->TimeInMilliseconds(), filter_mode, fraction_lost); - // For now use the filtered loss for computing the robustness settings. loss_prot_logic_->UpdateFilteredLossPr(packet_loss_enc); - if (loss_prot_logic_->SelectedType() == media_optimization::kNone) { return estimated_bitrate_bps; } - // Update method will compute the robustness settings for the given // protection method and the overhead cost // the protection method is set by the user via SetVideoProtection. loss_prot_logic_->UpdateMethod(); - // Get the bit cost of protection method, based on the amount of // overhead data actually transmitted (including headers) the last // second. - // Get the FEC code rate for Key frames (set to 0 when NA). key_fec_params.fec_rate = loss_prot_logic_->SelectedMethod()->RequiredProtectionFactorK(); - // Get the FEC code rate for Delta frames (set to 0 when NA). delta_fec_params.fec_rate = loss_prot_logic_->SelectedMethod()->RequiredProtectionFactorD(); - // The RTP module currently requires the same |max_fec_frames| for both // key and delta frames. delta_fec_params.max_fec_frames = @@ -101,26 +100,22 @@ uint32_t ProtectionBitrateCalculator::SetTargetRates( key_fec_params.max_fec_frames = loss_prot_logic_->SelectedMethod()->MaxFramesFec(); } - // Set the FEC packet mask type. |kFecMaskBursty| is more effective for // consecutive losses and little/no packet re-ordering. As we currently // do not have feedback data on the degree of correlated losses and packet // re-ordering, we keep default setting to |kFecMaskRandom| for now. delta_fec_params.fec_mask_type = kFecMaskRandom; key_fec_params.fec_mask_type = kFecMaskRandom; - // Update protection callback with protection settings. uint32_t sent_video_rate_bps = 0; uint32_t sent_nack_rate_bps = 0; uint32_t sent_fec_rate_bps = 0; // Rate cost of the protection methods. float protection_overhead_rate = 0.0f; - // TODO(Marco): Pass FEC protection values per layer. protection_callback_->ProtectionRequest( &delta_fec_params, &key_fec_params, &sent_video_rate_bps, &sent_nack_rate_bps, &sent_fec_rate_bps); - uint32_t sent_total_rate_bps = sent_video_rate_bps + sent_nack_rate_bps + sent_fec_rate_bps; // Estimate the overhead costs of the next second as staying the same @@ -133,13 +128,11 @@ uint32_t ProtectionBitrateCalculator::SetTargetRates( // Cap the overhead estimate to 50%. if (protection_overhead_rate > 0.5) protection_overhead_rate = 0.5; - // Source coding rate: total rate - protection overhead. return estimated_bitrate_bps * (1.0 - protection_overhead_rate); } - -void ProtectionBitrateCalculator::SetProtectionMethod(bool enable_fec, - bool enable_nack) { +void FecControllerDefault::SetProtectionMethod(bool enable_fec, + bool enable_nack) { media_optimization::VCMProtectionMethodEnum method(media_optimization::kNone); if (enable_fec && enable_nack) { method = media_optimization::kNackFec; @@ -151,14 +144,12 @@ void ProtectionBitrateCalculator::SetProtectionMethod(bool enable_fec, CritScope lock(&crit_sect_); loss_prot_logic_->SetMethod(method); } - -void ProtectionBitrateCalculator::UpdateWithEncodedData( +void FecControllerDefault::UpdateWithEncodedData( const EncodedImage& encoded_image) { const size_t encoded_length = encoded_image._length; CritScope lock(&crit_sect_); if (encoded_length > 0) { const bool delta_frame = encoded_image._frameType != kVideoFrameKey; - if (max_payload_size_ > 0 && encoded_length > 0) { const float min_packets_per_frame = encoded_length / static_cast(max_payload_size_); @@ -175,5 +166,4 @@ void ProtectionBitrateCalculator::UpdateWithEncodedData( } } } - } // namespace webrtc diff --git a/modules/video_coding/fec_controller_default.h b/modules/video_coding/fec_controller_default.h new file mode 100644 index 0000000000..8c795cb6bd --- /dev/null +++ b/modules/video_coding/fec_controller_default.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_FEC_CONTROLLER_DEFAULT_H_ +#define MODULES_VIDEO_CODING_FEC_CONTROLLER_DEFAULT_H_ + +#include +#include +#include "api/fec_controller.h" +#include "modules/include/module_common_types.h" +#include "modules/video_coding/media_opt_util.h" +#include "rtc_base/criticalsection.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { + +class FecControllerDefault : public FecController { + public: + FecControllerDefault(Clock* clock, + VCMProtectionCallback* protection_callback); + explicit FecControllerDefault(Clock* clock); + ~FecControllerDefault(); + void SetProtectionCallback( + VCMProtectionCallback* protection_callback) override; + void SetProtectionMethod(bool enable_fec, bool enable_nack) override; + void SetEncodingData(size_t width, + size_t height, + size_t num_temporal_layers, + size_t max_payload_size) override; + uint32_t UpdateFecRates(uint32_t estimated_bitrate_bps, + int actual_framerate_fps, + uint8_t fraction_lost, + std::vector loss_mask_vector, + int64_t round_trip_time_ms) override; + void UpdateWithEncodedData(const EncodedImage& encoded_image) override; + + private: + enum { kBitrateAverageWinMs = 1000 }; + Clock* const clock_; + VCMProtectionCallback* protection_callback_; + rtc::CriticalSection crit_sect_; + std::unique_ptr loss_prot_logic_ + RTC_GUARDED_BY(crit_sect_); + size_t max_payload_size_ RTC_GUARDED_BY(crit_sect_); + RTC_DISALLOW_COPY_AND_ASSIGN(FecControllerDefault); +}; + +} // namespace webrtc +#endif // MODULES_VIDEO_CODING_FEC_CONTROLLER_DEFAULT_H_ diff --git a/modules/video_coding/protection_bitrate_calculator_unittest.cc b/modules/video_coding/fec_controller_unittest.cc similarity index 67% rename from modules/video_coding/protection_bitrate_calculator_unittest.cc rename to modules/video_coding/fec_controller_unittest.cc index 66e2fa6f84..51ae15808f 100644 --- a/modules/video_coding/protection_bitrate_calculator_unittest.cc +++ b/modules/video_coding/fec_controller_unittest.cc @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/video_coding/protection_bitrate_calculator.h" +#include "modules/video_coding/fec_controller_default.h" #include "system_wrappers/include/clock.h" #include "test/gtest.h" @@ -42,22 +42,24 @@ class ProtectionBitrateCalculatorTest : public ::testing::Test { // Note: simulated clock starts at 1 seconds, since parts of webrtc use 0 as // a special case (e.g. frame rate in media optimization). ProtectionBitrateCalculatorTest() - : clock_(1000), media_opt_(&clock_, &protection_callback_) {} + : clock_(1000), fec_controller_(&clock_, &protection_callback_) {} SimulatedClock clock_; ProtectionCallback protection_callback_; - ProtectionBitrateCalculator media_opt_; + FecControllerDefault fec_controller_; }; TEST_F(ProtectionBitrateCalculatorTest, ProtectsUsingFecBitrate) { static const uint32_t kMaxBitrateBps = 130000; - media_opt_.SetProtectionMethod(true /*enable_fec*/, false /* enable_nack */); - media_opt_.SetEncodingData(640, 480, 1, 1000); + fec_controller_.SetProtectionMethod(true /*enable_fec*/, + false /* enable_nack */); + fec_controller_.SetEncodingData(640, 480, 1, 1000); // Using 10% of codec bitrate for FEC. protection_callback_.fec_rate_bps_ = kCodecBitrateBps / 10; - uint32_t target_bitrate = media_opt_.SetTargetRates(kMaxBitrateBps, 30, 0, 0); + uint32_t target_bitrate = fec_controller_.UpdateFecRates( + kMaxBitrateBps, 30, 0, std::vector(1, false), 0); EXPECT_GT(target_bitrate, 0u); EXPECT_GT(kMaxBitrateBps, target_bitrate); @@ -65,35 +67,40 @@ TEST_F(ProtectionBitrateCalculatorTest, ProtectsUsingFecBitrate) { // Using as much for codec bitrate as fec rate, new target rate should share // both equally, but only be half of max (since that ceiling should be hit). protection_callback_.fec_rate_bps_ = kCodecBitrateBps; - target_bitrate = media_opt_.SetTargetRates(kMaxBitrateBps, 30, 128, 100); + target_bitrate = fec_controller_.UpdateFecRates( + kMaxBitrateBps, 30, 128, std::vector(1, false), 100); EXPECT_EQ(kMaxBitrateBps / 2, target_bitrate); } TEST_F(ProtectionBitrateCalculatorTest, ProtectsUsingNackBitrate) { static const uint32_t kMaxBitrateBps = 130000; - media_opt_.SetProtectionMethod(false /*enable_fec*/, true /* enable_nack */); - media_opt_.SetEncodingData(640, 480, 1, 1000); + fec_controller_.SetProtectionMethod(false /*enable_fec*/, + true /* enable_nack */); + fec_controller_.SetEncodingData(640, 480, 1, 1000); - uint32_t target_bitrate = media_opt_.SetTargetRates(kMaxBitrateBps, 30, 0, 0); + uint32_t target_bitrate = fec_controller_.UpdateFecRates( + kMaxBitrateBps, 30, 0, std::vector(1, false), 0); EXPECT_EQ(kMaxBitrateBps, target_bitrate); // Using as much for codec bitrate as nack rate, new target rate should share // both equally, but only be half of max (since that ceiling should be hit). protection_callback_.nack_rate_bps_ = kMaxBitrateBps; - target_bitrate = media_opt_.SetTargetRates(kMaxBitrateBps, 30, 128, 100); + target_bitrate = fec_controller_.UpdateFecRates( + kMaxBitrateBps, 30, 128, std::vector(1, false), 100); EXPECT_EQ(kMaxBitrateBps / 2, target_bitrate); } TEST_F(ProtectionBitrateCalculatorTest, NoProtection) { static const uint32_t kMaxBitrateBps = 130000; - media_opt_.SetProtectionMethod(false /*enable_fec*/, false /* enable_nack */); - media_opt_.SetEncodingData(640, 480, 1, 1000); + fec_controller_.SetProtectionMethod(false /*enable_fec*/, + false /* enable_nack */); + fec_controller_.SetEncodingData(640, 480, 1, 1000); - uint32_t target_bitrate = - media_opt_.SetTargetRates(kMaxBitrateBps, 30, 128, 100); + uint32_t target_bitrate = fec_controller_.UpdateFecRates( + kMaxBitrateBps, 30, 128, std::vector(1, false), 100); EXPECT_EQ(kMaxBitrateBps, target_bitrate); } diff --git a/modules/video_coding/include/video_coding.h b/modules/video_coding/include/video_coding.h index d204f192fe..5173710b2a 100644 --- a/modules/video_coding/include/video_coding.h +++ b/modules/video_coding/include/video_coding.h @@ -21,6 +21,7 @@ #include #endif +#include "api/fec_controller.h" #include "api/video/video_frame.h" #include "modules/include/module.h" #include "modules/include/module_common_types.h" diff --git a/modules/video_coding/include/video_coding_defines.h b/modules/video_coding/include/video_coding_defines.h index 08497b08cb..e000bc6dde 100644 --- a/modules/video_coding/include/video_coding_defines.h +++ b/modules/video_coding/include/video_coding_defines.h @@ -108,20 +108,6 @@ class VCMReceiveStatisticsCallback { virtual ~VCMReceiveStatisticsCallback() {} }; -// Callback class used for telling the user about how to configure the FEC, -// and the rates sent the last second is returned to the VCM. -class VCMProtectionCallback { - public: - virtual int ProtectionRequest(const FecProtectionParams* delta_params, - const FecProtectionParams* key_params, - uint32_t* sent_video_rate_bps, - uint32_t* sent_nack_rate_bps, - uint32_t* sent_fec_rate_bps) = 0; - - protected: - virtual ~VCMProtectionCallback() {} -}; - // Callback class used for telling the user about what frame type needed to // continue decoding. // Typically a key frame when the stream has been corrupted in some way. diff --git a/modules/video_coding/protection_bitrate_calculator.h b/modules/video_coding/protection_bitrate_calculator.h deleted file mode 100644 index bf50589fe8..0000000000 --- a/modules/video_coding/protection_bitrate_calculator.h +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_VIDEO_CODING_PROTECTION_BITRATE_CALCULATOR_H_ -#define MODULES_VIDEO_CODING_PROTECTION_BITRATE_CALCULATOR_H_ - -#include -#include - -#include "modules/include/module_common_types.h" -#include "modules/video_coding/include/video_coding.h" -#include "modules/video_coding/media_opt_util.h" -#include "rtc_base/criticalsection.h" -#include "system_wrappers/include/clock.h" - -namespace webrtc { - -// ProtectionBitrateCalculator calculates how much of the allocated network -// capacity that can be used by an encoder and how much that -// is needed for redundant packets such as FEC and NACK. It uses an -// implementation of |VCMProtectionCallback| to set new FEC parameters and get -// the bitrate currently used for FEC and NACK. -// Usage: -// Setup by calling SetProtectionMethod and SetEncodingData. -// For each encoded image, call UpdateWithEncodedData. -// Each time the bandwidth estimate change, call SetTargetRates. SetTargetRates -// will return the bitrate that can be used by an encoder. -// A lock is used to protect internal states, so methods can be called on an -// arbitrary thread. -class ProtectionBitrateCalculator { - public: - ProtectionBitrateCalculator(Clock* clock, - VCMProtectionCallback* protection_callback); - ~ProtectionBitrateCalculator(); - - void SetProtectionMethod(bool enable_fec, bool enable_nack); - - // Informs media optimization of initial encoding state. - void SetEncodingData(size_t width, - size_t height, - size_t num_temporal_layers, - size_t max_payload_size); - - // Returns target rate for the encoder given the channel parameters. - // Inputs: estimated_bitrate_bps - the estimated network bitrate in bits/s. - // actual_framerate - encoder frame rate. - // fraction_lost - packet loss rate in % in the network. - // round_trip_time_ms - round trip time in milliseconds. - uint32_t SetTargetRates(uint32_t estimated_bitrate_bps, - int actual_framerate, - uint8_t fraction_lost, - int64_t round_trip_time_ms); - // Informs of encoded output. - void UpdateWithEncodedData(const EncodedImage& encoded_image); - - private: - enum { kBitrateAverageWinMs = 1000 }; - - Clock* const clock_; - VCMProtectionCallback* const protection_callback_; - - rtc::CriticalSection crit_sect_; - std::unique_ptr loss_prot_logic_ - RTC_GUARDED_BY(crit_sect_); - size_t max_payload_size_ RTC_GUARDED_BY(crit_sect_); - - RTC_DISALLOW_COPY_AND_ASSIGN(ProtectionBitrateCalculator); -}; - -} // namespace webrtc -#endif // MODULES_VIDEO_CODING_PROTECTION_BITRATE_CALCULATOR_H_ diff --git a/video/BUILD.gn b/video/BUILD.gn index 3d2d5e904d..1ec0e6a23e 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -56,6 +56,7 @@ rtc_static_library("video") { deps = [ "..:webrtc_common", "../:typedefs", + "../api:fec_controller_api", "../api:libjingle_peerconnection_api", "../api:optional", "../api:transport_api", diff --git a/video/video_quality_test.cc b/video/video_quality_test.cc index 5a204715ca..97cdced9d4 100644 --- a/video/video_quality_test.cc +++ b/video/video_quality_test.cc @@ -1060,6 +1060,12 @@ VideoQualityTest::VideoQualityTest() payload_type_map_[kPayloadTypeVP9] = webrtc::MediaType::VIDEO; } +VideoQualityTest::VideoQualityTest( + std::unique_ptr fec_controller) + : VideoQualityTest() { + fec_controller_ = std::move(fec_controller); +} + VideoQualityTest::Params::Params() : call({false, Call::Config::BitrateConfig(), 0}), video{{false, 640, 480, 30, 50, 800, 800, false, "VP8", 1, -1, 0, false, @@ -1803,6 +1809,24 @@ void VideoQualityTest::CreateVideoStreams() { AssociateFlexfecStreamsWithVideoStreams(); } +void VideoQualityTest::CreateVideoStreamsWithProtectionBitrateCalculator( + std::unique_ptr fec_controller) { + RTC_DCHECK(video_send_streams_.empty()); + RTC_DCHECK(video_receive_streams_.empty()); + RTC_DCHECK_EQ(video_send_configs_.size(), num_video_streams_); + for (size_t i = 0; i < video_send_configs_.size(); ++i) { + video_send_streams_.push_back(sender_call_->CreateVideoSendStream( + video_send_configs_[i].Copy(), video_encoder_configs_[i].Copy(), + std::move(fec_controller))); + } + for (size_t i = 0; i < video_receive_configs_.size(); ++i) { + video_receive_streams_.push_back(receiver_call_->CreateVideoReceiveStream( + video_receive_configs_[i].Copy())); + } + + AssociateFlexfecStreamsWithVideoStreams(); +} + void VideoQualityTest::DestroyStreams() { CallTest::DestroyStreams(); diff --git a/video/video_quality_test.h b/video/video_quality_test.h index 20f899d124..f1b44fe54c 100644 --- a/video/video_quality_test.h +++ b/video/video_quality_test.h @@ -96,6 +96,7 @@ class VideoQualityTest : public test::CallTest { }; VideoQualityTest(); + explicit VideoQualityTest(std::unique_ptr fec_controller); void RunWithAnalyzer(const Params& params); void RunWithRenderers(const Params& params); @@ -111,6 +112,7 @@ class VideoQualityTest : public test::CallTest { protected: std::map payload_type_map_; + std::unique_ptr fec_controller_; // No-op implementation to be able to instantiate this class from non-TEST_F // locations. @@ -127,6 +129,8 @@ class VideoQualityTest : public test::CallTest { // Helper methods for setting up the call. void CreateVideoStreams(); + void CreateVideoStreamsWithProtectionBitrateCalculator( + std::unique_ptr fec_controller); void DestroyStreams(); void CreateCapturers(); std::unique_ptr CreateFrameGenerator(size_t video_idx); diff --git a/video/video_send_stream.cc b/video/video_send_stream.cc index a64efa73df..1e749155d9 100644 --- a/video/video_send_stream.cc +++ b/video/video_send_stream.cc @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -26,6 +27,7 @@ #include "modules/rtp_rtcp/include/rtp_rtcp.h" #include "modules/rtp_rtcp/source/rtp_sender.h" #include "modules/utility/include/process_thread.h" +#include "modules/video_coding/fec_controller_default.h" #include "modules/video_coding/utility/ivf_file_writer.h" #include "rtc_base/checks.h" #include "rtc_base/experiments/alr_experiment.h" @@ -41,6 +43,8 @@ namespace webrtc { static const int kMinSendSidePacketHistorySize = 600; +static const int kSendSideSeqNumSetMaxSize = 15000; + namespace { // We don't do MTU discovery, so assume that we have the standard ethernet MTU. @@ -244,7 +248,8 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, public webrtc::OverheadObserver, public webrtc::VCMProtectionCallback, public VideoStreamEncoder::EncoderSink, - public VideoBitrateAllocationObserver { + public VideoBitrateAllocationObserver, + public webrtc::PacketFeedbackObserver { public: VideoSendStreamImpl( SendStatisticsProxy* stats_proxy, @@ -260,7 +265,8 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, double initial_encoder_bitrate_priority, std::map suspended_ssrcs, std::map suspended_payload_states, - VideoEncoderConfig::ContentType content_type); + VideoEncoderConfig::ContentType content_type, + std::unique_ptr fec_controller); ~VideoSendStreamImpl() override; // RegisterProcessThread register |module_process_thread| with those objects @@ -286,6 +292,11 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, rtc::Optional configured_pacing_factor_; + // From PacketFeedbackObserver. + void OnPacketAdded(uint32_t ssrc, uint16_t seq_num) override; + void OnPacketFeedbackVector( + const std::vector& packet_feedback_vector) override; + private: class CheckEncoderActivityTask; class EncoderReconfiguredTask; @@ -331,6 +342,7 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, const VideoSendStream::Config* const config_; std::map suspended_ssrcs_; + std::unique_ptr fec_controller_; ProcessThread* module_process_thread_; rtc::ThreadChecker module_process_thread_checker_; rtc::TaskQueue* const worker_queue_; @@ -358,7 +370,6 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, VideoStreamEncoder* const video_stream_encoder_; EncoderRtcpFeedback encoder_feedback_; - ProtectionBitrateCalculator protection_bitrate_calculator_; RtcpBandwidthObserver* const bandwidth_observer_; // RtpRtcp modules, declared here as they use other members on construction. @@ -377,6 +388,9 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, size_t overhead_bytes_per_packet_ RTC_GUARDED_BY(overhead_bytes_per_packet_crit_); size_t transport_overhead_bytes_per_packet_; + + std::set feedback_packet_seq_num_set_; + std::vector loss_mask_vector_; }; // TODO(tommi): See if there's a more elegant way to create a task that creates @@ -399,7 +413,8 @@ class VideoSendStream::ConstructionTask : public rtc::QueuedTask { double initial_encoder_bitrate_priority, const std::map& suspended_ssrcs, const std::map& suspended_payload_states, - VideoEncoderConfig::ContentType content_type) + VideoEncoderConfig::ContentType content_type, + std::unique_ptr fec_controller) : send_stream_(send_stream), done_event_(done_event), stats_proxy_(stats_proxy), @@ -414,7 +429,8 @@ class VideoSendStream::ConstructionTask : public rtc::QueuedTask { initial_encoder_bitrate_priority_(initial_encoder_bitrate_priority), suspended_ssrcs_(suspended_ssrcs), suspended_payload_states_(suspended_payload_states), - content_type_(content_type) {} + content_type_(content_type), + fec_controller_(std::move(fec_controller)) {} ~ConstructionTask() override { done_event_->Set(); } @@ -425,7 +441,8 @@ class VideoSendStream::ConstructionTask : public rtc::QueuedTask { bitrate_allocator_, send_delay_stats_, video_stream_encoder_, event_log_, config_, initial_encoder_max_bitrate_, initial_encoder_bitrate_priority_, std::move(suspended_ssrcs_), - std::move(suspended_payload_states_), content_type_)); + std::move(suspended_payload_states_), content_type_, + std::move(fec_controller_))); return true; } @@ -444,6 +461,7 @@ class VideoSendStream::ConstructionTask : public rtc::QueuedTask { std::map suspended_ssrcs_; std::map suspended_payload_states_; const VideoEncoderConfig::ContentType content_type_; + std::unique_ptr fec_controller_; }; class VideoSendStream::DestructAndGetRtpStateTask : public rtc::QueuedTask { @@ -562,7 +580,8 @@ VideoSendStream::VideoSendStream( VideoSendStream::Config config, VideoEncoderConfig encoder_config, const std::map& suspended_ssrcs, - const std::map& suspended_payload_states) + const std::map& suspended_payload_states, + std::unique_ptr fec_controller) : worker_queue_(worker_queue), thread_sync_event_(false /* manual_reset */, false), stats_proxy_(Clock::GetRealTimeClock(), @@ -580,7 +599,8 @@ VideoSendStream::VideoSendStream( video_stream_encoder_.get(), module_process_thread, call_stats, transport, bitrate_allocator, send_delay_stats, event_log, &config_, encoder_config.max_bitrate_bps, encoder_config.bitrate_priority, - suspended_ssrcs, suspended_payload_states, encoder_config.content_type))); + suspended_ssrcs, suspended_payload_states, encoder_config.content_type, + std::move(fec_controller)))); // Wait for ConstructionTask to complete so that |send_stream_| can be used. // |module_process_thread| must be registered and deregistered on the thread @@ -705,12 +725,14 @@ VideoSendStreamImpl::VideoSendStreamImpl( double initial_encoder_bitrate_priority, std::map suspended_ssrcs, std::map suspended_payload_states, - VideoEncoderConfig::ContentType content_type) + VideoEncoderConfig::ContentType content_type, + std::unique_ptr fec_controller) : send_side_bwe_with_overhead_( webrtc::field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead")), stats_proxy_(stats_proxy), config_(config), suspended_ssrcs_(std::move(suspended_ssrcs)), + fec_controller_(std::move(fec_controller)), module_process_thread_(nullptr), worker_queue_(worker_queue), check_encoder_activity_task_(nullptr), @@ -727,7 +749,6 @@ VideoSendStreamImpl::VideoSendStreamImpl( encoder_feedback_(Clock::GetRealTimeClock(), config_->rtp.ssrcs, video_stream_encoder), - protection_bitrate_calculator_(Clock::GetRealTimeClock(), this), bandwidth_observer_(transport->send_side_cc()->GetBandwidthObserver()), rtp_rtcp_modules_(CreateRtpRtcpModules( config_->send_transport, @@ -823,6 +844,10 @@ VideoSendStreamImpl::VideoSendStreamImpl( config_->encoder_settings.payload_name.c_str()); } + fec_controller_->SetProtectionCallback(this); + // Signal congestion controller this object is ready for OnPacket* callbacks. + transport_->send_side_cc()->RegisterPacketFeedbackObserver(this); + RTC_DCHECK(config_->encoder_settings.encoder); RTC_DCHECK_GE(config_->encoder_settings.payload_type, 0); RTC_DCHECK_LE(config_->encoder_settings.payload_type, 127); @@ -865,7 +890,7 @@ VideoSendStreamImpl::~VideoSendStreamImpl() { RTC_DCHECK(!payload_router_.IsActive()) << "VideoSendStreamImpl::Stop not called"; RTC_LOG(LS_INFO) << "~VideoSendStreamInternal: " << config_->ToString(); - + transport_->send_side_cc()->DeRegisterPacketFeedbackObserver(this); for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { transport_->packet_router()->RemoveSendRtpModule(rtp_rtcp); delete rtp_rtcp; @@ -989,9 +1014,9 @@ void VideoSendStreamImpl::OnEncoderConfigurationChanged( size_t number_of_temporal_layers = streams.back().temporal_layer_thresholds_bps.size() + 1; - protection_bitrate_calculator_.SetEncodingData( - streams[0].width, streams[0].height, number_of_temporal_layers, - config_->rtp.max_packet_size); + fec_controller_->SetEncodingData(streams[0].width, streams[0].height, + number_of_temporal_layers, + config_->rtp.max_packet_size); if (payload_router_.IsActive()) { // The send stream is started already. Update the allocator with new bitrate @@ -1025,7 +1050,7 @@ EncodedImageCallback::Result VideoSendStreamImpl::OnEncodedImage( check_encoder_activity_task_->UpdateEncoderActivity(); } - protection_bitrate_calculator_.UpdateWithEncodedData(encoded_image); + fec_controller_->UpdateWithEncodedData(encoded_image); EncodedImageCallback::Result result = payload_router_.OnEncodedImage( encoded_image, codec_specific_info, fragmentation); @@ -1131,8 +1156,8 @@ void VideoSendStreamImpl::ConfigureProtection() { // Currently, both ULPFEC and FlexFEC use the same FEC rate calculation logic, // so enable that logic if either of those FEC schemes are enabled. - protection_bitrate_calculator_.SetProtectionMethod( - flexfec_enabled || IsUlpfecEnabled(), nack_enabled); + fec_controller_->SetProtectionMethod(flexfec_enabled || IsUlpfecEnabled(), + nack_enabled); } void VideoSendStreamImpl::ConfigureSsrcs() { @@ -1239,9 +1264,10 @@ uint32_t VideoSendStreamImpl::OnBitrateUpdated(uint32_t bitrate_bps, // Get the encoder target rate. It is the estimated network rate - // protection overhead. - encoder_target_rate_bps_ = protection_bitrate_calculator_.SetTargetRates( + encoder_target_rate_bps_ = fec_controller_->UpdateFecRates( payload_bitrate_bps, stats_proxy_->GetSendFrameRate(), fraction_loss, - rtt); + loss_mask_vector_, rtt); + loss_mask_vector_.clear(); uint32_t encoder_overhead_rate_bps = send_side_bwe_with_overhead_ @@ -1340,5 +1366,47 @@ void VideoSendStreamImpl::SetTransportOverhead( } } +void VideoSendStreamImpl::OnPacketAdded(uint32_t ssrc, uint16_t seq_num) { + if (!worker_queue_->IsCurrent()) { + auto ptr = weak_ptr_; + worker_queue_->PostTask([=] { + if (!ptr.get()) + return; + ptr->OnPacketAdded(ssrc, seq_num); + }); + return; + } + const auto ssrcs = config_->rtp.ssrcs; + if (std::find(ssrcs.begin(), ssrcs.end(), ssrc) != ssrcs.end()) { + feedback_packet_seq_num_set_.insert(seq_num); + if (feedback_packet_seq_num_set_.size() > kSendSideSeqNumSetMaxSize) { + RTC_LOG(LS_WARNING) << "Feedback packet sequence number set exceed it's " + "max size', will get reset."; + feedback_packet_seq_num_set_.clear(); + } + } +} + +void VideoSendStreamImpl::OnPacketFeedbackVector( + const std::vector& packet_feedback_vector) { + if (!worker_queue_->IsCurrent()) { + auto ptr = weak_ptr_; + worker_queue_->PostTask([=] { + if (!ptr.get()) + return; + ptr->OnPacketFeedbackVector(packet_feedback_vector); + }); + return; + } + // Lost feedbacks are not considered to be lost packets. + for (const PacketFeedback& packet : packet_feedback_vector) { + if (auto it = feedback_packet_seq_num_set_.find(packet.sequence_number) != + feedback_packet_seq_num_set_.end()) { + const bool lost = packet.arrival_time_ms == PacketFeedback::kNotReceived; + loss_mask_vector_.push_back(lost); + feedback_packet_seq_num_set_.erase(it); + } + } +} } // namespace internal } // namespace webrtc diff --git a/video/video_send_stream.h b/video/video_send_stream.h index 204a3cee1c..d62c302a85 100644 --- a/video/video_send_stream.h +++ b/video/video_send_stream.h @@ -15,11 +15,11 @@ #include #include +#include "api/fec_controller.h" #include "call/bitrate_allocator.h" #include "call/video_receive_stream.h" #include "call/video_send_stream.h" #include "common_video/libyuv/include/webrtc_libyuv.h" -#include "modules/video_coding/protection_bitrate_calculator.h" #include "rtc_base/criticalsection.h" #include "rtc_base/event.h" #include "rtc_base/task_queue.h" @@ -63,7 +63,8 @@ class VideoSendStream : public webrtc::VideoSendStream { VideoSendStream::Config config, VideoEncoderConfig encoder_config, const std::map& suspended_ssrcs, - const std::map& suspended_payload_states); + const std::map& suspended_payload_states, + std::unique_ptr fec_controller); ~VideoSendStream() override;