admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make fec controller plug-able.

Browse Source 
Bug: webrtc:8656
Change-Id: I3d42ffc92a7c95266e5d53bab03f388bd0de2592
Reviewed-on: https://webrtc-review.googlesource.com/39760
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Commit-Queue: Ying Wang <yinwa@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#21710}
This commit is contained in:
Ying Wang 2018-01-19 17:58:57 +01:00 committed by Commit Bot
parent a6e7b88198
commit 3b790f316c
18 changed files with 348 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
api/BUILD.gn
View File

@ -217,6 +217,18 @@ rtc_source_set("transport_api") {
]
}
rtc_source_set("fec_controller_api") {
visibility = [ "*" ]
sources = [
"fec_controller.h",
]
deps = [
"../common_video:common_video",
"../modules:module_api",
]
}
rtc_source_set("video_frame_api") {
visibility = [ "*" ]
sources = [

80
api/fec_controller.h Normal file
View File

@ -0,0 +1,80 @@
/*
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef API_FEC_CONTROLLER_H_
#define API_FEC_CONTROLLER_H_
#include <vector>
#include "common_video/include/video_frame.h"
#include "modules/include/module_common_types.h"
namespace webrtc {
// TODO(yinwa): work in progress. API in class FecController should not be
// used by other users until this comment is removed.
// Callback class used for telling the user about how to configure the FEC,
// and the rates sent the last second is returned to the VCM.
class VCMProtectionCallback {
public:
virtual int ProtectionRequest(const FecProtectionParams* delta_params,
const FecProtectionParams* key_params,
uint32_t* sent_video_rate_bps,
uint32_t* sent_nack_rate_bps,
uint32_t* sent_fec_rate_bps) = 0;
protected:
virtual ~VCMProtectionCallback() {}
};
// FecController calculates how much of the allocated network
// capacity that can be used by an encoder and how much that
// is needed for redundant packets such as FEC and NACK. It uses an
// implementation of |VCMProtectionCallback| to set new FEC parameters and get
// the bitrate currently used for FEC and NACK.
// Usage:
// Setup by calling SetProtectionMethod and SetEncodingData.
// For each encoded image, call UpdateWithEncodedData.
// Each time the bandwidth estimate change, call UpdateFecRates. UpdateFecRates
// will return the bitrate that can be used by an encoder.
// A lock is used to protect internal states, so methods can be called on an
// arbitrary thread.
class FecController {
public:
virtual ~FecController() {}
virtual void SetProtectionCallback(
VCMProtectionCallback* protection_callback) = 0;
virtual void SetProtectionMethod(bool enable_fec, bool enable_nack) = 0;
// Informs loss protectoin logic of initial encoding state.
virtual void SetEncodingData(size_t width,
size_t height,
size_t num_temporal_layers,
size_t max_payload_size) = 0;
// Returns target rate for the encoder given the channel parameters.
// Inputs: estimated_bitrate_bps - the estimated network bitrate in bits/s.
// actual_framerate - encoder frame rate.
// fraction_lost - packet loss rate in % in the network.
// loss_mask_vector - packet loss mask since last time this method
// was called. round_trip_time_ms - round trip time in milliseconds.
virtual uint32_t UpdateFecRates(uint32_t estimated_bitrate_bps,
int actual_framerate,
uint8_t fraction_lost,
std::vector<bool> loss_mask_vector,
int64_t round_trip_time_ms) = 0;
// Informs of encoded output.
virtual void UpdateWithEncodedData(const EncodedImage& encoded_image) = 0;
};
} // namespace webrtc
#endif // API_FEC_CONTROLLER_H_

2
call/BUILD.gn
View File

@ -28,6 +28,7 @@ rtc_source_set("call_interfaces") {
"..:webrtc_common",
"../:typedefs",
"../api:audio_mixer_api",
"../api:fec_controller_api",
"../api:libjingle_peerconnection_api",
"../api:optional",
"../api:transport_api",
@ -149,6 +150,7 @@ rtc_static_library("call") {
"../modules/rtp_rtcp",
"../modules/rtp_rtcp:rtp_rtcp_format",
"../modules/utility",
"../modules/video_coding:video_coding",
"../rtc_base:checks",
"../rtc_base:rtc_base_approved",
"../rtc_base:rtc_task_queue",

1
call/DEPS
View File

@ -7,6 +7,7 @@ include_rules = [
"+modules/audio_processing",
"+modules/bitrate_controller",
"+modules/congestion_controller",
"+modules/video_coding",
"+modules/pacing",
"+modules/rtp_rtcp",
"+modules/utility",

23
call/call.cc
View File

@ -42,6 +42,7 @@
#include "modules/rtp_rtcp/source/byte_io.h"
#include "modules/rtp_rtcp/source/rtp_packet_received.h"
#include "modules/utility/include/process_thread.h"
#include "modules/video_coding/fec_controller_default.h"
#include "rtc_base/basictypes.h"
#include "rtc_base/checks.h"
#include "rtc_base/constructormagic.h"
@ -184,6 +185,10 @@ class Call : public webrtc::Call,
webrtc::VideoSendStream* CreateVideoSendStream(
webrtc::VideoSendStream::Config config,
VideoEncoderConfig encoder_config) override;
webrtc::VideoSendStream* CreateVideoSendStream(
webrtc::VideoSendStream::Config config,
VideoEncoderConfig encoder_config,
std::unique_ptr<FecController> fec_controller) override;
void DestroyVideoSendStream(webrtc::VideoSendStream* send_stream) override;
webrtc::VideoReceiveStream* CreateVideoReceiveStream(
@ -402,6 +407,13 @@ Call* Call::Create(
return new internal::Call(config, std::move(transport_send));
}
VideoSendStream* Call::CreateVideoSendStream(
VideoSendStream::Config config,
VideoEncoderConfig encoder_config,
std::unique_ptr<FecController> fec_controller) {
return nullptr;
}
namespace internal {
Call::Call(const Call::Config& config,
@ -714,6 +726,15 @@ void Call::DestroyAudioReceiveStream(
webrtc::VideoSendStream* Call::CreateVideoSendStream(
webrtc::VideoSendStream::Config config,
VideoEncoderConfig encoder_config) {
return CreateVideoSendStream(
std::move(config), std::move(encoder_config),
rtc::MakeUnique<FecControllerDefault>(Clock::GetRealTimeClock()));
}
webrtc::VideoSendStream* Call::CreateVideoSendStream(
webrtc::VideoSendStream::Config config,
VideoEncoderConfig encoder_config,
std::unique_ptr<FecController> fec_controller) {
TRACE_EVENT0("webrtc", "Call::CreateVideoSendStream");
RTC_DCHECK_CALLED_SEQUENTIALLY(&configuration_sequence_checker_);
@ -733,7 +754,7 @@ webrtc::VideoSendStream* Call::CreateVideoSendStream(
call_stats_.get(), transport_send_.get(), bitrate_allocator_.get(),
video_send_delay_stats_.get(), event_log_, std::move(config),
std::move(encoder_config), suspended_video_send_ssrcs_,
suspended_video_payload_states_);
suspended_video_payload_states_, std::move(fec_controller));
{
WriteLockScoped write_lock(*send_crit_);

5
call/call.h
View File

@ -15,6 +15,7 @@
#include <string>
#include <vector>
#include "api/fec_controller.h"
#include "api/rtcerror.h"
#include "call/audio_receive_stream.h"
#include "call/audio_send_stream.h"
@ -144,6 +145,10 @@ class Call {
virtual VideoSendStream* CreateVideoSendStream(
VideoSendStream::Config config,
VideoEncoderConfig encoder_config) = 0;
virtual VideoSendStream* CreateVideoSendStream(
VideoSendStream::Config config,
VideoEncoderConfig encoder_config,
std::unique_ptr<FecController> fec_controller);
virtual void DestroyVideoSendStream(VideoSendStream* send_stream) = 0;
virtual VideoReceiveStream* CreateVideoReceiveStream(

8
modules/video_coding/BUILD.gn
View File

@ -19,6 +19,8 @@ rtc_static_library("video_coding") {
"decoding_state.h",
"encoded_frame.cc",
"encoded_frame.h",
"fec_controller_default.cc",
"fec_controller_default.h",
"fec_rate_table.h",
"frame_buffer.cc",
"frame_buffer.h",
@ -37,6 +39,7 @@ rtc_static_library("video_coding") {
"histogram.cc",
"histogram.h",
"include/video_codec_initializer.h",
"include/video_coding.h",
"inter_frame_delay.cc",
"inter_frame_delay.h",
"internal_defines.h",
@ -56,8 +59,6 @@ rtc_static_library("video_coding") {
"packet.h",
"packet_buffer.cc",
"packet_buffer.h",
"protection_bitrate_calculator.cc",
"protection_bitrate_calculator.h",
"qp_parser.cc",
"qp_parser.h",
"receiver.cc",
@ -101,6 +102,7 @@ rtc_static_library("video_coding") {
"..:module_api_public",
"../..:webrtc_common",
"../../:typedefs",
"../../api:fec_controller_api",
"../../api:optional",
"../../api:video_frame_api",
"../../api:video_frame_api_i420",
@ -635,6 +637,7 @@ if (rtc_include_tests) {
"codecs/vp8/screenshare_layers_unittest.cc",
"codecs/vp8/simulcast_unittest.cc",
"decoding_state_unittest.cc",
"fec_controller_unittest.cc",
"frame_buffer2_unittest.cc",
"generic_encoder_unittest.cc",
"h264_sprop_parameter_sets_unittest.cc",
@ -644,7 +647,6 @@ if (rtc_include_tests) {
"jitter_buffer_unittest.cc",
"jitter_estimator_tests.cc",
"nack_module_unittest.cc",
"protection_bitrate_calculator_unittest.cc",
"receiver_unittest.cc",
"rtp_frame_reference_finder_unittest.cc",
"session_info_unittest.cc",

56
modules/video_coding/protection_bitrate_calculator.cc → modules/video_coding/fec_controller_default.cc
View File

@ -8,13 +8,11 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/protection_bitrate_calculator.h"
#include "modules/video_coding/fec_controller_default.h"
namespace webrtc {
using rtc::CritScope;
ProtectionBitrateCalculator::ProtectionBitrateCalculator(
FecControllerDefault::FecControllerDefault(
Clock* clock,
VCMProtectionCallback* protection_callback)
: clock_(clock),
@ -23,24 +21,36 @@ ProtectionBitrateCalculator::ProtectionBitrateCalculator(
clock_->TimeInMilliseconds())),
max_payload_size_(1460) {}
ProtectionBitrateCalculator::~ProtectionBitrateCalculator(void) {
FecControllerDefault::FecControllerDefault(Clock* clock)
: clock_(clock),
loss_prot_logic_(new media_optimization::VCMLossProtectionLogic(
clock_->TimeInMilliseconds())),
max_payload_size_(1460) {}
FecControllerDefault::~FecControllerDefault(void) {
loss_prot_logic_->Release();
}
void ProtectionBitrateCalculator::SetEncodingData(size_t width,
size_t height,
size_t num_temporal_layers,
size_t max_payload_size) {
void FecControllerDefault::SetProtectionCallback(
VCMProtectionCallback* protection_callback) {
protection_callback_ = protection_callback;
}
void FecControllerDefault::SetEncodingData(size_t width,
size_t height,
size_t num_temporal_layers,
size_t max_payload_size) {
CritScope lock(&crit_sect_);
loss_prot_logic_->UpdateFrameSize(width, height);
loss_prot_logic_->UpdateNumLayers(num_temporal_layers);
max_payload_size_ = max_payload_size;
}
uint32_t ProtectionBitrateCalculator::SetTargetRates(
uint32_t FecControllerDefault::UpdateFecRates(
uint32_t estimated_bitrate_bps,
int actual_framerate_fps,
uint8_t fraction_lost,
std::vector<bool> loss_mask_vector,
int64_t round_trip_time_ms) {
float target_bitrate_kbps =
static_cast<float>(estimated_bitrate_bps) / 1000.0f;
@ -48,19 +58,15 @@ uint32_t ProtectionBitrateCalculator::SetTargetRates(
if (actual_framerate_fps < 1.0) {
actual_framerate_fps = 1.0;
}
FecProtectionParams delta_fec_params;
FecProtectionParams key_fec_params;
{
CritScope lock(&crit_sect_);
loss_prot_logic_->UpdateBitRate(target_bitrate_kbps);
loss_prot_logic_->UpdateRtt(round_trip_time_ms);
// Update frame rate for the loss protection logic class: frame rate should
// be the actual/sent rate.
loss_prot_logic_->UpdateFrameRate(actual_framerate_fps);
// Returns the filtered packet loss, used for the protection setting.
// The filtered loss may be the received loss (no filter), or some
// filtered value (average or max window filter).
@ -69,31 +75,24 @@ uint32_t ProtectionBitrateCalculator::SetTargetRates(
media_optimization::kMaxFilter;
uint8_t packet_loss_enc = loss_prot_logic_->FilteredLoss(
clock_->TimeInMilliseconds(), filter_mode, fraction_lost);
// For now use the filtered loss for computing the robustness settings.
loss_prot_logic_->UpdateFilteredLossPr(packet_loss_enc);
if (loss_prot_logic_->SelectedType() == media_optimization::kNone) {
return estimated_bitrate_bps;
}
// Update method will compute the robustness settings for the given
// protection method and the overhead cost
// the protection method is set by the user via SetVideoProtection.
loss_prot_logic_->UpdateMethod();
// Get the bit cost of protection method, based on the amount of
// overhead data actually transmitted (including headers) the last
// second.
// Get the FEC code rate for Key frames (set to 0 when NA).
key_fec_params.fec_rate =
loss_prot_logic_->SelectedMethod()->RequiredProtectionFactorK();
// Get the FEC code rate for Delta frames (set to 0 when NA).
delta_fec_params.fec_rate =
loss_prot_logic_->SelectedMethod()->RequiredProtectionFactorD();
// The RTP module currently requires the same |max_fec_frames| for both
// key and delta frames.
delta_fec_params.max_fec_frames =
@ -101,26 +100,22 @@ uint32_t ProtectionBitrateCalculator::SetTargetRates(
key_fec_params.max_fec_frames =
loss_prot_logic_->SelectedMethod()->MaxFramesFec();
}
// Set the FEC packet mask type. |kFecMaskBursty| is more effective for
// consecutive losses and little/no packet re-ordering. As we currently
// do not have feedback data on the degree of correlated losses and packet
// re-ordering, we keep default setting to |kFecMaskRandom| for now.
delta_fec_params.fec_mask_type = kFecMaskRandom;
key_fec_params.fec_mask_type = kFecMaskRandom;
// Update protection callback with protection settings.
uint32_t sent_video_rate_bps = 0;
uint32_t sent_nack_rate_bps = 0;
uint32_t sent_fec_rate_bps = 0;
// Rate cost of the protection methods.
float protection_overhead_rate = 0.0f;
// TODO(Marco): Pass FEC protection values per layer.
protection_callback_->ProtectionRequest(
&delta_fec_params, &key_fec_params, &sent_video_rate_bps,
&sent_nack_rate_bps, &sent_fec_rate_bps);
uint32_t sent_total_rate_bps =
sent_video_rate_bps + sent_nack_rate_bps + sent_fec_rate_bps;
// Estimate the overhead costs of the next second as staying the same
@ -133,13 +128,11 @@ uint32_t ProtectionBitrateCalculator::SetTargetRates(
// Cap the overhead estimate to 50%.
if (protection_overhead_rate > 0.5)
protection_overhead_rate = 0.5;
// Source coding rate: total rate - protection overhead.
return estimated_bitrate_bps * (1.0 - protection_overhead_rate);
}
void ProtectionBitrateCalculator::SetProtectionMethod(bool enable_fec,
bool enable_nack) {
void FecControllerDefault::SetProtectionMethod(bool enable_fec,
bool enable_nack) {
media_optimization::VCMProtectionMethodEnum method(media_optimization::kNone);
if (enable_fec && enable_nack) {
method = media_optimization::kNackFec;
@ -151,14 +144,12 @@ void ProtectionBitrateCalculator::SetProtectionMethod(bool enable_fec,
CritScope lock(&crit_sect_);
loss_prot_logic_->SetMethod(method);
}
void ProtectionBitrateCalculator::UpdateWithEncodedData(
void FecControllerDefault::UpdateWithEncodedData(
const EncodedImage& encoded_image) {
const size_t encoded_length = encoded_image._length;
CritScope lock(&crit_sect_);
if (encoded_length > 0) {
const bool delta_frame = encoded_image._frameType != kVideoFrameKey;
if (max_payload_size_ > 0 && encoded_length > 0) {
const float min_packets_per_frame =
encoded_length / static_cast<float>(max_payload_size_);
@ -175,5 +166,4 @@ void ProtectionBitrateCalculator::UpdateWithEncodedData(
}
}
}
} // namespace webrtc

56
modules/video_coding/fec_controller_default.h Normal file
View File

@ -0,0 +1,56 @@
/*
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_FEC_CONTROLLER_DEFAULT_H_
#define MODULES_VIDEO_CODING_FEC_CONTROLLER_DEFAULT_H_
#include <memory>
#include <vector>
#include "api/fec_controller.h"
#include "modules/include/module_common_types.h"
#include "modules/video_coding/media_opt_util.h"
#include "rtc_base/criticalsection.h"
#include "system_wrappers/include/clock.h"
namespace webrtc {
class FecControllerDefault : public FecController {
public:
FecControllerDefault(Clock* clock,
VCMProtectionCallback* protection_callback);
explicit FecControllerDefault(Clock* clock);
~FecControllerDefault();
void SetProtectionCallback(
VCMProtectionCallback* protection_callback) override;
void SetProtectionMethod(bool enable_fec, bool enable_nack) override;
void SetEncodingData(size_t width,
size_t height,
size_t num_temporal_layers,
size_t max_payload_size) override;
uint32_t UpdateFecRates(uint32_t estimated_bitrate_bps,
int actual_framerate_fps,
uint8_t fraction_lost,
std::vector<bool> loss_mask_vector,
int64_t round_trip_time_ms) override;
void UpdateWithEncodedData(const EncodedImage& encoded_image) override;
private:
enum { kBitrateAverageWinMs = 1000 };
Clock* const clock_;
VCMProtectionCallback* protection_callback_;
rtc::CriticalSection crit_sect_;
std::unique_ptr<media_optimization::VCMLossProtectionLogic> loss_prot_logic_
RTC_GUARDED_BY(crit_sect_);
size_t max_payload_size_ RTC_GUARDED_BY(crit_sect_);
RTC_DISALLOW_COPY_AND_ASSIGN(FecControllerDefault);
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_FEC_CONTROLLER_DEFAULT_H_

37
modules/video_coding/protection_bitrate_calculator_unittest.cc → modules/video_coding/fec_controller_unittest.cc
View File

@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/protection_bitrate_calculator.h"
#include "modules/video_coding/fec_controller_default.h"
#include "system_wrappers/include/clock.h"
#include "test/gtest.h"
@ -42,22 +42,24 @@ class ProtectionBitrateCalculatorTest : public ::testing::Test {
// Note: simulated clock starts at 1 seconds, since parts of webrtc use 0 as
// a special case (e.g. frame rate in media optimization).
ProtectionBitrateCalculatorTest()
: clock_(1000), media_opt_(&clock_, &protection_callback_) {}
: clock_(1000), fec_controller_(&clock_, &protection_callback_) {}
SimulatedClock clock_;
ProtectionCallback protection_callback_;
ProtectionBitrateCalculator media_opt_;
FecControllerDefault fec_controller_;
};
TEST_F(ProtectionBitrateCalculatorTest, ProtectsUsingFecBitrate) {
static const uint32_t kMaxBitrateBps = 130000;
media_opt_.SetProtectionMethod(true /*enable_fec*/, false /* enable_nack */);
media_opt_.SetEncodingData(640, 480, 1, 1000);
fec_controller_.SetProtectionMethod(true /*enable_fec*/,
false /* enable_nack */);
fec_controller_.SetEncodingData(640, 480, 1, 1000);
// Using 10% of codec bitrate for FEC.
protection_callback_.fec_rate_bps_ = kCodecBitrateBps / 10;
uint32_t target_bitrate = media_opt_.SetTargetRates(kMaxBitrateBps, 30, 0, 0);
uint32_t target_bitrate = fec_controller_.UpdateFecRates(
kMaxBitrateBps, 30, 0, std::vector<bool>(1, false), 0);
EXPECT_GT(target_bitrate, 0u);
EXPECT_GT(kMaxBitrateBps, target_bitrate);
@ -65,35 +67,40 @@ TEST_F(ProtectionBitrateCalculatorTest, ProtectsUsingFecBitrate) {
// Using as much for codec bitrate as fec rate, new target rate should share
// both equally, but only be half of max (since that ceiling should be hit).
protection_callback_.fec_rate_bps_ = kCodecBitrateBps;
target_bitrate = media_opt_.SetTargetRates(kMaxBitrateBps, 30, 128, 100);
target_bitrate = fec_controller_.UpdateFecRates(
kMaxBitrateBps, 30, 128, std::vector<bool>(1, false), 100);
EXPECT_EQ(kMaxBitrateBps / 2, target_bitrate);
}
TEST_F(ProtectionBitrateCalculatorTest, ProtectsUsingNackBitrate) {
static const uint32_t kMaxBitrateBps = 130000;
media_opt_.SetProtectionMethod(false /*enable_fec*/, true /* enable_nack */);
media_opt_.SetEncodingData(640, 480, 1, 1000);
fec_controller_.SetProtectionMethod(false /*enable_fec*/,
true /* enable_nack */);
fec_controller_.SetEncodingData(640, 480, 1, 1000);
uint32_t target_bitrate = media_opt_.SetTargetRates(kMaxBitrateBps, 30, 0, 0);
uint32_t target_bitrate = fec_controller_.UpdateFecRates(
kMaxBitrateBps, 30, 0, std::vector<bool>(1, false), 0);
EXPECT_EQ(kMaxBitrateBps, target_bitrate);
// Using as much for codec bitrate as nack rate, new target rate should share
// both equally, but only be half of max (since that ceiling should be hit).
protection_callback_.nack_rate_bps_ = kMaxBitrateBps;
target_bitrate = media_opt_.SetTargetRates(kMaxBitrateBps, 30, 128, 100);
target_bitrate = fec_controller_.UpdateFecRates(
kMaxBitrateBps, 30, 128, std::vector<bool>(1, false), 100);
EXPECT_EQ(kMaxBitrateBps / 2, target_bitrate);
}
TEST_F(ProtectionBitrateCalculatorTest, NoProtection) {
static const uint32_t kMaxBitrateBps = 130000;
media_opt_.SetProtectionMethod(false /*enable_fec*/, false /* enable_nack */);
media_opt_.SetEncodingData(640, 480, 1, 1000);
fec_controller_.SetProtectionMethod(false /*enable_fec*/,
false /* enable_nack */);
fec_controller_.SetEncodingData(640, 480, 1, 1000);
uint32_t target_bitrate =
media_opt_.SetTargetRates(kMaxBitrateBps, 30, 128, 100);
uint32_t target_bitrate = fec_controller_.UpdateFecRates(
kMaxBitrateBps, 30, 128, std::vector<bool>(1, false), 100);
EXPECT_EQ(kMaxBitrateBps, target_bitrate);
}

1
modules/video_coding/include/video_coding.h
View File

@ -21,6 +21,7 @@
#include <windows.h>
#endif
#include "api/fec_controller.h"
#include "api/video/video_frame.h"
#include "modules/include/module.h"
#include "modules/include/module_common_types.h"

14
modules/video_coding/include/video_coding_defines.h
View File

@ -108,20 +108,6 @@ class VCMReceiveStatisticsCallback {
virtual ~VCMReceiveStatisticsCallback() {}
};
// Callback class used for telling the user about how to configure the FEC,
// and the rates sent the last second is returned to the VCM.
class VCMProtectionCallback {
public:
virtual int ProtectionRequest(const FecProtectionParams* delta_params,
const FecProtectionParams* key_params,
uint32_t* sent_video_rate_bps,
uint32_t* sent_nack_rate_bps,
uint32_t* sent_fec_rate_bps) = 0;
protected:
virtual ~VCMProtectionCallback() {}
};
// Callback class used for telling the user about what frame type needed to
// continue decoding.
// Typically a key frame when the stream has been corrupted in some way.

78
modules/video_coding/protection_bitrate_calculator.h
View File

@ -1,78 +0,0 @@
/*
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_PROTECTION_BITRATE_CALCULATOR_H_
#define MODULES_VIDEO_CODING_PROTECTION_BITRATE_CALCULATOR_H_
#include <list>
#include <memory>
#include "modules/include/module_common_types.h"
#include "modules/video_coding/include/video_coding.h"
#include "modules/video_coding/media_opt_util.h"
#include "rtc_base/criticalsection.h"
#include "system_wrappers/include/clock.h"
namespace webrtc {
// ProtectionBitrateCalculator calculates how much of the allocated network
// capacity that can be used by an encoder and how much that
// is needed for redundant packets such as FEC and NACK. It uses an
// implementation of |VCMProtectionCallback| to set new FEC parameters and get
// the bitrate currently used for FEC and NACK.
// Usage:
// Setup by calling SetProtectionMethod and SetEncodingData.
// For each encoded image, call UpdateWithEncodedData.
// Each time the bandwidth estimate change, call SetTargetRates. SetTargetRates
// will return the bitrate that can be used by an encoder.
// A lock is used to protect internal states, so methods can be called on an
// arbitrary thread.
class ProtectionBitrateCalculator {
public:
ProtectionBitrateCalculator(Clock* clock,
VCMProtectionCallback* protection_callback);
~ProtectionBitrateCalculator();
void SetProtectionMethod(bool enable_fec, bool enable_nack);
// Informs media optimization of initial encoding state.
void SetEncodingData(size_t width,
size_t height,
size_t num_temporal_layers,
size_t max_payload_size);
// Returns target rate for the encoder given the channel parameters.
// Inputs: estimated_bitrate_bps - the estimated network bitrate in bits/s.
// actual_framerate - encoder frame rate.
// fraction_lost - packet loss rate in % in the network.
// round_trip_time_ms - round trip time in milliseconds.
uint32_t SetTargetRates(uint32_t estimated_bitrate_bps,
int actual_framerate,
uint8_t fraction_lost,
int64_t round_trip_time_ms);
// Informs of encoded output.
void UpdateWithEncodedData(const EncodedImage& encoded_image);
private:
enum { kBitrateAverageWinMs = 1000 };
Clock* const clock_;
VCMProtectionCallback* const protection_callback_;
rtc::CriticalSection crit_sect_;
std::unique_ptr<media_optimization::VCMLossProtectionLogic> loss_prot_logic_
RTC_GUARDED_BY(crit_sect_);
size_t max_payload_size_ RTC_GUARDED_BY(crit_sect_);
RTC_DISALLOW_COPY_AND_ASSIGN(ProtectionBitrateCalculator);
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_PROTECTION_BITRATE_CALCULATOR_H_

1
video/BUILD.gn
View File

@ -56,6 +56,7 @@ rtc_static_library("video") {
deps = [
"..:webrtc_common",
"../:typedefs",
"../api:fec_controller_api",
"../api:libjingle_peerconnection_api",
"../api:optional",
"../api:transport_api",

24
video/video_quality_test.cc
View File

@ -1060,6 +1060,12 @@ VideoQualityTest::VideoQualityTest()
payload_type_map_[kPayloadTypeVP9] = webrtc::MediaType::VIDEO;
}
VideoQualityTest::VideoQualityTest(
std::unique_ptr<FecController> fec_controller)
: VideoQualityTest() {
fec_controller_ = std::move(fec_controller);
}
VideoQualityTest::Params::Params()
: call({false, Call::Config::BitrateConfig(), 0}),
video{{false, 640, 480, 30, 50, 800, 800, false, "VP8", 1, -1, 0, false,
@ -1803,6 +1809,24 @@ void VideoQualityTest::CreateVideoStreams() {
AssociateFlexfecStreamsWithVideoStreams();
}
void VideoQualityTest::CreateVideoStreamsWithProtectionBitrateCalculator(
std::unique_ptr<FecController> fec_controller) {
RTC_DCHECK(video_send_streams_.empty());
RTC_DCHECK(video_receive_streams_.empty());
RTC_DCHECK_EQ(video_send_configs_.size(), num_video_streams_);
for (size_t i = 0; i < video_send_configs_.size(); ++i) {
video_send_streams_.push_back(sender_call_->CreateVideoSendStream(
video_send_configs_[i].Copy(), video_encoder_configs_[i].Copy(),
std::move(fec_controller)));
}
for (size_t i = 0; i < video_receive_configs_.size(); ++i) {
video_receive_streams_.push_back(receiver_call_->CreateVideoReceiveStream(
video_receive_configs_[i].Copy()));
}
AssociateFlexfecStreamsWithVideoStreams();
}
void VideoQualityTest::DestroyStreams() {
CallTest::DestroyStreams();

4
video/video_quality_test.h
View File

@ -96,6 +96,7 @@ class VideoQualityTest : public test::CallTest {
};
VideoQualityTest();
explicit VideoQualityTest(std::unique_ptr<FecController> fec_controller);
void RunWithAnalyzer(const Params& params);
void RunWithRenderers(const Params& params);
@ -111,6 +112,7 @@ class VideoQualityTest : public test::CallTest {
protected:
std::map<uint8_t, webrtc::MediaType> payload_type_map_;
std::unique_ptr<FecController> fec_controller_;
// No-op implementation to be able to instantiate this class from non-TEST_F
// locations.
@ -127,6 +129,8 @@ class VideoQualityTest : public test::CallTest {
// Helper methods for setting up the call.
void CreateVideoStreams();
void CreateVideoStreamsWithProtectionBitrateCalculator(
std::unique_ptr<FecController> fec_controller);
void DestroyStreams();
void CreateCapturers();
std::unique_ptr<test::FrameGenerator> CreateFrameGenerator(size_t video_idx);

106
video/video_send_stream.cc
View File

@ -11,6 +11,7 @@
#include <algorithm>
#include <cmath>
#include <set>
#include <sstream>
#include <string>
#include <utility>
@ -26,6 +27,7 @@
#include "modules/rtp_rtcp/include/rtp_rtcp.h"
#include "modules/rtp_rtcp/source/rtp_sender.h"
#include "modules/utility/include/process_thread.h"
#include "modules/video_coding/fec_controller_default.h"
#include "modules/video_coding/utility/ivf_file_writer.h"
#include "rtc_base/checks.h"
#include "rtc_base/experiments/alr_experiment.h"
@ -41,6 +43,8 @@
namespace webrtc {
static const int kMinSendSidePacketHistorySize = 600;
static const int kSendSideSeqNumSetMaxSize = 15000;
namespace {
// We don't do MTU discovery, so assume that we have the standard ethernet MTU.
@ -244,7 +248,8 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver,
public webrtc::OverheadObserver,
public webrtc::VCMProtectionCallback,
public VideoStreamEncoder::EncoderSink,
public VideoBitrateAllocationObserver {
public VideoBitrateAllocationObserver,
public webrtc::PacketFeedbackObserver {
public:
VideoSendStreamImpl(
SendStatisticsProxy* stats_proxy,
@ -260,7 +265,8 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver,
double initial_encoder_bitrate_priority,
std::map<uint32_t, RtpState> suspended_ssrcs,
std::map<uint32_t, RtpPayloadState> suspended_payload_states,
VideoEncoderConfig::ContentType content_type);
VideoEncoderConfig::ContentType content_type,
std::unique_ptr<FecController> fec_controller);
~VideoSendStreamImpl() override;
// RegisterProcessThread register |module_process_thread| with those objects
@ -286,6 +292,11 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver,
rtc::Optional<float> configured_pacing_factor_;
// From PacketFeedbackObserver.
void OnPacketAdded(uint32_t ssrc, uint16_t seq_num) override;
void OnPacketFeedbackVector(
const std::vector<PacketFeedback>& packet_feedback_vector) override;
private:
class CheckEncoderActivityTask;
class EncoderReconfiguredTask;
@ -331,6 +342,7 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver,
const VideoSendStream::Config* const config_;
std::map<uint32_t, RtpState> suspended_ssrcs_;
std::unique_ptr<FecController> fec_controller_;
ProcessThread* module_process_thread_;
rtc::ThreadChecker module_process_thread_checker_;
rtc::TaskQueue* const worker_queue_;
@ -358,7 +370,6 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver,
VideoStreamEncoder* const video_stream_encoder_;
EncoderRtcpFeedback encoder_feedback_;
ProtectionBitrateCalculator protection_bitrate_calculator_;
RtcpBandwidthObserver* const bandwidth_observer_;
// RtpRtcp modules, declared here as they use other members on construction.
@ -377,6 +388,9 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver,
size_t overhead_bytes_per_packet_
RTC_GUARDED_BY(overhead_bytes_per_packet_crit_);
size_t transport_overhead_bytes_per_packet_;
std::set<uint16_t> feedback_packet_seq_num_set_;
std::vector<bool> loss_mask_vector_;
};
// TODO(tommi): See if there's a more elegant way to create a task that creates
@ -399,7 +413,8 @@ class VideoSendStream::ConstructionTask : public rtc::QueuedTask {
double initial_encoder_bitrate_priority,
const std::map<uint32_t, RtpState>& suspended_ssrcs,
const std::map<uint32_t, RtpPayloadState>& suspended_payload_states,
VideoEncoderConfig::ContentType content_type)
VideoEncoderConfig::ContentType content_type,
std::unique_ptr<FecController> fec_controller)
: send_stream_(send_stream),
done_event_(done_event),
stats_proxy_(stats_proxy),
@ -414,7 +429,8 @@ class VideoSendStream::ConstructionTask : public rtc::QueuedTask {
initial_encoder_bitrate_priority_(initial_encoder_bitrate_priority),
suspended_ssrcs_(suspended_ssrcs),
suspended_payload_states_(suspended_payload_states),
content_type_(content_type) {}
content_type_(content_type),
fec_controller_(std::move(fec_controller)) {}
~ConstructionTask() override { done_event_->Set(); }
@ -425,7 +441,8 @@ class VideoSendStream::ConstructionTask : public rtc::QueuedTask {
bitrate_allocator_, send_delay_stats_, video_stream_encoder_,
event_log_, config_, initial_encoder_max_bitrate_,
initial_encoder_bitrate_priority_, std::move(suspended_ssrcs_),
std::move(suspended_payload_states_), content_type_));
std::move(suspended_payload_states_), content_type_,
std::move(fec_controller_)));
return true;
}
@ -444,6 +461,7 @@ class VideoSendStream::ConstructionTask : public rtc::QueuedTask {
std::map<uint32_t, RtpState> suspended_ssrcs_;
std::map<uint32_t, RtpPayloadState> suspended_payload_states_;
const VideoEncoderConfig::ContentType content_type_;
std::unique_ptr<FecController> fec_controller_;
};
class VideoSendStream::DestructAndGetRtpStateTask : public rtc::QueuedTask {
@ -562,7 +580,8 @@ VideoSendStream::VideoSendStream(
VideoSendStream::Config config,
VideoEncoderConfig encoder_config,
const std::map<uint32_t, RtpState>& suspended_ssrcs,
const std::map<uint32_t, RtpPayloadState>& suspended_payload_states)
const std::map<uint32_t, RtpPayloadState>& suspended_payload_states,
std::unique_ptr<FecController> fec_controller)
: worker_queue_(worker_queue),
thread_sync_event_(false /* manual_reset */, false),
stats_proxy_(Clock::GetRealTimeClock(),
@ -580,7 +599,8 @@ VideoSendStream::VideoSendStream(
video_stream_encoder_.get(), module_process_thread, call_stats, transport,
bitrate_allocator, send_delay_stats, event_log, &config_,
encoder_config.max_bitrate_bps, encoder_config.bitrate_priority,
suspended_ssrcs, suspended_payload_states, encoder_config.content_type)));
suspended_ssrcs, suspended_payload_states, encoder_config.content_type,
std::move(fec_controller))));
// Wait for ConstructionTask to complete so that |send_stream_| can be used.
// |module_process_thread| must be registered and deregistered on the thread
@ -705,12 +725,14 @@ VideoSendStreamImpl::VideoSendStreamImpl(
double initial_encoder_bitrate_priority,
std::map<uint32_t, RtpState> suspended_ssrcs,
std::map<uint32_t, RtpPayloadState> suspended_payload_states,
VideoEncoderConfig::ContentType content_type)
VideoEncoderConfig::ContentType content_type,
std::unique_ptr<FecController> fec_controller)
: send_side_bwe_with_overhead_(
webrtc::field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead")),
stats_proxy_(stats_proxy),
config_(config),
suspended_ssrcs_(std::move(suspended_ssrcs)),
fec_controller_(std::move(fec_controller)),
module_process_thread_(nullptr),
worker_queue_(worker_queue),
check_encoder_activity_task_(nullptr),
@ -727,7 +749,6 @@ VideoSendStreamImpl::VideoSendStreamImpl(
encoder_feedback_(Clock::GetRealTimeClock(),
config_->rtp.ssrcs,
video_stream_encoder),
protection_bitrate_calculator_(Clock::GetRealTimeClock(), this),
bandwidth_observer_(transport->send_side_cc()->GetBandwidthObserver()),
rtp_rtcp_modules_(CreateRtpRtcpModules(
config_->send_transport,
@ -823,6 +844,10 @@ VideoSendStreamImpl::VideoSendStreamImpl(
config_->encoder_settings.payload_name.c_str());
}
fec_controller_->SetProtectionCallback(this);
// Signal congestion controller this object is ready for OnPacket* callbacks.
transport_->send_side_cc()->RegisterPacketFeedbackObserver(this);
RTC_DCHECK(config_->encoder_settings.encoder);
RTC_DCHECK_GE(config_->encoder_settings.payload_type, 0);
RTC_DCHECK_LE(config_->encoder_settings.payload_type, 127);
@ -865,7 +890,7 @@ VideoSendStreamImpl::~VideoSendStreamImpl() {
RTC_DCHECK(!payload_router_.IsActive())
<< "VideoSendStreamImpl::Stop not called";
RTC_LOG(LS_INFO) << "~VideoSendStreamInternal: " << config_->ToString();
transport_->send_side_cc()->DeRegisterPacketFeedbackObserver(this);
for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
transport_->packet_router()->RemoveSendRtpModule(rtp_rtcp);
delete rtp_rtcp;
@ -989,9 +1014,9 @@ void VideoSendStreamImpl::OnEncoderConfigurationChanged(
size_t number_of_temporal_layers =
streams.back().temporal_layer_thresholds_bps.size() + 1;
protection_bitrate_calculator_.SetEncodingData(
streams[0].width, streams[0].height, number_of_temporal_layers,
config_->rtp.max_packet_size);
fec_controller_->SetEncodingData(streams[0].width, streams[0].height,
number_of_temporal_layers,
config_->rtp.max_packet_size);
if (payload_router_.IsActive()) {
// The send stream is started already. Update the allocator with new bitrate
@ -1025,7 +1050,7 @@ EncodedImageCallback::Result VideoSendStreamImpl::OnEncodedImage(
check_encoder_activity_task_->UpdateEncoderActivity();
}
protection_bitrate_calculator_.UpdateWithEncodedData(encoded_image);
fec_controller_->UpdateWithEncodedData(encoded_image);
EncodedImageCallback::Result result = payload_router_.OnEncodedImage(
encoded_image, codec_specific_info, fragmentation);
@ -1131,8 +1156,8 @@ void VideoSendStreamImpl::ConfigureProtection() {
// Currently, both ULPFEC and FlexFEC use the same FEC rate calculation logic,
// so enable that logic if either of those FEC schemes are enabled.
protection_bitrate_calculator_.SetProtectionMethod(
flexfec_enabled || IsUlpfecEnabled(), nack_enabled);
fec_controller_->SetProtectionMethod(flexfec_enabled || IsUlpfecEnabled(),
nack_enabled);
}
void VideoSendStreamImpl::ConfigureSsrcs() {
@ -1239,9 +1264,10 @@ uint32_t VideoSendStreamImpl::OnBitrateUpdated(uint32_t bitrate_bps,
// Get the encoder target rate. It is the estimated network rate -
// protection overhead.
encoder_target_rate_bps_ = protection_bitrate_calculator_.SetTargetRates(
encoder_target_rate_bps_ = fec_controller_->UpdateFecRates(
payload_bitrate_bps, stats_proxy_->GetSendFrameRate(), fraction_loss,
rtt);
loss_mask_vector_, rtt);
loss_mask_vector_.clear();
uint32_t encoder_overhead_rate_bps =
send_side_bwe_with_overhead_
@ -1340,5 +1366,47 @@ void VideoSendStreamImpl::SetTransportOverhead(
}
}
void VideoSendStreamImpl::OnPacketAdded(uint32_t ssrc, uint16_t seq_num) {
if (!worker_queue_->IsCurrent()) {
auto ptr = weak_ptr_;
worker_queue_->PostTask([=] {
if (!ptr.get())
return;
ptr->OnPacketAdded(ssrc, seq_num);
});
return;
}
const auto ssrcs = config_->rtp.ssrcs;
if (std::find(ssrcs.begin(), ssrcs.end(), ssrc) != ssrcs.end()) {
feedback_packet_seq_num_set_.insert(seq_num);
if (feedback_packet_seq_num_set_.size() > kSendSideSeqNumSetMaxSize) {
RTC_LOG(LS_WARNING) << "Feedback packet sequence number set exceed it's "
"max size', will get reset.";
feedback_packet_seq_num_set_.clear();
}
}
}
void VideoSendStreamImpl::OnPacketFeedbackVector(
const std::vector<PacketFeedback>& packet_feedback_vector) {
if (!worker_queue_->IsCurrent()) {
auto ptr = weak_ptr_;
worker_queue_->PostTask([=] {
if (!ptr.get())
return;
ptr->OnPacketFeedbackVector(packet_feedback_vector);
});
return;
}
// Lost feedbacks are not considered to be lost packets.
for (const PacketFeedback& packet : packet_feedback_vector) {
if (auto it = feedback_packet_seq_num_set_.find(packet.sequence_number) !=
feedback_packet_seq_num_set_.end()) {
const bool lost = packet.arrival_time_ms == PacketFeedback::kNotReceived;
loss_mask_vector_.push_back(lost);
feedback_packet_seq_num_set_.erase(it);
}
}
}
} // namespace internal
} // namespace webrtc

5
video/video_send_stream.h
View File

@ -15,11 +15,11 @@
#include <memory>
#include <vector>
#include "api/fec_controller.h"
#include "call/bitrate_allocator.h"
#include "call/video_receive_stream.h"
#include "call/video_send_stream.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "modules/video_coding/protection_bitrate_calculator.h"
#include "rtc_base/criticalsection.h"
#include "rtc_base/event.h"
#include "rtc_base/task_queue.h"
@ -63,7 +63,8 @@ class VideoSendStream : public webrtc::VideoSendStream {
VideoSendStream::Config config,
VideoEncoderConfig encoder_config,
const std::map<uint32_t, RtpState>& suspended_ssrcs,
const std::map<uint32_t, RtpPayloadState>& suspended_payload_states);
const std::map<uint32_t, RtpPayloadState>& suspended_payload_states,
std::unique_ptr<FecController> fec_controller);
~VideoSendStream() override;