6ba7feb302
logging the configuration, in particular the content type which together with RTP configuration information like the ssrcs helps differentiating between encoders. BUG=None Change-Id: I1b4b2ec2bffea338cc73c3a9c6a3f775d8f1c26b Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/319560 Reviewed-by: Henrik Boström <hbos@webrtc.org> Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org> Commit-Queue: Philipp Hancke <phancke@microsoft.com> Cr-Commit-Position: refs/heads/main@{#40744}
346 lines
12 KiB
C++
346 lines
12 KiB
C++
/*
|
|
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
#include "video/video_send_stream.h"
|
|
|
|
#include <utility>
|
|
|
|
#include "api/array_view.h"
|
|
#include "api/task_queue/task_queue_base.h"
|
|
#include "api/video/video_stream_encoder_settings.h"
|
|
#include "modules/rtp_rtcp/include/rtp_header_extension_map.h"
|
|
#include "modules/rtp_rtcp/source/rtp_header_extension_size.h"
|
|
#include "modules/rtp_rtcp/source/rtp_sender.h"
|
|
#include "rtc_base/checks.h"
|
|
#include "rtc_base/logging.h"
|
|
#include "rtc_base/strings/string_builder.h"
|
|
#include "system_wrappers/include/clock.h"
|
|
#include "video/adaptation/overuse_frame_detector.h"
|
|
#include "video/frame_cadence_adapter.h"
|
|
#include "video/video_stream_encoder.h"
|
|
|
|
namespace webrtc {
|
|
|
|
namespace {
|
|
|
|
size_t CalculateMaxHeaderSize(const RtpConfig& config) {
|
|
size_t header_size = kRtpHeaderSize;
|
|
size_t extensions_size = 0;
|
|
size_t fec_extensions_size = 0;
|
|
if (!config.extensions.empty()) {
|
|
RtpHeaderExtensionMap extensions_map(config.extensions);
|
|
extensions_size = RtpHeaderExtensionSize(RTPSender::VideoExtensionSizes(),
|
|
extensions_map);
|
|
fec_extensions_size =
|
|
RtpHeaderExtensionSize(RTPSender::FecExtensionSizes(), extensions_map);
|
|
}
|
|
header_size += extensions_size;
|
|
if (config.flexfec.payload_type >= 0) {
|
|
// All FEC extensions again plus maximum FlexFec overhead.
|
|
header_size += fec_extensions_size + 32;
|
|
} else {
|
|
if (config.ulpfec.ulpfec_payload_type >= 0) {
|
|
// Header with all the FEC extensions will be repeated plus maximum
|
|
// UlpFec overhead.
|
|
header_size += fec_extensions_size + 18;
|
|
}
|
|
if (config.ulpfec.red_payload_type >= 0) {
|
|
header_size += 1; // RED header.
|
|
}
|
|
}
|
|
// Additional room for Rtx.
|
|
if (config.rtx.payload_type >= 0)
|
|
header_size += kRtxHeaderSize;
|
|
return header_size;
|
|
}
|
|
|
|
VideoStreamEncoder::BitrateAllocationCallbackType
|
|
GetBitrateAllocationCallbackType(const VideoSendStream::Config& config,
|
|
const FieldTrialsView& field_trials) {
|
|
if (webrtc::RtpExtension::FindHeaderExtensionByUri(
|
|
config.rtp.extensions,
|
|
webrtc::RtpExtension::kVideoLayersAllocationUri,
|
|
config.crypto_options.srtp.enable_encrypted_rtp_header_extensions
|
|
? RtpExtension::Filter::kPreferEncryptedExtension
|
|
: RtpExtension::Filter::kDiscardEncryptedExtension)) {
|
|
return VideoStreamEncoder::BitrateAllocationCallbackType::
|
|
kVideoLayersAllocation;
|
|
}
|
|
if (field_trials.IsEnabled("WebRTC-Target-Bitrate-Rtcp")) {
|
|
return VideoStreamEncoder::BitrateAllocationCallbackType::
|
|
kVideoBitrateAllocation;
|
|
}
|
|
return VideoStreamEncoder::BitrateAllocationCallbackType::
|
|
kVideoBitrateAllocationWhenScreenSharing;
|
|
}
|
|
|
|
RtpSenderFrameEncryptionConfig CreateFrameEncryptionConfig(
|
|
const VideoSendStream::Config* config) {
|
|
RtpSenderFrameEncryptionConfig frame_encryption_config;
|
|
frame_encryption_config.frame_encryptor = config->frame_encryptor.get();
|
|
frame_encryption_config.crypto_options = config->crypto_options;
|
|
return frame_encryption_config;
|
|
}
|
|
|
|
RtpSenderObservers CreateObservers(RtcpRttStats* call_stats,
|
|
EncoderRtcpFeedback* encoder_feedback,
|
|
SendStatisticsProxy* stats_proxy,
|
|
SendPacketObserver* send_packet_observer) {
|
|
RtpSenderObservers observers;
|
|
observers.rtcp_rtt_stats = call_stats;
|
|
observers.intra_frame_callback = encoder_feedback;
|
|
observers.rtcp_loss_notification_observer = encoder_feedback;
|
|
observers.report_block_data_observer = stats_proxy;
|
|
observers.rtp_stats = stats_proxy;
|
|
observers.bitrate_observer = stats_proxy;
|
|
observers.frame_count_observer = stats_proxy;
|
|
observers.rtcp_type_observer = stats_proxy;
|
|
observers.send_delay_observer = nullptr;
|
|
observers.send_packet_observer = send_packet_observer;
|
|
return observers;
|
|
}
|
|
|
|
std::unique_ptr<VideoStreamEncoder> CreateVideoStreamEncoder(
|
|
Clock* clock,
|
|
int num_cpu_cores,
|
|
TaskQueueFactory* task_queue_factory,
|
|
SendStatisticsProxy* stats_proxy,
|
|
const VideoStreamEncoderSettings& encoder_settings,
|
|
VideoStreamEncoder::BitrateAllocationCallbackType
|
|
bitrate_allocation_callback_type,
|
|
const FieldTrialsView& field_trials,
|
|
webrtc::VideoEncoderFactory::EncoderSelectorInterface* encoder_selector) {
|
|
std::unique_ptr<TaskQueueBase, TaskQueueDeleter> encoder_queue =
|
|
task_queue_factory->CreateTaskQueue("EncoderQueue",
|
|
TaskQueueFactory::Priority::NORMAL);
|
|
TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
|
|
return std::make_unique<VideoStreamEncoder>(
|
|
clock, num_cpu_cores, stats_proxy, encoder_settings,
|
|
std::make_unique<OveruseFrameDetector>(stats_proxy),
|
|
FrameCadenceAdapterInterface::Create(clock, encoder_queue_ptr,
|
|
field_trials),
|
|
std::move(encoder_queue), bitrate_allocation_callback_type, field_trials,
|
|
encoder_selector);
|
|
}
|
|
|
|
} // namespace
|
|
|
|
namespace internal {
|
|
|
|
VideoSendStream::VideoSendStream(
|
|
Clock* clock,
|
|
int num_cpu_cores,
|
|
TaskQueueFactory* task_queue_factory,
|
|
TaskQueueBase* network_queue,
|
|
RtcpRttStats* call_stats,
|
|
RtpTransportControllerSendInterface* transport,
|
|
BitrateAllocatorInterface* bitrate_allocator,
|
|
SendDelayStats* send_delay_stats,
|
|
RtcEventLog* event_log,
|
|
VideoSendStream::Config config,
|
|
VideoEncoderConfig encoder_config,
|
|
const std::map<uint32_t, RtpState>& suspended_ssrcs,
|
|
const std::map<uint32_t, RtpPayloadState>& suspended_payload_states,
|
|
std::unique_ptr<FecController> fec_controller,
|
|
const FieldTrialsView& field_trials)
|
|
: transport_(transport),
|
|
stats_proxy_(clock, config, encoder_config.content_type, field_trials),
|
|
send_packet_observer_(&stats_proxy_, send_delay_stats),
|
|
config_(std::move(config)),
|
|
content_type_(encoder_config.content_type),
|
|
video_stream_encoder_(CreateVideoStreamEncoder(
|
|
clock,
|
|
num_cpu_cores,
|
|
task_queue_factory,
|
|
&stats_proxy_,
|
|
config_.encoder_settings,
|
|
GetBitrateAllocationCallbackType(config_, field_trials),
|
|
field_trials,
|
|
config_.encoder_selector)),
|
|
encoder_feedback_(
|
|
clock,
|
|
config_.rtp.ssrcs,
|
|
video_stream_encoder_.get(),
|
|
[this](uint32_t ssrc, const std::vector<uint16_t>& seq_nums) {
|
|
return rtp_video_sender_->GetSentRtpPacketInfos(ssrc, seq_nums);
|
|
}),
|
|
rtp_video_sender_(transport->CreateRtpVideoSender(
|
|
suspended_ssrcs,
|
|
suspended_payload_states,
|
|
config_.rtp,
|
|
config_.rtcp_report_interval_ms,
|
|
config_.send_transport,
|
|
CreateObservers(call_stats,
|
|
&encoder_feedback_,
|
|
&stats_proxy_,
|
|
&send_packet_observer_),
|
|
event_log,
|
|
std::move(fec_controller),
|
|
CreateFrameEncryptionConfig(&config_),
|
|
config_.frame_transformer)),
|
|
send_stream_(clock,
|
|
&stats_proxy_,
|
|
transport,
|
|
bitrate_allocator,
|
|
video_stream_encoder_.get(),
|
|
&config_,
|
|
encoder_config.max_bitrate_bps,
|
|
encoder_config.bitrate_priority,
|
|
encoder_config.content_type,
|
|
rtp_video_sender_,
|
|
field_trials) {
|
|
RTC_DCHECK(config_.encoder_settings.encoder_factory);
|
|
RTC_DCHECK(config_.encoder_settings.bitrate_allocator_factory);
|
|
|
|
video_stream_encoder_->SetFecControllerOverride(rtp_video_sender_);
|
|
|
|
ReconfigureVideoEncoder(std::move(encoder_config));
|
|
}
|
|
|
|
VideoSendStream::~VideoSendStream() {
|
|
RTC_DCHECK_RUN_ON(&thread_checker_);
|
|
RTC_DCHECK(!running_);
|
|
transport_->DestroyRtpVideoSender(rtp_video_sender_);
|
|
}
|
|
|
|
void VideoSendStream::Start() {
|
|
const std::vector<bool> active_layers(config_.rtp.ssrcs.size(), true);
|
|
StartPerRtpStream(active_layers);
|
|
}
|
|
|
|
void VideoSendStream::StartPerRtpStream(const std::vector<bool> active_layers) {
|
|
RTC_DCHECK_RUN_ON(&thread_checker_);
|
|
|
|
// Keep our `running_` flag expected state in sync with active layers since
|
|
// the `send_stream_` will be implicitly stopped/started depending on the
|
|
// state of the layers.
|
|
bool running = false;
|
|
|
|
rtc::StringBuilder active_layers_string;
|
|
active_layers_string << "{";
|
|
for (size_t i = 0; i < active_layers.size(); ++i) {
|
|
if (active_layers[i]) {
|
|
running = true;
|
|
active_layers_string << "1";
|
|
} else {
|
|
active_layers_string << "0";
|
|
}
|
|
if (i < active_layers.size() - 1) {
|
|
active_layers_string << ", ";
|
|
}
|
|
}
|
|
active_layers_string << "}";
|
|
RTC_LOG(LS_INFO) << "StartPerRtpStream: " << active_layers_string.str();
|
|
send_stream_.StartPerRtpStream(active_layers);
|
|
running_ = running;
|
|
}
|
|
|
|
void VideoSendStream::Stop() {
|
|
RTC_DCHECK_RUN_ON(&thread_checker_);
|
|
if (!running_)
|
|
return;
|
|
RTC_DLOG(LS_INFO) << "VideoSendStream::Stop";
|
|
running_ = false;
|
|
send_stream_.Stop();
|
|
}
|
|
|
|
bool VideoSendStream::started() {
|
|
RTC_DCHECK_RUN_ON(&thread_checker_);
|
|
return running_;
|
|
}
|
|
|
|
void VideoSendStream::AddAdaptationResource(
|
|
rtc::scoped_refptr<Resource> resource) {
|
|
RTC_DCHECK_RUN_ON(&thread_checker_);
|
|
video_stream_encoder_->AddAdaptationResource(resource);
|
|
}
|
|
|
|
std::vector<rtc::scoped_refptr<Resource>>
|
|
VideoSendStream::GetAdaptationResources() {
|
|
RTC_DCHECK_RUN_ON(&thread_checker_);
|
|
return video_stream_encoder_->GetAdaptationResources();
|
|
}
|
|
|
|
void VideoSendStream::SetSource(
|
|
rtc::VideoSourceInterface<webrtc::VideoFrame>* source,
|
|
const DegradationPreference& degradation_preference) {
|
|
RTC_DCHECK_RUN_ON(&thread_checker_);
|
|
video_stream_encoder_->SetSource(source, degradation_preference);
|
|
}
|
|
|
|
void VideoSendStream::ReconfigureVideoEncoder(VideoEncoderConfig config) {
|
|
ReconfigureVideoEncoder(std::move(config), nullptr);
|
|
}
|
|
|
|
void VideoSendStream::ReconfigureVideoEncoder(VideoEncoderConfig config,
|
|
SetParametersCallback callback) {
|
|
RTC_DCHECK_RUN_ON(&thread_checker_);
|
|
RTC_DCHECK_EQ(content_type_, config.content_type);
|
|
RTC_LOG(LS_VERBOSE) << "Encoder config: " << config.ToString()
|
|
<< " VideoSendStream config: " << config_.ToString();
|
|
video_stream_encoder_->ConfigureEncoder(
|
|
std::move(config),
|
|
config_.rtp.max_packet_size - CalculateMaxHeaderSize(config_.rtp),
|
|
std::move(callback));
|
|
}
|
|
|
|
VideoSendStream::Stats VideoSendStream::GetStats() {
|
|
RTC_DCHECK_RUN_ON(&thread_checker_);
|
|
return stats_proxy_.GetStats();
|
|
}
|
|
|
|
absl::optional<float> VideoSendStream::GetPacingFactorOverride() const {
|
|
return send_stream_.configured_pacing_factor();
|
|
}
|
|
|
|
void VideoSendStream::StopPermanentlyAndGetRtpStates(
|
|
VideoSendStream::RtpStateMap* rtp_state_map,
|
|
VideoSendStream::RtpPayloadStateMap* payload_state_map) {
|
|
RTC_DCHECK_RUN_ON(&thread_checker_);
|
|
video_stream_encoder_->Stop();
|
|
|
|
running_ = false;
|
|
// Always run these cleanup steps regardless of whether running_ was set
|
|
// or not. This will unregister callbacks before destruction.
|
|
// See `VideoSendStreamImpl::StopVideoSendStream` for more.
|
|
send_stream_.Stop();
|
|
*rtp_state_map = send_stream_.GetRtpStates();
|
|
*payload_state_map = send_stream_.GetRtpPayloadStates();
|
|
}
|
|
|
|
void VideoSendStream::DeliverRtcp(const uint8_t* packet, size_t length) {
|
|
RTC_DCHECK_RUN_ON(&thread_checker_);
|
|
send_stream_.DeliverRtcp(packet, length);
|
|
}
|
|
|
|
void VideoSendStream::GenerateKeyFrame(const std::vector<std::string>& rids) {
|
|
RTC_DCHECK_RUN_ON(&thread_checker_);
|
|
// Map rids to layers. If rids is empty, generate a keyframe for all layers.
|
|
std::vector<VideoFrameType> next_frames(config_.rtp.ssrcs.size(),
|
|
VideoFrameType::kVideoFrameKey);
|
|
if (!config_.rtp.rids.empty() && !rids.empty()) {
|
|
std::fill(next_frames.begin(), next_frames.end(),
|
|
VideoFrameType::kVideoFrameDelta);
|
|
for (const auto& rid : rids) {
|
|
for (size_t i = 0; i < config_.rtp.rids.size(); i++) {
|
|
if (config_.rtp.rids[i] == rid) {
|
|
next_frames[i] = VideoFrameType::kVideoFrameKey;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (video_stream_encoder_) {
|
|
video_stream_encoder_->SendKeyFrame(next_frames);
|
|
}
|
|
}
|
|
|
|
} // namespace internal
|
|
} // namespace webrtc
|