diff --git a/api/video/video_stream_encoder_settings.h b/api/video/video_stream_encoder_settings.h index 0183d4cac1..9a9889a084 100644 --- a/api/video/video_stream_encoder_settings.h +++ b/api/video/video_stream_encoder_settings.h @@ -17,6 +17,12 @@ namespace webrtc { +class EncoderFailureCallback { + public: + virtual ~EncoderFailureCallback() {} + virtual void OnEncoderFailure() = 0; +}; + struct VideoStreamEncoderSettings { explicit VideoStreamEncoderSettings( const VideoEncoder::Capabilities& capabilities) @@ -29,6 +35,9 @@ struct VideoStreamEncoderSettings { // Ownership stays with WebrtcVideoEngine (delegated from PeerConnection). VideoEncoderFactory* encoder_factory = nullptr; + // Notifies the WebRtcVideoChannel that the currently used encoder is broken. + EncoderFailureCallback* encoder_failure_callback = nullptr; + // Ownership stays with WebrtcVideoEngine (delegated from PeerConnection). VideoBitrateAllocatorFactory* bitrate_allocator_factory = nullptr; diff --git a/api/video_codecs/sdp_video_format.cc b/api/video_codecs/sdp_video_format.cc index 909904c19c..167d26a699 100644 --- a/api/video_codecs/sdp_video_format.cc +++ b/api/video_codecs/sdp_video_format.cc @@ -9,6 +9,7 @@ */ #include "api/video_codecs/sdp_video_format.h" +#include "rtc_base/strings/string_builder.h" namespace webrtc { @@ -25,6 +26,16 @@ SdpVideoFormat& SdpVideoFormat::operator=(SdpVideoFormat&&) = default; SdpVideoFormat::~SdpVideoFormat() = default; +std::string SdpVideoFormat::ToString() const { + rtc::StringBuilder builder; + builder << "Codec name: " << name << ", parameters: {"; + for (const auto& kv : parameters) + builder << " " << kv.first << "=" << kv.second; + builder << " }"; + + return builder.str(); +} + bool operator==(const SdpVideoFormat& a, const SdpVideoFormat& b) { return a.name == b.name && a.parameters == b.parameters; } diff --git a/api/video_codecs/sdp_video_format.h b/api/video_codecs/sdp_video_format.h index da1ff0baba..97bb75489d 100644 --- a/api/video_codecs/sdp_video_format.h +++ b/api/video_codecs/sdp_video_format.h @@ -32,6 +32,8 @@ struct RTC_EXPORT SdpVideoFormat { ~SdpVideoFormat(); + std::string ToString() const; + friend RTC_EXPORT bool operator==(const SdpVideoFormat& a, const SdpVideoFormat& b); friend RTC_EXPORT bool operator!=(const SdpVideoFormat& a, diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index 4400aef4dc..83366836e3 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -543,6 +543,7 @@ WebRtcVideoChannel::WebRtcVideoChannel( webrtc::VideoDecoderFactory* decoder_factory, webrtc::VideoBitrateAllocatorFactory* bitrate_allocator_factory) : VideoMediaChannel(config), + worker_thread_(rtc::Thread::Current()), call_(call), unsignalled_ssrc_handler_(&default_unsignalled_ssrc_handler_), video_config_(config.video), @@ -575,23 +576,42 @@ WebRtcVideoChannel::~WebRtcVideoChannel() { delete kv.second; } -absl::optional -WebRtcVideoChannel::SelectSendVideoCodec( +std::vector +WebRtcVideoChannel::SelectSendVideoCodecs( const std::vector& remote_mapped_codecs) const { - const std::vector local_supported_codecs = - AssignPayloadTypesAndDefaultCodecs(encoder_factory_); - // Select the first remote codec that is supported locally. - for (const VideoCodecSettings& remote_mapped_codec : remote_mapped_codecs) { - // For H264, we will limit the encode level to the remote offered level - // regardless if level asymmetry is allowed or not. This is strictly not - // following the spec in https://tools.ietf.org/html/rfc6184#section-8.2.2 - // since we should limit the encode level to the lower of local and remote - // level when level asymmetry is not allowed. - if (FindMatchingCodec(local_supported_codecs, remote_mapped_codec.codec)) - return remote_mapped_codec; + std::vector sdp_formats = + encoder_factory_->GetSupportedFormats(); + + // The returned vector holds the VideoCodecSettings in term of preference. + // They are orderd by receive codec preference first and local implementation + // preference second. + std::vector encoders; + for (const VideoCodecSettings& remote_codec : remote_mapped_codecs) { + for (auto format_it = sdp_formats.begin(); + format_it != sdp_formats.end();) { + // For H264, we will limit the encode level to the remote offered level + // regardless if level asymmetry is allowed or not. This is strictly not + // following the spec in https://tools.ietf.org/html/rfc6184#section-8.2.2 + // since we should limit the encode level to the lower of local and remote + // level when level asymmetry is not allowed. + if (IsSameCodec(format_it->name, format_it->parameters, + remote_codec.codec.name, remote_codec.codec.params)) { + encoders.push_back(remote_codec); + + // To allow the VideoEncoderFactory to keep information about which + // implementation to instantitate when CreateEncoder is called the two + // parmeter sets are merged. + encoders.back().codec.params.insert(format_it->parameters.begin(), + format_it->parameters.end()); + + format_it = sdp_formats.erase(format_it); + } else { + ++format_it; + } + } } - // No remote codec was supported. - return absl::nullopt; + + return encoders; } bool WebRtcVideoChannel::NonFlexfecReceiveCodecsHaveChanged( @@ -627,27 +647,27 @@ bool WebRtcVideoChannel::GetChangedSendParameters( return false; } - // Select one of the remote codecs that will be used as send codec. - absl::optional selected_send_codec = - SelectSendVideoCodec(MapCodecs(params.codecs)); + std::vector negotiated_codecs = + SelectSendVideoCodecs(MapCodecs(params.codecs)); - if (!selected_send_codec) { + if (negotiated_codecs.empty()) { RTC_LOG(LS_ERROR) << "No video codecs supported."; return false; } // Never enable sending FlexFEC, unless we are in the experiment. if (!IsFlexfecFieldTrialEnabled()) { - if (selected_send_codec->flexfec_payload_type != -1) { - RTC_LOG(LS_INFO) - << "Remote supports flexfec-03, but we will not send since " - << "WebRTC-FlexFEC-03 field trial is not enabled."; - } - selected_send_codec->flexfec_payload_type = -1; + RTC_LOG(LS_INFO) << "WebRTC-FlexFEC-03 field trial is not enabled."; + for (VideoCodecSettings& codec : negotiated_codecs) + codec.flexfec_payload_type = -1; } - if (!send_codec_ || *selected_send_codec != *send_codec_) - changed_params->codec = selected_send_codec; + if (negotiated_codecs_ != negotiated_codecs) { + if (send_codec_ != negotiated_codecs.front()) { + changed_params->send_codec = negotiated_codecs.front(); + } + changed_params->negotiated_codecs = std::move(negotiated_codecs); + } // Handle RTP header extensions. if (params.extmap_allow_mixed != ExtmapAllowMixed()) { @@ -698,12 +718,44 @@ bool WebRtcVideoChannel::SetSendParameters(const VideoSendParameters& params) { return false; } - if (changed_params.codec) { - const VideoCodecSettings& codec_settings = *changed_params.codec; - send_codec_ = codec_settings; - RTC_LOG(LS_INFO) << "Using codec: " << codec_settings.codec.ToString(); + if (changed_params.negotiated_codecs) { + for (const auto& send_codec : *changed_params.negotiated_codecs) + RTC_LOG(LS_INFO) << "Negotiated codec: " << send_codec.codec.ToString(); } + send_params_ = params; + return ApplyChangedParams(changed_params); +} + +void WebRtcVideoChannel::OnEncoderFailure() { + invoker_.AsyncInvoke( + RTC_FROM_HERE, worker_thread_, [this] { + RTC_DCHECK_RUN_ON(&thread_checker_); + if (negotiated_codecs_.size() <= 1) { + RTC_LOG(LS_WARNING) + << "Encoder failed but no fallback codec is available"; + return; + } + + ChangedSendParameters params; + params.negotiated_codecs = negotiated_codecs_; + params.negotiated_codecs->erase(params.negotiated_codecs->begin()); + params.send_codec = params.negotiated_codecs->front(); + ApplyChangedParams(params); + }); +} + +bool WebRtcVideoChannel::ApplyChangedParams( + const ChangedSendParameters& changed_params) { + RTC_DCHECK_RUN_ON(&thread_checker_); + if (changed_params.negotiated_codecs) + negotiated_codecs_ = *changed_params.negotiated_codecs; + + if (changed_params.send_codec) + send_codec_ = changed_params.send_codec; + + RTC_DCHECK(send_codec_); + if (changed_params.extmap_allow_mixed) { SetExtmapAllowMixed(*changed_params.extmap_allow_mixed); } @@ -711,8 +763,8 @@ bool WebRtcVideoChannel::SetSendParameters(const VideoSendParameters& params) { send_rtp_extensions_ = changed_params.rtp_header_extensions; } - if (changed_params.codec || changed_params.max_bandwidth_bps) { - if (params.max_bandwidth_bps == -1) { + if (changed_params.send_codec || changed_params.max_bandwidth_bps) { + if (send_params_.max_bandwidth_bps == -1) { // Unset the global max bitrate (max_bitrate_bps) if max_bandwidth_bps is // -1, which corresponds to no "b=AS" attribute in SDP. Note that the // global max bitrate may be set below in GetBitrateConfigForCodec, from @@ -721,17 +773,19 @@ bool WebRtcVideoChannel::SetSendParameters(const VideoSendParameters& params) { // probably not affect global call max bitrate). bitrate_config_.max_bitrate_bps = -1; } + if (send_codec_) { // TODO(holmer): Changing the codec parameters shouldn't necessarily mean // that we change the min/max of bandwidth estimation. Reevaluate this. bitrate_config_ = GetBitrateConfigForCodec(send_codec_->codec); - if (!changed_params.codec) { + if (!changed_params.send_codec) { // If the codec isn't changing, set the start bitrate to -1 which means // "unchanged" so that BWE isn't affected. bitrate_config_.start_bitrate_bps = -1; } } - if (params.max_bandwidth_bps >= 0) { + + if (send_params_.max_bandwidth_bps >= 0) { // Note that max_bandwidth_bps intentionally takes priority over the // bitrate config for the codec. This allows FEC to be applied above the // codec target bitrate. @@ -739,8 +793,9 @@ bool WebRtcVideoChannel::SetSendParameters(const VideoSendParameters& params) { // WebRtcVideoChannel (in which case we're good), or per sender (SSRC), // in which case this should not set a BitrateConstraints but rather // reconfigure all senders. - bitrate_config_.max_bitrate_bps = - params.max_bandwidth_bps == 0 ? -1 : params.max_bandwidth_bps; + bitrate_config_.max_bitrate_bps = send_params_.max_bandwidth_bps == 0 + ? -1 + : send_params_.max_bandwidth_bps; } if (media_transport()) { @@ -767,7 +822,7 @@ bool WebRtcVideoChannel::SetSendParameters(const VideoSendParameters& params) { for (auto& kv : send_streams_) { kv.second->SetSendParameters(changed_params); } - if (changed_params.codec || changed_params.rtcp_mode) { + if (changed_params.send_codec || changed_params.rtcp_mode) { // Update receive feedback parameters from new codec or RTCP mode. RTC_LOG(LS_INFO) << "SetFeedbackOptions on all the receive streams because the send " @@ -777,11 +832,10 @@ bool WebRtcVideoChannel::SetSendParameters(const VideoSendParameters& params) { kv.second->SetFeedbackParameters( HasLntf(send_codec_->codec), HasNack(send_codec_->codec), HasRemb(send_codec_->codec), HasTransportCc(send_codec_->codec), - params.rtcp.reduced_size ? webrtc::RtcpMode::kReducedSize - : webrtc::RtcpMode::kCompound); + send_params_.rtcp.reduced_size ? webrtc::RtcpMode::kReducedSize + : webrtc::RtcpMode::kCompound); } } - send_params_ = params; return true; } @@ -1107,6 +1161,7 @@ bool WebRtcVideoChannel::AddSendStream(const StreamParams& sp) { config.encoder_settings.encoder_factory = encoder_factory_; config.encoder_settings.bitrate_allocator_factory = bitrate_allocator_factory_; + config.encoder_settings.encoder_failure_callback = this; config.crypto_options = crypto_options_; config.rtp.extmap_allow_mixed = ExtmapAllowMixed(); config.rtcp_report_interval_ms = video_config_.rtcp_report_interval_ms; @@ -1949,8 +2004,8 @@ void WebRtcVideoChannel::WebRtcVideoSendStream::SetSendParameters( } // Set codecs and options. - if (params.codec) { - SetCodec(*params.codec); + if (params.send_codec) { + SetCodec(*params.send_codec); recreate_stream = false; // SetCodec has already recreated the stream. } else if (params.conference_mode && parameters_.codec_settings) { SetCodec(*parameters_.codec_settings); diff --git a/media/engine/webrtc_video_engine.h b/media/engine/webrtc_video_engine.h index da078354f5..9882fd7ac8 100644 --- a/media/engine/webrtc_video_engine.h +++ b/media/engine/webrtc_video_engine.h @@ -106,7 +106,9 @@ class WebRtcVideoEngine : public VideoEngineInterface { bitrate_allocator_factory_; }; -class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport { +class WebRtcVideoChannel : public VideoMediaChannel, + public webrtc::Transport, + public webrtc::EncoderFailureCallback { public: WebRtcVideoChannel( webrtc::Call* call, @@ -205,6 +207,9 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport { // This method does nothing unless unknown_ssrc_packet_buffer_ is configured. void BackfillBufferedPackets(rtc::ArrayView ssrcs); + // Implements webrtc::EncoderFailureCallback. + void OnEncoderFailure() override; + private: class WebRtcVideoReceiveStream; struct VideoCodecSettings { @@ -228,7 +233,8 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport { struct ChangedSendParameters { // These optionals are unset if not changed. - absl::optional codec; + absl::optional send_codec; + absl::optional> negotiated_codecs; absl::optional> rtp_header_extensions; absl::optional mid; absl::optional extmap_allow_mixed; @@ -250,6 +256,7 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport { bool GetChangedSendParameters(const VideoSendParameters& params, ChangedSendParameters* changed_params) const RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + bool ApplyChangedParams(const ChangedSendParameters& changed_params); bool GetChangedRecvParameters(const VideoRecvParameters& params, ChangedRecvParameters* changed_params) const RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); @@ -474,10 +481,8 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport { static std::vector MapCodecs( const std::vector& codecs); - // Select what video codec will be used for sending, i.e. what codec is used - // for local encoding, based on supported remote codecs. The first remote - // codec that is supported locally will be selected. - absl::optional SelectSendVideoCodec( + // Get all codecs that are compatible with the receiver. + std::vector SelectSendVideoCodecs( const std::vector& remote_mapped_codecs) const RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); @@ -495,6 +500,7 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport { void FillSendAndReceiveCodecStats(VideoMediaInfo* video_media_info) RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + rtc::Thread* worker_thread_; rtc::ThreadChecker thread_checker_; uint32_t rtcp_receiver_report_ssrc_ RTC_GUARDED_BY(thread_checker_); @@ -521,6 +527,9 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport { absl::optional send_codec_ RTC_GUARDED_BY(thread_checker_); + std::vector negotiated_codecs_ + RTC_GUARDED_BY(thread_checker_); + absl::optional> send_rtp_extensions_ RTC_GUARDED_BY(thread_checker_); @@ -556,6 +565,10 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport { // Buffer for unhandled packets. std::unique_ptr unknown_ssrc_packet_buffer_ RTC_GUARDED_BY(thread_checker_); + + // In order for the |invoker_| to protect other members from being destructed + // as they are used in asynchronous tasks it has to be destructed first. + rtc::AsyncInvoker invoker_; }; class EncoderStreamFactory diff --git a/media/engine/webrtc_video_engine_unittest.cc b/media/engine/webrtc_video_engine_unittest.cc index 3702064685..ff5f9e54db 100644 --- a/media/engine/webrtc_video_engine_unittest.cc +++ b/media/engine/webrtc_video_engine_unittest.cc @@ -2082,6 +2082,30 @@ TEST_F(WebRtcVideoChannelBaseTest, TwoStreamsSendAndReceive) { TwoStreamsSendAndReceive(codec); } +TEST_F(WebRtcVideoChannelBaseTest, OnEncoderFailure) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP9")); + parameters.codecs.push_back(GetEngineCodec("VP8")); + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + + VideoCodec codec; + ASSERT_TRUE(channel_->GetSendCodec(&codec)); + EXPECT_EQ("VP9", codec.name); + + // OnEncoderFailure will post a task to the worker thread (which is also + // the current thread), hence the ProcessMessages call. + channel_->OnEncoderFailure(); + rtc::Thread::Current()->ProcessMessages(30); + ASSERT_TRUE(channel_->GetSendCodec(&codec)); + EXPECT_EQ("VP8", codec.name); + + // No other codec to fall back to, keep using VP8. + channel_->OnEncoderFailure(); + rtc::Thread::Current()->ProcessMessages(30); + ASSERT_TRUE(channel_->GetSendCodec(&codec)); + EXPECT_EQ("VP8", codec.name); +} + class WebRtcVideoChannelTest : public WebRtcVideoEngineTest { public: WebRtcVideoChannelTest() : WebRtcVideoChannelTest("") {} diff --git a/modules/video_coding/include/video_error_codes.h b/modules/video_coding/include/video_error_codes.h index 005c0e23b2..4ae0ca127d 100644 --- a/modules/video_coding/include/video_error_codes.h +++ b/modules/video_coding/include/video_error_codes.h @@ -25,5 +25,6 @@ #define WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE -13 #define WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT -14 #define WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED -15 +#define WEBRTC_VIDEO_CODEC_ENCODER_FAILURE -16 #endif // MODULES_VIDEO_CODING_INCLUDE_VIDEO_ERROR_CODES_H_ diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 3dba91e107..005d49b735 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -491,6 +491,7 @@ VideoStreamEncoder::VideoStreamEncoder( max_data_payload_length_(0), encoder_paused_and_dropped_frame_(false), was_encode_called_since_last_initialization_(false), + encoder_failed_(false), clock_(clock), degradation_preference_(DegradationPreference::DISABLED), posted_frames_waiting_for_encode_(0), @@ -1262,6 +1263,13 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame, int64_t time_when_posted_us) { RTC_DCHECK_RUN_ON(&encoder_queue_); + + // If the encoder fail we can't continue to encode frames. When this happens + // the WebrtcVideoSender is notified and the whole VideoSendStream is + // recreated. + if (encoder_failed_) + return; + TraceFrameDropEnd(); VideoFrame out_frame(video_frame); @@ -1387,8 +1395,21 @@ void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame, was_encode_called_since_last_initialization_ = true; if (encode_status < 0) { - RTC_LOG(LS_ERROR) << "Failed to encode frame. Error code: " - << encode_status; + if (encode_status == WEBRTC_VIDEO_CODEC_ENCODER_FAILURE) { + RTC_LOG(LS_ERROR) << "Encoder failed, failing encoder format: " + << encoder_config_.video_format.ToString(); + if (settings_.encoder_failure_callback) { + encoder_failed_ = true; + settings_.encoder_failure_callback->OnEncoderFailure(); + } else { + RTC_LOG(LS_ERROR) + << "Encoder failed but no encoder fallback callback is registered"; + } + } else { + RTC_LOG(LS_ERROR) << "Failed to encode frame. Error code: " + << encode_status; + } + return; } diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 87c8547bf4..3faa9538de 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -281,6 +281,7 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, bool was_encode_called_since_last_initialization_ RTC_GUARDED_BY(&encoder_queue_); + bool encoder_failed_ RTC_GUARDED_BY(&encoder_queue_); Clock* const clock_; // Counters used for deciding if the video resolution or framerate is // currently restricted, and if so, why, on a per degradation preference