diff --git a/api/BUILD.gn b/api/BUILD.gn index 144ee0371e..8d34b581cb 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -101,6 +101,7 @@ rtc_static_library("libjingle_peerconnection_api") { ":array_view", ":audio_options_api", ":callfactory_api", + ":fec_controller_api", ":libjingle_logging_api", ":optional", ":rtc_stats_api", @@ -211,8 +212,8 @@ rtc_source_set("fec_controller_api") { ] deps = [ - "../common_video:common_video", - "../modules:module_api", + "..:webrtc_common", + "../modules:module_fec_api", ] } diff --git a/api/fec_controller.h b/api/fec_controller.h index aad2316afb..59e86ccedb 100644 --- a/api/fec_controller.h +++ b/api/fec_controller.h @@ -11,10 +11,11 @@ #ifndef API_FEC_CONTROLLER_H_ #define API_FEC_CONTROLLER_H_ +#include #include -#include "common_video/include/video_frame.h" -#include "modules/include/module_common_types.h" +#include "common_types.h" // NOLINT(build/include) +#include "modules/include/module_fec_types.h" namespace webrtc { // TODO(yinwa): work in progress. API in class FecController should not be @@ -73,11 +74,18 @@ class FecController { int64_t round_trip_time_ms) = 0; // Informs of encoded output. - virtual void UpdateWithEncodedData(const EncodedImage& encoded_image) = 0; + virtual void UpdateWithEncodedData(size_t encoded_image_length, + FrameType encoded_image_frametype) = 0; // Returns whether this FEC Controller needs Loss Vector Mask as input. virtual bool UseLossVectorMask() = 0; }; +class FecControllerFactoryInterface { + public: + virtual std::unique_ptr CreateFecController() = 0; + virtual ~FecControllerFactoryInterface() = default; +}; + } // namespace webrtc #endif // API_FEC_CONTROLLER_H_ diff --git a/api/peerconnectioninterface.h b/api/peerconnectioninterface.h index 5784d9500a..63912b22e7 100644 --- a/api/peerconnectioninterface.h +++ b/api/peerconnectioninterface.h @@ -83,6 +83,7 @@ #include "api/call/callfactoryinterface.h" #include "api/datachannelinterface.h" #include "api/dtmfsenderinterface.h" +#include "api/fec_controller.h" #include "api/jsep.h" #include "api/mediastreaminterface.h" #include "api/rtcerror.h" @@ -1296,6 +1297,27 @@ rtc::scoped_refptr CreatePeerConnectionFactory( rtc::scoped_refptr audio_mixer, rtc::scoped_refptr audio_processing); +// Create a new instance of PeerConnectionFactoryInterface with optional +// external audio mixer, audio processing, and fec controller modules. +// +// If |audio_mixer| is null, an internal audio mixer will be created and used. +// If |audio_processing| is null, an internal audio processing module will be +// created and used. +// If |fec_controller_factory| is null, an internal fec controller module will +// be created and used. +rtc::scoped_refptr CreatePeerConnectionFactory( + rtc::Thread* network_thread, + rtc::Thread* worker_thread, + rtc::Thread* signaling_thread, + AudioDeviceModule* default_adm, + rtc::scoped_refptr audio_encoder_factory, + rtc::scoped_refptr audio_decoder_factory, + cricket::WebRtcVideoEncoderFactory* video_encoder_factory, + cricket::WebRtcVideoDecoderFactory* video_decoder_factory, + rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr audio_processing, + std::unique_ptr fec_controller_factory); + // Create a new instance of PeerConnectionFactoryInterface with optional video // codec factories. These video factories represents all video codecs, i.e. no // extra internal video codecs will be added. @@ -1385,6 +1407,16 @@ CreateModularPeerConnectionFactory( std::unique_ptr call_factory, std::unique_ptr event_log_factory); +rtc::scoped_refptr +CreateModularPeerConnectionFactory( + rtc::Thread* network_thread, + rtc::Thread* worker_thread, + rtc::Thread* signaling_thread, + std::unique_ptr media_engine, + std::unique_ptr call_factory, + std::unique_ptr event_log_factory, + std::unique_ptr fec_controller_factory); + } // namespace webrtc #endif // API_PEERCONNECTIONINTERFACE_H_ diff --git a/call/call.cc b/call/call.cc index ddc43cbbd9..3a72040a9a 100644 --- a/call/call.cc +++ b/call/call.cc @@ -408,6 +408,9 @@ Call* Call::Create( return new internal::Call(config, std::move(transport_send)); } +// This method here to avoid subclasses has to implement this method. +// Call perf test will use Internal::Call::CreateVideoSendStream() to inject +// FecController. VideoSendStream* Call::CreateVideoSendStream( VideoSendStream::Config config, VideoEncoderConfig encoder_config, @@ -723,14 +726,7 @@ void Call::DestroyAudioReceiveStream( delete audio_receive_stream; } -webrtc::VideoSendStream* Call::CreateVideoSendStream( - webrtc::VideoSendStream::Config config, - VideoEncoderConfig encoder_config) { - return CreateVideoSendStream( - std::move(config), std::move(encoder_config), - rtc::MakeUnique(Clock::GetRealTimeClock())); -} - +// This method can be used for Call tests with external fec controller factory. webrtc::VideoSendStream* Call::CreateVideoSendStream( webrtc::VideoSendStream::Config config, VideoEncoderConfig encoder_config, @@ -749,6 +745,7 @@ webrtc::VideoSendStream* Call::CreateVideoSendStream( // the call has already started. // Copy ssrcs from |config| since |config| is moved. std::vector ssrcs = config.rtp.ssrcs; + VideoSendStream* send_stream = new VideoSendStream( num_cpu_cores_, module_process_thread_.get(), &worker_queue_, call_stats_.get(), transport_send_.get(), bitrate_allocator_.get(), @@ -770,6 +767,17 @@ webrtc::VideoSendStream* Call::CreateVideoSendStream( return send_stream; } +webrtc::VideoSendStream* Call::CreateVideoSendStream( + webrtc::VideoSendStream::Config config, + VideoEncoderConfig encoder_config) { + std::unique_ptr fec_controller = + config_.fec_controller_factory + ? config_.fec_controller_factory->CreateFecController() + : rtc::MakeUnique(Clock::GetRealTimeClock()); + return CreateVideoSendStream(std::move(config), std::move(encoder_config), + std::move(fec_controller)); +} + void Call::DestroyVideoSendStream(webrtc::VideoSendStream* send_stream) { TRACE_EVENT0("webrtc", "Call::DestroyVideoSendStream"); RTC_DCHECK(send_stream != nullptr); diff --git a/call/call.h b/call/call.h index 884a21b8b1..1155aaf4b2 100644 --- a/call/call.h +++ b/call/call.h @@ -109,6 +109,9 @@ struct CallConfig { // RtcEventLog to use for this call. Required. // Use webrtc::RtcEventLog::CreateNull() for a null implementation. RtcEventLog* event_log = nullptr; + + // FecController to use for this call. + FecControllerFactoryInterface* fec_controller_factory = nullptr; }; // A Call instance can contain several send and/or receive streams. All streams diff --git a/modules/BUILD.gn b/modules/BUILD.gn index d912366183..c4010a28b0 100644 --- a/modules/BUILD.gn +++ b/modules/BUILD.gn @@ -49,6 +49,7 @@ rtc_source_set("module_api") { ] deps = [ ":module_api_public", + ":module_fec_api", "..:webrtc_common", "../:typedefs", "../api:libjingle_peerconnection_api", @@ -62,6 +63,13 @@ rtc_source_set("module_api") { ] } +rtc_source_set("module_fec_api") { + visibility = [ "*" ] + sources = [ + "include/module_fec_types.h", + ] +} + if (rtc_include_tests) { modules_tests_resources = [ "../resources/audio_coding/testfile32kHz.pcm", diff --git a/modules/include/module_common_types.h b/modules/include/module_common_types.h index 08423703f3..12900757aa 100644 --- a/modules/include/module_common_types.h +++ b/modules/include/module_common_types.h @@ -25,6 +25,7 @@ #include "api/video/video_rotation.h" #include "common_types.h" // NOLINT(build/include) #include "modules/include/module_common_types_public.h" +#include "modules/include/module_fec_types.h" #include "modules/video_coding/codecs/h264/include/h264_globals.h" #include "modules/video_coding/codecs/vp8/include/vp8_globals.h" #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" @@ -265,22 +266,6 @@ struct RTCPVoIPMetric { uint16_t JBabsMax; }; -// Types for the FEC packet masks. The type |kFecMaskRandom| is based on a -// random loss model. The type |kFecMaskBursty| is based on a bursty/consecutive -// loss model. The packet masks are defined in -// modules/rtp_rtcp/fec_private_tables_random(bursty).h -enum FecMaskType { - kFecMaskRandom, - kFecMaskBursty, -}; - -// Struct containing forward error correction settings. -struct FecProtectionParams { - int fec_rate; - int max_fec_frames; - FecMaskType fec_mask_type; -}; - // Interface used by the CallStats class to distribute call statistics. // Callbacks will be triggered as soon as the class has been registered to a // CallStats object using RegisterStatsObserver. diff --git a/modules/include/module_fec_types.h b/modules/include/module_fec_types.h new file mode 100644 index 0000000000..25d6bc5714 --- /dev/null +++ b/modules/include/module_fec_types.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_INCLUDE_MODULE_FEC_TYPES_H_ +#define MODULES_INCLUDE_MODULE_FEC_TYPES_H_ + +namespace webrtc { + +// Types for the FEC packet masks. The type |kFecMaskRandom| is based on a +// random loss model. The type |kFecMaskBursty| is based on a bursty/consecutive +// loss model. The packet masks are defined in +// modules/rtp_rtcp/fec_private_tables_random(bursty).h +enum FecMaskType { + kFecMaskRandom, + kFecMaskBursty, +}; + +// Struct containing forward error correction settings. +struct FecProtectionParams { + int fec_rate; + int max_fec_frames; + FecMaskType fec_mask_type; +}; + +} // namespace webrtc + +#endif // MODULES_INCLUDE_MODULE_FEC_TYPES_H_ diff --git a/modules/video_coding/fec_controller_default.cc b/modules/video_coding/fec_controller_default.cc index 9d26aace56..84e8a7bfcc 100644 --- a/modules/video_coding/fec_controller_default.cc +++ b/modules/video_coding/fec_controller_default.cc @@ -145,11 +145,12 @@ void FecControllerDefault::SetProtectionMethod(bool enable_fec, loss_prot_logic_->SetMethod(method); } void FecControllerDefault::UpdateWithEncodedData( - const EncodedImage& encoded_image) { - const size_t encoded_length = encoded_image._length; + const size_t encoded_image_length, + const FrameType encoded_image_frametype) { + const size_t encoded_length = encoded_image_length; CritScope lock(&crit_sect_); if (encoded_length > 0) { - const bool delta_frame = encoded_image._frameType != kVideoFrameKey; + const bool delta_frame = encoded_image_frametype != kVideoFrameKey; if (max_payload_size_ > 0 && encoded_length > 0) { const float min_packets_per_frame = encoded_length / static_cast(max_payload_size_); diff --git a/modules/video_coding/fec_controller_default.h b/modules/video_coding/fec_controller_default.h index a7f445fa24..776977a27d 100644 --- a/modules/video_coding/fec_controller_default.h +++ b/modules/video_coding/fec_controller_default.h @@ -39,7 +39,8 @@ class FecControllerDefault : public FecController { uint8_t fraction_lost, std::vector loss_mask_vector, int64_t round_trip_time_ms) override; - void UpdateWithEncodedData(const EncodedImage& encoded_image) override; + void UpdateWithEncodedData(const size_t encoded_image_length, + const FrameType encoded_image_frametype) override; bool UseLossVectorMask() override { return false; } private: diff --git a/modules/video_coding/generic_encoder.cc b/modules/video_coding/generic_encoder.cc index bb5f75e11b..d1816e0acc 100644 --- a/modules/video_coding/generic_encoder.cc +++ b/modules/video_coding/generic_encoder.cc @@ -412,7 +412,8 @@ EncodedImageCallback::Result VCMEncodedFrameCallback::OnEncodedImage( return result; if (media_opt_) { - media_opt_->UpdateWithEncodedData(image_copy); + media_opt_->UpdateWithEncodedData(image_copy._length, + image_copy._frameType); if (internal_source_) { // Signal to encoder to drop next frame. result.drop_next_frame = media_opt_->DropFrame(); diff --git a/modules/video_coding/media_optimization.cc b/modules/video_coding/media_optimization.cc index 51c5bcb6a6..ea70f3f47c 100644 --- a/modules/video_coding/media_optimization.cc +++ b/modules/video_coding/media_optimization.cc @@ -90,11 +90,12 @@ uint32_t MediaOptimization::InputFrameRateInternal() { } void MediaOptimization::UpdateWithEncodedData( - const EncodedImage& encoded_image) { - size_t encoded_length = encoded_image._length; + const size_t encoded_image_length, + const FrameType encoded_image_frametype) { + size_t encoded_length = encoded_image_length; rtc::CritScope lock(&crit_sect_); if (encoded_length > 0) { - const bool delta_frame = encoded_image._frameType != kVideoFrameKey; + const bool delta_frame = encoded_image_frametype != kVideoFrameKey; frame_dropper_->Fill(encoded_length, delta_frame); } } diff --git a/modules/video_coding/media_optimization.h b/modules/video_coding/media_optimization.h index 5fa025453c..411f86ce51 100644 --- a/modules/video_coding/media_optimization.h +++ b/modules/video_coding/media_optimization.h @@ -51,7 +51,8 @@ class MediaOptimization { // Informs Media Optimization of encoded output. // TODO(perkj): Deprecate SetEncodingData once its not used for stats in // VideoStreamEncoder. - void UpdateWithEncodedData(const EncodedImage& encoded_image); + void UpdateWithEncodedData(const size_t encoded_image_length, + const FrameType encoded_image_frametype); // InputFrameRate 0 = no frame rate estimate available. uint32_t InputFrameRate(); diff --git a/pc/BUILD.gn b/pc/BUILD.gn index 1eb0f72dcf..d536109aa8 100644 --- a/pc/BUILD.gn +++ b/pc/BUILD.gn @@ -181,6 +181,7 @@ rtc_static_library("peerconnection") { ":rtc_pc_base", "..:webrtc_common", "../api:call_api", + "../api:fec_controller_api", "../api:libjingle_peerconnection_api", "../api:optional", "../api:rtc_stats_api", diff --git a/pc/createpeerconnectionfactory.cc b/pc/createpeerconnectionfactory.cc index 197df8ac73..11650ec0fb 100644 --- a/pc/createpeerconnectionfactory.cc +++ b/pc/createpeerconnectionfactory.cc @@ -69,6 +69,40 @@ rtc::scoped_refptr CreatePeerConnectionFactory( } #endif +rtc::scoped_refptr CreatePeerConnectionFactory( + rtc::Thread* network_thread, + rtc::Thread* worker_thread, + rtc::Thread* signaling_thread, + AudioDeviceModule* default_adm, + rtc::scoped_refptr audio_encoder_factory, + rtc::scoped_refptr audio_decoder_factory, + cricket::WebRtcVideoEncoderFactory* video_encoder_factory, + cricket::WebRtcVideoDecoderFactory* video_decoder_factory, + rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr audio_processing, + std::unique_ptr fec_controller_factory) { + rtc::scoped_refptr audio_processing_use = audio_processing; + if (!audio_processing_use) { + audio_processing_use = AudioProcessingBuilder().Create(); + } + + std::unique_ptr media_engine( + cricket::WebRtcMediaEngineFactory::Create( + default_adm, audio_encoder_factory, audio_decoder_factory, + video_encoder_factory, video_decoder_factory, audio_mixer, + audio_processing_use)); + + std::unique_ptr call_factory = CreateCallFactory(); + + std::unique_ptr event_log_factory = + CreateRtcEventLogFactory(); + + return CreateModularPeerConnectionFactory( + network_thread, worker_thread, signaling_thread, std::move(media_engine), + std::move(call_factory), std::move(event_log_factory), + std::move(fec_controller_factory)); +} + rtc::scoped_refptr CreatePeerConnectionFactory( rtc::Thread* network_thread, rtc::Thread* worker_thread, diff --git a/pc/peerconnectionfactory.cc b/pc/peerconnectionfactory.cc index af8f541bd7..a6b889b7a7 100644 --- a/pc/peerconnectionfactory.cc +++ b/pc/peerconnectionfactory.cc @@ -12,6 +12,7 @@ #include +#include "api/fec_controller.h" #include "api/mediaconstraintsinterface.h" #include "api/mediastreamproxy.h" #include "api/mediastreamtrackproxy.h" @@ -52,11 +53,25 @@ CreateModularPeerConnectionFactory( std::unique_ptr media_engine, std::unique_ptr call_factory, std::unique_ptr event_log_factory) { + return CreateModularPeerConnectionFactory( + network_thread, worker_thread, signaling_thread, std::move(media_engine), + std::move(call_factory), std::move(event_log_factory), nullptr); +} + +rtc::scoped_refptr +CreateModularPeerConnectionFactory( + rtc::Thread* network_thread, + rtc::Thread* worker_thread, + rtc::Thread* signaling_thread, + std::unique_ptr media_engine, + std::unique_ptr call_factory, + std::unique_ptr event_log_factory, + std::unique_ptr fec_controller_factory) { rtc::scoped_refptr pc_factory( new rtc::RefCountedObject( network_thread, worker_thread, signaling_thread, std::move(media_engine), std::move(call_factory), - std::move(event_log_factory))); + std::move(event_log_factory), std::move(fec_controller_factory))); // Call Initialize synchronously but make sure it is executed on // |signaling_thread|. @@ -78,13 +93,30 @@ PeerConnectionFactory::PeerConnectionFactory( std::unique_ptr media_engine, std::unique_ptr call_factory, std::unique_ptr event_log_factory) + : PeerConnectionFactory(network_thread, + worker_thread, + signaling_thread, + std::move(media_engine), + std::move(call_factory), + std::move(event_log_factory), + nullptr) {} + +PeerConnectionFactory::PeerConnectionFactory( + rtc::Thread* network_thread, + rtc::Thread* worker_thread, + rtc::Thread* signaling_thread, + std::unique_ptr media_engine, + std::unique_ptr call_factory, + std::unique_ptr event_log_factory, + std::unique_ptr fec_controller_factory) : wraps_current_thread_(false), network_thread_(network_thread), worker_thread_(worker_thread), signaling_thread_(signaling_thread), media_engine_(std::move(media_engine)), call_factory_(std::move(call_factory)), - event_log_factory_(std::move(event_log_factory)) { + event_log_factory_(std::move(event_log_factory)), + fec_controller_factory_(std::move(fec_controller_factory)) { if (!network_thread_) { owned_network_thread_ = rtc::Thread::CreateWithSocketServer(); owned_network_thread_->SetName("pc_network_thread", nullptr); @@ -355,6 +387,8 @@ std::unique_ptr PeerConnectionFactory::CreateCall_w( call_config.bitrate_config.start_bitrate_bps = kStartBandwidthBps; call_config.bitrate_config.max_bitrate_bps = kMaxBandwidthBps; + call_config.fec_controller_factory = fec_controller_factory_.get(); + return std::unique_ptr(call_factory_->CreateCall(call_config)); } diff --git a/pc/peerconnectionfactory.h b/pc/peerconnectionfactory.h index 652f76ad61..60facec103 100644 --- a/pc/peerconnectionfactory.h +++ b/pc/peerconnectionfactory.h @@ -109,6 +109,14 @@ class PeerConnectionFactory : public PeerConnectionFactoryInterface { std::unique_ptr media_engine, std::unique_ptr call_factory, std::unique_ptr event_log_factory); + PeerConnectionFactory( + rtc::Thread* network_thread, + rtc::Thread* worker_thread, + rtc::Thread* signaling_thread, + std::unique_ptr media_engine, + std::unique_ptr call_factory, + std::unique_ptr event_log_factory, + std::unique_ptr fec_controller_factory); virtual ~PeerConnectionFactory(); private: @@ -128,6 +136,7 @@ class PeerConnectionFactory : public PeerConnectionFactoryInterface { std::unique_ptr media_engine_; std::unique_ptr call_factory_; std::unique_ptr event_log_factory_; + std::unique_ptr fec_controller_factory_; }; } // namespace webrtc diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index a73bed2514..8aa39e228c 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -642,6 +642,7 @@ rtc_android_library("libjingle_peerconnection_java") { "api/org/webrtc/EglBase.java", "api/org/webrtc/EglRenderer.java", "api/org/webrtc/EncodedImage.java", + "api/org/webrtc/FecControllerFactoryFactoryInterface.java", "api/org/webrtc/FileVideoCapturer.java", "api/org/webrtc/GlRectDrawer.java", "api/org/webrtc/GlShader.java", diff --git a/sdk/android/api/org/webrtc/FecControllerFactoryFactoryInterface.java b/sdk/android/api/org/webrtc/FecControllerFactoryFactoryInterface.java new file mode 100644 index 0000000000..6d39390f72 --- /dev/null +++ b/sdk/android/api/org/webrtc/FecControllerFactoryFactoryInterface.java @@ -0,0 +1,22 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Factory for creating webrtc::FecControllerFactory instances. + */ +public interface FecControllerFactoryFactoryInterface { + /** + * Dynamically allocates a webrtc::FecControllerFactory instance and returns a pointer to it. + * The caller takes ownership of the object. + */ + public long createNative(); +} diff --git a/sdk/android/api/org/webrtc/PeerConnectionFactory.java b/sdk/android/api/org/webrtc/PeerConnectionFactory.java index 79eca7e5d3..21729a57a6 100644 --- a/sdk/android/api/org/webrtc/PeerConnectionFactory.java +++ b/sdk/android/api/org/webrtc/PeerConnectionFactory.java @@ -124,6 +124,55 @@ public class PeerConnectionFactory { } } + public static class Builder { + private Options options; + private VideoEncoderFactory encoderFactory; + private VideoDecoderFactory decoderFactory; + private AudioProcessingFactory audioProcessingFactory; + private FecControllerFactoryFactoryInterface fecControllerFactoryFactory; + + private Builder() {} + + public Builder setOptions(Options options) { + this.options = options; + return this; + } + + public Builder setVideoEncoderFactory(VideoEncoderFactory encoderFactory) { + this.encoderFactory = encoderFactory; + return this; + } + + public Builder setVideoDecoderFactory(VideoDecoderFactory decoderFactory) { + this.decoderFactory = decoderFactory; + return this; + } + + public Builder setAudioProcessingFactory(AudioProcessingFactory audioProcessingFactory) { + if (audioProcessingFactory == null) { + throw new NullPointerException( + "PeerConnectionFactory builder does not accept a null AudioProcessingFactory."); + } + this.audioProcessingFactory = audioProcessingFactory; + return this; + } + + public Builder setFecControllerFactoryFactoryInterface( + FecControllerFactoryFactoryInterface fecControllerFactoryFactory) { + this.fecControllerFactoryFactory = fecControllerFactoryFactory; + return this; + } + + public PeerConnectionFactory createPeerConnectionFactory() { + return new PeerConnectionFactory(options, encoderFactory, decoderFactory, + audioProcessingFactory, fecControllerFactoryFactory); + } + } + + public static Builder builder() { + return new Builder(); + } + /** * Loads and initializes WebRTC. This must be called at least once before creating a * PeerConnectionFactory. Replaces all the old initialization methods. Must not be called while @@ -191,28 +240,36 @@ public class PeerConnectionFactory { // Note: initializeAndroidGlobals must be called at least once before // constructing a PeerConnectionFactory. + @Deprecated public PeerConnectionFactory(Options options) { this(options, null /* encoderFactory */, null /* decoderFactory */); } + @Deprecated public PeerConnectionFactory( Options options, VideoEncoderFactory encoderFactory, VideoDecoderFactory decoderFactory) { checkInitializeHasBeenCalled(); - nativeFactory = nativeCreatePeerConnectionFactory(options, encoderFactory, decoderFactory); + nativeFactory = + nativeCreatePeerConnectionFactory(options, encoderFactory, decoderFactory, 0, 0); if (nativeFactory == 0) { throw new RuntimeException("Failed to initialize PeerConnectionFactory!"); } } + @Deprecated public PeerConnectionFactory(Options options, VideoEncoderFactory encoderFactory, VideoDecoderFactory decoderFactory, AudioProcessingFactory audioProcessingFactory) { + this(options, encoderFactory, decoderFactory, audioProcessingFactory, + null /* fecControllerFactoryFactory */); + } + + private PeerConnectionFactory(Options options, VideoEncoderFactory encoderFactory, + VideoDecoderFactory decoderFactory, AudioProcessingFactory audioProcessingFactory, + FecControllerFactoryFactoryInterface fecControllerFactoryFactory) { checkInitializeHasBeenCalled(); - if (audioProcessingFactory == null) { - throw new NullPointerException( - "PeerConnectionFactory constructor does not accept a null AudioProcessingFactory."); - } - nativeFactory = nativeCreatePeerConnectionFactoryWithAudioProcessing( - options, encoderFactory, decoderFactory, audioProcessingFactory.createNative()); + nativeFactory = nativeCreatePeerConnectionFactory(options, encoderFactory, decoderFactory, + audioProcessingFactory == null ? 0 : audioProcessingFactory.createNative(), + fecControllerFactoryFactory == null ? 0 : fecControllerFactoryFactory.createNative()); if (nativeFactory == 0) { throw new RuntimeException("Failed to initialize PeerConnectionFactory!"); } @@ -405,11 +462,9 @@ public class PeerConnectionFactory { private static native void nativeShutdownInternalTracer(); private static native boolean nativeStartInternalTracingCapture(String tracingFilename); private static native void nativeStopInternalTracingCapture(); - private static native long nativeCreatePeerConnectionFactory( - Options options, VideoEncoderFactory encoderFactory, VideoDecoderFactory decoderFactory); - private static native long nativeCreatePeerConnectionFactoryWithAudioProcessing(Options options, + private static native long nativeCreatePeerConnectionFactory(Options options, VideoEncoderFactory encoderFactory, VideoDecoderFactory decoderFactory, - long nativeAudioProcessor); + long nativeAudioProcessor, long nativeFecControllerFactory); private static native long nativeCreatePeerConnection(long factory, PeerConnection.RTCConfiguration rtcConfig, MediaConstraints constraints, long nativeObserver); private static native long nativeCreateLocalMediaStream(long factory, String label); diff --git a/sdk/android/src/jni/pc/peerconnectionfactory.cc b/sdk/android/src/jni/pc/peerconnectionfactory.cc index 1a7008a426..b251a17bfb 100644 --- a/sdk/android/src/jni/pc/peerconnectionfactory.cc +++ b/sdk/android/src/jni/pc/peerconnectionfactory.cc @@ -172,7 +172,8 @@ jlong CreatePeerConnectionFactoryForJava( const JavaParamRef& joptions, const JavaParamRef& jencoder_factory, const JavaParamRef& jdecoder_factory, - rtc::scoped_refptr audio_processor) { + rtc::scoped_refptr audio_processor, + std::unique_ptr fec_controller_factory) { // talk/ assumes pretty widely that the current Thread is ThreadManager'd, but // ThreadManager only WrapCurrentThread()s the thread where it is first // created. Since the semantics around when auto-wrapping happens in @@ -269,7 +270,7 @@ jlong CreatePeerConnectionFactoryForJava( CreateModularPeerConnectionFactory( network_thread.get(), worker_thread.get(), signaling_thread.get(), std::move(media_engine), std::move(call_factory), - std::move(rtc_event_log_factory))); + std::move(rtc_event_log_factory), std::move(fec_controller_factory))); RTC_CHECK(factory) << "Failed to create the peer connection factory; " << "WebRTC/libjingle init likely failed on this device"; // TODO(honghaiz): Maybe put the options as the argument of @@ -286,29 +287,22 @@ jlong CreatePeerConnectionFactoryForJava( } static jlong JNI_PeerConnectionFactory_CreatePeerConnectionFactory( - JNIEnv* jni, - const JavaParamRef&, - const JavaParamRef& joptions, - const JavaParamRef& jencoder_factory, - const JavaParamRef& jdecoder_factory) { - return CreatePeerConnectionFactoryForJava(jni, joptions, jencoder_factory, - jdecoder_factory, - CreateAudioProcessing()); -} - -static jlong -JNI_PeerConnectionFactory_CreatePeerConnectionFactoryWithAudioProcessing( JNIEnv* jni, const JavaParamRef&, const JavaParamRef& joptions, const JavaParamRef& jencoder_factory, const JavaParamRef& jdecoder_factory, - jlong native_audio_processor) { + jlong native_audio_processor, + jlong native_fec_controller_factory) { rtc::scoped_refptr audio_processor = reinterpret_cast(native_audio_processor); - RTC_DCHECK(audio_processor); - return CreatePeerConnectionFactoryForJava(jni, joptions, jencoder_factory, - jdecoder_factory, audio_processor); + std::unique_ptr fec_controller_factory( + reinterpret_cast( + native_fec_controller_factory)); + return CreatePeerConnectionFactoryForJava( + jni, joptions, jencoder_factory, jdecoder_factory, + audio_processor ? audio_processor : CreateAudioProcessing(), + std::move(fec_controller_factory)); } static void JNI_PeerConnectionFactory_FreeFactory(JNIEnv*, diff --git a/video/video_quality_test.cc b/video/video_quality_test.cc index ef853a53f0..ed3531dafc 100644 --- a/video/video_quality_test.cc +++ b/video/video_quality_test.cc @@ -1087,9 +1087,9 @@ VideoQualityTest::VideoQualityTest() } VideoQualityTest::VideoQualityTest( - std::unique_ptr fec_controller) + std::unique_ptr fec_controller_factory) : VideoQualityTest() { - fec_controller_ = std::move(fec_controller); + fec_controller_factory_ = std::move(fec_controller_factory); } VideoQualityTest::Params::Params() @@ -1827,16 +1827,17 @@ void VideoQualityTest::CreateVideoStreams() { RTC_DCHECK(video_send_streams_.empty()); RTC_DCHECK(video_receive_streams_.empty()); RTC_DCHECK_EQ(video_send_configs_.size(), num_video_streams_); + // We currently only support testing external fec controllers with a single // VideoSendStream. - if (fec_controller_.get()) { + if (fec_controller_factory_.get()) { RTC_DCHECK_LE(video_send_configs_.size(), 1); } for (size_t i = 0; i < video_send_configs_.size(); ++i) { - if (fec_controller_.get()) { + if (fec_controller_factory_.get()) { video_send_streams_.push_back(sender_call_->CreateVideoSendStream( video_send_configs_[i].Copy(), video_encoder_configs_[i].Copy(), - std::move(fec_controller_))); + fec_controller_factory_->CreateFecController())); } else { video_send_streams_.push_back(sender_call_->CreateVideoSendStream( video_send_configs_[i].Copy(), video_encoder_configs_[i].Copy())); @@ -1850,24 +1851,6 @@ void VideoQualityTest::CreateVideoStreams() { AssociateFlexfecStreamsWithVideoStreams(); } -void VideoQualityTest::CreateVideoStreamsWithProtectionBitrateCalculator( - std::unique_ptr fec_controller) { - RTC_DCHECK(video_send_streams_.empty()); - RTC_DCHECK(video_receive_streams_.empty()); - RTC_DCHECK_EQ(video_send_configs_.size(), num_video_streams_); - for (size_t i = 0; i < video_send_configs_.size(); ++i) { - video_send_streams_.push_back(sender_call_->CreateVideoSendStream( - video_send_configs_[i].Copy(), video_encoder_configs_[i].Copy(), - std::move(fec_controller))); - } - for (size_t i = 0; i < video_receive_configs_.size(); ++i) { - video_receive_streams_.push_back(receiver_call_->CreateVideoReceiveStream( - video_receive_configs_[i].Copy())); - } - - AssociateFlexfecStreamsWithVideoStreams(); -} - void VideoQualityTest::DestroyStreams() { CallTest::DestroyStreams(); diff --git a/video/video_quality_test.h b/video/video_quality_test.h index f1b44fe54c..da79894163 100644 --- a/video/video_quality_test.h +++ b/video/video_quality_test.h @@ -96,7 +96,8 @@ class VideoQualityTest : public test::CallTest { }; VideoQualityTest(); - explicit VideoQualityTest(std::unique_ptr fec_controller); + explicit VideoQualityTest( + std::unique_ptr fec_controller_factory); void RunWithAnalyzer(const Params& params); void RunWithRenderers(const Params& params); @@ -112,7 +113,7 @@ class VideoQualityTest : public test::CallTest { protected: std::map payload_type_map_; - std::unique_ptr fec_controller_; + std::unique_ptr fec_controller_factory_; // No-op implementation to be able to instantiate this class from non-TEST_F // locations. @@ -129,8 +130,6 @@ class VideoQualityTest : public test::CallTest { // Helper methods for setting up the call. void CreateVideoStreams(); - void CreateVideoStreamsWithProtectionBitrateCalculator( - std::unique_ptr fec_controller); void DestroyStreams(); void CreateCapturers(); std::unique_ptr CreateFrameGenerator(size_t video_idx); diff --git a/video/video_send_stream.cc b/video/video_send_stream.cc index c1fb177428..4960a87e3c 100644 --- a/video/video_send_stream.cc +++ b/video/video_send_stream.cc @@ -1115,7 +1115,8 @@ EncodedImageCallback::Result VideoSendStreamImpl::OnEncodedImage( check_encoder_activity_task_->UpdateEncoderActivity(); } - fec_controller_->UpdateWithEncodedData(encoded_image); + fec_controller_->UpdateWithEncodedData(encoded_image._length, + encoded_image._frameType); EncodedImageCallback::Result result = payload_router_.OnEncodedImage( encoded_image, codec_specific_info, fragmentation);