diff --git a/api/BUILD.gn b/api/BUILD.gn index f8b282f633..dd779879d3 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -87,8 +87,6 @@ rtc_static_library("libjingle_peerconnection_api") { "turncustomizer.h", "umametrics.cc", "umametrics.h", - "videosourceinterface.cc", - "videosourceinterface.h", "videosourceproxy.h", ] @@ -227,6 +225,7 @@ rtc_source_set("video_frame_api") { visibility = [ "*" ] sources = [ "videosinkinterface.h", + "videosourceinterface.h", ] public_deps = [ # no-presubmit-check TODO(webrtc:8603) diff --git a/api/mediastreaminterface.h b/api/mediastreaminterface.h index 34a3143fd3..195ecc6a50 100644 --- a/api/mediastreaminterface.h +++ b/api/mediastreaminterface.h @@ -28,7 +28,7 @@ // relying on them; they were previously transitively included by // mediachannel.h, which is no longer a dependency of this file. #include "api/video/video_sink_interface.h" -#include "api/videosourceinterface.h" +#include "api/video/video_source_interface.h" #include "modules/audio_processing/include/audio_processing_statistics.h" #include "rtc_base/ratetracker.h" #include "rtc_base/refcount.h" diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn index 14f68e6174..0b34945b4d 100644 --- a/api/video/BUILD.gn +++ b/api/video/BUILD.gn @@ -19,11 +19,14 @@ rtc_source_set("video_frame") { "video_frame_buffer.h", "video_rotation.h", "video_sink_interface.h", + "video_source_interface.cc", + "video_source_interface.h", "video_timing.cc", "video_timing.h", ] deps = [ + "..:optional", "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", ] @@ -99,3 +102,18 @@ rtc_source_set("video_stream_decoder_create") { "../../video:video_stream_decoder_impl", ] } + +rtc_source_set("video_stream_encoder") { + visibility = [ "*" ] + sources = [ + "video_stream_encoder_interface.h", + ] + + deps = [ + ":video_frame", + + # For rtpparameters.h + "..:libjingle_peerconnection_api", + "../video_codecs:video_codecs_api", + ] +} diff --git a/api/videosourceinterface.cc b/api/video/video_source_interface.cc similarity index 92% rename from api/videosourceinterface.cc rename to api/video/video_source_interface.cc index 8cf8202911..70a86c3d64 100644 --- a/api/videosourceinterface.cc +++ b/api/video/video_source_interface.cc @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "api/videosourceinterface.h" +#include "api/video/video_source_interface.h" namespace rtc { diff --git a/api/video/video_source_interface.h b/api/video/video_source_interface.h new file mode 100644 index 0000000000..d4e2d3ad9e --- /dev/null +++ b/api/video/video_source_interface.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_VIDEO_SOURCE_INTERFACE_H_ +#define API_VIDEO_VIDEO_SOURCE_INTERFACE_H_ + +#include + +#include "api/optional.h" +#include "api/video/video_sink_interface.h" + +namespace rtc { + +// VideoSinkWants is used for notifying the source of properties a video frame +// should have when it is delivered to a certain sink. +struct VideoSinkWants { + VideoSinkWants(); + VideoSinkWants(const VideoSinkWants&); + ~VideoSinkWants(); + // Tells the source whether the sink wants frames with rotation applied. + // By default, any rotation must be applied by the sink. + bool rotation_applied = false; + + // Tells the source that the sink only wants black frames. + bool black_frames = false; + + // Tells the source the maximum number of pixels the sink wants. + int max_pixel_count = std::numeric_limits::max(); + // Tells the source the desired number of pixels the sinks wants. This will + // typically be used when stepping the resolution up again when conditions + // have improved after an earlier downgrade. The source should select the + // closest resolution to this pixel count, but if max_pixel_count is set, it + // still sets the absolute upper bound. + rtc::Optional target_pixel_count; + // Tells the source the maximum framerate the sink wants. + int max_framerate_fps = std::numeric_limits::max(); +}; + +template +class VideoSourceInterface { + public: + virtual void AddOrUpdateSink(VideoSinkInterface* sink, + const VideoSinkWants& wants) = 0; + // RemoveSink must guarantee that at the time the method returns, + // there is no current and no future calls to VideoSinkInterface::OnFrame. + virtual void RemoveSink(VideoSinkInterface* sink) = 0; + + protected: + // Non-public, since one shouldn't own sources via this interface. + virtual ~VideoSourceInterface() {} +}; + +} // namespace rtc +#endif // API_VIDEO_VIDEO_SOURCE_INTERFACE_H_ diff --git a/api/video/video_stream_encoder_interface.h b/api/video/video_stream_encoder_interface.h new file mode 100644 index 0000000000..fad3bbdfcb --- /dev/null +++ b/api/video/video_stream_encoder_interface.h @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_VIDEO_STREAM_ENCODER_INTERFACE_H_ +#define API_VIDEO_VIDEO_STREAM_ENCODER_INTERFACE_H_ + +#include + +#include "api/rtpparameters.h" // For DegradationPreference. +#include "api/video/video_sink_interface.h" +#include "api/video/video_source_interface.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_config.h" + +namespace webrtc { + +// TODO(nisse): Move full declaration to api/. +class VideoBitrateAllocationObserver; + +// This interface represents a class responsible for creating and driving the +// encoder(s) for a single video stream. It is also responsible for adaptation +// decisions related to video quality, requesting reduced frame rate or +// resolution from the VideoSource when needed. +// TODO(bugs.webrtc.org/8830): This interface is under development. Changes +// under consideration include: +// +// 1. Taking out responsibility for adaptation decisions, instead only reporting +// per-frame measurements to the decision maker. +// +// 2. Moving responsibility for simulcast and for software fallback into this +// class. +class VideoStreamEncoderInterface : public rtc::VideoSinkInterface { + public: + // Interface for receiving encoded video frames and notifications about + // configuration changes. + class EncoderSink : public EncodedImageCallback { + public: + virtual void OnEncoderConfigurationChanged( + std::vector streams, + int min_transmit_bitrate_bps) = 0; + }; + + virtual ~VideoStreamEncoderInterface() = default; + + // Sets the source that will provide video frames to the VideoStreamEncoder's + // OnFrame method. |degradation_preference| control whether or not resolution + // or frame rate may be reduced. The VideoStreamEncoder registers itself with + // |source|, and signals adaptation decisions to the source in the form of + // VideoSinkWants. + // TODO(nisse): When adaptation logic is extracted from this class, + // it no longer needs to know the source. + virtual void SetSource( + rtc::VideoSourceInterface* source, + const DegradationPreference& degradation_preference) = 0; + + // Sets the |sink| that gets the encoded frames. |rotation_applied| means + // that the source must support rotation. Only set |rotation_applied| if the + // remote side does not support the rotation extension. + virtual void SetSink(EncoderSink* sink, bool rotation_applied) = 0; + + // Sets an initial bitrate, later overriden by OnBitrateUpdated. Mainly + // affects the resolution of the initial key frame: If incoming frames are + // larger than reasonable for the start bitrate, and scaling is enabled, + // VideoStreamEncoder asks the source to scale down and drops a few initial + // frames. + // TODO(nisse): This is a poor interface, and mixes bandwidth estimation and + // codec configuration in an undesired way. For the actual send bandwidth, we + // should always be somewhat conservative, but we may nevertheless want to let + // the application configure a more optimistic quality for the initial + // resolution. Should be replaced by a construction time setting. + virtual void SetStartBitrate(int start_bitrate_bps) = 0; + + // Request a key frame. Used for signalling from the remote receiver. + virtual void SendKeyFrame() = 0; + + // Set the currently estimated network properties. A |bitrate_bps| + // of zero pauses the encoder. + virtual void OnBitrateUpdated(uint32_t bitrate_bps, + uint8_t fraction_lost, + int64_t round_trip_time_ms) = 0; + + // Register observer for the bitrate allocation between the temporal + // and spatial layers. + virtual void SetBitrateAllocationObserver( + VideoBitrateAllocationObserver* bitrate_observer) = 0; + + // Creates and configures an encoder with the given |config|. The + // |max_data_payload_length| is used to support single NAL unit + // packetization for H.264. + virtual void ConfigureEncoder(VideoEncoderConfig config, + size_t max_data_payload_length) = 0; + + // Permanently stop encoding. After this method has returned, it is + // guaranteed that no encoded frames will be delivered to the sink. + virtual void Stop() = 0; +}; + +} // namespace webrtc + +#endif // API_VIDEO_VIDEO_STREAM_ENCODER_INTERFACE_H_ diff --git a/api/videosourceinterface.h b/api/videosourceinterface.h index 17f16046c0..aabe3798a0 100644 --- a/api/videosourceinterface.h +++ b/api/videosourceinterface.h @@ -11,50 +11,8 @@ #ifndef API_VIDEOSOURCEINTERFACE_H_ #define API_VIDEOSOURCEINTERFACE_H_ -#include +// TODO(nisse): Place holder for moved file. Delete after applications are +// updated. +#include "api/video/video_source_interface.h" -#include "api/optional.h" -#include "api/video/video_sink_interface.h" - -namespace rtc { - -// VideoSinkWants is used for notifying the source of properties a video frame -// should have when it is delivered to a certain sink. -struct VideoSinkWants { - VideoSinkWants(); - VideoSinkWants(const VideoSinkWants&); - ~VideoSinkWants(); - // Tells the source whether the sink wants frames with rotation applied. - // By default, any rotation must be applied by the sink. - bool rotation_applied = false; - - // Tells the source that the sink only wants black frames. - bool black_frames = false; - - // Tells the source the maximum number of pixels the sink wants. - int max_pixel_count = std::numeric_limits::max(); - // Tells the source the desired number of pixels the sinks wants. This will - // typically be used when stepping the resolution up again when conditions - // have improved after an earlier downgrade. The source should select the - // closest resolution to this pixel count, but if max_pixel_count is set, it - // still sets the absolute upper bound. - rtc::Optional target_pixel_count; - // Tells the source the maximum framerate the sink wants. - int max_framerate_fps = std::numeric_limits::max(); -}; - -template -class VideoSourceInterface { - public: - virtual void AddOrUpdateSink(VideoSinkInterface* sink, - const VideoSinkWants& wants) = 0; - // RemoveSink must guarantee that at the time the method returns, - // there is no current and no future calls to VideoSinkInterface::OnFrame. - virtual void RemoveSink(VideoSinkInterface* sink) = 0; - - protected: - virtual ~VideoSourceInterface() {} -}; - -} // namespace rtc #endif // API_VIDEOSOURCEINTERFACE_H_ diff --git a/call/video_send_stream.h b/call/video_send_stream.h index 65e16873ff..620719efbb 100644 --- a/call/video_send_stream.h +++ b/call/video_send_stream.h @@ -17,10 +17,10 @@ #include #include "api/call/transport.h" -#include "api/rtpparameters.h" #include "api/rtp_headers.h" +#include "api/rtpparameters.h" #include "api/video/video_sink_interface.h" -#include "api/videosourceinterface.h" +#include "api/video/video_source_interface.h" #include "api/video_codecs/video_encoder_config.h" #include "api/video_codecs/video_encoder_factory.h" #include "call/rtp_config.h" diff --git a/media/base/mediachannel.h b/media/base/mediachannel.h index e5a6df2090..2580c0d392 100644 --- a/media/base/mediachannel.h +++ b/media/base/mediachannel.h @@ -24,9 +24,10 @@ #include "api/rtpparameters.h" #include "api/rtpreceiverinterface.h" #include "api/video/video_content_type.h" -#include "api/video/video_timing.h" #include "api/video/video_sink_interface.h" -#include "api/videosourceinterface.h" +#include "api/video/video_source_interface.h" +#include "api/video/video_timing.h" +#include "api/video_codecs/video_encoder_config.h" #include "media/base/codec.h" #include "media/base/mediaconfig.h" #include "media/base/mediaconstants.h" @@ -42,7 +43,6 @@ #include "rtc_base/socket.h" #include "rtc_base/stringencode.h" - namespace rtc { class Timing; } diff --git a/media/base/videocapturer.h b/media/base/videocapturer.h index 219e03643f..0470b4d17c 100644 --- a/media/base/videocapturer.h +++ b/media/base/videocapturer.h @@ -20,7 +20,7 @@ #include #include -#include "api/videosourceinterface.h" +#include "api/video/video_source_interface.h" #include "media/base/videoadapter.h" #include "media/base/videobroadcaster.h" #include "media/base/videocommon.h" diff --git a/media/base/videosourcebase.h b/media/base/videosourcebase.h index 6222e61c14..74ad280343 100644 --- a/media/base/videosourcebase.h +++ b/media/base/videosourcebase.h @@ -14,7 +14,7 @@ #include #include "api/video/video_frame.h" -#include "api/videosourceinterface.h" +#include "api/video/video_source_interface.h" #include "rtc_base/thread_checker.h" namespace rtc { diff --git a/media/engine/webrtcvideoengine.h b/media/engine/webrtcvideoengine.h index f75be26227..5a4eab3c3b 100644 --- a/media/engine/webrtcvideoengine.h +++ b/media/engine/webrtcvideoengine.h @@ -20,9 +20,9 @@ #include "api/call/transport.h" #include "api/optional.h" #include "api/video/video_frame.h" -#include "api/video_codecs/sdp_video_format.h" #include "api/video/video_sink_interface.h" -#include "api/videosourceinterface.h" +#include "api/video/video_source_interface.h" +#include "api/video_codecs/sdp_video_format.h" #include "call/call.h" #include "call/flexfec_receive_stream.h" #include "call/video_receive_stream.h" diff --git a/pc/BUILD.gn b/pc/BUILD.gn index 2c564cc85f..068156438b 100644 --- a/pc/BUILD.gn +++ b/pc/BUILD.gn @@ -378,6 +378,7 @@ if (rtc_include_tests) { "../api:libjingle_peerconnection_api", "../api:libjingle_peerconnection_test_api", "../api:rtc_stats_api", + "../api/video:video_frame", "../api/video_codecs:builtin_video_decoder_factory", "../api/video_codecs:builtin_video_encoder_factory", "../call:call_interfaces", diff --git a/pc/channel.h b/pc/channel.h index 86f9aa677f..cb068a2051 100644 --- a/pc/channel.h +++ b/pc/channel.h @@ -22,7 +22,7 @@ #include "api/jsep.h" #include "api/rtpreceiverinterface.h" #include "api/video/video_sink_interface.h" -#include "api/videosourceinterface.h" +#include "api/video/video_source_interface.h" #include "call/rtp_packet_sink_interface.h" #include "media/base/mediachannel.h" #include "media/base/mediaengine.h" diff --git a/pc/test/fakeperiodicvideosource.h b/pc/test/fakeperiodicvideosource.h index 62076350db..2cdd104e0d 100644 --- a/pc/test/fakeperiodicvideosource.h +++ b/pc/test/fakeperiodicvideosource.h @@ -13,7 +13,7 @@ #include -#include "api/videosourceinterface.h" +#include "api/video/video_source_interface.h" #include "media/base/fakeframesource.h" #include "media/base/videobroadcaster.h" #include "rtc_base/ptr_util.h" diff --git a/pc/test/fakevideotracksource.h b/pc/test/fakevideotracksource.h index 62e3e50cf6..9cc5ef185a 100644 --- a/pc/test/fakevideotracksource.h +++ b/pc/test/fakevideotracksource.h @@ -12,7 +12,7 @@ #define PC_TEST_FAKEVIDEOTRACKSOURCE_H_ #include "api/mediastreaminterface.h" -#include "api/videosourceinterface.h" +#include "api/video/video_source_interface.h" #include "pc/videotracksource.h" namespace webrtc { diff --git a/test/frame_generator.h b/test/frame_generator.h index 5728cfa032..049b5e735c 100644 --- a/test/frame_generator.h +++ b/test/frame_generator.h @@ -15,7 +15,7 @@ #include #include "api/video/video_frame.h" -#include "api/videosourceinterface.h" +#include "api/video/video_source_interface.h" #include "rtc_base/criticalsection.h" #include "typedefs.h" // NOLINT(build/include) diff --git a/test/video_capturer.h b/test/video_capturer.h index 7e640c93a5..63e1cd8e7e 100644 --- a/test/video_capturer.h +++ b/test/video_capturer.h @@ -17,7 +17,7 @@ #include "api/optional.h" #include "api/video/i420_buffer.h" #include "api/video/video_frame.h" -#include "api/videosourceinterface.h" +#include "api/video/video_source_interface.h" #include "media/base/videoadapter.h" #include "rtc_base/criticalsection.h" diff --git a/video/BUILD.gn b/video/BUILD.gn index 25ec808073..378221773f 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -64,6 +64,7 @@ rtc_static_library("video") { "../api:transport_api", "../api/video:video_frame", "../api/video:video_frame_i420", + "../api/video:video_stream_encoder", "../api/video_codecs:video_codecs_api", "../call:bitrate_allocator", "../call:call_interfaces", diff --git a/video/test/mock_video_stream_encoder.h b/video/test/mock_video_stream_encoder.h index efba400c03..37337f7fff 100644 --- a/video/test/mock_video_stream_encoder.h +++ b/video/test/mock_video_stream_encoder.h @@ -25,7 +25,8 @@ class MockVideoStreamEncoder : public VideoStreamEncoderInterface { MOCK_METHOD0(SendKeyFrame, void()); MOCK_METHOD3(OnBitrateUpdated, void(uint32_t, uint8_t, int64_t)); MOCK_METHOD1(OnFrame, void(const VideoFrame&)); - MOCK_METHOD1(SetBitrateObserver, void(VideoBitrateAllocationObserver*)); + MOCK_METHOD1(SetBitrateAllocationObserver, + void(VideoBitrateAllocationObserver*)); MOCK_METHOD0(Stop, void()); MOCK_METHOD2(MockedConfigureEncoder, diff --git a/video/video_send_stream.cc b/video/video_send_stream.cc index 88362f6288..e3fd64277f 100644 --- a/video/video_send_stream.cc +++ b/video/video_send_stream.cc @@ -109,7 +109,7 @@ VideoSendStream::VideoSendStream( // TODO(sprang): Enable this also for regular video calls if it works well. if (encoder_config.content_type == VideoEncoderConfig::ContentType::kScreen) { // Only signal target bitrate for screenshare streams, for now. - video_stream_encoder_->SetBitrateObserver(send_stream_.get()); + video_stream_encoder_->SetBitrateAllocationObserver(send_stream_.get()); } ReconfigureVideoEncoder(std::move(encoder_config)); diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 48d99cc8c3..1c9ad9fbd6 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -374,7 +374,7 @@ void VideoStreamEncoder::Stop() { shutdown_event_.Wait(rtc::Event::kForever); } -void VideoStreamEncoder::SetBitrateObserver( +void VideoStreamEncoder::SetBitrateAllocationObserver( VideoBitrateAllocationObserver* bitrate_observer) { RTC_DCHECK_RUN_ON(&thread_checker_); encoder_queue_.PostTask([this, bitrate_observer] { diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 84c054ff0a..7a48a07964 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -17,11 +17,12 @@ #include #include -#include "api/rtpparameters.h" // For DegradationPreference. #include "api/video/video_rotation.h" -#include "api/video_codecs/video_encoder.h" #include "api/video/video_sink_interface.h" +#include "api/video/video_stream_encoder_interface.h" +#include "api/video_codecs/video_encoder.h" #include "call/call.h" +#include "call/video_send_stream.h" #include "common_types.h" // NOLINT(build/include) #include "common_video/include/video_bitrate_allocator.h" #include "modules/video_coding/include/video_coding_defines.h" @@ -37,38 +38,7 @@ namespace webrtc { class SendStatisticsProxy; -class VideoBitrateAllocationObserver; -class VideoStreamEncoderInterface : public rtc::VideoSinkInterface { - public: - // Interface for receiving encoded video frames and notifications about - // configuration changes. - class EncoderSink : public EncodedImageCallback { - public: - virtual void OnEncoderConfigurationChanged( - std::vector streams, - int min_transmit_bitrate_bps) = 0; - }; - virtual void SetSource( - rtc::VideoSourceInterface* source, - const DegradationPreference& degradation_preference) = 0; - virtual void SetSink(EncoderSink* sink, bool rotation_applied) = 0; - - virtual void SetStartBitrate(int start_bitrate_bps) = 0; - virtual void SendKeyFrame() = 0; - virtual void OnBitrateUpdated(uint32_t bitrate_bps, - uint8_t fraction_lost, - int64_t round_trip_time_ms) = 0; - - virtual void SetBitrateObserver( - VideoBitrateAllocationObserver* bitrate_observer) = 0; - virtual void ConfigureEncoder(VideoEncoderConfig config, - size_t max_data_payload_length) = 0; - virtual void Stop() = 0; - - protected: - ~VideoStreamEncoderInterface() = default; -}; // VideoStreamEncoder represent a video encoder that accepts raw video frames as // input and produces an encoded bit stream. // Usage: @@ -95,21 +65,15 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, std::unique_ptr overuse_detector); ~VideoStreamEncoder(); - // Sets the source that will provide I420 video frames. - // |degradation_preference| control whether or not resolution or frame rate - // may be reduced. void SetSource(rtc::VideoSourceInterface* source, const DegradationPreference& degradation_preference) override; - // Sets the |sink| that gets the encoded frames. |rotation_applied| means - // that the source must support rotation. Only set |rotation_applied| if the - // remote side does not support the rotation extension. void SetSink(EncoderSink* sink, bool rotation_applied) override; // TODO(perkj): Can we remove VideoCodec.startBitrate ? void SetStartBitrate(int start_bitrate_bps) override; - void SetBitrateObserver( + void SetBitrateAllocationObserver( VideoBitrateAllocationObserver* bitrate_observer) override; void ConfigureEncoder(VideoEncoderConfig config, diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index b18abbe488..239470e0d5 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -2043,7 +2043,7 @@ TEST_F(VideoStreamEncoderTest, TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) { MockBitrateObserver bitrate_observer; - video_stream_encoder_->SetBitrateObserver(&bitrate_observer); + video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer); const int kDefaultFps = 30; const VideoBitrateAllocation expected_bitrate = @@ -3090,7 +3090,7 @@ TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) { const int kTargetBitrateBps = 1000000; MockBitrateObserver bitrate_observer; - video_stream_encoder_->SetBitrateObserver(&bitrate_observer); + video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer); EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1); // Initial bitrate update.