From 529d0d9795b81dbed5e4231f15d3752a5fc0df32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20M=C3=B6ller?= Date: Thu, 27 Sep 2018 12:43:52 +0200 Subject: [PATCH] Replace VideoDecoder with VideoDecoderFactory in VideoReceiveStream config. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Preparation for deleting EnableFrameRecordning, and also a step towards landing of the new VideoStreamDecoder. Bug: webrtc:9106 Change-Id: I50964ee458b08a702ec69b82a62e4995c57cee82 Reviewed-on: https://webrtc-review.googlesource.com/97660 Commit-Queue: Niels Moller Reviewed-by: Rasmus Brandt Reviewed-by: Anders Carlsson Reviewed-by: Sebastian Jansson Reviewed-by: Erik Språng Cr-Commit-Position: refs/heads/master@{#24861} --- call/bitrate_estimator_tests.cc | 8 +- call/call_perf_tests.cc | 4 +- call/video_receive_stream.cc | 5 +- call/video_receive_stream.h | 12 ++- media/engine/webrtcvideoengine.cc | 84 ++----------------- media/engine/webrtcvideoengine.h | 22 +---- test/BUILD.gn | 3 +- test/call_test.cc | 6 +- test/call_test.h | 3 +- test/encoder_settings.cc | 18 ---- test/video_decoder_proxy_factory.h | 81 ++++++++++++++++++ ...actory.h => video_encoder_proxy_factory.h} | 10 +-- video/BUILD.gn | 1 + video/end_to_end_tests/bandwidth_tests.cc | 4 +- video/end_to_end_tests/codec_tests.cc | 67 ++++++++++----- video/end_to_end_tests/fec_tests.cc | 13 ++- video/end_to_end_tests/log_tests.cc | 7 +- .../multi_codec_receive_tests.cc | 83 +++++++++++------- video/end_to_end_tests/multi_stream_tester.cc | 6 +- video/end_to_end_tests/network_state_tests.cc | 6 +- video/replay.cc | 24 ++++-- video/video_quality_test.cc | 1 - video/video_receive_stream.cc | 48 ++++++++++- video/video_receive_stream.h | 4 + video/video_receive_stream_unittest.cc | 11 ++- video/video_send_stream_tests.cc | 26 +++--- video/video_stream_encoder_unittest.cc | 4 +- 27 files changed, 327 insertions(+), 234 deletions(-) create mode 100644 test/video_decoder_proxy_factory.h rename test/{encoder_proxy_factory.h => video_encoder_proxy_factory.h} (92%) diff --git a/call/bitrate_estimator_tests.cc b/call/bitrate_estimator_tests.cc index af0e4c9f22..766e38c978 100644 --- a/call/bitrate_estimator_tests.cc +++ b/call/bitrate_estimator_tests.cc @@ -170,7 +170,8 @@ class BitrateEstimatorTest : public test::CallTest { is_sending_receiving_(false), send_stream_(nullptr), frame_generator_capturer_(), - fake_decoder_() { + decoder_factory_( + []() { return absl::make_unique(); }) { test_->GetVideoSendConfig()->rtp.ssrcs[0]++; send_stream_ = test_->sender_call_->CreateVideoSendStream( test_->GetVideoSendConfig()->Copy(), @@ -185,7 +186,7 @@ class BitrateEstimatorTest : public test::CallTest { frame_generator_capturer_->Start(); VideoReceiveStream::Decoder decoder; - decoder.decoder = &fake_decoder_; + decoder.decoder_factory = &decoder_factory_; decoder.payload_type = test_->GetVideoSendConfig()->rtp.payload_type; decoder.video_format = SdpVideoFormat(test_->GetVideoSendConfig()->rtp.payload_name); @@ -229,7 +230,8 @@ class BitrateEstimatorTest : public test::CallTest { VideoSendStream* send_stream_; VideoReceiveStream* video_receive_stream_; std::unique_ptr frame_generator_capturer_; - test::FakeDecoder fake_decoder_; + + test::FunctionVideoDecoderFactory decoder_factory_; }; LogObserver receiver_log_; diff --git a/call/call_perf_tests.cc b/call/call_perf_tests.cc index b8b460135e..05c39feb8f 100644 --- a/call/call_perf_tests.cc +++ b/call/call_perf_tests.cc @@ -33,7 +33,6 @@ #include "test/call_test.h" #include "test/direct_transport.h" #include "test/drifting_clock.h" -#include "test/encoder_proxy_factory.h" #include "test/encoder_settings.h" #include "test/fake_encoder.h" #include "test/field_trial.h" @@ -44,6 +43,7 @@ #include "test/single_threaded_task_queue.h" #include "test/testsupport/fileutils.h" #include "test/testsupport/perf_test.h" +#include "test/video_encoder_proxy_factory.h" #include "video/transport_adapter.h" using webrtc::test::DriftingClock; @@ -807,7 +807,7 @@ TEST_F(CallPerfTest, MAYBE_KeepsHighBitrateWhenReconfiguringSender) { uint32_t last_set_bitrate_kbps_; VideoSendStream* send_stream_; test::FrameGeneratorCapturer* frame_generator_; - test::EncoderProxyFactory encoder_factory_; + test::VideoEncoderProxyFactory encoder_factory_; VideoEncoderConfig encoder_config_; } test; diff --git a/call/video_receive_stream.cc b/call/video_receive_stream.cc index c4cdd49277..672abd83bb 100644 --- a/call/video_receive_stream.cc +++ b/call/video_receive_stream.cc @@ -20,9 +20,8 @@ VideoReceiveStream::Decoder::~Decoder() = default; std::string VideoReceiveStream::Decoder::ToString() const { char buf[1024]; rtc::SimpleStringBuilder ss(buf); - ss << "{decoder: " << (decoder ? "(VideoDecoder)" : "nullptr"); - ss << ", payload_type: " << payload_type; - ss << ", name: " << video_format.name; + ss << "{payload_type: " << payload_type; + ss << ", payload_name: " << video_format.name; ss << ", codec_params: {"; for (const auto& it : video_format.parameters) ss << it.first << ": " << it.second; diff --git a/call/video_receive_stream.h b/call/video_receive_stream.h index b5c1649aa9..d299cd8bde 100644 --- a/call/video_receive_stream.h +++ b/call/video_receive_stream.h @@ -33,7 +33,7 @@ namespace webrtc { class RtpPacketSinkInterface; -class VideoDecoder; +class VideoDecoderFactory; class VideoReceiveStream { public: @@ -45,8 +45,10 @@ class VideoReceiveStream { ~Decoder(); std::string ToString() const; - // The actual decoder instance. - VideoDecoder* decoder = nullptr; + // Ownership stays with WebrtcVideoEngine (delegated from PeerConnection). + // TODO(nisse): Move one level out, to VideoReceiveStream::Config, and later + // to the configuration of VideoStreamDecoder. + VideoDecoderFactory* decoder_factory = nullptr; SdpVideoFormat video_format; // Received RTP packets with this payload type will be sent to this decoder @@ -210,6 +212,10 @@ class VideoReceiveStream { // Target delay in milliseconds. A positive value indicates this stream is // used for streaming instead of a real-time call. int target_delay_ms = 0; + + // TODO(nisse): Used with VideoDecoderFactory::LegacyCreateVideoDecoder. + // Delete when that method is retired. + std::string stream_id; }; // Starts stream activity. diff --git a/media/engine/webrtcvideoengine.cc b/media/engine/webrtcvideoengine.cc index 501f5de89d..73585a84c8 100644 --- a/media/engine/webrtcvideoengine.cc +++ b/media/engine/webrtcvideoengine.cc @@ -16,14 +16,11 @@ #include #include -#include "api/video/i420_buffer.h" #include "api/video_codecs/sdp_video_format.h" -#include "api/video_codecs/video_decoder.h" #include "api/video_codecs/video_decoder_factory.h" #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder_factory.h" #include "call/call.h" -#include "common_video/h264/profile_level_id.h" #include "media/engine/constants.h" #if defined(USE_BUILTIN_SW_CODECS) #include "media/engine/convert_legacy_video_factory.h" // nogncheck @@ -31,7 +28,6 @@ #include "media/engine/simulcast.h" #include "media/engine/webrtcmediaengine.h" #include "media/engine/webrtcvoiceengine.h" -#include "modules/video_coding/include/video_error_codes.h" #include "rtc_base/copyonwritebuffer.h" #include "rtc_base/logging.h" #include "rtc_base/strings/string_builder.h" @@ -44,36 +40,6 @@ namespace cricket { namespace { -// Video decoder class to be used for unknown codecs. Doesn't support decoding -// but logs messages to LS_ERROR. -class NullVideoDecoder : public webrtc::VideoDecoder { - public: - int32_t InitDecode(const webrtc::VideoCodec* codec_settings, - int32_t number_of_cores) override { - RTC_LOG(LS_ERROR) << "Can't initialize NullVideoDecoder."; - return WEBRTC_VIDEO_CODEC_OK; - } - - int32_t Decode(const webrtc::EncodedImage& input_image, - bool missing_frames, - const webrtc::CodecSpecificInfo* codec_specific_info, - int64_t render_time_ms) override { - RTC_LOG(LS_ERROR) << "The NullVideoDecoder doesn't support decoding."; - return WEBRTC_VIDEO_CODEC_OK; - } - - int32_t RegisterDecodeCompleteCallback( - webrtc::DecodedImageCallback* callback) override { - RTC_LOG(LS_ERROR) - << "Can't register decode complete callback on NullVideoDecoder."; - return WEBRTC_VIDEO_CODEC_OK; - } - - int32_t Release() override { return WEBRTC_VIDEO_CODEC_OK; } - - const char* ImplementationName() const override { return "NullVideoDecoder"; } -}; - // If this field trial is enabled, we will enable sending FlexFEC and disable // sending ULPFEC whenever the former has been negotiated in the SDPs. bool IsFlexfecFieldTrialEnabled() { @@ -2194,12 +2160,10 @@ WebRtcVideoChannel::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream( first_frame_timestamp_(-1), estimated_remote_start_ntp_time_ms_(0) { config_.renderer = this; - DecoderMap old_decoders; - ConfigureCodecs(recv_codecs, &old_decoders); + ConfigureCodecs(recv_codecs); ConfigureFlexfecCodec(flexfec_config.payload_type); MaybeRecreateWebRtcFlexfecStream(); RecreateWebRtcVideoStream(); - RTC_DCHECK(old_decoders.empty()); } WebRtcVideoChannel::WebRtcVideoReceiveStream::~WebRtcVideoReceiveStream() { @@ -2208,7 +2172,6 @@ WebRtcVideoChannel::WebRtcVideoReceiveStream::~WebRtcVideoReceiveStream() { call_->DestroyFlexfecReceiveStream(flexfec_stream_); } call_->DestroyVideoReceiveStream(stream_); - allocated_decoders_.clear(); } const std::vector& @@ -2247,58 +2210,23 @@ WebRtcVideoChannel::WebRtcVideoReceiveStream::GetRtpParameters() const { } void WebRtcVideoChannel::WebRtcVideoReceiveStream::ConfigureCodecs( - const std::vector& recv_codecs, - DecoderMap* old_decoders) { + const std::vector& recv_codecs) { RTC_DCHECK(!recv_codecs.empty()); - *old_decoders = std::move(allocated_decoders_); config_.decoders.clear(); config_.rtp.rtx_associated_payload_types.clear(); for (const auto& recv_codec : recv_codecs) { webrtc::SdpVideoFormat video_format(recv_codec.codec.name, recv_codec.codec.params); - std::unique_ptr new_decoder; - - if (allocated_decoders_.count(video_format) > 0) { - RTC_LOG(LS_WARNING) - << "VideoReceiveStream configured with duplicate codecs: " - << video_format.name; - continue; - } - - auto it = old_decoders->find(video_format); - if (it != old_decoders->end()) { - new_decoder = std::move(it->second); - old_decoders->erase(it); - } - - if (!new_decoder && decoder_factory_) { - new_decoder = decoder_factory_->LegacyCreateVideoDecoder( - webrtc::SdpVideoFormat(recv_codec.codec.name, - recv_codec.codec.params), - stream_params_.id); - } - - // If we still have no valid decoder, we have to create a "Null" decoder - // that ignores all calls. The reason we can get into this state is that - // the old decoder factory interface doesn't have a way to query supported - // codecs. - if (!new_decoder) - new_decoder.reset(new NullVideoDecoder()); webrtc::VideoReceiveStream::Decoder decoder; - decoder.decoder = new_decoder.get(); + decoder.decoder_factory = decoder_factory_; + decoder.video_format = video_format; decoder.payload_type = recv_codec.codec.id; decoder.video_format = webrtc::SdpVideoFormat(recv_codec.codec.name, recv_codec.codec.params); config_.decoders.push_back(decoder); config_.rtp.rtx_associated_payload_types[recv_codec.rtx_payload_type] = recv_codec.codec.id; - - const bool did_insert = - allocated_decoders_ - .insert(std::make_pair(video_format, std::move(new_decoder))) - .second; - RTC_CHECK(did_insert); } const auto& codec = recv_codecs.front(); @@ -2378,9 +2306,8 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetRecvParameters( const ChangedRecvParameters& params) { bool video_needs_recreation = false; bool flexfec_needs_recreation = false; - DecoderMap old_decoders; if (params.codec_settings) { - ConfigureCodecs(*params.codec_settings, &old_decoders); + ConfigureCodecs(*params.codec_settings); video_needs_recreation = true; } if (params.rtp_header_extensions) { @@ -2413,6 +2340,7 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::RecreateWebRtcVideoStream() { } webrtc::VideoReceiveStream::Config config = config_.Copy(); config.rtp.protected_by_flexfec = (flexfec_stream_ != nullptr); + config.stream_id = stream_params_.id; stream_ = call_->CreateVideoReceiveStream(std::move(config)); MaybeAssociateFlexfecWithVideo(); stream_->Start(); diff --git a/media/engine/webrtcvideoengine.h b/media/engine/webrtcvideoengine.h index d949d786b7..40c39afd5e 100644 --- a/media/engine/webrtcvideoengine.h +++ b/media/engine/webrtcvideoengine.h @@ -37,9 +37,7 @@ #include "rtc_base/thread_checker.h" namespace webrtc { -class VideoDecoder; class VideoDecoderFactory; -class VideoEncoder; class VideoEncoderFactory; struct MediaConfig; } // namespace webrtc @@ -380,26 +378,13 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport { VideoReceiverInfo GetVideoReceiverInfo(bool log_stats); private: - struct SdpVideoFormatCompare { - bool operator()(const webrtc::SdpVideoFormat& lhs, - const webrtc::SdpVideoFormat& rhs) const { - return std::tie(lhs.name, lhs.parameters) < - std::tie(rhs.name, rhs.parameters); - } - }; - typedef std::map, - SdpVideoFormatCompare> - DecoderMap; - void RecreateWebRtcVideoStream(); void MaybeRecreateWebRtcFlexfecStream(); void MaybeAssociateFlexfecWithVideo(); void MaybeDissociateFlexfecFromVideo(); - void ConfigureCodecs(const std::vector& recv_codecs, - DecoderMap* old_codecs); + void ConfigureCodecs(const std::vector& recv_codecs); void ConfigureFlexfecCodec(int flexfec_payload_type); std::string GetCodecNameFromPayloadType(int payload_type); @@ -407,7 +392,7 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport { absl::optional GetFirstPrimarySsrc() const; webrtc::Call* const call_; - StreamParams stream_params_; + const StreamParams stream_params_; // Both |stream_| and |flexfec_stream_| are managed by |this|. They are // destroyed by calling call_->DestroyVideoReceiveStream and @@ -418,8 +403,7 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport { webrtc::FlexfecReceiveStream::Config flexfec_config_; webrtc::FlexfecReceiveStream* flexfec_stream_; - webrtc::VideoDecoderFactory* decoder_factory_; - DecoderMap allocated_decoders_; + webrtc::VideoDecoderFactory* const decoder_factory_; rtc::CriticalSection sink_lock_; rtc::VideoSinkInterface* sink_ diff --git a/test/BUILD.gn b/test/BUILD.gn index 6e18988958..3fa25cf877 100644 --- a/test/BUILD.gn +++ b/test/BUILD.gn @@ -540,7 +540,6 @@ rtc_source_set("test_common") { "constants.h", "drifting_clock.cc", "drifting_clock.h", - "encoder_proxy_factory.h", "encoder_settings.cc", "encoder_settings.h", "fake_videorenderer.h", @@ -555,6 +554,8 @@ rtc_source_set("test_common") { "rtp_rtcp_observer.h", "statistics.cc", "statistics.h", + "video_decoder_proxy_factory.h", + "video_encoder_proxy_factory.h", "win/run_loop_win.cc", ] if (!is_win) { diff --git a/test/call_test.cc b/test/call_test.cc index 89ac329302..1b1d356fd0 100644 --- a/test/call_test.cc +++ b/test/call_test.cc @@ -52,6 +52,7 @@ CallTest::CallTest() fake_encoder->SetMaxBitrate(fake_encoder_max_bitrate_); return fake_encoder; }), + fake_decoder_factory_([]() { return absl::make_unique(); }), num_video_streams_(1), num_audio_streams_(0), num_flexfec_streams_(0), @@ -366,11 +367,10 @@ void CallTest::AddMatchingVideoReceiveConfigs( if (!decode_sub_stream || i == *decode_sub_stream) { decoder = test::CreateMatchingDecoder(video_send_config); } else { - decoder.decoder = new test::FakeDecoder(); decoder.payload_type = video_send_config.rtp.payload_type; decoder.video_format = SdpVideoFormat(video_send_config.rtp.payload_name); } - allocated_decoders_.emplace_back(decoder.decoder); + decoder.decoder_factory = &fake_decoder_factory_; video_recv_config.decoders.push_back(decoder); receive_configs->emplace_back(std::move(video_recv_config)); } @@ -433,7 +433,6 @@ void CallTest::CreateMatchingFecConfig( void CallTest::CreateMatchingReceiveConfigs(Transport* rtcp_send_transport) { video_receive_configs_.clear(); - allocated_decoders_.clear(); for (VideoSendStream::Config& video_send_config : video_send_configs_) { CreateMatchingVideoReceiveConfigs(video_send_config, rtcp_send_transport); } @@ -635,7 +634,6 @@ void CallTest::DestroyStreams() { receiver_call_->DestroyFlexfecReceiveStream(flexfec_recv_stream); video_receive_streams_.clear(); - allocated_decoders_.clear(); } void CallTest::DestroyVideoSendStreams() { diff --git a/test/call_test.h b/test/call_test.h index eb96cfde48..0b2058b9ec 100644 --- a/test/call_test.h +++ b/test/call_test.h @@ -22,6 +22,7 @@ #include "test/fake_videorenderer.h" #include "test/fake_vp8_encoder.h" #include "test/frame_generator_capturer.h" +#include "test/function_video_decoder_factory.h" #include "test/function_video_encoder_factory.h" #include "test/rtp_rtcp_observer.h" #include "test/single_threaded_task_queue.h" @@ -201,7 +202,7 @@ class CallTest : public ::testing::Test { test::FunctionVideoEncoderFactory fake_encoder_factory_; int fake_encoder_max_bitrate_ = -1; - std::vector> allocated_decoders_; + test::FunctionVideoDecoderFactory fake_decoder_factory_; // Number of simulcast substreams. size_t num_video_streams_; size_t num_audio_streams_; diff --git a/test/encoder_settings.cc b/test/encoder_settings.cc index 1013433917..945559b4ee 100644 --- a/test/encoder_settings.cc +++ b/test/encoder_settings.cc @@ -12,13 +12,7 @@ #include #include -#include "media/engine/internaldecoderfactory.h" -#include "modules/video_coding/codecs/h264/include/h264.h" -#include "modules/video_coding/codecs/multiplex/include/multiplex_decoder_adapter.h" -#include "modules/video_coding/codecs/vp8/include/vp8.h" -#include "modules/video_coding/codecs/vp9/include/vp9.h" #include "rtc_base/refcountedobject.h" -#include "test/fake_decoder.h" namespace webrtc { namespace test { @@ -128,18 +122,6 @@ VideoReceiveStream::Decoder CreateMatchingDecoder( VideoReceiveStream::Decoder decoder; decoder.payload_type = payload_type; decoder.video_format = SdpVideoFormat(payload_name); - if (payload_name == "H264") { - decoder.decoder = H264Decoder::Create().release(); - } else if (payload_name == "VP8") { - decoder.decoder = VP8Decoder::Create().release(); - } else if (payload_name == "VP9") { - decoder.decoder = VP9Decoder::Create().release(); - } else if (payload_name == "multiplex") { - decoder.decoder = new MultiplexDecoderAdapter( - new InternalDecoderFactory(), SdpVideoFormat(cricket::kVp9CodecName)); - } else { - decoder.decoder = new FakeDecoder(); - } return decoder; } diff --git a/test/video_decoder_proxy_factory.h b/test/video_decoder_proxy_factory.h new file mode 100644 index 0000000000..250750cb11 --- /dev/null +++ b/test/video_decoder_proxy_factory.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_VIDEO_DECODER_PROXY_FACTORY_H_ +#define TEST_VIDEO_DECODER_PROXY_FACTORY_H_ + +#include +#include + +#include "absl/memory/memory.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" + +namespace webrtc { +namespace test { + +// An decoder factory with a single underlying VideoDecoder object, intended for +// test purposes. Each call to CreateVideoDecoder returns a proxy for the same +// decoder, typically an instance of FakeDecoder or MockEncoder. +class VideoDecoderProxyFactory final : public VideoDecoderFactory { + public: + explicit VideoDecoderProxyFactory(VideoDecoder* decoder) + : decoder_(decoder) {} + + // Unused by tests. + std::vector GetSupportedFormats() const override { + RTC_NOTREACHED(); + return {}; + } + + std::unique_ptr CreateVideoDecoder( + const SdpVideoFormat& format) override { + return absl::make_unique(decoder_); + } + + private: + // Wrapper class, since CreateVideoDecoder needs to surrender + // ownership to the object it returns. + class DecoderProxy final : public VideoDecoder { + public: + explicit DecoderProxy(VideoDecoder* decoder) : decoder_(decoder) {} + + private: + int32_t Decode(const EncodedImage& input_image, + bool missing_frames, + const CodecSpecificInfo* codec_specific_info, + int64_t render_time_ms) override { + return decoder_->Decode(input_image, missing_frames, codec_specific_info, + render_time_ms); + } + int32_t InitDecode(const VideoCodec* config, + int32_t number_of_cores) override { + return decoder_->InitDecode(config, number_of_cores); + } + int32_t RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) override { + return decoder_->RegisterDecodeCompleteCallback(callback); + } + int32_t Release() override { return decoder_->Release(); } + bool PrefersLateDecoding() const { return decoder_->PrefersLateDecoding(); } + const char* ImplementationName() const override { + return decoder_->ImplementationName(); + } + + VideoDecoder* const decoder_; + }; + + VideoDecoder* const decoder_; +}; + +} // namespace test +} // namespace webrtc + +#endif // TEST_VIDEO_DECODER_PROXY_FACTORY_H_ diff --git a/test/encoder_proxy_factory.h b/test/video_encoder_proxy_factory.h similarity index 92% rename from test/encoder_proxy_factory.h rename to test/video_encoder_proxy_factory.h index 8672a46492..e7289c0399 100644 --- a/test/encoder_proxy_factory.h +++ b/test/video_encoder_proxy_factory.h @@ -8,8 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef TEST_ENCODER_PROXY_FACTORY_H_ -#define TEST_ENCODER_PROXY_FACTORY_H_ +#ifndef TEST_VIDEO_ENCODER_PROXY_FACTORY_H_ +#define TEST_VIDEO_ENCODER_PROXY_FACTORY_H_ #include #include @@ -24,9 +24,9 @@ namespace test { // An encoder factory with a single underlying VideoEncoder object, // intended for test purposes. Each call to CreateVideoEncoder returns // a proxy for the same encoder, typically an instance of FakeEncoder. -class EncoderProxyFactory final : public VideoEncoderFactory { +class VideoEncoderProxyFactory final : public VideoEncoderFactory { public: - explicit EncoderProxyFactory(VideoEncoder* encoder) : encoder_(encoder) { + explicit VideoEncoderProxyFactory(VideoEncoder* encoder) : encoder_(encoder) { codec_info_.is_hardware_accelerated = false; codec_info_.has_internal_source = false; } @@ -100,4 +100,4 @@ class EncoderProxyFactory final : public VideoEncoderFactory { } // namespace test } // namespace webrtc -#endif // TEST_ENCODER_PROXY_FACTORY_H_ +#endif // TEST_VIDEO_ENCODER_PROXY_FACTORY_H_ diff --git a/video/BUILD.gn b/video/BUILD.gn index cecdd38ad3..624d278cb8 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -364,6 +364,7 @@ if (rtc_include_tests) { "../call:call_interfaces", "../common_video", "../logging:rtc_event_log_api", + "../media:rtc_internal_video_codecs", "../modules/rtp_rtcp", "../rtc_base:checks", "../rtc_base:rtc_base_approved", diff --git a/video/end_to_end_tests/bandwidth_tests.cc b/video/end_to_end_tests/bandwidth_tests.cc index 5ab24fa853..ab5e5687b9 100644 --- a/video/end_to_end_tests/bandwidth_tests.cc +++ b/video/end_to_end_tests/bandwidth_tests.cc @@ -15,12 +15,12 @@ #include "rtc_base/rate_limiter.h" #include "system_wrappers/include/sleep.h" #include "test/call_test.h" -#include "test/encoder_proxy_factory.h" #include "test/fake_encoder.h" #include "test/field_trial.h" #include "test/gtest.h" #include "test/rtcp_packet_parser.h" #include "test/rtp_rtcp_observer.h" +#include "test/video_encoder_proxy_factory.h" namespace webrtc { @@ -344,7 +344,7 @@ TEST_F(BandwidthEndToEndTest, ReportsSetEncoderRates) { test::SingleThreadedTaskQueueForTesting* const task_queue_; rtc::CriticalSection crit_; VideoSendStream* send_stream_; - test::EncoderProxyFactory encoder_factory_; + test::VideoEncoderProxyFactory encoder_factory_; uint32_t bitrate_kbps_ RTC_GUARDED_BY(crit_); } test(&task_queue_); diff --git a/video/end_to_end_tests/codec_tests.cc b/video/end_to_end_tests/codec_tests.cc index 5c5255567a..fad3ac7ede 100644 --- a/video/end_to_end_tests/codec_tests.cc +++ b/video/end_to_end_tests/codec_tests.cc @@ -39,7 +39,7 @@ class CodecObserver : public test::EndToEndTest, VideoRotation rotation_to_test, const std::string& payload_name, VideoEncoderFactory* encoder_factory, - std::unique_ptr decoder) + VideoDecoderFactory* decoder_factory) : EndToEndTest(4 * CodecEndToEndTest::kDefaultTimeoutMs), // TODO(hta): This timeout (120 seconds) is excessive. // https://bugs.webrtc.org/6830 @@ -47,7 +47,7 @@ class CodecObserver : public test::EndToEndTest, expected_rotation_(rotation_to_test), payload_name_(payload_name), encoder_factory_(encoder_factory), - decoder_(std::move(decoder)), + decoder_factory_(decoder_factory), frame_counter_(0) {} void PerformTest() override { @@ -70,7 +70,7 @@ class CodecObserver : public test::EndToEndTest, send_config->rtp.payload_type; (*receive_configs)[0].decoders[0].video_format = SdpVideoFormat(send_config->rtp.payload_name); - (*receive_configs)[0].decoders[0].decoder = decoder_.get(); + (*receive_configs)[0].decoders[0].decoder_factory = decoder_factory_; } void OnFrame(const VideoFrame& video_frame) override { @@ -89,7 +89,7 @@ class CodecObserver : public test::EndToEndTest, VideoRotation expected_rotation_; std::string payload_name_; VideoEncoderFactory* encoder_factory_; - std::unique_ptr decoder_; + VideoDecoderFactory* decoder_factory_; int frame_counter_; }; @@ -101,16 +101,20 @@ INSTANTIATE_TEST_CASE_P(GenericDescriptor, TEST_P(CodecEndToEndTest, SendsAndReceivesVP8) { test::FunctionVideoEncoderFactory encoder_factory( []() { return VP8Encoder::Create(); }); + test::FunctionVideoDecoderFactory decoder_factory( + []() { return VP8Decoder::Create(); }); CodecObserver test(5, kVideoRotation_0, "VP8", &encoder_factory, - VP8Decoder::Create()); + &decoder_factory); RunBaseTest(&test); } TEST_P(CodecEndToEndTest, SendsAndReceivesVP8Rotation90) { test::FunctionVideoEncoderFactory encoder_factory( []() { return VP8Encoder::Create(); }); + test::FunctionVideoDecoderFactory decoder_factory( + []() { return VP8Decoder::Create(); }); CodecObserver test(5, kVideoRotation_90, "VP8", &encoder_factory, - VP8Decoder::Create()); + &decoder_factory); RunBaseTest(&test); } @@ -118,47 +122,58 @@ TEST_P(CodecEndToEndTest, SendsAndReceivesVP8Rotation90) { TEST_P(CodecEndToEndTest, SendsAndReceivesVP9) { test::FunctionVideoEncoderFactory encoder_factory( []() { return VP9Encoder::Create(); }); + test::FunctionVideoDecoderFactory decoder_factory( + []() { return VP9Decoder::Create(); }); CodecObserver test(500, kVideoRotation_0, "VP9", &encoder_factory, - VP9Decoder::Create()); + &decoder_factory); RunBaseTest(&test); } TEST_P(CodecEndToEndTest, SendsAndReceivesVP9VideoRotation90) { test::FunctionVideoEncoderFactory encoder_factory( []() { return VP9Encoder::Create(); }); + test::FunctionVideoDecoderFactory decoder_factory( + []() { return VP9Decoder::Create(); }); CodecObserver test(5, kVideoRotation_90, "VP9", &encoder_factory, - VP9Decoder::Create()); + &decoder_factory); RunBaseTest(&test); } // Mutiplex tests are using VP9 as the underlying implementation. TEST_P(CodecEndToEndTest, SendsAndReceivesMultiplex) { InternalEncoderFactory internal_encoder_factory; - InternalDecoderFactory decoder_factory; + InternalDecoderFactory internal_decoder_factory; test::FunctionVideoEncoderFactory encoder_factory( [&internal_encoder_factory]() { return absl::make_unique( &internal_encoder_factory, SdpVideoFormat(cricket::kVp9CodecName)); }); - CodecObserver test( - 5, kVideoRotation_0, "multiplex", &encoder_factory, - absl::make_unique( - &decoder_factory, SdpVideoFormat(cricket::kVp9CodecName))); + test::FunctionVideoDecoderFactory decoder_factory( + [&internal_decoder_factory]() { + return absl::make_unique( + &internal_decoder_factory, SdpVideoFormat(cricket::kVp9CodecName)); + }); + + CodecObserver test(5, kVideoRotation_0, "multiplex", &encoder_factory, + &decoder_factory); RunBaseTest(&test); } TEST_P(CodecEndToEndTest, SendsAndReceivesMultiplexVideoRotation90) { InternalEncoderFactory internal_encoder_factory; - InternalDecoderFactory decoder_factory; + InternalDecoderFactory internal_decoder_factory; test::FunctionVideoEncoderFactory encoder_factory( [&internal_encoder_factory]() { return absl::make_unique( &internal_encoder_factory, SdpVideoFormat(cricket::kVp9CodecName)); }); - CodecObserver test( - 5, kVideoRotation_90, "multiplex", &encoder_factory, - absl::make_unique( - &decoder_factory, SdpVideoFormat(cricket::kVp9CodecName))); + test::FunctionVideoDecoderFactory decoder_factory( + [&internal_decoder_factory]() { + return absl::make_unique( + &internal_decoder_factory, SdpVideoFormat(cricket::kVp9CodecName)); + }); + CodecObserver test(5, kVideoRotation_90, "multiplex", &encoder_factory, + &decoder_factory); RunBaseTest(&test); } @@ -183,16 +198,20 @@ INSTANTIATE_TEST_CASE_P( TEST_P(EndToEndTestH264, SendsAndReceivesH264) { test::FunctionVideoEncoderFactory encoder_factory( []() { return H264Encoder::Create(cricket::VideoCodec("H264")); }); + test::FunctionVideoDecoderFactory decoder_factory( + []() { return H264Decoder::Create(); }); CodecObserver test(500, kVideoRotation_0, "H264", &encoder_factory, - H264Decoder::Create()); + &decoder_factory); RunBaseTest(&test); } TEST_P(EndToEndTestH264, SendsAndReceivesH264VideoRotation90) { test::FunctionVideoEncoderFactory encoder_factory( []() { return H264Encoder::Create(cricket::VideoCodec("H264")); }); + test::FunctionVideoDecoderFactory decoder_factory( + []() { return H264Decoder::Create(); }); CodecObserver test(5, kVideoRotation_90, "H264", &encoder_factory, - H264Decoder::Create()); + &decoder_factory); RunBaseTest(&test); } @@ -201,8 +220,10 @@ TEST_P(EndToEndTestH264, SendsAndReceivesH264PacketizationMode0) { codec.SetParam(cricket::kH264FmtpPacketizationMode, "0"); test::FunctionVideoEncoderFactory encoder_factory( [codec]() { return H264Encoder::Create(codec); }); + test::FunctionVideoDecoderFactory decoder_factory( + []() { return H264Decoder::Create(); }); CodecObserver test(500, kVideoRotation_0, "H264", &encoder_factory, - H264Decoder::Create()); + &decoder_factory); RunBaseTest(&test); } @@ -211,8 +232,10 @@ TEST_P(EndToEndTestH264, SendsAndReceivesH264PacketizationMode1) { codec.SetParam(cricket::kH264FmtpPacketizationMode, "1"); test::FunctionVideoEncoderFactory encoder_factory( [codec]() { return H264Encoder::Create(codec); }); + test::FunctionVideoDecoderFactory decoder_factory( + []() { return H264Decoder::Create(); }); CodecObserver test(500, kVideoRotation_0, "H264", &encoder_factory, - H264Decoder::Create()); + &decoder_factory); RunBaseTest(&test); } #endif // defined(WEBRTC_USE_H264) diff --git a/video/end_to_end_tests/fec_tests.cc b/video/end_to_end_tests/fec_tests.cc index 000f6d7e26..0300d12f8b 100644 --- a/video/end_to_end_tests/fec_tests.cc +++ b/video/end_to_end_tests/fec_tests.cc @@ -11,6 +11,7 @@ #include "api/test/simulated_network.h" #include "call/fake_network_pipe.h" #include "call/simulated_network.h" +#include "media/engine/internaldecoderfactory.h" #include "modules/rtp_rtcp/source/byte_io.h" #include "modules/video_coding/codecs/vp8/include/vp8.h" #include "test/call_test.h" @@ -98,7 +99,7 @@ TEST_F(FecEndToEndTest, ReceivesUlpfec) { encoder_config->codec_type = kVideoCodecVP8; VideoReceiveStream::Decoder decoder = test::CreateMatchingDecoder(*send_config); - decoder_.reset(decoder.decoder); + decoder.decoder_factory = &decoder_factory_; (*receive_configs)[0].decoders.clear(); (*receive_configs)[0].decoders.push_back(decoder); @@ -119,7 +120,7 @@ TEST_F(FecEndToEndTest, ReceivesUlpfec) { rtc::CriticalSection crit_; std::unique_ptr encoder_; test::FunctionVideoEncoderFactory encoder_factory_; - std::unique_ptr decoder_; + InternalDecoderFactory decoder_factory_; std::set dropped_sequence_numbers_ RTC_GUARDED_BY(crit_); // Several packets can have the same timestamp. std::multiset dropped_timestamps_ RTC_GUARDED_BY(crit_); @@ -329,8 +330,7 @@ TEST_F(FecEndToEndTest, ReceivedUlpfecPacketsNotNacked) { ulpfec_sequence_number_(0), has_last_sequence_number_(false), last_sequence_number_(0), - encoder_factory_([]() { return VP8Encoder::Create(); }), - decoder_(VP8Decoder::Create()) {} + encoder_factory_([]() { return VP8Encoder::Create(); }) {} private: Action OnSendRtp(const uint8_t* packet, size_t length) override { @@ -461,7 +461,7 @@ TEST_F(FecEndToEndTest, ReceivedUlpfecPacketsNotNacked) { send_config->rtp.payload_type; (*receive_configs)[0].decoders[0].video_format = SdpVideoFormat(send_config->rtp.payload_name); - (*receive_configs)[0].decoders[0].decoder = decoder_.get(); + (*receive_configs)[0].decoders[0].decoder_factory = &decoder_factory_; } void PerformTest() override { @@ -482,9 +482,8 @@ TEST_F(FecEndToEndTest, ReceivedUlpfecPacketsNotNacked) { uint16_t ulpfec_sequence_number_ RTC_GUARDED_BY(&crit_); bool has_last_sequence_number_; uint16_t last_sequence_number_; - std::unique_ptr encoder_; test::FunctionVideoEncoderFactory encoder_factory_; - std::unique_ptr decoder_; + InternalDecoderFactory decoder_factory_; } test; RunBaseTest(&test); diff --git a/video/end_to_end_tests/log_tests.cc b/video/end_to_end_tests/log_tests.cc index aa62383240..77466b6d8d 100644 --- a/video/end_to_end_tests/log_tests.cc +++ b/video/end_to_end_tests/log_tests.cc @@ -8,6 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include "media/engine/internaldecoderfactory.h" #include "modules/video_coding/codecs/vp8/include/vp8.h" #include "rtc_base/file.h" #include "test/call_test.h" @@ -66,8 +67,6 @@ TEST_F(LogEndToEndTest, LogsEncodedFramesWhenRequested) { VideoSendStream::Config* send_config, std::vector* receive_configs, VideoEncoderConfig* encoder_config) override { - decoder_ = VP8Decoder::Create(); - send_config->post_encode_callback = this; send_config->rtp.payload_name = "VP8"; send_config->encoder_settings.encoder_factory = &encoder_factory_; @@ -78,7 +77,7 @@ TEST_F(LogEndToEndTest, LogsEncodedFramesWhenRequested) { send_config->rtp.payload_type; (*receive_configs)[0].decoders[0].video_format = SdpVideoFormat(send_config->rtp.payload_name); - (*receive_configs)[0].decoders[0].decoder = decoder_.get(); + (*receive_configs)[0].decoders[0].decoder_factory = &decoder_factory_; } void EncodedFrameCallback(const EncodedFrame& encoded_frame) override { @@ -97,7 +96,7 @@ TEST_F(LogEndToEndTest, LogsEncodedFramesWhenRequested) { private: LogEndToEndTest* const fixture_; test::FunctionVideoEncoderFactory encoder_factory_; - std::unique_ptr decoder_; + InternalDecoderFactory decoder_factory_; rtc::CriticalSection crit_; int recorded_frames_ RTC_GUARDED_BY(crit_); } test(this); diff --git a/video/end_to_end_tests/multi_codec_receive_tests.cc b/video/end_to_end_tests/multi_codec_receive_tests.cc index 7399ef81a4..e13bc351ca 100644 --- a/video/end_to_end_tests/multi_codec_receive_tests.cc +++ b/video/end_to_end_tests/multi_codec_receive_tests.cc @@ -180,6 +180,7 @@ class MultiCodecReceiveTest : public test::CallTest { struct CodecConfig { std::string payload_name; VideoEncoderFactory* encoder_factory; + VideoDecoderFactory* decoder_factory; size_t num_temporal_layers; }; @@ -197,18 +198,16 @@ class MultiCodecReceiveTest : public test::CallTest { void MultiCodecReceiveTest::ConfigureDecoders( const std::vector& configs) { + video_receive_configs_[0].decoders.clear(); // Placing the payload names in a std::set retains the unique names only. std::set unique_payload_names; for (const auto& config : configs) - unique_payload_names.insert(config.payload_name); + if (unique_payload_names.insert(config.payload_name).second) { + VideoReceiveStream::Decoder decoder = test::CreateMatchingDecoder( + PayloadNameToPayloadType(config.payload_name), config.payload_name); + decoder.decoder_factory = config.decoder_factory; - video_receive_configs_[0].decoders.clear(); - for (const auto& payload_name : unique_payload_names) { - VideoReceiveStream::Decoder decoder = test::CreateMatchingDecoder( - PayloadNameToPayloadType(payload_name), payload_name); - allocated_decoders_.push_back( - std::unique_ptr(decoder.decoder)); - video_receive_configs_[0].decoders.push_back(decoder); + video_receive_configs_[0].decoders.push_back(decoder); } } @@ -270,9 +269,13 @@ TEST_F(MultiCodecReceiveTest, SingleStreamReceivesVp8Vp9) { []() { return VP8Encoder::Create(); }); test::FunctionVideoEncoderFactory vp9_encoder_factory( []() { return VP9Encoder::Create(); }); - RunTestWithCodecs({{"VP8", &vp8_encoder_factory, 1}, - {"VP9", &vp9_encoder_factory, 1}, - {"VP8", &vp8_encoder_factory, 1}}); + test::FunctionVideoDecoderFactory vp8_decoder_factory( + []() { return VP8Decoder::Create(); }); + test::FunctionVideoDecoderFactory vp9_decoder_factory( + []() { return VP9Decoder::Create(); }); + RunTestWithCodecs({{"VP8", &vp8_encoder_factory, &vp8_decoder_factory, 1}, + {"VP9", &vp9_encoder_factory, &vp9_decoder_factory, 1}, + {"VP8", &vp8_encoder_factory, &vp8_decoder_factory, 1}}); } TEST_F(MultiCodecReceiveTest, SingleStreamReceivesVp8Vp9WithTl) { @@ -280,9 +283,13 @@ TEST_F(MultiCodecReceiveTest, SingleStreamReceivesVp8Vp9WithTl) { []() { return VP8Encoder::Create(); }); test::FunctionVideoEncoderFactory vp9_encoder_factory( []() { return VP9Encoder::Create(); }); - RunTestWithCodecs({{"VP8", &vp8_encoder_factory, 2}, - {"VP9", &vp9_encoder_factory, 2}, - {"VP8", &vp8_encoder_factory, 2}}); + test::FunctionVideoDecoderFactory vp8_decoder_factory( + []() { return VP8Decoder::Create(); }); + test::FunctionVideoDecoderFactory vp9_decoder_factory( + []() { return VP9Decoder::Create(); }); + RunTestWithCodecs({{"VP8", &vp8_encoder_factory, &vp8_decoder_factory, 2}, + {"VP9", &vp9_encoder_factory, &vp9_decoder_factory, 2}, + {"VP8", &vp8_encoder_factory, &vp8_decoder_factory, 2}}); } #if defined(WEBRTC_USE_H264) @@ -291,9 +298,13 @@ TEST_F(MultiCodecReceiveTest, SingleStreamReceivesVp8H264) { []() { return VP8Encoder::Create(); }); test::FunctionVideoEncoderFactory h264_encoder_factory( []() { return H264Encoder::Create(cricket::VideoCodec("H264")); }); - RunTestWithCodecs({{"VP8", &vp8_encoder_factory, 1}, - {"H264", &h264_encoder_factory, 1}, - {"VP8", &vp8_encoder_factory, 1}}); + test::FunctionVideoDecoderFactory vp8_decoder_factory( + []() { return VP8Decoder::Create(); }); + test::FunctionVideoDecoderFactory h264_decoder_factory( + []() { return H264Decoder::Create(); }); + RunTestWithCodecs({{"VP8", &vp8_encoder_factory, &vp8_decoder_factory, 1}, + {"H264", &h264_encoder_factory, &h264_decoder_factory, 1}, + {"VP8", &vp8_encoder_factory, &vp8_decoder_factory, 1}}); } TEST_F(MultiCodecReceiveTest, SingleStreamReceivesVp8H264WithTl) { @@ -301,9 +312,13 @@ TEST_F(MultiCodecReceiveTest, SingleStreamReceivesVp8H264WithTl) { []() { return VP8Encoder::Create(); }); test::FunctionVideoEncoderFactory h264_encoder_factory( []() { return H264Encoder::Create(cricket::VideoCodec("H264")); }); - RunTestWithCodecs({{"VP8", &vp8_encoder_factory, 3}, - {"H264", &h264_encoder_factory, 1}, - {"VP8", &vp8_encoder_factory, 3}}); + test::FunctionVideoDecoderFactory vp8_decoder_factory( + []() { return VP8Decoder::Create(); }); + test::FunctionVideoDecoderFactory h264_decoder_factory( + []() { return H264Decoder::Create(); }); + RunTestWithCodecs({{"VP8", &vp8_encoder_factory, &vp8_decoder_factory, 3}, + {"H264", &h264_encoder_factory, &h264_decoder_factory, 1}, + {"VP8", &vp8_encoder_factory, &vp8_decoder_factory, 3}}); } TEST_F(MultiCodecReceiveTest, SingleStreamReceivesVp8Vp9H264) { @@ -313,10 +328,16 @@ TEST_F(MultiCodecReceiveTest, SingleStreamReceivesVp8Vp9H264) { []() { return VP9Encoder::Create(); }); test::FunctionVideoEncoderFactory h264_encoder_factory( []() { return H264Encoder::Create(cricket::VideoCodec("H264")); }); - RunTestWithCodecs({{"VP8", &vp8_encoder_factory, 1}, - {"VP9", &vp9_encoder_factory, 1}, - {"H264", &h264_encoder_factory, 1}, - {"VP9", &vp9_encoder_factory, 1}}); + test::FunctionVideoDecoderFactory vp8_decoder_factory( + []() { return VP8Decoder::Create(); }); + test::FunctionVideoDecoderFactory vp9_decoder_factory( + []() { return VP9Decoder::Create(); }); + test::FunctionVideoDecoderFactory h264_decoder_factory( + []() { return H264Decoder::Create(); }); + RunTestWithCodecs({{"VP8", &vp8_encoder_factory, &vp8_decoder_factory, 1}, + {"VP9", &vp9_encoder_factory, &vp9_decoder_factory, 1}, + {"H264", &h264_encoder_factory, &h264_decoder_factory, 1}, + {"VP9", &vp9_encoder_factory, &vp9_decoder_factory, 1}}); } TEST_F(MultiCodecReceiveTest, SingleStreamReceivesVp8Vp9H264WithTl) { @@ -326,10 +347,16 @@ TEST_F(MultiCodecReceiveTest, SingleStreamReceivesVp8Vp9H264WithTl) { []() { return VP9Encoder::Create(); }); test::FunctionVideoEncoderFactory h264_encoder_factory( []() { return H264Encoder::Create(cricket::VideoCodec("H264")); }); - RunTestWithCodecs({{"VP8", &vp8_encoder_factory, 3}, - {"VP9", &vp9_encoder_factory, 2}, - {"H264", &h264_encoder_factory, 1}, - {"VP9", &vp9_encoder_factory, 3}}); + test::FunctionVideoDecoderFactory vp8_decoder_factory( + []() { return VP8Decoder::Create(); }); + test::FunctionVideoDecoderFactory vp9_decoder_factory( + []() { return VP9Decoder::Create(); }); + test::FunctionVideoDecoderFactory h264_decoder_factory( + []() { return H264Decoder::Create(); }); + RunTestWithCodecs({{"VP8", &vp8_encoder_factory, &vp8_decoder_factory, 3}, + {"VP9", &vp9_encoder_factory, &vp9_decoder_factory, 2}, + {"H264", &h264_encoder_factory, &h264_decoder_factory, 1}, + {"VP9", &vp9_encoder_factory, &vp9_decoder_factory, 3}}); } #endif // defined(WEBRTC_USE_H264) diff --git a/video/end_to_end_tests/multi_stream_tester.cc b/video/end_to_end_tests/multi_stream_tester.cc index c4f99e68c1..7d66976001 100644 --- a/video/end_to_end_tests/multi_stream_tester.cc +++ b/video/end_to_end_tests/multi_stream_tester.cc @@ -18,6 +18,7 @@ #include "call/fake_network_pipe.h" #include "call/simulated_network.h" #include "logging/rtc_event_log/rtc_event_log.h" +#include "media/engine/internaldecoderfactory.h" #include "modules/video_coding/codecs/vp8/include/vp8.h" #include "test/call_test.h" #include "test/encoder_settings.h" @@ -51,9 +52,9 @@ void MultiStreamTester::RunTest() { VideoSendStream* send_streams[kNumStreams]; VideoReceiveStream* receive_streams[kNumStreams]; test::FrameGeneratorCapturer* frame_generators[kNumStreams]; - std::vector> allocated_decoders; test::FunctionVideoEncoderFactory encoder_factory( []() { return VP8Encoder::Create(); }); + InternalDecoderFactory decoder_factory; task_queue_->SendTask([&]() { sender_call = absl::WrapUnique(Call::Create(config)); @@ -91,8 +92,7 @@ void MultiStreamTester::RunTest() { receive_config.rtp.local_ssrc = test::CallTest::kReceiverLocalVideoSsrc; VideoReceiveStream::Decoder decoder = test::CreateMatchingDecoder(send_config); - allocated_decoders.push_back( - std::unique_ptr(decoder.decoder)); + decoder.decoder_factory = &decoder_factory; receive_config.decoders.push_back(decoder); UpdateReceiveConfig(i, &receive_config); diff --git a/video/end_to_end_tests/network_state_tests.cc b/video/end_to_end_tests/network_state_tests.cc index 59c2b51027..937777f6b3 100644 --- a/video/end_to_end_tests/network_state_tests.cc +++ b/video/end_to_end_tests/network_state_tests.cc @@ -13,9 +13,9 @@ #include "call/simulated_network.h" #include "system_wrappers/include/sleep.h" #include "test/call_test.h" -#include "test/encoder_proxy_factory.h" #include "test/fake_encoder.h" #include "test/gtest.h" +#include "test/video_encoder_proxy_factory.h" namespace webrtc { namespace { @@ -82,7 +82,7 @@ void NetworkStateEndToEndTest::VerifyNewVideoSendStreamsRespectNetworkState( MediaType network_to_bring_up, VideoEncoder* encoder, Transport* transport) { - test::EncoderProxyFactory encoder_factory(encoder); + test::VideoEncoderProxyFactory encoder_factory(encoder); task_queue_.SendTask([this, network_to_bring_up, &encoder_factory, transport]() { @@ -341,7 +341,7 @@ TEST_F(NetworkStateEndToEndTest, RespectsNetworkState) { rtc::Event packet_event_; Call* sender_call_; Call* receiver_call_; - test::EncoderProxyFactory encoder_factory_; + test::VideoEncoderProxyFactory encoder_factory_; NetworkState sender_state_ RTC_GUARDED_BY(test_crit_); int sender_rtp_ RTC_GUARDED_BY(test_crit_); int sender_padding_ RTC_GUARDED_BY(test_crit_); diff --git a/video/replay.cc b/video/replay.cc index a5eafae4e7..10aec6f944 100644 --- a/video/replay.cc +++ b/video/replay.cc @@ -19,6 +19,7 @@ #include "call/call.h" #include "common_video/libyuv/include/webrtc_libyuv.h" #include "logging/rtc_event_log/rtc_event_log.h" +#include "media/engine/internaldecoderfactory.h" #include "modules/rtp_rtcp/include/rtp_header_parser.h" #include "rtc_base/checks.h" #include "rtc_base/file.h" @@ -31,6 +32,7 @@ #include "test/call_test.h" #include "test/encoder_settings.h" #include "test/fake_decoder.h" +#include "test/function_video_decoder_factory.h" #include "test/gtest.h" #include "test/null_transport.h" #include "test/rtp_file_reader.h" @@ -330,6 +332,7 @@ class RtpReplayer final { test::NullTransport transport; std::vector>> sinks; std::vector receive_streams; + std::unique_ptr decoder_factory; }; // Loads multiple configurations from the provided configuration file. @@ -350,6 +353,7 @@ class RtpReplayer final { return nullptr; } + stream_state->decoder_factory = absl::make_unique(); size_t config_count = 0; for (const auto& json : json_configs) { // Create the configuration and parse the JSON into the config. @@ -357,9 +361,9 @@ class RtpReplayer final { &(stream_state->transport), json); // Instantiate the underlying decoder. for (auto& decoder : receive_config.decoders) { - decoder.decoder = test::CreateMatchingDecoder(decoder.payload_type, - decoder.video_format.name) - .decoder; + decoder = test::CreateMatchingDecoder(decoder.payload_type, + decoder.video_format.name); + decoder.decoder_factory = stream_state->decoder_factory.get(); } // Create a window for this config. std::stringstream window_title; @@ -417,13 +421,19 @@ class RtpReplayer final { VideoReceiveStream::Decoder decoder; decoder = test::CreateMatchingDecoder(flags::MediaPayloadType(), flags::Codec()); - if (!flags::DecoderBitstreamFilename().empty()) { + if (flags::DecoderBitstreamFilename().empty()) { + stream_state->decoder_factory = + absl::make_unique(); + } else { // Replace decoder with file writer if we're writing the bitstream to a // file instead. - delete decoder.decoder; - decoder.decoder = new DecoderBitstreamFileWriter( - flags::DecoderBitstreamFilename().c_str()); + stream_state->decoder_factory = + absl::make_unique([]() { + return absl::make_unique( + flags::DecoderBitstreamFilename().c_str()); + }); } + decoder.decoder_factory = stream_state->decoder_factory.get(); receive_config.decoders.push_back(decoder); stream_state->receive_streams.emplace_back( diff --git a/video/video_quality_test.cc b/video/video_quality_test.cc index cd4c5f7608..838825e53b 100644 --- a/video/video_quality_test.cc +++ b/video/video_quality_test.cc @@ -495,7 +495,6 @@ void VideoQualityTest::SetupVideo(Transport* send_transport, video_receive_configs_.clear(); video_send_configs_.clear(); video_encoder_configs_.clear(); - allocated_decoders_.clear(); bool decode_all_receive_streams = true; size_t num_video_substreams = params_.ss[0].streams.size(); RTC_CHECK(num_video_streams_ > 0); diff --git a/video/video_receive_stream.cc b/video/video_receive_stream.cc index 7facf85dda..186ce4beb5 100644 --- a/video/video_receive_stream.cc +++ b/video/video_receive_stream.cc @@ -18,6 +18,7 @@ #include "absl/memory/memory.h" #include "absl/types/optional.h" +#include "api/video_codecs/video_decoder_factory.h" #include "call/rtp_stream_receiver_controller_interface.h" #include "call/rtx_receive_stream.h" #include "common_types.h" // NOLINT(build/include) @@ -75,6 +76,37 @@ VideoCodec CreateDecoderVideoCodec(const VideoReceiveStream::Decoder& decoder) { return codec; } + +// Video decoder class to be used for unknown codecs. Doesn't support decoding +// but logs messages to LS_ERROR. +class NullVideoDecoder : public webrtc::VideoDecoder { + public: + int32_t InitDecode(const webrtc::VideoCodec* codec_settings, + int32_t number_of_cores) override { + RTC_LOG(LS_ERROR) << "Can't initialize NullVideoDecoder."; + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t Decode(const webrtc::EncodedImage& input_image, + bool missing_frames, + const webrtc::CodecSpecificInfo* codec_specific_info, + int64_t render_time_ms) override { + RTC_LOG(LS_ERROR) << "The NullVideoDecoder doesn't support decoding."; + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t RegisterDecodeCompleteCallback( + webrtc::DecodedImageCallback* callback) override { + RTC_LOG(LS_ERROR) + << "Can't register decode complete callback on NullVideoDecoder."; + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t Release() override { return WEBRTC_VIDEO_CODEC_OK; } + + const char* ImplementationName() const override { return "NullVideoDecoder"; } +}; + } // namespace namespace internal { @@ -121,7 +153,7 @@ VideoReceiveStream::VideoReceiveStream( RTC_DCHECK(!config_.decoders.empty()); std::set decoder_payload_types; for (const Decoder& decoder : config_.decoders) { - RTC_CHECK(decoder.decoder); + RTC_CHECK(decoder.decoder_factory); RTC_CHECK(decoder_payload_types.find(decoder.payload_type) == decoder_payload_types.end()) << "Duplicate payload type (" << decoder.payload_type @@ -203,7 +235,19 @@ void VideoReceiveStream::Start() { RTC_DCHECK(renderer != nullptr); for (const Decoder& decoder : config_.decoders) { - video_receiver_.RegisterExternalDecoder(decoder.decoder, + std::unique_ptr video_decoder = + decoder.decoder_factory->LegacyCreateVideoDecoder(decoder.video_format, + config_.stream_id); + // If we still have no valid decoder, we have to create a "Null" decoder + // that ignores all calls. The reason we can get into this state is that the + // old decoder factory interface doesn't have a way to query supported + // codecs. + if (!video_decoder) { + video_decoder = absl::make_unique(); + } + video_decoders_.push_back(std::move(video_decoder)); + + video_receiver_.RegisterExternalDecoder(video_decoders_.back().get(), decoder.payload_type); VideoCodec codec = CreateDecoderVideoCodec(decoder); rtp_video_stream_receiver_.AddReceiveCodec(codec, diff --git a/video/video_receive_stream.h b/video/video_receive_stream.h index 76ee2ed0fd..fb0ee3bd34 100644 --- a/video/video_receive_stream.h +++ b/video/video_receive_stream.h @@ -143,6 +143,10 @@ class VideoReceiveStream : public webrtc::VideoReceiveStream, std::unique_ptr video_stream_decoder_; RtpStreamsSynchronizer rtp_stream_sync_; + // TODO(nisse, philipel): Creation and ownership of video encoders should be + // moved to the new VideoStreamDecoder. + std::vector> video_decoders_; + rtc::CriticalSection ivf_writer_lock_; std::unique_ptr ivf_writer_ RTC_GUARDED_BY(ivf_writer_lock_); diff --git a/video/video_receive_stream_unittest.cc b/video/video_receive_stream_unittest.cc index 4b433d28e8..e4ba69b7b9 100644 --- a/video/video_receive_stream_unittest.cc +++ b/video/video_receive_stream_unittest.cc @@ -23,6 +23,7 @@ #include "rtc_base/event.h" #include "system_wrappers/include/clock.h" #include "test/field_trial.h" +#include "test/video_decoder_proxy_factory.h" #include "video/call_stats.h" #include "video/video_receive_stream.h" @@ -65,7 +66,9 @@ class VideoReceiveStreamTest : public testing::Test { VideoReceiveStreamTest() : process_thread_(ProcessThread::Create("TestThread")), config_(&mock_transport_), - call_stats_(Clock::GetRealTimeClock(), process_thread_.get()) {} + call_stats_(Clock::GetRealTimeClock(), process_thread_.get()), + h264_decoder_factory_(&mock_h264_video_decoder_), + null_decoder_factory_(&mock_null_video_decoder_) {} void SetUp() { constexpr int kDefaultNumCpuCores = 2; @@ -77,12 +80,12 @@ class VideoReceiveStreamTest : public testing::Test { h264_decoder.video_format = SdpVideoFormat("H264"); h264_decoder.video_format.parameters.insert( {"sprop-parameter-sets", "Z0IACpZTBYmI,aMljiA=="}); - h264_decoder.decoder = &mock_h264_video_decoder_; + h264_decoder.decoder_factory = &h264_decoder_factory_; config_.decoders.push_back(h264_decoder); VideoReceiveStream::Decoder null_decoder; null_decoder.payload_type = 98; null_decoder.video_format = SdpVideoFormat("null"); - null_decoder.decoder = &mock_null_video_decoder_; + null_decoder.decoder_factory = &null_decoder_factory_; config_.decoders.push_back(null_decoder); video_receive_stream_.reset(new webrtc::internal::VideoReceiveStream( @@ -96,6 +99,8 @@ class VideoReceiveStreamTest : public testing::Test { CallStats call_stats_; MockVideoDecoder mock_h264_video_decoder_; MockVideoDecoder mock_null_video_decoder_; + test::VideoDecoderProxyFactory h264_decoder_factory_; + test::VideoDecoderProxyFactory null_decoder_factory_; cricket::FakeVideoRenderer fake_renderer_; MockTransport mock_transport_; PacketRouter packet_router_; diff --git a/video/video_send_stream_tests.cc b/video/video_send_stream_tests.cc index c015f56142..a4d5e195e4 100644 --- a/video/video_send_stream_tests.cc +++ b/video/video_send_stream_tests.cc @@ -36,7 +36,6 @@ #include "system_wrappers/include/sleep.h" #include "test/call_test.h" #include "test/configurable_frame_size_encoder.h" -#include "test/encoder_proxy_factory.h" #include "test/fake_encoder.h" #include "test/fake_texture_frame.h" #include "test/field_trial.h" @@ -48,6 +47,7 @@ #include "test/null_transport.h" #include "test/rtcp_packet_parser.h" #include "test/testsupport/perf_test.h" +#include "test/video_encoder_proxy_factory.h" #include "call/video_send_stream.h" #include "video/send_statistics_proxy.h" @@ -1173,7 +1173,7 @@ void VideoSendStreamTest::TestPacketFragmentationSize(VideoFormat format, std::unique_ptr transport_adapter_; test::ConfigurableFrameSizeEncoder encoder_; - test::EncoderProxyFactory encoder_factory_; + test::VideoEncoderProxyFactory encoder_factory_; const size_t max_packet_size_; const size_t stop_size_; @@ -1961,7 +1961,7 @@ TEST_P(VideoSendStreamTest, test::NullTransport transport; EncoderObserver encoder; - test::EncoderProxyFactory encoder_factory(&encoder); + test::VideoEncoderProxyFactory encoder_factory(&encoder); task_queue_.SendTask([this, &transport, &encoder_factory]() { CreateSenderCall(); @@ -2039,7 +2039,7 @@ TEST_P(VideoSendStreamTest, CanReconfigureToUseStartBitrateAbovePreviousMax) { bitrate_config); StartBitrateObserver encoder; - test::EncoderProxyFactory encoder_factory(&encoder); + test::VideoEncoderProxyFactory encoder_factory(&encoder); // Since this test does not use a capturer, set |internal_source| = true. // Encoder configuration is otherwise updated on the next video frame. encoder_factory.SetHasInternalSource(true); @@ -2124,7 +2124,7 @@ class StartStopBitrateObserver : public test::FakeEncoder { TEST_P(VideoSendStreamTest, VideoSendStreamStopSetEncoderRateToZero) { test::NullTransport transport; StartStopBitrateObserver encoder; - test::EncoderProxyFactory encoder_factory(&encoder); + test::VideoEncoderProxyFactory encoder_factory(&encoder); encoder_factory.SetHasInternalSource(true); test::FrameForwarder forwarder; @@ -2167,7 +2167,7 @@ TEST_P(VideoSendStreamTest, VideoSendStreamStopSetEncoderRateToZero) { TEST_P(VideoSendStreamTest, VideoSendStreamUpdateActiveSimulcastLayers) { test::NullTransport transport; StartStopBitrateObserver encoder; - test::EncoderProxyFactory encoder_factory(&encoder); + test::VideoEncoderProxyFactory encoder_factory(&encoder); encoder_factory.SetHasInternalSource(true); test::FrameForwarder forwarder; @@ -2441,7 +2441,7 @@ TEST_P(VideoSendStreamTest, EncoderIsProperlyInitializedAndDestroyed) { bool callback_registered_ RTC_GUARDED_BY(crit_); size_t num_releases_ RTC_GUARDED_BY(crit_); bool released_ RTC_GUARDED_BY(crit_); - test::EncoderProxyFactory encoder_factory_; + test::VideoEncoderProxyFactory encoder_factory_; VideoEncoderConfig encoder_config_; } test_encoder(&task_queue_); @@ -2512,7 +2512,7 @@ TEST_P(VideoSendStreamTest, EncoderSetupPropagatesCommonEncoderConfigValues) { rtc::Event init_encode_event_; size_t num_initializations_; VideoSendStream* stream_; - test::EncoderProxyFactory encoder_factory_; + test::VideoEncoderProxyFactory encoder_factory_; VideoEncoderConfig encoder_config_; } test; @@ -2622,7 +2622,7 @@ class VideoCodecConfigObserver : public test::SendTest, rtc::Event init_encode_event_; size_t num_initializations_; VideoSendStream* stream_; - test::EncoderProxyFactory encoder_factory_; + test::VideoEncoderProxyFactory encoder_factory_; VideoEncoderConfig encoder_config_; }; @@ -2840,7 +2840,7 @@ TEST_P(VideoSendStreamTest, TranslatesTwoLayerScreencastToTargetBitrate) { EXPECT_TRUE(Wait()) << "Timed out while waiting for the encoder to be initialized."; } - test::EncoderProxyFactory encoder_factory_; + test::VideoEncoderProxyFactory encoder_factory_; } test; RunBaseTest(&test); @@ -3020,7 +3020,7 @@ TEST_P(VideoSendStreamTest, ReconfigureBitratesSetsEncoderBitratesCorrectly) { int num_initializations_; webrtc::Call* call_; webrtc::VideoSendStream* send_stream_; - test::EncoderProxyFactory encoder_factory_; + test::VideoEncoderProxyFactory encoder_factory_; webrtc::VideoEncoderConfig encoder_config_; } test(&task_queue_); @@ -3109,7 +3109,7 @@ TEST_P(VideoSendStreamTest, ReportsSentResolution) { } VideoSendStream* send_stream_; - test::EncoderProxyFactory encoder_factory_; + test::VideoEncoderProxyFactory encoder_factory_; } test; RunBaseTest(&test); @@ -3741,7 +3741,7 @@ TEST_P(VideoSendStreamTest, RemoveOverheadFromBandwidth) { private: test::SingleThreadedTaskQueueForTesting* const task_queue_; - test::EncoderProxyFactory encoder_factory_; + test::VideoEncoderProxyFactory encoder_factory_; Call* call_; rtc::CriticalSection crit_; uint32_t max_bitrate_bps_ RTC_GUARDED_BY(&crit_); diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index b97d2cf621..bcccbca7b6 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -22,13 +22,13 @@ #include "rtc_base/refcountedobject.h" #include "system_wrappers/include/metrics_default.h" #include "system_wrappers/include/sleep.h" -#include "test/encoder_proxy_factory.h" #include "test/encoder_settings.h" #include "test/fake_encoder.h" #include "test/field_trial.h" #include "test/frame_generator.h" #include "test/gmock.h" #include "test/gtest.h" +#include "test/video_encoder_proxy_factory.h" #include "video/send_statistics_proxy.h" #include "video/video_stream_encoder.h" @@ -691,7 +691,7 @@ class VideoStreamEncoderTest : public ::testing::Test { int codec_height_; int max_framerate_; TestEncoder fake_encoder_; - test::EncoderProxyFactory encoder_factory_; + test::VideoEncoderProxyFactory encoder_factory_; std::unique_ptr stats_proxy_; TestSink sink_; AdaptingFrameForwarder video_source_;