From 060ae594b385e79df4c250a8b60821374eef6b56 Mon Sep 17 00:00:00 2001 From: Per Kjellander Date: Wed, 30 Oct 2024 15:06:34 +0000 Subject: [PATCH] Modify Demo for Linux Fallback to squared pixel generator if no camera exist. Remove render scaling and rendering of local preview since contains bugs that crash the demo. Use rtc::Buffer for storing the rendered frames. Run build cleaner Bug: none Change-Id: I46dc972eaa50069433d8afaa1fe38380edd3d337 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/367120 Reviewed-by: Harald Alvestrand Commit-Queue: Per Kjellander Cr-Commit-Position: refs/heads/main@{#43350} --- examples/BUILD.gn | 12 ++ examples/DEPS | 1 + examples/peerconnection/client/conductor.cc | 115 +++++++++++------- examples/peerconnection/client/conductor.h | 8 +- .../peerconnection/client/linux/main_wnd.cc | 79 ++++-------- .../peerconnection/client/linux/main_wnd.h | 12 +- 6 files changed, 123 insertions(+), 104 deletions(-) diff --git a/examples/BUILD.gn b/examples/BUILD.gn index b793c8bff8..2f731ced22 100644 --- a/examples/BUILD.gn +++ b/examples/BUILD.gn @@ -683,17 +683,24 @@ if (is_linux || is_chromeos || is_win) { ] deps = [ + "../api:array_view", "../api:async_dns_resolver", "../api:audio_options_api", + "../api:create_frame_generator", "../api:create_peerconnection_factory", + "../api:enable_media", "../api:libjingle_peerconnection_api", + "../api:make_ref_counted", "../api:media_stream_interface", + "../api:rtc_error", "../api:rtp_sender_interface", "../api:scoped_refptr", "../api/audio:audio_device", "../api/audio:audio_mixer_api", "../api/audio:audio_processing", "../api/audio_codecs:audio_codecs_api", + "../api/task_queue", + "../api/task_queue:default_task_queue_factory", "../api/task_queue:pending_task_safety_flag", "../api/units:time_delta", "../api/video:video_frame", @@ -706,6 +713,7 @@ if (is_linux || is_chromeos || is_win) { "../p2p:rtc_p2p", "../pc:video_track_source", "../rtc_base:async_dns_resolver", + "../rtc_base:buffer", "../rtc_base:checks", "../rtc_base:logging", "../rtc_base:macromagic", @@ -716,11 +724,15 @@ if (is_linux || is_chromeos || is_win) { "../rtc_base:stringutils", "../rtc_base:threading", "../rtc_base/third_party/sigslot", + "../system_wrappers", "../system_wrappers:field_trial", "../test:field_trial", + "../test:frame_generator_capturer", "../test:platform_video_capturer", "../test:rtp_test_utils", + "../test:test_video_capturer", "//third_party/abseil-cpp/absl/memory", + "//third_party/jsoncpp", ] if (is_win) { sources += [ diff --git a/examples/DEPS b/examples/DEPS index 114cda384b..8d512c2146 100644 --- a/examples/DEPS +++ b/examples/DEPS @@ -1,5 +1,6 @@ include_rules = [ "+common_video", + "+json", "+logging/rtc_event_log/rtc_event_log_factory.h", "+media", "+modules/audio_device", diff --git a/examples/peerconnection/client/conductor.cc b/examples/peerconnection/client/conductor.cc index cbe11ebf70..36d38161bd 100644 --- a/examples/peerconnection/client/conductor.cc +++ b/examples/peerconnection/client/conductor.cc @@ -11,48 +11,62 @@ #include "examples/peerconnection/client/conductor.h" #include -#include #include #include +#include #include #include #include "absl/memory/memory.h" -#include "api/audio/audio_device.h" -#include "api/audio/audio_mixer.h" -#include "api/audio/audio_processing.h" -#include "api/audio_codecs/audio_decoder_factory.h" -#include "api/audio_codecs/audio_encoder_factory.h" #include "api/audio_codecs/builtin_audio_decoder_factory.h" #include "api/audio_codecs/builtin_audio_encoder_factory.h" #include "api/audio_options.h" #include "api/create_peerconnection_factory.h" +#include "api/enable_media.h" +#include "api/jsep.h" +#include "api/make_ref_counted.h" +#include "api/media_stream_interface.h" +#include "api/peer_connection_interface.h" +#include "api/rtc_error.h" +#include "api/rtp_receiver_interface.h" #include "api/rtp_sender_interface.h" -#include "api/video_codecs/video_decoder_factory.h" +#include "api/scoped_refptr.h" +#include "api/task_queue/default_task_queue_factory.h" +#include "api/task_queue/task_queue_factory.h" +#include "api/test/create_frame_generator.h" +#include "api/video/video_frame.h" +#include "api/video/video_source_interface.h" #include "api/video_codecs/video_decoder_factory_template.h" #include "api/video_codecs/video_decoder_factory_template_dav1d_adapter.h" #include "api/video_codecs/video_decoder_factory_template_libvpx_vp8_adapter.h" #include "api/video_codecs/video_decoder_factory_template_libvpx_vp9_adapter.h" #include "api/video_codecs/video_decoder_factory_template_open_h264_adapter.h" -#include "api/video_codecs/video_encoder_factory.h" #include "api/video_codecs/video_encoder_factory_template.h" #include "api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h" #include "api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h" #include "api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h" #include "api/video_codecs/video_encoder_factory_template_open_h264_adapter.h" #include "examples/peerconnection/client/defaults.h" +#include "examples/peerconnection/client/main_wnd.h" +#include "examples/peerconnection/client/peer_connection_client.h" +#include "json/reader.h" +#include "json/value.h" +#include "json/writer.h" #include "modules/video_capture/video_capture.h" #include "modules/video_capture/video_capture_factory.h" -#include "p2p/base/port_allocator.h" #include "pc/video_track_source.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" -#include "rtc_base/rtc_certificate_generator.h" #include "rtc_base/strings/json.h" -#include "test/vcm_capturer.h" +#include "system_wrappers/include/clock.h" +#include "test/frame_generator_capturer.h" +#include "test/platform_video_capturer.h" +#include "test/test_video_capturer.h" namespace { +using webrtc::test::TestVideoCapturer; + // Names used for a IceCandidate JSON object. const char kCandidateSdpMidName[] = "sdpMid"; const char kCandidateSdpMlineIndexName[] = "sdpMLineIndex"; @@ -75,40 +89,53 @@ class DummySetSessionDescriptionObserver } }; +std::unique_ptr CreateCapturer( + webrtc::TaskQueueFactory& task_queue_factory) { + const size_t kWidth = 640; + const size_t kHeight = 480; + const size_t kFps = 30; + std::unique_ptr info( + webrtc::VideoCaptureFactory::CreateDeviceInfo()); + if (!info) { + return nullptr; + } + int num_devices = info->NumberOfDevices(); + for (int i = 0; i < num_devices; ++i) { + std::unique_ptr capturer = + webrtc::test::CreateVideoCapturer(kWidth, kHeight, kFps, i); + if (capturer) { + return capturer; + } + } + auto frame_generator = webrtc::test::CreateSquareFrameGenerator( + kWidth, kHeight, std::nullopt, std::nullopt); + return std::make_unique( + webrtc::Clock::GetRealTimeClock(), std::move(frame_generator), kFps, + task_queue_factory); +} class CapturerTrackSource : public webrtc::VideoTrackSource { public: - static rtc::scoped_refptr Create() { - const size_t kWidth = 640; - const size_t kHeight = 480; - const size_t kFps = 30; - std::unique_ptr capturer; - std::unique_ptr info( - webrtc::VideoCaptureFactory::CreateDeviceInfo()); - if (!info) { - return nullptr; + static rtc::scoped_refptr Create( + webrtc::TaskQueueFactory& task_queue_factory) { + std::unique_ptr capturer = + CreateCapturer(task_queue_factory); + if (capturer) { + capturer->Start(); + return rtc::make_ref_counted(std::move(capturer)); } - int num_devices = info->NumberOfDevices(); - for (int i = 0; i < num_devices; ++i) { - capturer = absl::WrapUnique( - webrtc::test::VcmCapturer::Create(kWidth, kHeight, kFps, i)); - if (capturer) { - return rtc::make_ref_counted(std::move(capturer)); - } - } - return nullptr; } protected: - explicit CapturerTrackSource( - std::unique_ptr capturer) + explicit CapturerTrackSource(std::unique_ptr capturer) : VideoTrackSource(/*remote=*/false), capturer_(std::move(capturer)) {} private: rtc::VideoSourceInterface* source() override { return capturer_.get(); } - std::unique_ptr capturer_; + + std::unique_ptr capturer_; }; } // namespace @@ -140,22 +167,28 @@ bool Conductor::InitializePeerConnection() { signaling_thread_ = rtc::Thread::CreateWithSocketServer(); signaling_thread_->Start(); } - peer_connection_factory_ = webrtc::CreatePeerConnectionFactory( - nullptr /* network_thread */, nullptr /* worker_thread */, - signaling_thread_.get(), nullptr /* default_adm */, - webrtc::CreateBuiltinAudioEncoderFactory(), - webrtc::CreateBuiltinAudioDecoderFactory(), + + webrtc::PeerConnectionFactoryDependencies deps; + deps.signaling_thread = signaling_thread_.get(); + deps.task_queue_factory = webrtc::CreateDefaultTaskQueueFactory(), + deps.audio_encoder_factory = webrtc::CreateBuiltinAudioEncoderFactory(); + deps.audio_decoder_factory = webrtc::CreateBuiltinAudioDecoderFactory(); + deps.video_encoder_factory = std::make_unique>(), + webrtc::LibaomAv1EncoderTemplateAdapter>>(); + deps.video_decoder_factory = std::make_unique>(), - nullptr /* audio_mixer */, nullptr /* audio_processing */); + webrtc::Dav1dDecoderTemplateAdapter>>(); + webrtc::EnableMedia(deps); + task_queue_factory_ = deps.task_queue_factory.get(); + peer_connection_factory_ = + webrtc::CreateModularPeerConnectionFactory(std::move(deps)); if (!peer_connection_factory_) { main_wnd_->MessageBox("Error", "Failed to initialize PeerConnectionFactory", @@ -481,7 +514,7 @@ void Conductor::AddTracks() { } rtc::scoped_refptr video_device = - CapturerTrackSource::Create(); + CapturerTrackSource::Create(*task_queue_factory_); if (video_device) { rtc::scoped_refptr video_track_( peer_connection_factory_->CreateVideoTrack(video_device, kVideoLabel)); diff --git a/examples/peerconnection/client/conductor.h b/examples/peerconnection/client/conductor.h index 80617d3cf4..dbde199377 100644 --- a/examples/peerconnection/client/conductor.h +++ b/examples/peerconnection/client/conductor.h @@ -12,13 +12,18 @@ #define EXAMPLES_PEERCONNECTION_CLIENT_CONDUCTOR_H_ #include -#include #include #include #include +#include "api/data_channel_interface.h" +#include "api/jsep.h" #include "api/media_stream_interface.h" #include "api/peer_connection_interface.h" +#include "api/rtc_error.h" +#include "api/rtp_receiver_interface.h" +#include "api/scoped_refptr.h" +#include "api/task_queue/task_queue_factory.h" #include "examples/peerconnection/client/main_wnd.h" #include "examples/peerconnection/client/peer_connection_client.h" #include "rtc_base/thread.h" @@ -124,6 +129,7 @@ class Conductor : public webrtc::PeerConnectionObserver, int peer_id_; bool loopback_; std::unique_ptr signaling_thread_; + webrtc::TaskQueueFactory* task_queue_factory_ = nullptr; rtc::scoped_refptr peer_connection_; rtc::scoped_refptr peer_connection_factory_; diff --git a/examples/peerconnection/client/linux/main_wnd.cc b/examples/peerconnection/client/linux/main_wnd.cc index 2be75d8f8d..c926d34eec 100644 --- a/examples/peerconnection/client/linux/main_wnd.cc +++ b/examples/peerconnection/client/linux/main_wnd.cc @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -22,17 +23,19 @@ #include #include -#include #include -#include +#include "api/media_stream_interface.h" +#include "api/scoped_refptr.h" #include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" #include "api/video/video_frame_buffer.h" #include "api/video/video_rotation.h" #include "api/video/video_source_interface.h" +#include "examples/peerconnection/client/main_wnd.h" +#include "examples/peerconnection/client/peer_connection_client.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" -#include "third_party/libyuv/include/libyuv/convert.h" #include "third_party/libyuv/include/libyuv/convert_from.h" namespace { @@ -312,7 +315,7 @@ void GtkMainWnd::SwitchToPeerList(const Peers& peers) { } else if (draw_area_) { gtk_widget_destroy(draw_area_); draw_area_ = NULL; - draw_buffer_.reset(); + draw_buffer_.SetSize(0); } peer_list_ = gtk_tree_view_new(); @@ -424,68 +427,29 @@ void GtkMainWnd::OnRedraw() { gdk_threads_enter(); VideoRenderer* remote_renderer = remote_renderer_.get(); - if (remote_renderer && remote_renderer->image() != NULL && + if (remote_renderer && !remote_renderer->image().empty() && draw_area_ != NULL) { - width_ = remote_renderer->width(); - height_ = remote_renderer->height(); - - if (!draw_buffer_.get()) { - draw_buffer_size_ = (width_ * height_ * 4) * 4; - draw_buffer_.reset(new uint8_t[draw_buffer_size_]); - gtk_widget_set_size_request(draw_area_, width_ * 2, height_ * 2); + if (width_ != remote_renderer->width() || + height_ != remote_renderer->height()) { + width_ = remote_renderer->width(); + height_ = remote_renderer->height(); + gtk_widget_set_size_request(draw_area_, remote_renderer->width(), + remote_renderer->height()); } - - const uint32_t* image = - reinterpret_cast(remote_renderer->image()); - uint32_t* scaled = reinterpret_cast(draw_buffer_.get()); - for (int r = 0; r < height_; ++r) { - for (int c = 0; c < width_; ++c) { - int x = c * 2; - scaled[x] = scaled[x + 1] = image[c]; - } - - uint32_t* prev_line = scaled; - scaled += width_ * 2; - memcpy(scaled, prev_line, (width_ * 2) * 4); - - image += width_; - scaled += width_ * 2; - } - - VideoRenderer* local_renderer = local_renderer_.get(); - if (local_renderer && local_renderer->image()) { - image = reinterpret_cast(local_renderer->image()); - scaled = reinterpret_cast(draw_buffer_.get()); - // Position the local preview on the right side. - scaled += (width_ * 2) - (local_renderer->width() / 2); - // right margin... - scaled -= 10; - // ... towards the bottom. - scaled += (height_ * width_ * 4) - ((local_renderer->height() / 2) * - (local_renderer->width() / 2) * 4); - // bottom margin... - scaled -= (width_ * 2) * 5; - for (int r = 0; r < local_renderer->height(); r += 2) { - for (int c = 0; c < local_renderer->width(); c += 2) { - scaled[c / 2] = image[c + r * local_renderer->width()]; - } - scaled += width_ * 2; - } - } - + draw_buffer_.SetData(remote_renderer->image()); gtk_widget_queue_draw(draw_area_); } - + // Here we can draw the local preview as well if we want.... gdk_threads_leave(); } void GtkMainWnd::Draw(GtkWidget* widget, cairo_t* cr) { cairo_format_t format = CAIRO_FORMAT_ARGB32; cairo_surface_t* surface = cairo_image_surface_create_for_data( - draw_buffer_.get(), format, width_ * 2, height_ * 2, - cairo_format_stride_for_width(format, width_ * 2)); + draw_buffer_.data(), format, width_, height_, + cairo_format_stride_for_width(format, width_)); cairo_set_source_surface(cr, surface, 0, 0); - cairo_rectangle(cr, 0, 0, width_ * 2, height_ * 2); + cairo_rectangle(cr, 0, 0, width_, height_); cairo_fill(cr); cairo_surface_destroy(surface); } @@ -513,7 +477,8 @@ void GtkMainWnd::VideoRenderer::SetSize(int width, int height) { width_ = width; height_ = height; - image_.reset(new uint8_t[width * height * 4]); + // ARGB + image_.SetSize(width * height * 4); gdk_threads_leave(); } @@ -536,7 +501,7 @@ void GtkMainWnd::VideoRenderer::OnFrame(const webrtc::VideoFrame& video_frame) { // native endianness. libyuv::I420ToARGB(buffer->DataY(), buffer->StrideY(), buffer->DataU(), buffer->StrideU(), buffer->DataV(), buffer->StrideV(), - image_.get(), width_ * 4, buffer->width(), + image_.data(), width_ * 4, buffer->width(), buffer->height()); gdk_threads_leave(); diff --git a/examples/peerconnection/client/linux/main_wnd.h b/examples/peerconnection/client/linux/main_wnd.h index 3b31e1be3b..aec76e816e 100644 --- a/examples/peerconnection/client/linux/main_wnd.h +++ b/examples/peerconnection/client/linux/main_wnd.h @@ -16,12 +16,14 @@ #include #include +#include "api/array_view.h" #include "api/media_stream_interface.h" #include "api/scoped_refptr.h" #include "api/video/video_frame.h" #include "api/video/video_sink_interface.h" #include "examples/peerconnection/client/main_wnd.h" #include "examples/peerconnection/client/peer_connection_client.h" +#include "rtc_base/buffer.h" // Forward declarations. typedef struct _GtkWidget GtkWidget; @@ -90,7 +92,7 @@ class GtkMainWnd : public MainWindow { // VideoSinkInterface implementation void OnFrame(const webrtc::VideoFrame& frame) override; - const uint8_t* image() const { return image_.get(); } + rtc::ArrayView image() const { return image_; } int width() const { return width_; } @@ -98,7 +100,7 @@ class GtkMainWnd : public MainWindow { protected: void SetSize(int width, int height); - std::unique_ptr image_; + rtc::Buffer image_; int width_; int height_; GtkMainWnd* main_wnd_; @@ -119,9 +121,9 @@ class GtkMainWnd : public MainWindow { bool autocall_; std::unique_ptr local_renderer_; std::unique_ptr remote_renderer_; - int width_; - int height_; - std::unique_ptr draw_buffer_; + int width_ = 0; + int height_ = 0; + rtc::Buffer draw_buffer_; int draw_buffer_size_; };