Modify Demo for Linux

Fallback to squared pixel generator if no camera exist. Remove render scaling and rendering of local preview since contains bugs that crash the demo. Use rtc::Buffer for storing the rendered frames. Run build cleaner Bug: none Change-Id: I46dc972eaa50069433d8afaa1fe38380edd3d337 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/367120 Reviewed-by: Harald Alvestrand <hta@webrtc.org> Commit-Queue: Per Kjellander <perkj@webrtc.org> Cr-Commit-Position: refs/heads/main@{#43350}
2024-10-30 15:06:34 +00:00 · 2024-10-30 15:06:34 +00:00 · 060ae594b3
commit 060ae594b3
parent 397645b2bb
6 changed files with 123 additions and 104 deletions
--- a/examples/BUILD.gn
+++ b/examples/BUILD.gn
@ -683,17 +683,24 @@ if (is_linux || is_chromeos || is_win) {
    ]

    deps = [
+      "../api:array_view",
      "../api:async_dns_resolver",
      "../api:audio_options_api",
+      "../api:create_frame_generator",
      "../api:create_peerconnection_factory",
+      "../api:enable_media",
      "../api:libjingle_peerconnection_api",
+      "../api:make_ref_counted",
      "../api:media_stream_interface",
+      "../api:rtc_error",
      "../api:rtp_sender_interface",
      "../api:scoped_refptr",
      "../api/audio:audio_device",
      "../api/audio:audio_mixer_api",
      "../api/audio:audio_processing",
      "../api/audio_codecs:audio_codecs_api",
+      "../api/task_queue",
+      "../api/task_queue:default_task_queue_factory",
      "../api/task_queue:pending_task_safety_flag",
      "../api/units:time_delta",
      "../api/video:video_frame",
@ -706,6 +713,7 @@ if (is_linux || is_chromeos || is_win) {
      "../p2p:rtc_p2p",
      "../pc:video_track_source",
      "../rtc_base:async_dns_resolver",
+      "../rtc_base:buffer",
      "../rtc_base:checks",
      "../rtc_base:logging",
      "../rtc_base:macromagic",
@ -716,11 +724,15 @@ if (is_linux || is_chromeos || is_win) {
      "../rtc_base:stringutils",
      "../rtc_base:threading",
      "../rtc_base/third_party/sigslot",
+      "../system_wrappers",
      "../system_wrappers:field_trial",
      "../test:field_trial",
+      "../test:frame_generator_capturer",
      "../test:platform_video_capturer",
      "../test:rtp_test_utils",
+      "../test:test_video_capturer",
      "//third_party/abseil-cpp/absl/memory",
+      "//third_party/jsoncpp",
    ]
    if (is_win) {
      sources += [
--- a/examples/DEPS
+++ b/examples/DEPS
@ -1,5 +1,6 @@
 include_rules = [
  "+common_video",
+  "+json",
  "+logging/rtc_event_log/rtc_event_log_factory.h",
  "+media",
  "+modules/audio_device",
--- a/examples/peerconnection/client/conductor.cc
+++ b/examples/peerconnection/client/conductor.cc
@ -11,48 +11,62 @@
 #include "examples/peerconnection/client/conductor.h"

 #include <stddef.h>
-#include <stdint.h>

 #include <memory>
 #include <optional>
+#include <string>
 #include <utility>
 #include <vector>

 #include "absl/memory/memory.h"
-#include "api/audio/audio_device.h"
-#include "api/audio/audio_mixer.h"
-#include "api/audio/audio_processing.h"
-#include "api/audio_codecs/audio_decoder_factory.h"
-#include "api/audio_codecs/audio_encoder_factory.h"
 #include "api/audio_codecs/builtin_audio_decoder_factory.h"
 #include "api/audio_codecs/builtin_audio_encoder_factory.h"
 #include "api/audio_options.h"
 #include "api/create_peerconnection_factory.h"
+#include "api/enable_media.h"
+#include "api/jsep.h"
+#include "api/make_ref_counted.h"
+#include "api/media_stream_interface.h"
+#include "api/peer_connection_interface.h"
+#include "api/rtc_error.h"
+#include "api/rtp_receiver_interface.h"
 #include "api/rtp_sender_interface.h"
-#include "api/video_codecs/video_decoder_factory.h"
+#include "api/scoped_refptr.h"
+#include "api/task_queue/default_task_queue_factory.h"
+#include "api/task_queue/task_queue_factory.h"
+#include "api/test/create_frame_generator.h"
+#include "api/video/video_frame.h"
+#include "api/video/video_source_interface.h"
 #include "api/video_codecs/video_decoder_factory_template.h"
 #include "api/video_codecs/video_decoder_factory_template_dav1d_adapter.h"
 #include "api/video_codecs/video_decoder_factory_template_libvpx_vp8_adapter.h"
 #include "api/video_codecs/video_decoder_factory_template_libvpx_vp9_adapter.h"
 #include "api/video_codecs/video_decoder_factory_template_open_h264_adapter.h"
-#include "api/video_codecs/video_encoder_factory.h"
 #include "api/video_codecs/video_encoder_factory_template.h"
 #include "api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h"
 #include "api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h"
 #include "api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h"
 #include "api/video_codecs/video_encoder_factory_template_open_h264_adapter.h"
 #include "examples/peerconnection/client/defaults.h"
+#include "examples/peerconnection/client/main_wnd.h"
+#include "examples/peerconnection/client/peer_connection_client.h"
+#include "json/reader.h"
+#include "json/value.h"
+#include "json/writer.h"
 #include "modules/video_capture/video_capture.h"
 #include "modules/video_capture/video_capture_factory.h"
-#include "p2p/base/port_allocator.h"
 #include "pc/video_track_source.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/logging.h"
-#include "rtc_base/rtc_certificate_generator.h"
 #include "rtc_base/strings/json.h"
-#include "test/vcm_capturer.h"
+#include "system_wrappers/include/clock.h"
+#include "test/frame_generator_capturer.h"
+#include "test/platform_video_capturer.h"
+#include "test/test_video_capturer.h"

 namespace {
+using webrtc::test::TestVideoCapturer;
+
 // Names used for a IceCandidate JSON object.
 const char kCandidateSdpMidName[] = "sdpMid";
 const char kCandidateSdpMlineIndexName[] = "sdpMLineIndex";
@ -75,40 +89,53 @@ class DummySetSessionDescriptionObserver
  }
 };

+std::unique_ptr<TestVideoCapturer> CreateCapturer(
+    webrtc::TaskQueueFactory& task_queue_factory) {
+  const size_t kWidth = 640;
+  const size_t kHeight = 480;
+  const size_t kFps = 30;
+  std::unique_ptr<webrtc::VideoCaptureModule::DeviceInfo> info(
+      webrtc::VideoCaptureFactory::CreateDeviceInfo());
+  if (!info) {
+    return nullptr;
+  }
+  int num_devices = info->NumberOfDevices();
+  for (int i = 0; i < num_devices; ++i) {
+    std::unique_ptr<TestVideoCapturer> capturer =
+        webrtc::test::CreateVideoCapturer(kWidth, kHeight, kFps, i);
+    if (capturer) {
+      return capturer;
+    }
+  }
+  auto frame_generator = webrtc::test::CreateSquareFrameGenerator(
+      kWidth, kHeight, std::nullopt, std::nullopt);
+  return std::make_unique<webrtc::test::FrameGeneratorCapturer>(
+      webrtc::Clock::GetRealTimeClock(), std::move(frame_generator), kFps,
+      task_queue_factory);
+}
 class CapturerTrackSource : public webrtc::VideoTrackSource {
 public:
-  static rtc::scoped_refptr<CapturerTrackSource> Create() {
-    const size_t kWidth = 640;
-    const size_t kHeight = 480;
-    const size_t kFps = 30;
-    std::unique_ptr<webrtc::test::VcmCapturer> capturer;
-    std::unique_ptr<webrtc::VideoCaptureModule::DeviceInfo> info(
-        webrtc::VideoCaptureFactory::CreateDeviceInfo());
-    if (!info) {
-      return nullptr;
+  static rtc::scoped_refptr<CapturerTrackSource> Create(
+      webrtc::TaskQueueFactory& task_queue_factory) {
+    std::unique_ptr<TestVideoCapturer> capturer =
+        CreateCapturer(task_queue_factory);
+    if (capturer) {
+      capturer->Start();
+      return rtc::make_ref_counted<CapturerTrackSource>(std::move(capturer));
    }
-    int num_devices = info->NumberOfDevices();
-    for (int i = 0; i < num_devices; ++i) {
-      capturer = absl::WrapUnique(
-          webrtc::test::VcmCapturer::Create(kWidth, kHeight, kFps, i));
-      if (capturer) {
-        return rtc::make_ref_counted<CapturerTrackSource>(std::move(capturer));
-      }
-    }
-
    return nullptr;
  }

 protected:
-  explicit CapturerTrackSource(
-      std::unique_ptr<webrtc::test::VcmCapturer> capturer)
+  explicit CapturerTrackSource(std::unique_ptr<TestVideoCapturer> capturer)
      : VideoTrackSource(/*remote=*/false), capturer_(std::move(capturer)) {}

 private:
  rtc::VideoSourceInterface<webrtc::VideoFrame>* source() override {
    return capturer_.get();
  }
-  std::unique_ptr<webrtc::test::VcmCapturer> capturer_;
+
+  std::unique_ptr<TestVideoCapturer> capturer_;
 };

 }  // namespace
@ -140,22 +167,28 @@ bool Conductor::InitializePeerConnection() {
    signaling_thread_ = rtc::Thread::CreateWithSocketServer();
    signaling_thread_->Start();
  }
-  peer_connection_factory_ = webrtc::CreatePeerConnectionFactory(
-      nullptr /* network_thread */, nullptr /* worker_thread */,
-      signaling_thread_.get(), nullptr /* default_adm */,
-      webrtc::CreateBuiltinAudioEncoderFactory(),
-      webrtc::CreateBuiltinAudioDecoderFactory(),
+
+  webrtc::PeerConnectionFactoryDependencies deps;
+  deps.signaling_thread = signaling_thread_.get();
+  deps.task_queue_factory = webrtc::CreateDefaultTaskQueueFactory(),
+  deps.audio_encoder_factory = webrtc::CreateBuiltinAudioEncoderFactory();
+  deps.audio_decoder_factory = webrtc::CreateBuiltinAudioDecoderFactory();
+  deps.video_encoder_factory =
      std::make_unique<webrtc::VideoEncoderFactoryTemplate<
          webrtc::LibvpxVp8EncoderTemplateAdapter,
          webrtc::LibvpxVp9EncoderTemplateAdapter,
          webrtc::OpenH264EncoderTemplateAdapter,
-          webrtc::LibaomAv1EncoderTemplateAdapter>>(),
+          webrtc::LibaomAv1EncoderTemplateAdapter>>();
+  deps.video_decoder_factory =
      std::make_unique<webrtc::VideoDecoderFactoryTemplate<
          webrtc::LibvpxVp8DecoderTemplateAdapter,
          webrtc::LibvpxVp9DecoderTemplateAdapter,
          webrtc::OpenH264DecoderTemplateAdapter,
-          webrtc::Dav1dDecoderTemplateAdapter>>(),
-      nullptr /* audio_mixer */, nullptr /* audio_processing */);
+          webrtc::Dav1dDecoderTemplateAdapter>>();
+  webrtc::EnableMedia(deps);
+  task_queue_factory_ = deps.task_queue_factory.get();
+  peer_connection_factory_ =
+      webrtc::CreateModularPeerConnectionFactory(std::move(deps));

  if (!peer_connection_factory_) {
    main_wnd_->MessageBox("Error", "Failed to initialize PeerConnectionFactory",
@ -481,7 +514,7 @@ void Conductor::AddTracks() {
  }

  rtc::scoped_refptr<CapturerTrackSource> video_device =
-      CapturerTrackSource::Create();
+      CapturerTrackSource::Create(*task_queue_factory_);
  if (video_device) {
    rtc::scoped_refptr<webrtc::VideoTrackInterface> video_track_(
        peer_connection_factory_->CreateVideoTrack(video_device, kVideoLabel));
--- a/examples/peerconnection/client/conductor.h
+++ b/examples/peerconnection/client/conductor.h
@ -12,13 +12,18 @@
 #define EXAMPLES_PEERCONNECTION_CLIENT_CONDUCTOR_H_

 #include <deque>
-#include <map>
 #include <memory>
 #include <string>
 #include <vector>

+#include "api/data_channel_interface.h"
+#include "api/jsep.h"
 #include "api/media_stream_interface.h"
 #include "api/peer_connection_interface.h"
+#include "api/rtc_error.h"
+#include "api/rtp_receiver_interface.h"
+#include "api/scoped_refptr.h"
+#include "api/task_queue/task_queue_factory.h"
 #include "examples/peerconnection/client/main_wnd.h"
 #include "examples/peerconnection/client/peer_connection_client.h"
 #include "rtc_base/thread.h"
@ -124,6 +129,7 @@ class Conductor : public webrtc::PeerConnectionObserver,
  int peer_id_;
  bool loopback_;
  std::unique_ptr<rtc::Thread> signaling_thread_;
+  webrtc::TaskQueueFactory* task_queue_factory_ = nullptr;
  rtc::scoped_refptr<webrtc::PeerConnectionInterface> peer_connection_;
  rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface>
      peer_connection_factory_;
--- a/examples/peerconnection/client/linux/main_wnd.cc
+++ b/examples/peerconnection/client/linux/main_wnd.cc
@ -15,6 +15,7 @@
 #include <gdk/gdkkeysyms.h>
 #include <glib-object.h>
 #include <glib.h>
+#include <glibconfig.h>
 #include <gobject/gclosure.h>
 #include <gtk/gtk.h>
 #include <stddef.h>
@ -22,17 +23,19 @@
 #include <stdlib.h>
 #include <string.h>

-#include <cstdint>
 #include <map>
-#include <utility>

+#include "api/media_stream_interface.h"
+#include "api/scoped_refptr.h"
 #include "api/video/i420_buffer.h"
+#include "api/video/video_frame.h"
 #include "api/video/video_frame_buffer.h"
 #include "api/video/video_rotation.h"
 #include "api/video/video_source_interface.h"
+#include "examples/peerconnection/client/main_wnd.h"
+#include "examples/peerconnection/client/peer_connection_client.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/logging.h"
-#include "third_party/libyuv/include/libyuv/convert.h"
 #include "third_party/libyuv/include/libyuv/convert_from.h"

 namespace {
@ -312,7 +315,7 @@ void GtkMainWnd::SwitchToPeerList(const Peers& peers) {
    } else if (draw_area_) {
      gtk_widget_destroy(draw_area_);
      draw_area_ = NULL;
-      draw_buffer_.reset();
+      draw_buffer_.SetSize(0);
    }

    peer_list_ = gtk_tree_view_new();
@ -424,68 +427,29 @@ void GtkMainWnd::OnRedraw() {
  gdk_threads_enter();

  VideoRenderer* remote_renderer = remote_renderer_.get();
-  if (remote_renderer && remote_renderer->image() != NULL &&
+  if (remote_renderer && !remote_renderer->image().empty() &&
      draw_area_ != NULL) {
-    width_ = remote_renderer->width();
-    height_ = remote_renderer->height();
-
-    if (!draw_buffer_.get()) {
-      draw_buffer_size_ = (width_ * height_ * 4) * 4;
-      draw_buffer_.reset(new uint8_t[draw_buffer_size_]);
-      gtk_widget_set_size_request(draw_area_, width_ * 2, height_ * 2);
+    if (width_ != remote_renderer->width() ||
+        height_ != remote_renderer->height()) {
+      width_ = remote_renderer->width();
+      height_ = remote_renderer->height();
+      gtk_widget_set_size_request(draw_area_, remote_renderer->width(),
+                                  remote_renderer->height());
    }
-
-    const uint32_t* image =
-        reinterpret_cast<const uint32_t*>(remote_renderer->image());
-    uint32_t* scaled = reinterpret_cast<uint32_t*>(draw_buffer_.get());
-    for (int r = 0; r < height_; ++r) {
-      for (int c = 0; c < width_; ++c) {
-        int x = c * 2;
-        scaled[x] = scaled[x + 1] = image[c];
-      }
-
-      uint32_t* prev_line = scaled;
-      scaled += width_ * 2;
-      memcpy(scaled, prev_line, (width_ * 2) * 4);
-
-      image += width_;
-      scaled += width_ * 2;
-    }
-
-    VideoRenderer* local_renderer = local_renderer_.get();
-    if (local_renderer && local_renderer->image()) {
-      image = reinterpret_cast<const uint32_t*>(local_renderer->image());
-      scaled = reinterpret_cast<uint32_t*>(draw_buffer_.get());
-      // Position the local preview on the right side.
-      scaled += (width_ * 2) - (local_renderer->width() / 2);
-      // right margin...
-      scaled -= 10;
-      // ... towards the bottom.
-      scaled += (height_ * width_ * 4) - ((local_renderer->height() / 2) *
-                                          (local_renderer->width() / 2) * 4);
-      // bottom margin...
-      scaled -= (width_ * 2) * 5;
-      for (int r = 0; r < local_renderer->height(); r += 2) {
-        for (int c = 0; c < local_renderer->width(); c += 2) {
-          scaled[c / 2] = image[c + r * local_renderer->width()];
-        }
-        scaled += width_ * 2;
-      }
-    }
-
+    draw_buffer_.SetData(remote_renderer->image());
    gtk_widget_queue_draw(draw_area_);
  }
-
+  // Here we can draw the local preview as well if we want....
  gdk_threads_leave();
 }

 void GtkMainWnd::Draw(GtkWidget* widget, cairo_t* cr) {
  cairo_format_t format = CAIRO_FORMAT_ARGB32;
  cairo_surface_t* surface = cairo_image_surface_create_for_data(
-      draw_buffer_.get(), format, width_ * 2, height_ * 2,
-      cairo_format_stride_for_width(format, width_ * 2));
+      draw_buffer_.data(), format, width_, height_,
+      cairo_format_stride_for_width(format, width_));
  cairo_set_source_surface(cr, surface, 0, 0);
-  cairo_rectangle(cr, 0, 0, width_ * 2, height_ * 2);
+  cairo_rectangle(cr, 0, 0, width_, height_);
  cairo_fill(cr);
  cairo_surface_destroy(surface);
 }
@ -513,7 +477,8 @@ void GtkMainWnd::VideoRenderer::SetSize(int width, int height) {

  width_ = width;
  height_ = height;
-  image_.reset(new uint8_t[width * height * 4]);
+  // ARGB
+  image_.SetSize(width * height * 4);
  gdk_threads_leave();
 }

@ -536,7 +501,7 @@ void GtkMainWnd::VideoRenderer::OnFrame(const webrtc::VideoFrame& video_frame) {
  // native endianness.
  libyuv::I420ToARGB(buffer->DataY(), buffer->StrideY(), buffer->DataU(),
                     buffer->StrideU(), buffer->DataV(), buffer->StrideV(),
-                     image_.get(), width_ * 4, buffer->width(),
+                     image_.data(), width_ * 4, buffer->width(),
                     buffer->height());

  gdk_threads_leave();
--- a/examples/peerconnection/client/linux/main_wnd.h
+++ b/examples/peerconnection/client/linux/main_wnd.h
@ -16,12 +16,14 @@
 #include <memory>
 #include <string>

+#include "api/array_view.h"
 #include "api/media_stream_interface.h"
 #include "api/scoped_refptr.h"
 #include "api/video/video_frame.h"
 #include "api/video/video_sink_interface.h"
 #include "examples/peerconnection/client/main_wnd.h"
 #include "examples/peerconnection/client/peer_connection_client.h"
+#include "rtc_base/buffer.h"

 // Forward declarations.
 typedef struct _GtkWidget GtkWidget;
@ -90,7 +92,7 @@ class GtkMainWnd : public MainWindow {
    // VideoSinkInterface implementation
    void OnFrame(const webrtc::VideoFrame& frame) override;

-    const uint8_t* image() const { return image_.get(); }
+    rtc::ArrayView<const uint8_t> image() const { return image_; }

    int width() const { return width_; }

@ -98,7 +100,7 @@ class GtkMainWnd : public MainWindow {

   protected:
    void SetSize(int width, int height);
-    std::unique_ptr<uint8_t[]> image_;
+    rtc::Buffer image_;
    int width_;
    int height_;
    GtkMainWnd* main_wnd_;
@ -119,9 +121,9 @@ class GtkMainWnd : public MainWindow {
  bool autocall_;
  std::unique_ptr<VideoRenderer> local_renderer_;
  std::unique_ptr<VideoRenderer> remote_renderer_;
-  int width_;
-  int height_;
-  std::unique_ptr<uint8_t[]> draw_buffer_;
+  int width_ = 0;
+  int height_ = 0;
+  rtc::Buffer draw_buffer_;
  int draw_buffer_size_;
 };