Modify Demo for Linux

Fallback to squared pixel generator if no camera exist.

Remove render scaling and rendering of local preview since contains
bugs that crash the demo. Use rtc::Buffer for storing the rendered
frames.

Run build cleaner

Bug: none
Change-Id: I46dc972eaa50069433d8afaa1fe38380edd3d337
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/367120
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Commit-Queue: Per Kjellander <perkj@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#43350}

examples/BUILD.gn

@@ -683,17 +683,24 @@ if (is_linux || is_chromeos || is_win) {
     ]
 
     deps = [
+      "../api:array_view",
       "../api:async_dns_resolver",
       "../api:audio_options_api",
+      "../api:create_frame_generator",
       "../api:create_peerconnection_factory",
+      "../api:enable_media",
       "../api:libjingle_peerconnection_api",
+      "../api:make_ref_counted",
       "../api:media_stream_interface",
+      "../api:rtc_error",
       "../api:rtp_sender_interface",
       "../api:scoped_refptr",
       "../api/audio:audio_device",
       "../api/audio:audio_mixer_api",
       "../api/audio:audio_processing",
       "../api/audio_codecs:audio_codecs_api",
+      "../api/task_queue",
+      "../api/task_queue:default_task_queue_factory",
       "../api/task_queue:pending_task_safety_flag",
       "../api/units:time_delta",
       "../api/video:video_frame",
@@ -706,6 +713,7 @@ if (is_linux || is_chromeos || is_win) {
       "../p2p:rtc_p2p",
       "../pc:video_track_source",
       "../rtc_base:async_dns_resolver",
+      "../rtc_base:buffer",
       "../rtc_base:checks",
       "../rtc_base:logging",
       "../rtc_base:macromagic",
@@ -716,11 +724,15 @@ if (is_linux || is_chromeos || is_win) {
       "../rtc_base:stringutils",
       "../rtc_base:threading",
       "../rtc_base/third_party/sigslot",
+      "../system_wrappers",
       "../system_wrappers:field_trial",
       "../test:field_trial",
+      "../test:frame_generator_capturer",
       "../test:platform_video_capturer",
       "../test:rtp_test_utils",
+      "../test:test_video_capturer",
       "//third_party/abseil-cpp/absl/memory",
+      "//third_party/jsoncpp",
     ]
     if (is_win) {
       sources += [

examples/DEPS

@@ -1,5 +1,6 @@
 include_rules = [
   "+common_video",
+  "+json",
   "+logging/rtc_event_log/rtc_event_log_factory.h",
   "+media",
   "+modules/audio_device",

examples/peerconnection/client/conductor.cc

@@ -11,48 +11,62 @@
 #include "examples/peerconnection/client/conductor.h"
 
 #include <stddef.h>
-#include <stdint.h>
 
 #include <memory>
 #include <optional>
+#include <string>
 #include <utility>
 #include <vector>
 
 #include "absl/memory/memory.h"
-#include "api/audio/audio_device.h"
-#include "api/audio/audio_mixer.h"
-#include "api/audio/audio_processing.h"
-#include "api/audio_codecs/audio_decoder_factory.h"
-#include "api/audio_codecs/audio_encoder_factory.h"
 #include "api/audio_codecs/builtin_audio_decoder_factory.h"
 #include "api/audio_codecs/builtin_audio_encoder_factory.h"
 #include "api/audio_options.h"
 #include "api/create_peerconnection_factory.h"
+#include "api/enable_media.h"
+#include "api/jsep.h"
+#include "api/make_ref_counted.h"
+#include "api/media_stream_interface.h"
+#include "api/peer_connection_interface.h"
+#include "api/rtc_error.h"
+#include "api/rtp_receiver_interface.h"
 #include "api/rtp_sender_interface.h"
-#include "api/video_codecs/video_decoder_factory.h"
+#include "api/scoped_refptr.h"
+#include "api/task_queue/default_task_queue_factory.h"
+#include "api/task_queue/task_queue_factory.h"
+#include "api/test/create_frame_generator.h"
+#include "api/video/video_frame.h"
+#include "api/video/video_source_interface.h"
 #include "api/video_codecs/video_decoder_factory_template.h"
 #include "api/video_codecs/video_decoder_factory_template_dav1d_adapter.h"
 #include "api/video_codecs/video_decoder_factory_template_libvpx_vp8_adapter.h"
 #include "api/video_codecs/video_decoder_factory_template_libvpx_vp9_adapter.h"
 #include "api/video_codecs/video_decoder_factory_template_open_h264_adapter.h"
-#include "api/video_codecs/video_encoder_factory.h"
 #include "api/video_codecs/video_encoder_factory_template.h"
 #include "api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h"
 #include "api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h"
 #include "api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h"
 #include "api/video_codecs/video_encoder_factory_template_open_h264_adapter.h"
 #include "examples/peerconnection/client/defaults.h"
+#include "examples/peerconnection/client/main_wnd.h"
+#include "examples/peerconnection/client/peer_connection_client.h"
+#include "json/reader.h"
+#include "json/value.h"
+#include "json/writer.h"
 #include "modules/video_capture/video_capture.h"
 #include "modules/video_capture/video_capture_factory.h"
-#include "p2p/base/port_allocator.h"
 #include "pc/video_track_source.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/logging.h"
-#include "rtc_base/rtc_certificate_generator.h"
 #include "rtc_base/strings/json.h"
-#include "test/vcm_capturer.h"
+#include "system_wrappers/include/clock.h"
+#include "test/frame_generator_capturer.h"
+#include "test/platform_video_capturer.h"
+#include "test/test_video_capturer.h"
 
 namespace {
+using webrtc::test::TestVideoCapturer;
+
 // Names used for a IceCandidate JSON object.
 const char kCandidateSdpMidName[] = "sdpMid";
 const char kCandidateSdpMlineIndexName[] = "sdpMLineIndex";
@@ -75,13 +89,11 @@ class DummySetSessionDescriptionObserver
   }
 };
 
-class CapturerTrackSource : public webrtc::VideoTrackSource {
+std::unique_ptr<TestVideoCapturer> CreateCapturer(
- public:
+    webrtc::TaskQueueFactory& task_queue_factory) {
-  static rtc::scoped_refptr<CapturerTrackSource> Create() {
   const size_t kWidth = 640;
   const size_t kHeight = 480;
   const size_t kFps = 30;
-    std::unique_ptr<webrtc::test::VcmCapturer> capturer;
   std::unique_ptr<webrtc::VideoCaptureModule::DeviceInfo> info(
       webrtc::VideoCaptureFactory::CreateDeviceInfo());
   if (!info) {
@@ -89,26 +101,41 @@ class CapturerTrackSource : public webrtc::VideoTrackSource {
   }
   int num_devices = info->NumberOfDevices();
   for (int i = 0; i < num_devices; ++i) {
-      capturer = absl::WrapUnique(
+    std::unique_ptr<TestVideoCapturer> capturer =
-          webrtc::test::VcmCapturer::Create(kWidth, kHeight, kFps, i));
+        webrtc::test::CreateVideoCapturer(kWidth, kHeight, kFps, i);
     if (capturer) {
+      return capturer;
+    }
+  }
+  auto frame_generator = webrtc::test::CreateSquareFrameGenerator(
+      kWidth, kHeight, std::nullopt, std::nullopt);
+  return std::make_unique<webrtc::test::FrameGeneratorCapturer>(
+      webrtc::Clock::GetRealTimeClock(), std::move(frame_generator), kFps,
+      task_queue_factory);
+}
+class CapturerTrackSource : public webrtc::VideoTrackSource {
+ public:
+  static rtc::scoped_refptr<CapturerTrackSource> Create(
+      webrtc::TaskQueueFactory& task_queue_factory) {
+    std::unique_ptr<TestVideoCapturer> capturer =
+        CreateCapturer(task_queue_factory);
+    if (capturer) {
+      capturer->Start();
       return rtc::make_ref_counted<CapturerTrackSource>(std::move(capturer));
     }
-    }
-
     return nullptr;
   }
 
  protected:
-  explicit CapturerTrackSource(
+  explicit CapturerTrackSource(std::unique_ptr<TestVideoCapturer> capturer)
-      std::unique_ptr<webrtc::test::VcmCapturer> capturer)
       : VideoTrackSource(/*remote=*/false), capturer_(std::move(capturer)) {}
 
  private:
   rtc::VideoSourceInterface<webrtc::VideoFrame>* source() override {
     return capturer_.get();
   }
-  std::unique_ptr<webrtc::test::VcmCapturer> capturer_;
+
+  std::unique_ptr<TestVideoCapturer> capturer_;
 };
 
 }  // namespace
@@ -140,22 +167,28 @@ bool Conductor::InitializePeerConnection() {
     signaling_thread_ = rtc::Thread::CreateWithSocketServer();
     signaling_thread_->Start();
   }
-  peer_connection_factory_ = webrtc::CreatePeerConnectionFactory(
+
-      nullptr /* network_thread */, nullptr /* worker_thread */,
+  webrtc::PeerConnectionFactoryDependencies deps;
-      signaling_thread_.get(), nullptr /* default_adm */,
+  deps.signaling_thread = signaling_thread_.get();
-      webrtc::CreateBuiltinAudioEncoderFactory(),
+  deps.task_queue_factory = webrtc::CreateDefaultTaskQueueFactory(),
-      webrtc::CreateBuiltinAudioDecoderFactory(),
+  deps.audio_encoder_factory = webrtc::CreateBuiltinAudioEncoderFactory();
+  deps.audio_decoder_factory = webrtc::CreateBuiltinAudioDecoderFactory();
+  deps.video_encoder_factory =
       std::make_unique<webrtc::VideoEncoderFactoryTemplate<
           webrtc::LibvpxVp8EncoderTemplateAdapter,
           webrtc::LibvpxVp9EncoderTemplateAdapter,
           webrtc::OpenH264EncoderTemplateAdapter,
-          webrtc::LibaomAv1EncoderTemplateAdapter>>(),
+          webrtc::LibaomAv1EncoderTemplateAdapter>>();
+  deps.video_decoder_factory =
       std::make_unique<webrtc::VideoDecoderFactoryTemplate<
           webrtc::LibvpxVp8DecoderTemplateAdapter,
           webrtc::LibvpxVp9DecoderTemplateAdapter,
           webrtc::OpenH264DecoderTemplateAdapter,
-          webrtc::Dav1dDecoderTemplateAdapter>>(),
+          webrtc::Dav1dDecoderTemplateAdapter>>();
-      nullptr /* audio_mixer */, nullptr /* audio_processing */);
+  webrtc::EnableMedia(deps);
+  task_queue_factory_ = deps.task_queue_factory.get();
+  peer_connection_factory_ =
+      webrtc::CreateModularPeerConnectionFactory(std::move(deps));
 
   if (!peer_connection_factory_) {
     main_wnd_->MessageBox("Error", "Failed to initialize PeerConnectionFactory",
@@ -481,7 +514,7 @@ void Conductor::AddTracks() {
   }
 
   rtc::scoped_refptr<CapturerTrackSource> video_device =
-      CapturerTrackSource::Create();
+      CapturerTrackSource::Create(*task_queue_factory_);
   if (video_device) {
     rtc::scoped_refptr<webrtc::VideoTrackInterface> video_track_(
         peer_connection_factory_->CreateVideoTrack(video_device, kVideoLabel));

examples/peerconnection/client/conductor.h

@@ -12,13 +12,18 @@
 #define EXAMPLES_PEERCONNECTION_CLIENT_CONDUCTOR_H_
 
 #include <deque>
-#include <map>
 #include <memory>
 #include <string>
 #include <vector>
 
+#include "api/data_channel_interface.h"
+#include "api/jsep.h"
 #include "api/media_stream_interface.h"
 #include "api/peer_connection_interface.h"
+#include "api/rtc_error.h"
+#include "api/rtp_receiver_interface.h"
+#include "api/scoped_refptr.h"
+#include "api/task_queue/task_queue_factory.h"
 #include "examples/peerconnection/client/main_wnd.h"
 #include "examples/peerconnection/client/peer_connection_client.h"
 #include "rtc_base/thread.h"
@@ -124,6 +129,7 @@ class Conductor : public webrtc::PeerConnectionObserver,
   int peer_id_;
   bool loopback_;
   std::unique_ptr<rtc::Thread> signaling_thread_;
+  webrtc::TaskQueueFactory* task_queue_factory_ = nullptr;
   rtc::scoped_refptr<webrtc::PeerConnectionInterface> peer_connection_;
   rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface>
       peer_connection_factory_;

examples/peerconnection/client/linux/main_wnd.cc

@@ -15,6 +15,7 @@
 #include <gdk/gdkkeysyms.h>
 #include <glib-object.h>
 #include <glib.h>
+#include <glibconfig.h>
 #include <gobject/gclosure.h>
 #include <gtk/gtk.h>
 #include <stddef.h>
@@ -22,17 +23,19 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include <cstdint>
 #include <map>
-#include <utility>
 
+#include "api/media_stream_interface.h"
+#include "api/scoped_refptr.h"
 #include "api/video/i420_buffer.h"
+#include "api/video/video_frame.h"
 #include "api/video/video_frame_buffer.h"
 #include "api/video/video_rotation.h"
 #include "api/video/video_source_interface.h"
+#include "examples/peerconnection/client/main_wnd.h"
+#include "examples/peerconnection/client/peer_connection_client.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/logging.h"
-#include "third_party/libyuv/include/libyuv/convert.h"
 #include "third_party/libyuv/include/libyuv/convert_from.h"
 
 namespace {
@@ -312,7 +315,7 @@ void GtkMainWnd::SwitchToPeerList(const Peers& peers) {
     } else if (draw_area_) {
       gtk_widget_destroy(draw_area_);
       draw_area_ = NULL;
-      draw_buffer_.reset();
+      draw_buffer_.SetSize(0);
     }
 
     peer_list_ = gtk_tree_view_new();
@@ -424,68 +427,29 @@ void GtkMainWnd::OnRedraw() {
   gdk_threads_enter();
 
   VideoRenderer* remote_renderer = remote_renderer_.get();
-  if (remote_renderer && remote_renderer->image() != NULL &&
+  if (remote_renderer && !remote_renderer->image().empty() &&
       draw_area_ != NULL) {
+    if (width_ != remote_renderer->width() ||
+        height_ != remote_renderer->height()) {
       width_ = remote_renderer->width();
       height_ = remote_renderer->height();
-
+      gtk_widget_set_size_request(draw_area_, remote_renderer->width(),
-    if (!draw_buffer_.get()) {
+                                  remote_renderer->height());
-      draw_buffer_size_ = (width_ * height_ * 4) * 4;
-      draw_buffer_.reset(new uint8_t[draw_buffer_size_]);
-      gtk_widget_set_size_request(draw_area_, width_ * 2, height_ * 2);
     }
-
+    draw_buffer_.SetData(remote_renderer->image());
-    const uint32_t* image =
-        reinterpret_cast<const uint32_t*>(remote_renderer->image());
-    uint32_t* scaled = reinterpret_cast<uint32_t*>(draw_buffer_.get());
-    for (int r = 0; r < height_; ++r) {
-      for (int c = 0; c < width_; ++c) {
-        int x = c * 2;
-        scaled[x] = scaled[x + 1] = image[c];
-      }
-
-      uint32_t* prev_line = scaled;
-      scaled += width_ * 2;
-      memcpy(scaled, prev_line, (width_ * 2) * 4);
-
-      image += width_;
-      scaled += width_ * 2;
-    }
-
-    VideoRenderer* local_renderer = local_renderer_.get();
-    if (local_renderer && local_renderer->image()) {
-      image = reinterpret_cast<const uint32_t*>(local_renderer->image());
-      scaled = reinterpret_cast<uint32_t*>(draw_buffer_.get());
-      // Position the local preview on the right side.
-      scaled += (width_ * 2) - (local_renderer->width() / 2);
-      // right margin...
-      scaled -= 10;
-      // ... towards the bottom.
-      scaled += (height_ * width_ * 4) - ((local_renderer->height() / 2) *
-                                          (local_renderer->width() / 2) * 4);
-      // bottom margin...
-      scaled -= (width_ * 2) * 5;
-      for (int r = 0; r < local_renderer->height(); r += 2) {
-        for (int c = 0; c < local_renderer->width(); c += 2) {
-          scaled[c / 2] = image[c + r * local_renderer->width()];
-        }
-        scaled += width_ * 2;
-      }
-    }
-
     gtk_widget_queue_draw(draw_area_);
   }
-
+  // Here we can draw the local preview as well if we want....
   gdk_threads_leave();
 }
 
 void GtkMainWnd::Draw(GtkWidget* widget, cairo_t* cr) {
   cairo_format_t format = CAIRO_FORMAT_ARGB32;
   cairo_surface_t* surface = cairo_image_surface_create_for_data(
-      draw_buffer_.get(), format, width_ * 2, height_ * 2,
+      draw_buffer_.data(), format, width_, height_,
-      cairo_format_stride_for_width(format, width_ * 2));
+      cairo_format_stride_for_width(format, width_));
   cairo_set_source_surface(cr, surface, 0, 0);
-  cairo_rectangle(cr, 0, 0, width_ * 2, height_ * 2);
+  cairo_rectangle(cr, 0, 0, width_, height_);
   cairo_fill(cr);
   cairo_surface_destroy(surface);
 }
@@ -513,7 +477,8 @@ void GtkMainWnd::VideoRenderer::SetSize(int width, int height) {
 
   width_ = width;
   height_ = height;
-  image_.reset(new uint8_t[width * height * 4]);
+  // ARGB
+  image_.SetSize(width * height * 4);
   gdk_threads_leave();
 }
 
@@ -536,7 +501,7 @@ void GtkMainWnd::VideoRenderer::OnFrame(const webrtc::VideoFrame& video_frame) {
   // native endianness.
   libyuv::I420ToARGB(buffer->DataY(), buffer->StrideY(), buffer->DataU(),
                      buffer->StrideU(), buffer->DataV(), buffer->StrideV(),
-                     image_.get(), width_ * 4, buffer->width(),
+                     image_.data(), width_ * 4, buffer->width(),
                      buffer->height());
 
   gdk_threads_leave();

examples/peerconnection/client/linux/main_wnd.h

@@ -16,12 +16,14 @@
 #include <memory>
 #include <string>
 
+#include "api/array_view.h"
 #include "api/media_stream_interface.h"
 #include "api/scoped_refptr.h"
 #include "api/video/video_frame.h"
 #include "api/video/video_sink_interface.h"
 #include "examples/peerconnection/client/main_wnd.h"
 #include "examples/peerconnection/client/peer_connection_client.h"
+#include "rtc_base/buffer.h"
 
 // Forward declarations.
 typedef struct _GtkWidget GtkWidget;
@@ -90,7 +92,7 @@ class GtkMainWnd : public MainWindow {
     // VideoSinkInterface implementation
     void OnFrame(const webrtc::VideoFrame& frame) override;
 
-    const uint8_t* image() const { return image_.get(); }
+    rtc::ArrayView<const uint8_t> image() const { return image_; }
 
     int width() const { return width_; }
 
@@ -98,7 +100,7 @@ class GtkMainWnd : public MainWindow {
 
    protected:
     void SetSize(int width, int height);
-    std::unique_ptr<uint8_t[]> image_;
+    rtc::Buffer image_;
     int width_;
     int height_;
     GtkMainWnd* main_wnd_;
@@ -119,9 +121,9 @@ class GtkMainWnd : public MainWindow {
   bool autocall_;
   std::unique_ptr<VideoRenderer> local_renderer_;
   std::unique_ptr<VideoRenderer> remote_renderer_;
-  int width_;
+  int width_ = 0;
-  int height_;
+  int height_ = 0;
-  std::unique_ptr<uint8_t[]> draw_buffer_;
+  rtc::Buffer draw_buffer_;
   int draw_buffer_size_;
 };