webrtc_m130/modules/video_capture/linux/video_capture_pipewire.cc

/*
 *  Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "modules/video_capture/linux/video_capture_pipewire.h"

#include <spa/param/format.h>
#include <spa/param/video/format-utils.h>
#include <spa/pod/builder.h>
#include <spa/utils/result.h>

#include <vector>

#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "modules/portal/pipewire_utils.h"
#include "rtc_base/logging.h"
#include "rtc_base/sanitizer.h"
#include "rtc_base/string_to_number.h"

namespace webrtc {
namespace videocapturemodule {

struct {
  uint32_t spa_format;
  VideoType video_type;
} constexpr kSupportedFormats[] = {
    {SPA_VIDEO_FORMAT_I420, VideoType::kI420},
    {SPA_VIDEO_FORMAT_NV12, VideoType::kNV12},
    {SPA_VIDEO_FORMAT_YUY2, VideoType::kYUY2},
    {SPA_VIDEO_FORMAT_UYVY, VideoType::kUYVY},
    // PipeWire is big-endian for the formats, while libyuv is little-endian
    // This means that BGRA == ARGB, RGBA == ABGR and similar
    // This follows mapping in libcamera PipeWire plugin:
    // https://gitlab.freedesktop.org/pipewire/pipewire/-/blob/master/spa/plugins/libcamera/libcamera-utils.cpp
    {SPA_VIDEO_FORMAT_BGRA, VideoType::kARGB},
    {SPA_VIDEO_FORMAT_RGBA, VideoType::kABGR},
    {SPA_VIDEO_FORMAT_ARGB, VideoType::kBGRA},
    {SPA_VIDEO_FORMAT_RGB, VideoType::kBGR24},
    {SPA_VIDEO_FORMAT_BGR, VideoType::kRGB24},
    {SPA_VIDEO_FORMAT_RGB16, VideoType::kRGB565},
};

VideoType VideoCaptureModulePipeWire::PipeWireRawFormatToVideoType(
    uint32_t spa_format) {
  for (const auto& spa_and_pixel_format : kSupportedFormats) {
    if (spa_and_pixel_format.spa_format == spa_format)
      return spa_and_pixel_format.video_type;
  }
  RTC_LOG(LS_WARNING) << "Unsupported pixel format: " << spa_format;
  return VideoType::kUnknown;
}

uint32_t VideoCaptureModulePipeWire::VideoTypeToPipeWireRawFormat(
    VideoType type) {
  for (const auto& spa_and_pixel_format : kSupportedFormats) {
    if (spa_and_pixel_format.video_type == type)
      return spa_and_pixel_format.spa_format;
  }
  RTC_LOG(LS_WARNING) << "Unsupported video type: " << static_cast<int>(type);
  return SPA_VIDEO_FORMAT_UNKNOWN;
}

VideoCaptureModulePipeWire::VideoCaptureModulePipeWire(
    VideoCaptureOptions* options)
    : VideoCaptureImpl(),
      session_(options->pipewire_session()),
      initialized_(false),
      started_(false) {}

VideoCaptureModulePipeWire::~VideoCaptureModulePipeWire() {
  RTC_DCHECK_RUN_ON(&api_checker_);

  StopCapture();
}

int32_t VideoCaptureModulePipeWire::Init(const char* deviceUniqueId) {
  RTC_CHECK_RUNS_SERIALIZED(&capture_checker_);
  RTC_DCHECK_RUN_ON(&api_checker_);

  std::optional<int> id;
  id = rtc::StringToNumber<int>(deviceUniqueId);
  if (id == std::nullopt)
    return -1;

  node_id_ = id.value();

  const int len = strlen(deviceUniqueId);
  _deviceUniqueId = new (std::nothrow) char[len + 1];
  memcpy(_deviceUniqueId, deviceUniqueId, len + 1);

  return 0;
}

static spa_pod* BuildFormat(spa_pod_builder* builder,
                            VideoType video_type,
                            uint32_t width,
                            uint32_t height,
                            float frame_rate) {
  spa_pod_frame frame;

  const uint32_t media_subtype = video_type == VideoType::kMJPEG
                                     ? SPA_MEDIA_SUBTYPE_mjpg
                                     : SPA_MEDIA_SUBTYPE_raw;

  spa_pod_builder_push_object(builder, &frame, SPA_TYPE_OBJECT_Format,
                              SPA_PARAM_EnumFormat);
  spa_pod_builder_add(builder, SPA_FORMAT_mediaType,
                      SPA_POD_Id(SPA_MEDIA_TYPE_video), SPA_FORMAT_mediaSubtype,
                      SPA_POD_Id(media_subtype), 0);

  if (media_subtype == SPA_MEDIA_SUBTYPE_raw) {
    const uint32_t format =
        VideoCaptureModulePipeWire::VideoTypeToPipeWireRawFormat(video_type);
    RTC_CHECK(format != SPA_VIDEO_FORMAT_UNKNOWN);
    spa_pod_builder_add(builder, SPA_FORMAT_VIDEO_format, SPA_POD_Id(format),
                        0);
  }

  spa_rectangle resolution = spa_rectangle{width, height};
  spa_pod_builder_add(builder, SPA_FORMAT_VIDEO_size,
                      SPA_POD_Rectangle(&resolution), 0);

  // Framerate can be also set to 0 to be unspecified
  if (frame_rate) {
    spa_fraction framerate = spa_fraction{static_cast<uint32_t>(frame_rate), 1};
    spa_pod_builder_add(builder, SPA_FORMAT_VIDEO_framerate,
                        SPA_POD_Fraction(&framerate), 0);
  } else {
    // Default to some reasonable values
    spa_fraction preferred_frame_rate =
        spa_fraction{static_cast<uint32_t>(30), 1};
    spa_fraction min_frame_rate = spa_fraction{1, 1};
    spa_fraction max_frame_rate = spa_fraction{30, 1};
    spa_pod_builder_add(
        builder, SPA_FORMAT_VIDEO_framerate,
        SPA_POD_CHOICE_RANGE_Fraction(&preferred_frame_rate, &min_frame_rate,
                                      &max_frame_rate),
        0);
  }

  return static_cast<spa_pod*>(spa_pod_builder_pop(builder, &frame));
}

RTC_NO_SANITIZE("cfi-icall")
int32_t VideoCaptureModulePipeWire::StartCapture(
    const VideoCaptureCapability& capability) {
  RTC_DCHECK_RUN_ON(&api_checker_);

  if (initialized_) {
    if (capability == _requestedCapability) {
      return 0;
    } else {
      StopCapture();
    }
  }

  uint8_t buffer[1024] = {};

  // We don't want members above to be guarded by capture_checker_ as
  // it's meant to be for members that are accessed on the API thread
  // only when we are not capturing. The code above can be called many
  // times while sharing instance of VideoCapturePipeWire between
  // websites and therefore it would not follow the requirements of this
  // checker.
  RTC_CHECK_RUNS_SERIALIZED(&capture_checker_);
  PipeWireThreadLoopLock thread_loop_lock(session_->pw_main_loop_);

  RTC_LOG(LS_VERBOSE) << "Creating new PipeWire stream for node " << node_id_;

  pw_properties* reuse_props =
      pw_properties_new_string("pipewire.client.reuse=1");
  stream_ = pw_stream_new(session_->pw_core_, "camera-stream", reuse_props);

  if (!stream_) {
    RTC_LOG(LS_ERROR) << "Failed to create camera stream!";
    return -1;
  }

  static const pw_stream_events stream_events{
      .version = PW_VERSION_STREAM_EVENTS,
      .state_changed = &OnStreamStateChanged,
      .param_changed = &OnStreamParamChanged,
      .process = &OnStreamProcess,
  };

  pw_stream_add_listener(stream_, &stream_listener_, &stream_events, this);

  spa_pod_builder builder = spa_pod_builder{buffer, sizeof(buffer)};
  std::vector<const spa_pod*> params;
  uint32_t width = capability.width;
  uint32_t height = capability.height;
  uint32_t frame_rate = capability.maxFPS;
  VideoType video_type = capability.videoType;

  params.push_back(
      BuildFormat(&builder, video_type, width, height, frame_rate));

  int res = pw_stream_connect(
      stream_, PW_DIRECTION_INPUT, node_id_,
      static_cast<enum pw_stream_flags>(PW_STREAM_FLAG_AUTOCONNECT |
                                        PW_STREAM_FLAG_DONT_RECONNECT),
      params.data(), params.size());
  if (res != 0) {
    RTC_LOG(LS_ERROR) << "Could not connect to camera stream: "
                      << spa_strerror(res);
    return -1;
  }

  _requestedCapability = capability;
  initialized_ = true;

  return 0;
}

int32_t VideoCaptureModulePipeWire::StopCapture() {
  RTC_DCHECK_RUN_ON(&api_checker_);

  PipeWireThreadLoopLock thread_loop_lock(session_->pw_main_loop_);
  // PipeWireSession is guarded by API checker so just make sure we do
  // race detection when the PipeWire loop is locked/stopped to not run
  // any callback at this point.
  RTC_CHECK_RUNS_SERIALIZED(&capture_checker_);
  if (stream_) {
    pw_stream_destroy(stream_);
    stream_ = nullptr;
  }

  _requestedCapability = VideoCaptureCapability();
  return 0;
}

bool VideoCaptureModulePipeWire::CaptureStarted() {
  RTC_DCHECK_RUN_ON(&api_checker_);
  MutexLock lock(&api_lock_);

  return started_;
}

int32_t VideoCaptureModulePipeWire::CaptureSettings(
    VideoCaptureCapability& settings) {
  RTC_DCHECK_RUN_ON(&api_checker_);

  settings = _requestedCapability;

  return 0;
}

void VideoCaptureModulePipeWire::OnStreamParamChanged(
    void* data,
    uint32_t id,
    const struct spa_pod* format) {
  VideoCaptureModulePipeWire* that =
      static_cast<VideoCaptureModulePipeWire*>(data);
  RTC_DCHECK(that);
  RTC_CHECK_RUNS_SERIALIZED(&that->capture_checker_);

  if (format && id == SPA_PARAM_Format)
    that->OnFormatChanged(format);
}

RTC_NO_SANITIZE("cfi-icall")
void VideoCaptureModulePipeWire::OnFormatChanged(const struct spa_pod* format) {
  RTC_CHECK_RUNS_SERIALIZED(&capture_checker_);

  uint32_t media_type, media_subtype;

  if (spa_format_parse(format, &media_type, &media_subtype) < 0) {
    RTC_LOG(LS_ERROR) << "Failed to parse video format.";
    return;
  }

  switch (media_subtype) {
    case SPA_MEDIA_SUBTYPE_raw: {
      struct spa_video_info_raw f;
      spa_format_video_raw_parse(format, &f);
      configured_capability_.width = f.size.width;
      configured_capability_.height = f.size.height;
      configured_capability_.videoType = PipeWireRawFormatToVideoType(f.format);
      configured_capability_.maxFPS = f.framerate.num / f.framerate.denom;
      break;
    }
    case SPA_MEDIA_SUBTYPE_mjpg: {
      struct spa_video_info_mjpg f;
      spa_format_video_mjpg_parse(format, &f);
      configured_capability_.width = f.size.width;
      configured_capability_.height = f.size.height;
      configured_capability_.videoType = VideoType::kMJPEG;
      configured_capability_.maxFPS = f.framerate.num / f.framerate.denom;
      break;
    }
    default:
      configured_capability_.videoType = VideoType::kUnknown;
  }

  if (configured_capability_.videoType == VideoType::kUnknown) {
    RTC_LOG(LS_ERROR) << "Unsupported video format.";
    return;
  }

  RTC_LOG(LS_VERBOSE) << "Configured capture format = "
                      << static_cast<int>(configured_capability_.videoType);

  uint8_t buffer[1024] = {};
  auto builder = spa_pod_builder{buffer, sizeof(buffer)};

  // Setup buffers and meta header for new format.
  std::vector<const spa_pod*> params;
  spa_pod_frame frame;
  spa_pod_builder_push_object(&builder, &frame, SPA_TYPE_OBJECT_ParamBuffers,
                              SPA_PARAM_Buffers);

  if (media_subtype == SPA_MEDIA_SUBTYPE_raw) {
    // Enforce stride without padding.
    size_t stride;
    switch (configured_capability_.videoType) {
      case VideoType::kI420:
      case VideoType::kNV12:
        stride = configured_capability_.width;
        break;
      case VideoType::kYUY2:
      case VideoType::kUYVY:
      case VideoType::kRGB565:
        stride = configured_capability_.width * 2;
        break;
      case VideoType::kRGB24:
      case VideoType::kBGR24:
        stride = configured_capability_.width * 3;
        break;
      case VideoType::kARGB:
      case VideoType::kABGR:
      case VideoType::kBGRA:
        stride = configured_capability_.width * 4;
        break;
      default:
        RTC_LOG(LS_ERROR) << "Unsupported video format.";
        return;
    }
    spa_pod_builder_add(&builder, SPA_PARAM_BUFFERS_stride, SPA_POD_Int(stride),
                        0);
  }

  const int buffer_types =
      (1 << SPA_DATA_DmaBuf) | (1 << SPA_DATA_MemFd) | (1 << SPA_DATA_MemPtr);
  spa_pod_builder_add(
      &builder, SPA_PARAM_BUFFERS_buffers, SPA_POD_CHOICE_RANGE_Int(8, 1, 32),
      SPA_PARAM_BUFFERS_dataType, SPA_POD_CHOICE_FLAGS_Int(buffer_types), 0);
  params.push_back(
      static_cast<spa_pod*>(spa_pod_builder_pop(&builder, &frame)));

  params.push_back(reinterpret_cast<spa_pod*>(spa_pod_builder_add_object(
      &builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type,
      SPA_POD_Id(SPA_META_Header), SPA_PARAM_META_size,
      SPA_POD_Int(sizeof(struct spa_meta_header)))));
  params.push_back(reinterpret_cast<spa_pod*>(spa_pod_builder_add_object(
      &builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type,
      SPA_POD_Id(SPA_META_VideoTransform), SPA_PARAM_META_size,
      SPA_POD_Int(sizeof(struct spa_meta_videotransform)))));
  pw_stream_update_params(stream_, params.data(), params.size());
}

void VideoCaptureModulePipeWire::OnStreamStateChanged(
    void* data,
    pw_stream_state old_state,
    pw_stream_state state,
    const char* error_message) {
  VideoCaptureModulePipeWire* that =
      static_cast<VideoCaptureModulePipeWire*>(data);
  RTC_DCHECK(that);

  MutexLock lock(&that->api_lock_);
  switch (state) {
    case PW_STREAM_STATE_STREAMING:
      that->started_ = true;
      break;
    case PW_STREAM_STATE_ERROR:
      RTC_LOG(LS_ERROR) << "PipeWire stream state error: " << error_message;
      [[fallthrough]];
    case PW_STREAM_STATE_PAUSED:
    case PW_STREAM_STATE_UNCONNECTED:
    case PW_STREAM_STATE_CONNECTING:
      that->started_ = false;
      break;
  }
  RTC_LOG(LS_VERBOSE) << "PipeWire stream state change: "
                      << pw_stream_state_as_string(old_state) << " -> "
                      << pw_stream_state_as_string(state);
}

void VideoCaptureModulePipeWire::OnStreamProcess(void* data) {
  VideoCaptureModulePipeWire* that =
      static_cast<VideoCaptureModulePipeWire*>(data);
  RTC_DCHECK(that);
  RTC_CHECK_RUNS_SERIALIZED(&that->capture_checker_);
  that->ProcessBuffers();
}

static VideoRotation VideorotationFromPipeWireTransform(uint32_t transform) {
  switch (transform) {
    case SPA_META_TRANSFORMATION_90:
      return kVideoRotation_90;
    case SPA_META_TRANSFORMATION_180:
      return kVideoRotation_180;
    case SPA_META_TRANSFORMATION_270:
      return kVideoRotation_270;
    default:
      return kVideoRotation_0;
  }
}

RTC_NO_SANITIZE("cfi-icall")
void VideoCaptureModulePipeWire::ProcessBuffers() {
  RTC_CHECK_RUNS_SERIALIZED(&capture_checker_);

  while (pw_buffer* buffer = pw_stream_dequeue_buffer(stream_)) {
    spa_buffer* spaBuffer = buffer->buffer;
    struct spa_meta_header* h;
    h = static_cast<struct spa_meta_header*>(
        spa_buffer_find_meta_data(spaBuffer, SPA_META_Header, sizeof(*h)));

    struct spa_meta_videotransform* videotransform;
    videotransform =
        static_cast<struct spa_meta_videotransform*>(spa_buffer_find_meta_data(
            spaBuffer, SPA_META_VideoTransform, sizeof(*videotransform)));
    if (videotransform) {
      VideoRotation rotation =
          VideorotationFromPipeWireTransform(videotransform->transform);
      SetCaptureRotation(rotation);
      SetApplyRotation(rotation != kVideoRotation_0);
    }

    if (h->flags & SPA_META_HEADER_FLAG_CORRUPTED) {
      RTC_LOG(LS_INFO) << "Dropping corruped frame.";
      pw_stream_queue_buffer(stream_, buffer);
      continue;
    }

    if (spaBuffer->datas[0].type == SPA_DATA_DmaBuf ||
        spaBuffer->datas[0].type == SPA_DATA_MemFd) {
      ScopedBuf frame;
      frame.initialize(
          static_cast<uint8_t*>(
              mmap(nullptr, spaBuffer->datas[0].maxsize, PROT_READ, MAP_SHARED,
                   spaBuffer->datas[0].fd, spaBuffer->datas[0].mapoffset)),
          spaBuffer->datas[0].maxsize, spaBuffer->datas[0].fd,
          spaBuffer->datas[0].type == SPA_DATA_DmaBuf);

      if (!frame) {
        RTC_LOG(LS_ERROR) << "Failed to mmap the memory: "
                          << std::strerror(errno);
        return;
      }

      IncomingFrame(
          SPA_MEMBER(frame.get(), spaBuffer->datas[0].mapoffset, uint8_t),
          spaBuffer->datas[0].chunk->size, configured_capability_);
    } else {  // SPA_DATA_MemPtr
      IncomingFrame(static_cast<uint8_t*>(spaBuffer->datas[0].data),
                    spaBuffer->datas[0].chunk->size, configured_capability_);
    }

    pw_stream_queue_buffer(stream_, buffer);
  }
}

}  // namespace videocapturemodule
}  // namespace webrtc