From 9982efa830b5b441ad8a84d59a79a8384344b648 Mon Sep 17 00:00:00 2001 From: Markus Handell Date: Thu, 21 Nov 2019 11:56:50 +0100 Subject: [PATCH] MediaStreamInterface: introduce encoded sinks. This change adds a new type of sink for consuming encoded data from a video source. Bug: chromium:1013590 Change-Id: Ia7c4e372190c3d6bc007a0d4deb05c2d1bce58d2 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/159927 Commit-Queue: Markus Handell Reviewed-by: Per Kjellander Reviewed-by: Niels Moller Cr-Commit-Position: refs/heads/master@{#29856} --- api/BUILD.gn | 1 + api/media_stream_interface.h | 26 ++++++++++++ api/video/BUILD.gn | 17 ++++++++ api/video/DEPS | 4 ++ api/video/recordable_encoded_frame.h | 59 ++++++++++++++++++++++++++++ api/video_track_source_proxy.h | 8 ++++ 6 files changed, 115 insertions(+) create mode 100644 api/video/recordable_encoded_frame.h diff --git a/api/BUILD.gn b/api/BUILD.gn index b97118b575..8bb4f24a16 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -115,6 +115,7 @@ rtc_library("media_stream_interface") { "../rtc_base:checks", "../rtc_base:refcount", "../rtc_base/system:rtc_export", + "video:recordable_encoded_frame", "video:video_frame", "//third_party/abseil-cpp/absl/types:optional", ] diff --git a/api/media_stream_interface.h b/api/media_stream_interface.h index 5fb73c94a4..dde6272fa5 100644 --- a/api/media_stream_interface.h +++ b/api/media_stream_interface.h @@ -24,6 +24,7 @@ #include "absl/types/optional.h" #include "api/audio_options.h" #include "api/scoped_refptr.h" +#include "api/video/recordable_encoded_frame.h" #include "api/video/video_frame.h" #include "api/video/video_sink_interface.h" #include "api/video/video_source_interface.h" @@ -135,6 +136,31 @@ class VideoTrackSourceInterface : public MediaSourceInterface, // Implementation should avoid blocking. virtual bool GetStats(Stats* stats) = 0; + // Returns true if encoded output can be enabled in the source. + // TODO(bugs.webrtc.org/11114): make pure virtual once downstream project + // adapts. + virtual bool SupportsEncodedOutput() const { return false; } + + // Reliably cause a key frame to be generated in encoded output. + // TODO(bugs.webrtc.org/11115): find optimal naming. + // TODO(bugs.webrtc.org/11114): make pure virtual once downstream project + // adapts. + virtual void GenerateKeyFrame() {} + + // Add an encoded video sink to the source and additionally cause + // a key frame to be generated from the source. The sink will be + // invoked from a decoder queue. + // TODO(bugs.webrtc.org/11114): make pure virtual once downstream project + // adapts. + virtual void AddEncodedSink( + rtc::VideoSinkInterface* sink) {} + + // Removes an encoded video sink from the source. + // TODO(bugs.webrtc.org/11114): make pure virtual once downstream project + // adapts. + virtual void RemoveEncodedSink( + rtc::VideoSinkInterface* sink) {} + protected: ~VideoTrackSourceInterface() override = default; }; diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn index 41698f089b..0124d10c2e 100644 --- a/api/video/BUILD.gn +++ b/api/video/BUILD.gn @@ -57,6 +57,23 @@ rtc_library("video_frame") { ] } +rtc_source_set("recordable_encoded_frame") { + visibility = [ "*" ] + sources = [ + "recordable_encoded_frame.h", + ] + + deps = [ + ":encoded_image", + ":video_frame", + ":video_rtp_headers", + "..:array_view", + "..:scoped_refptr", + "../../rtc_base:refcount", + "../units:timestamp", + ] +} + rtc_source_set("video_frame_type") { visibility = [ "*" ] sources = [ diff --git a/api/video/DEPS b/api/video/DEPS index 555f7e148a..85a4c01d2e 100644 --- a/api/video/DEPS +++ b/api/video/DEPS @@ -18,6 +18,10 @@ specific_include_rules = { "+rtc_base/memory/aligned_malloc.h", ], + "recordable_encoded_frame\.h": [ + "+rtc_base/ref_count.h", + ], + "video_frame\.h": [ "+rtc_base/ref_count.h", ], diff --git a/api/video/recordable_encoded_frame.h b/api/video/recordable_encoded_frame.h new file mode 100644 index 0000000000..db59964f26 --- /dev/null +++ b/api/video/recordable_encoded_frame.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_RECORDABLE_ENCODED_FRAME_H_ +#define API_VIDEO_RECORDABLE_ENCODED_FRAME_H_ + +#include "api/array_view.h" +#include "api/scoped_refptr.h" +#include "api/units/timestamp.h" +#include "api/video/color_space.h" +#include "api/video/encoded_image.h" +#include "api/video/video_codec_type.h" +#include "rtc_base/ref_count.h" + +namespace webrtc { + +// Interface for accessing recordable elements of an encoded frame. +class RecordableEncodedFrame { + public: + // Encoded resolution in pixels + struct EncodedResolution { + unsigned width; + unsigned height; + }; + + virtual ~RecordableEncodedFrame() = default; + + // Provides access to encoded data + virtual rtc::scoped_refptr encoded_buffer() + const = 0; + + // Optionally returns the colorspace of the encoded frame. This can differ + // from the eventually decoded frame's colorspace. + virtual absl::optional color_space() const = 0; + + // Returns the codec of the encoded frame + virtual VideoCodecType codec() const = 0; + + // Returns whether the encoded frame is a key frame + virtual bool is_key_frame() const = 0; + + // Returns the frame's encoded resolution. May be 0x0 if the frame + // doesn't contain resolution information + virtual EncodedResolution resolution() const = 0; + + // Returns the computed render time + virtual Timestamp render_time() const = 0; +}; + +} // namespace webrtc + +#endif // API_VIDEO_RECORDABLE_ENCODED_FRAME_H_ diff --git a/api/video_track_source_proxy.h b/api/video_track_source_proxy.h index 820cdcb286..85405f0a64 100644 --- a/api/video_track_source_proxy.h +++ b/api/video_track_source_proxy.h @@ -34,6 +34,14 @@ PROXY_WORKER_METHOD2(void, PROXY_WORKER_METHOD1(void, RemoveSink, rtc::VideoSinkInterface*) PROXY_METHOD1(void, RegisterObserver, ObserverInterface*) PROXY_METHOD1(void, UnregisterObserver, ObserverInterface*) +PROXY_CONSTMETHOD0(bool, SupportsEncodedOutput) +PROXY_METHOD0(void, GenerateKeyFrame) +PROXY_WORKER_METHOD1(void, + AddEncodedSink, + rtc::VideoSinkInterface*) +PROXY_WORKER_METHOD1(void, + RemoveEncodedSink, + rtc::VideoSinkInterface*) END_PROXY_MAP() } // namespace webrtc