From 6c371ca7005adde9812d9527bea64f552cf53867 Mon Sep 17 00:00:00 2001 From: Elad Alon Date: Thu, 4 Apr 2019 12:28:51 +0200 Subject: [PATCH] Add OnLossNotification() to VideoEncoder and Vp8FrameBufferController MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: webrtc:10501 Change-Id: I33e8bfcf16cf24aadcfdf214d7d9bcd495bf9348 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/131021 Commit-Queue: Elad Alon Reviewed-by: Erik Språng Cr-Commit-Position: refs/heads/master@{#27449} --- api/video_codecs/video_encoder.cc | 3 +++ api/video_codecs/video_encoder.h | 24 +++++++++++++++++++ .../vp8_frame_buffer_controller.h | 6 +++++ api/video_codecs/vp8_temporal_layers.cc | 7 ++++++ api/video_codecs/vp8_temporal_layers.h | 3 +++ .../codecs/vp8/default_temporal_layers.cc | 3 +++ .../codecs/vp8/default_temporal_layers.h | 3 +++ .../codecs/vp8/libvpx_vp8_encoder.cc | 7 ++++++ .../codecs/vp8/libvpx_vp8_encoder.h | 2 ++ .../codecs/vp8/screenshare_layers.cc | 3 +++ .../codecs/vp8/screenshare_layers.h | 3 +++ 11 files changed, 64 insertions(+) diff --git a/api/video_codecs/video_encoder.cc b/api/video_codecs/video_encoder.cc index 9856c08f1f..46496c7e0d 100644 --- a/api/video_codecs/video_encoder.cc +++ b/api/video_codecs/video_encoder.cc @@ -121,6 +121,9 @@ void VideoEncoder::OnPacketLossRateUpdate(float packet_loss_rate) {} void VideoEncoder::OnRttUpdate(int64_t rtt_ms) {} +void VideoEncoder::OnLossNotification( + const LossNotification& loss_notification) {} + // TODO(webrtc:9722): Remove and make pure virtual. VideoEncoder::EncoderInfo VideoEncoder::GetEncoderInfo() const { return EncoderInfo(); diff --git a/api/video_codecs/video_encoder.h b/api/video_codecs/video_encoder.h index 01082536b4..0546c7df65 100644 --- a/api/video_codecs/video_encoder.h +++ b/api/video_codecs/video_encoder.h @@ -208,6 +208,27 @@ class RTC_EXPORT VideoEncoder { DataRate bandwidth_allocation; }; + struct LossNotification { + // The timestamp of the last decodable frame *prior* to the last received. + // (The last received - described below - might itself be decodable or not.) + uint32_t timestamp_of_last_decodable; + // The timestamp of the last received frame. + uint32_t timestamp_of_last_received; + // Describes whether the dependencies of the last received frame were + // all decodable. + // |false| if some dependencies were undecodable, |true| if all dependencies + // were decodable, and |nullopt| if the dependencies are unknown. + absl::optional is_last_received_dependencies_decodable; + // Describes whether the received frame was decodable. + // |false| if some dependency was undecodable or if some packet belonging + // to the last received frame was missed. + // |true| if all dependencies were decodable and all packets belonging + // to the last received frame were received. + // |nullopt| if no packet belonging to the last frame was missed, but the + // last packet in the frame was not yet received. + absl::optional is_last_received_decodable; + }; + static VideoCodecVP8 GetDefaultVp8Settings(); static VideoCodecVP9 GetDefaultVp9Settings(); static VideoCodecH264 GetDefaultH264Settings(); @@ -291,6 +312,9 @@ class RTC_EXPORT VideoEncoder { // Input: - rtt_ms : The new RTT, in milliseconds. virtual void OnRttUpdate(int64_t rtt_ms); + // Called when a loss notification is received. + virtual void OnLossNotification(const LossNotification& loss_notification); + // Returns meta-data about the encoder, such as implementation name. // The output of this method may change during runtime. For instance if a // hardware encoder fails, it may fall back to doing software encoding using diff --git a/api/video_codecs/vp8_frame_buffer_controller.h b/api/video_codecs/vp8_frame_buffer_controller.h index 7f4c282ded..f3f6745504 100644 --- a/api/video_codecs/vp8_frame_buffer_controller.h +++ b/api/video_codecs/vp8_frame_buffer_controller.h @@ -14,7 +14,9 @@ #include #include +#include "absl/types/optional.h" #include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" #include "api/video_codecs/vp8_frame_config.h" namespace webrtc { @@ -137,6 +139,10 @@ class Vp8FrameBufferController { // Called by the encoder when the round trip time changes. virtual void OnRttUpdate(int64_t rtt_ms) = 0; + + // Called when a loss notification is received. + virtual void OnLossNotification( + const VideoEncoder::LossNotification loss_notification) = 0; }; // Interface for a factory of Vp8FrameBufferController instances. diff --git a/api/video_codecs/vp8_temporal_layers.cc b/api/video_codecs/vp8_temporal_layers.cc index df4dfa7c11..4c552852fc 100644 --- a/api/video_codecs/vp8_temporal_layers.cc +++ b/api/video_codecs/vp8_temporal_layers.cc @@ -82,4 +82,11 @@ void Vp8TemporalLayers::OnRttUpdate(int64_t rtt_ms) { } } +void Vp8TemporalLayers::OnLossNotification( + const VideoEncoder::LossNotification loss_notification) { + for (auto& controller : controllers_) { + controller->OnLossNotification(loss_notification); + } +} + } // namespace webrtc diff --git a/api/video_codecs/vp8_temporal_layers.h b/api/video_codecs/vp8_temporal_layers.h index d5d29cdb6c..c46699bbdd 100644 --- a/api/video_codecs/vp8_temporal_layers.h +++ b/api/video_codecs/vp8_temporal_layers.h @@ -59,6 +59,9 @@ class Vp8TemporalLayers final : public Vp8FrameBufferController { void OnRttUpdate(int64_t rtt_ms) override; + void OnLossNotification( + const VideoEncoder::LossNotification loss_notification) override; + private: std::vector> controllers_; }; diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers.cc b/modules/video_coding/codecs/vp8/default_temporal_layers.cc index 27a801b093..37363468c6 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers.cc +++ b/modules/video_coding/codecs/vp8/default_temporal_layers.cc @@ -542,6 +542,9 @@ void DefaultTemporalLayers::OnPacketLossRateUpdate(float packet_loss_rate) {} void DefaultTemporalLayers::OnRttUpdate(int64_t rtt_ms) {} +void DefaultTemporalLayers::OnLossNotification( + const VideoEncoder::LossNotification loss_notification) {} + TemplateStructure DefaultTemporalLayers::GetTemplateStructure( int num_layers) const { RTC_CHECK_LT(num_layers, 5); diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers.h b/modules/video_coding/codecs/vp8/default_temporal_layers.h index 9aa95db49b..1667b5cd39 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers.h +++ b/modules/video_coding/codecs/vp8/default_temporal_layers.h @@ -61,6 +61,9 @@ class DefaultTemporalLayers final : public Vp8FrameBufferController { void OnRttUpdate(int64_t rtt_ms) override; + void OnLossNotification( + const VideoEncoder::LossNotification loss_notification) override; + private: struct DependencyInfo { DependencyInfo() = default; diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc index 4e367d3e13..e2d4089534 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc @@ -330,6 +330,13 @@ void LibvpxVp8Encoder::OnRttUpdate(int64_t rtt_ms) { } } +void LibvpxVp8Encoder::OnLossNotification( + const LossNotification& loss_notification) { + if (frame_buffer_controller_) { + frame_buffer_controller_->OnLossNotification(loss_notification); + } +} + void LibvpxVp8Encoder::SetStreamState(bool send_stream, int stream_idx) { if (send_stream && !send_stream_[stream_idx]) { // Need a key frame if we have not sent this stream before. diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h index c6bd455189..535f032d76 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h @@ -61,6 +61,8 @@ class LibvpxVp8Encoder : public VideoEncoder { void OnRttUpdate(int64_t rtt_ms) override; + void OnLossNotification(const LossNotification& loss_notification) override; + EncoderInfo GetEncoderInfo() const override; static vpx_enc_frame_flags_t EncodeFlags(const Vp8FrameConfig& references); diff --git a/modules/video_coding/codecs/vp8/screenshare_layers.cc b/modules/video_coding/codecs/vp8/screenshare_layers.cc index 3f05341852..a58b98423e 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers.cc +++ b/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -388,6 +388,9 @@ void ScreenshareLayers::OnPacketLossRateUpdate(float packet_loss_rate) {} void ScreenshareLayers::OnRttUpdate(int64_t rtt_ms) {} +void ScreenshareLayers::OnLossNotification( + const VideoEncoder::LossNotification loss_notification) {} + TemplateStructure ScreenshareLayers::GetTemplateStructure( int num_layers) const { RTC_CHECK_LT(num_layers, 3); diff --git a/modules/video_coding/codecs/vp8/screenshare_layers.h b/modules/video_coding/codecs/vp8/screenshare_layers.h index 8fe5fa3556..111b1c1322 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers.h +++ b/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -65,6 +65,9 @@ class ScreenshareLayers final : public Vp8FrameBufferController { void OnRttUpdate(int64_t rtt_ms) override; + void OnLossNotification( + const VideoEncoder::LossNotification loss_notification) override; + private: enum class TemporalLayerState : int { kDrop, kTl0, kTl1, kTl1Sync };