From bf35298996a1ec1830f536bb9d572491198a574f Mon Sep 17 00:00:00 2001 From: Ilya Nikolaevskiy Date: Mon, 2 Oct 2017 10:08:25 +0200 Subject: [PATCH] Implement temporal layers checkers for vp8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All frames are checked against hard-coded dependency graph using new helper class. It's invoked in RTC_DCHECK(). Only DefaultTemporalLayers are fully implemented in this CL, checker for ScreenshareLayers is not doing anything for now. Bug: none Change-Id: I3ec017176d8c25f7572c8f161e52f2ebfac8220f Reviewed-on: https://webrtc-review.googlesource.com/3740 Reviewed-by: Per Kjellander Reviewed-by: Erik Språng Commit-Queue: Ilya Nikolaevskiy Cr-Commit-Position: refs/heads/master@{#20066} --- media/engine/simulcast_encoder_adapter.cc | 7 + .../codecs/vp8/default_temporal_layers.cc | 177 +++++++++++++++++- .../codecs/vp8/default_temporal_layers.h | 45 ++++- .../vp8/default_temporal_layers_unittest.cc | 31 ++- .../codecs/vp8/screenshare_layers.cc | 53 ++++-- .../codecs/vp8/screenshare_layers.h | 23 ++- .../video_coding/codecs/vp8/temporal_layers.h | 52 ++++- modules/video_coding/codecs/vp8/vp8_impl.cc | 42 +++-- modules/video_coding/codecs/vp8/vp8_impl.h | 5 +- 9 files changed, 360 insertions(+), 75 deletions(-) diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc index c931aba62f..808bc0fc15 100644 --- a/media/engine/simulcast_encoder_adapter.cc +++ b/media/engine/simulcast_encoder_adapter.cc @@ -118,6 +118,13 @@ class TemporalLayersFactoryAdapter : public webrtc::TemporalLayersFactory { return tl_factory_.Create(adapted_simulcast_id_, temporal_layers, initial_tl0_pic_idx); } + std::unique_ptr CreateChecker( + int simulcast_id, + int temporal_layers, + uint8_t initial_tl0_pic_idx) const override { + return tl_factory_.CreateChecker(adapted_simulcast_id_, temporal_layers, + initial_tl0_pic_idx); + } const int adapted_simulcast_id_; const TemporalLayersFactory& tl_factory_; diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers.cc b/modules/video_coding/codecs/vp8/default_temporal_layers.cc index 6757ba26b3..98306bb6a8 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers.cc +++ b/modules/video_coding/codecs/vp8/default_temporal_layers.cc @@ -1,11 +1,11 @@ /* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. -* -* Use of this source code is governed by a BSD-style license -* that can be found in the LICENSE file in the root of the source -* tree. An additional intellectual property rights grant can be found -* in the file PATENTS. All contributing project authors may -* be found in the AUTHORS file in the root of the source tree. -*/ + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ #include "modules/video_coding/codecs/vp8/default_temporal_layers.h" @@ -13,16 +13,19 @@ #include #include +#include +#include #include #include "modules/include/module_common_types.h" #include "modules/video_coding/codecs/vp8/include/vp8_common_types.h" #include "modules/video_coding/include/video_codec_interface.h" #include "rtc_base/checks.h" +#include "rtc_base/logging.h" #include "system_wrappers/include/field_trial.h" -#include "vpx/vpx_encoder.h" #include "vpx/vp8cx.h" +#include "vpx/vpx_encoder.h" namespace webrtc { @@ -424,8 +427,166 @@ TemporalLayers* TemporalLayersFactory::Create( return tl; } +std::unique_ptr TemporalLayersFactory::CreateChecker( + int /*simulcast_id*/, + int temporal_layers, + uint8_t initial_tl0_pic_idx) const { + TemporalLayersChecker* tlc = + new DefaultTemporalLayersChecker(temporal_layers, initial_tl0_pic_idx); + return std::unique_ptr(tlc); +} + void TemporalLayersFactory::SetListener(TemporalLayersListener* listener) { listener_ = listener; } +// Returns list of temporal dependencies for each frame in the temporal pattern. +// Values are lists of indecies in the pattern. +std::vector> GetTemporalDependencies( + int num_temporal_layers) { + switch (num_temporal_layers) { + case 1: + return {{0}}; + case 2: + return {{6}, {0}, {0}, {1, 2}, {2}, {3, 4}, {4}, {5, 6}}; + case 3: + if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) { + return {{0}, {0}, {0}, {0, 1, 2}}; + } else { + return {{4}, {0}, {0}, {0, 2}, {0}, {2, 4}, {2, 4}, {4, 6}}; + } + case 4: + return {{8}, {0}, {0}, {0, 2}, + {0}, {0, 2, 4}, {0, 2, 4}, {0, 4, 6}, + {0}, {4, 6, 8}, {4, 6, 8}, {4, 8, 10}, + {4, 8}, {8, 10, 12}, {8, 10, 12}, {8, 12, 14}}; + default: + RTC_NOTREACHED(); + return {}; + } +} + +DefaultTemporalLayersChecker::DefaultTemporalLayersChecker( + int num_temporal_layers, + uint8_t initial_tl0_pic_idx) + : num_layers_(std::max(1, num_temporal_layers)), + temporal_ids_(GetTemporalIds(num_layers_)), + temporal_dependencies_(GetTemporalDependencies(num_layers_)), + pattern_idx_(255) { + int i = 0; + while (temporal_ids_.size() < temporal_dependencies_.size()) { + temporal_ids_.push_back(temporal_ids_[i++]); + } +} + +bool DefaultTemporalLayersChecker::CheckTemporalConfig( + bool frame_is_keyframe, + const TemporalLayers::FrameConfig& frame_config) { + ++pattern_idx_; + if (pattern_idx_ == temporal_ids_.size()) { + // All non key-frame buffers should be updated each pattern cycle. + if (!last_.is_keyframe && !last_.is_updated_this_cycle) { + LOG(LS_ERROR) << "Last buffer was not updated during pattern cycle."; + return false; + } + if (!arf_.is_keyframe && !arf_.is_updated_this_cycle) { + LOG(LS_ERROR) << "Arf buffer was not updated during pattern cycle."; + return false; + } + if (!golden_.is_keyframe && !golden_.is_updated_this_cycle) { + LOG(LS_ERROR) << "Golden buffer was not updated during pattern cycle."; + return false; + } + last_.is_updated_this_cycle = false; + arf_.is_updated_this_cycle = false; + golden_.is_updated_this_cycle = false; + pattern_idx_ = 0; + } + uint8_t expected_tl_idx = temporal_ids_[pattern_idx_]; + if (frame_config.packetizer_temporal_idx != expected_tl_idx) { + LOG(LS_ERROR) << "Frame has an incorrect temporal index. Expected: " + << static_cast(expected_tl_idx) << " Actual: " + << static_cast(frame_config.packetizer_temporal_idx); + return false; + } + + bool need_sync = temporal_ids_[pattern_idx_] > 0 && + temporal_ids_[pattern_idx_] != kNoTemporalIdx; + std::vector dependencies; + + if (frame_config.last_buffer_flags & + TemporalLayers::BufferFlags::kReference) { + uint8_t referenced_layer = temporal_ids_[last_.pattern_idx]; + if (referenced_layer > 0) { + need_sync = false; + } + if (!last_.is_keyframe) { + dependencies.push_back(last_.pattern_idx); + } + } + + if (frame_config.arf_buffer_flags & TemporalLayers::BufferFlags::kReference) { + uint8_t referenced_layer = temporal_ids_[arf_.pattern_idx]; + if (referenced_layer > 0) { + need_sync = false; + } + if (!arf_.is_keyframe) { + dependencies.push_back(arf_.pattern_idx); + } + } + + if (frame_config.golden_buffer_flags & + TemporalLayers::BufferFlags::kReference) { + uint8_t referenced_layer = temporal_ids_[golden_.pattern_idx]; + if (referenced_layer > 0) { + need_sync = false; + } + if (!golden_.is_keyframe) { + dependencies.push_back(golden_.pattern_idx); + } + } + + if (need_sync != frame_config.layer_sync) { + LOG(LS_ERROR) << "Sync bit is set incorrectly on a frame. Expected: " + << need_sync << " Actual: " << frame_config.layer_sync; + return false; + } + + if (!frame_is_keyframe) { + size_t i; + for (i = 0; i < dependencies.size(); ++i) { + if (temporal_dependencies_[pattern_idx_].find(dependencies[i]) == + temporal_dependencies_[pattern_idx_].end()) { + LOG(LS_ERROR) << "Illegal temporal dependency out of defined pattern " + "from position " + << static_cast(pattern_idx_) << " to position " + << static_cast(dependencies[i]); + return false; + } + } + } + + if (frame_config.last_buffer_flags & TemporalLayers::BufferFlags::kUpdate) { + last_.is_updated_this_cycle = true; + last_.pattern_idx = pattern_idx_; + last_.is_keyframe = false; + } + if (frame_config.arf_buffer_flags & TemporalLayers::BufferFlags::kUpdate) { + arf_.is_updated_this_cycle = true; + arf_.pattern_idx = pattern_idx_; + arf_.is_keyframe = false; + } + if (frame_config.golden_buffer_flags & TemporalLayers::BufferFlags::kUpdate) { + golden_.is_updated_this_cycle = true; + golden_.pattern_idx = pattern_idx_; + golden_.is_keyframe = false; + } + if (frame_is_keyframe) { + last_.is_keyframe = true; + arf_.is_keyframe = true; + golden_.is_keyframe = true; + } + return true; +} + } // namespace webrtc diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers.h b/modules/video_coding/codecs/vp8/default_temporal_layers.h index e1a5987dec..f192556920 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers.h +++ b/modules/video_coding/codecs/vp8/default_temporal_layers.h @@ -1,17 +1,18 @@ /* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. -* -* Use of this source code is governed by a BSD-style license -* that can be found in the LICENSE file in the root of the source -* tree. An additional intellectual property rights grant can be found -* in the file PATENTS. All contributing project authors may -* be found in the AUTHORS file in the root of the source tree. -*/ + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ /* -* This file defines classes for doing temporal layers with VP8. -*/ + * This file defines classes for doing temporal layers with VP8. + */ #ifndef MODULES_VIDEO_CODING_CODECS_VP8_DEFAULT_TEMPORAL_LAYERS_H_ #define MODULES_VIDEO_CODING_CODECS_VP8_DEFAULT_TEMPORAL_LAYERS_H_ +#include #include #include "modules/video_coding/codecs/vp8/temporal_layers.h" @@ -59,5 +60,31 @@ class DefaultTemporalLayers : public TemporalLayers { rtc::Optional> new_bitrates_kbps_; }; +class DefaultTemporalLayersChecker : public TemporalLayersChecker { + public: + DefaultTemporalLayersChecker(int number_of_temporal_layers, + uint8_t initial_tl0_pic_idx); + bool CheckTemporalConfig( + bool frame_is_keyframe, + const TemporalLayers::FrameConfig& frame_config) override; + + private: + struct BufferState { + BufferState() + : is_updated_this_cycle(false), is_keyframe(true), pattern_idx(0) {} + + bool is_updated_this_cycle; + bool is_keyframe; + uint8_t pattern_idx; + }; + const size_t num_layers_; + std::vector temporal_ids_; + const std::vector> temporal_dependencies_; + BufferState last_; + BufferState arf_; + BufferState golden_; + uint8_t pattern_idx_; +}; + } // namespace webrtc #endif // MODULES_VIDEO_CODING_CODECS_VP8_DEFAULT_TEMPORAL_LAYERS_H_ diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc b/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc index d28b67ff89..1c1d63f1f8 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc +++ b/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc @@ -9,12 +9,12 @@ */ #include "modules/video_coding/codecs/vp8/default_temporal_layers.h" -#include "vpx/vp8cx.h" -#include "vpx/vpx_encoder.h" #include "modules/video_coding/codecs/vp8/vp8_impl.h" #include "modules/video_coding/include/video_codec_interface.h" #include "test/field_trial.h" #include "test/gtest.h" +#include "vpx/vp8cx.h" +#include "vpx/vpx_encoder.h" namespace webrtc { namespace test { @@ -53,6 +53,7 @@ enum { TEST(TemporalLayersTest, 2Layers) { DefaultTemporalLayers tl(2, 0); + DefaultTemporalLayersChecker checker(2, 0); vpx_codec_enc_cfg_t cfg; CodecSpecificInfoVP8 vp8_info; tl.OnRatesUpdated(500, 500, 30); @@ -87,11 +88,12 @@ TEST(TemporalLayersTest, 2Layers) { for (int i = 0; i < 16; ++i) { TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)) << i; - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, 0); + tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_info, 0); + EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config)); EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx); EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id); - EXPECT_EQ(expected_layer_sync[i], vp8_info.layerSync); + EXPECT_EQ(i == 0 || expected_layer_sync[i], vp8_info.layerSync); EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync); timestamp += 3000; } @@ -99,6 +101,7 @@ TEST(TemporalLayersTest, 2Layers) { TEST(TemporalLayersTest, 3Layers) { DefaultTemporalLayers tl(3, 0); + DefaultTemporalLayersChecker checker(3, 0); vpx_codec_enc_cfg_t cfg; CodecSpecificInfoVP8 vp8_info; tl.OnRatesUpdated(500, 500, 30); @@ -133,11 +136,12 @@ TEST(TemporalLayersTest, 3Layers) { for (int i = 0; i < 16; ++i) { TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)) << i; - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, 0); + tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_info, 0); + EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config)); EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx); EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id); - EXPECT_EQ(expected_layer_sync[i], vp8_info.layerSync); + EXPECT_EQ(i == 0 || expected_layer_sync[i], vp8_info.layerSync); EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync); timestamp += 3000; } @@ -146,6 +150,7 @@ TEST(TemporalLayersTest, 3Layers) { TEST(TemporalLayersTest, Alternative3Layers) { ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/"); DefaultTemporalLayers tl(3, 0); + DefaultTemporalLayersChecker checker(3, 0); vpx_codec_enc_cfg_t cfg; CodecSpecificInfoVP8 vp8_info; tl.OnRatesUpdated(500, 500, 30); @@ -168,11 +173,12 @@ TEST(TemporalLayersTest, Alternative3Layers) { for (int i = 0; i < 8; ++i) { TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)) << i; - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, 0); + tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_info, 0); + EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config)); EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx); EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id); - EXPECT_EQ(expected_layer_sync[i], vp8_info.layerSync); + EXPECT_EQ(i == 0 || expected_layer_sync[i], vp8_info.layerSync); EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync); timestamp += 3000; } @@ -180,6 +186,7 @@ TEST(TemporalLayersTest, Alternative3Layers) { TEST(TemporalLayersTest, 4Layers) { DefaultTemporalLayers tl(4, 0); + DefaultTemporalLayersChecker checker(4, 0); vpx_codec_enc_cfg_t cfg; CodecSpecificInfoVP8 vp8_info; tl.OnRatesUpdated(500, 500, 30); @@ -213,11 +220,12 @@ TEST(TemporalLayersTest, 4Layers) { for (int i = 0; i < 16; ++i) { TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)) << i; - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, 0); + tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_info, 0); + EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config)); EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx); EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id); - EXPECT_EQ(expected_layer_sync[i], vp8_info.layerSync); + EXPECT_EQ(i == 0 || expected_layer_sync[i], vp8_info.layerSync); EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync); timestamp += 3000; } @@ -225,6 +233,7 @@ TEST(TemporalLayersTest, 4Layers) { TEST(TemporalLayersTest, KeyFrame) { DefaultTemporalLayers tl(3, 0); + DefaultTemporalLayersChecker checker(3, 0); vpx_codec_enc_cfg_t cfg; CodecSpecificInfoVP8 vp8_info; tl.OnRatesUpdated(500, 500, 30); @@ -249,6 +258,7 @@ TEST(TemporalLayersTest, KeyFrame) { TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)) << i; tl.PopulateCodecSpecific(true, tl_config, &vp8_info, 0); + EXPECT_TRUE(checker.CheckTemporalConfig(true, tl_config)); EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx); EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id); EXPECT_EQ(0, vp8_info.temporalIdx) @@ -260,6 +270,7 @@ TEST(TemporalLayersTest, KeyFrame) { TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[7], VP8EncoderImpl::EncodeFlags(tl_config)); tl.PopulateCodecSpecific(false, tl_config, &vp8_info, 0); + EXPECT_TRUE(checker.CheckTemporalConfig(false, tl_config)); EXPECT_NE(0, vp8_info.temporalIdx) << "To test something useful, this frame should not use layer 0."; EXPECT_EQ(expected_temporal_idx[7], vp8_info.temporalIdx) diff --git a/modules/video_coding/codecs/vp8/screenshare_layers.cc b/modules/video_coding/codecs/vp8/screenshare_layers.cc index 1d8b6de7bd..dac16ebe70 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers.cc +++ b/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -1,24 +1,25 @@ /* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. -* -* Use of this source code is governed by a BSD-style license -* that can be found in the LICENSE file in the root of the source -* tree. An additional intellectual property rights grant can be found -* in the file PATENTS. All contributing project authors may -* be found in the AUTHORS file in the root of the source tree. -*/ + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ #include "modules/video_coding/codecs/vp8/screenshare_layers.h" #include #include +#include -#include "vpx/vp8cx.h" -#include "vpx/vpx_encoder.h" #include "modules/video_coding/include/video_codec_interface.h" #include "rtc_base/checks.h" #include "system_wrappers/include/clock.h" #include "system_wrappers/include/metrics.h" +#include "vpx/vp8cx.h" +#include "vpx/vpx_encoder.h" namespace webrtc { @@ -54,6 +55,23 @@ webrtc::TemporalLayers* ScreenshareTemporalLayersFactory::Create( return tl; } +std::unique_ptr +ScreenshareTemporalLayersFactory::CreateChecker( + int simulcast_id, + int temporal_layers, + uint8_t initial_tl0_pic_idx) const { + webrtc::TemporalLayersChecker* tlc; + if (simulcast_id == 0) { + tlc = new webrtc::ScreenshareTemporalLayersChecker(temporal_layers, + initial_tl0_pic_idx); + } else { + TemporalLayersFactory rt_tl_factory; + return rt_tl_factory.CreateChecker(simulcast_id, temporal_layers, + initial_tl0_pic_idx); + } + return std::unique_ptr(tlc); +} + ScreenshareLayers::ScreenshareLayers(int num_temporal_layers, uint8_t initial_tl0_pic_idx, Clock* clock) @@ -433,8 +451,9 @@ void ScreenshareLayers::UpdateHistograms() { int total_frames = stats_.num_tl0_frames_ + stats_.num_tl1_frames_; RTC_HISTOGRAM_COUNTS_10000( "WebRTC.Video.Screenshare.FramesPerDrop", - (stats_.num_dropped_frames_ == 0 ? 0 : total_frames / - stats_.num_dropped_frames_)); + (stats_.num_dropped_frames_ == 0 + ? 0 + : total_frames / stats_.num_dropped_frames_)); RTC_HISTOGRAM_COUNTS_10000( "WebRTC.Video.Screenshare.FramesPerOvershoot", (stats_.num_overshoots_ == 0 ? 0 @@ -456,4 +475,16 @@ void ScreenshareLayers::UpdateHistograms() { } } +ScreenshareTemporalLayersChecker::ScreenshareTemporalLayersChecker( + int /*num_temporal_layers*/, + uint8_t /*initial_tl0_pic_idx*/) {} + +// TODO(ilnik): Implement layers dependency checks here. Keep track of +// last/golden/arf buffers and sync bits. +bool ScreenshareTemporalLayersChecker::CheckTemporalConfig( + bool /*frame_is_keyframe*/, + const TemporalLayers::FrameConfig& /*frame_config*/) { + return true; +} + } // namespace webrtc diff --git a/modules/video_coding/codecs/vp8/screenshare_layers.h b/modules/video_coding/codecs/vp8/screenshare_layers.h index 5548c5a7db..9d7dbd2e04 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers.h +++ b/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -1,11 +1,11 @@ /* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. -* -* Use of this source code is governed by a BSD-style license -* that can be found in the LICENSE file in the root of the source -* tree. An additional intellectual property rights grant can be found -* in the file PATENTS. All contributing project authors may -* be found in the AUTHORS file in the root of the source tree. -*/ + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ #ifndef MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_ #define MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_ @@ -122,6 +122,15 @@ class ScreenshareLayers : public TemporalLayers { int64_t tl1_target_bitrate_sum_ = 0; } stats_; }; + +class ScreenshareTemporalLayersChecker : public TemporalLayersChecker { + public: + ScreenshareTemporalLayersChecker(int number_of_temporal_layers, + uint8_t initial_tl0_pic_idx); + bool CheckTemporalConfig( + bool frame_is_keyframe, + const TemporalLayers::FrameConfig& frame_config) override; +}; } // namespace webrtc #endif // MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_ diff --git a/modules/video_coding/codecs/vp8/temporal_layers.h b/modules/video_coding/codecs/vp8/temporal_layers.h index cedfad51c8..6a164f1319 100644 --- a/modules/video_coding/codecs/vp8/temporal_layers.h +++ b/modules/video_coding/codecs/vp8/temporal_layers.h @@ -1,21 +1,21 @@ /* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. -* -* Use of this source code is governed by a BSD-style license -* that can be found in the LICENSE file in the root of the source -* tree. An additional intellectual property rights grant can be found -* in the file PATENTS. All contributing project authors may -* be found in the AUTHORS file in the root of the source tree. -*/ + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ /* -* This file defines the interface for doing temporal layers with VP8. -*/ + * This file defines the interface for doing temporal layers with VP8. + */ #ifndef MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_ #define MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_ #include +#include #include "typedefs.h" // NOLINT(build/include) - struct vpx_codec_enc_cfg; typedef struct vpx_codec_enc_cfg vpx_codec_enc_cfg_t; @@ -115,6 +115,8 @@ class TemporalLayers { }; class TemporalLayersListener; +class TemporalLayersChecker; + class TemporalLayersFactory { public: TemporalLayersFactory() : listener_(nullptr) {} @@ -122,6 +124,15 @@ class TemporalLayersFactory { virtual TemporalLayers* Create(int simulcast_id, int temporal_layers, uint8_t initial_tl0_pic_idx) const; + + // Creates helper class which performs online checks of a correctness of + // temporal layers dependencies returned by TemporalLayers class created in + // the same factory. + virtual std::unique_ptr CreateChecker( + int simulcast_id, + int temporal_layers, + uint8_t initial_tl0_pic_idx) const; + void SetListener(TemporalLayersListener* listener); protected: @@ -136,6 +147,14 @@ class ScreenshareTemporalLayersFactory : public webrtc::TemporalLayersFactory { webrtc::TemporalLayers* Create(int simulcast_id, int num_temporal_layers, uint8_t initial_tl0_pic_idx) const override; + + // Creates helper class which performs online checks of a correctness of + // temporal layers dependencies returned by TemporalLayers class created in + // the same factory. + std::unique_ptr CreateChecker( + int simulcast_id, + int temporal_layers, + uint8_t initial_tl0_pic_idx) const override; }; class TemporalLayersListener { @@ -147,5 +166,18 @@ class TemporalLayersListener { TemporalLayers* layers) = 0; }; +// Used only inside RTC_DCHECK(). It checks correctness of temporal layers +// dependencies and sync bits. The only method of this class is called after +// each UpdateLayersConfig() of a corresponding TemporalLayers class. +class TemporalLayersChecker { + public: + TemporalLayersChecker() {} + virtual ~TemporalLayersChecker() {} + + virtual bool CheckTemporalConfig( + bool frame_is_keyframe, + const TemporalLayers::FrameConfig& frame_config) = 0; +}; + } // namespace webrtc #endif // MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_ diff --git a/modules/video_coding/codecs/vp8/vp8_impl.cc b/modules/video_coding/codecs/vp8/vp8_impl.cc index 463afead7e..bebc749e22 100644 --- a/modules/video_coding/codecs/vp8/vp8_impl.cc +++ b/modules/video_coding/codecs/vp8/vp8_impl.cc @@ -17,8 +17,8 @@ #include // NOTE(ajm): Path provided by gyp. -#include "libyuv/scale.h" // NOLINT #include "libyuv/convert.h" // NOLINT +#include "libyuv/scale.h" // NOLINT #include "common_types.h" // NOLINT(build/include) #include "common_video/libyuv/include/webrtc_libyuv.h" @@ -224,6 +224,7 @@ VP8EncoderImpl::VP8EncoderImpl() tl0_pic_idx_.push_back(random.Rand()); } temporal_layers_.reserve(kMaxSimulcastStreams); + temporal_layers_checkers_.reserve(kMaxSimulcastStreams); raw_images_.reserve(kMaxSimulcastStreams); encoded_images_.reserve(kMaxSimulcastStreams); send_stream_.reserve(kMaxSimulcastStreams); @@ -263,6 +264,7 @@ int VP8EncoderImpl::Release() { tl0_pic_idx_[i] = temporal_layers_[i]->Tl0PicIdx(); } temporal_layers_.clear(); + temporal_layers_checkers_.clear(); inited_ = false; return ret_val; } @@ -332,8 +334,7 @@ const char* VP8EncoderImpl::ImplementationName() const { return "libvpx"; } -void VP8EncoderImpl::SetStreamState(bool send_stream, - int stream_idx) { +void VP8EncoderImpl::SetStreamState(bool send_stream, int stream_idx) { if (send_stream && !send_stream_[stream_idx]) { // Need a key frame if we have not sent this stream before. key_frame_request_[stream_idx] = true; @@ -349,6 +350,8 @@ void VP8EncoderImpl::SetupTemporalLayers(int num_streams, if (num_streams == 1) { temporal_layers_.emplace_back( tl_factory->Create(0, num_temporal_layers, tl0_pic_idx_[0])); + temporal_layers_checkers_.emplace_back( + tl_factory->CreateChecker(0, num_temporal_layers, tl0_pic_idx_[0])); } else { for (int i = 0; i < num_streams; ++i) { RTC_CHECK_GT(num_temporal_layers, 0); @@ -356,6 +359,8 @@ void VP8EncoderImpl::SetupTemporalLayers(int num_streams, codec.simulcastStream[i].numberOfTemporalLayers); temporal_layers_.emplace_back( tl_factory->Create(i, layers, tl0_pic_idx_[i])); + temporal_layers_checkers_.emplace_back( + tl_factory->CreateChecker(i, layers, tl0_pic_idx_[i])); } } } @@ -754,17 +759,6 @@ int VP8EncoderImpl::Encode(const VideoFrame& frame, raw_images_[i].stride[VPX_PLANE_V], raw_images_[i].d_w, raw_images_[i].d_h, libyuv::kFilterBilinear); } - vpx_enc_frame_flags_t flags[kMaxSimulcastStreams]; - TemporalLayers::FrameConfig tl_configs[kMaxSimulcastStreams]; - for (size_t i = 0; i < encoders_.size(); ++i) { - tl_configs[i] = temporal_layers_[i]->UpdateLayerConfig(frame.timestamp()); - - if (tl_configs[i].drop_frame) { - // Drop this frame. - return WEBRTC_VIDEO_CODEC_OK; - } - flags[i] = EncodeFlags(tl_configs[i]); - } bool send_key_frame = false; for (size_t i = 0; i < key_frame_request_.size() && i < send_stream_.size(); ++i) { @@ -782,6 +776,18 @@ int VP8EncoderImpl::Encode(const VideoFrame& frame, } } } + vpx_enc_frame_flags_t flags[kMaxSimulcastStreams]; + TemporalLayers::FrameConfig tl_configs[kMaxSimulcastStreams]; + for (size_t i = 0; i < encoders_.size(); ++i) { + tl_configs[i] = temporal_layers_[i]->UpdateLayerConfig(frame.timestamp()); + RTC_DCHECK(temporal_layers_checkers_[i]->CheckTemporalConfig( + send_key_frame, tl_configs[i])); + if (tl_configs[i].drop_frame) { + // Drop this frame. + return WEBRTC_VIDEO_CODEC_OK; + } + flags[i] = EncodeFlags(tl_configs[i]); + } if (send_key_frame) { // Adapt the size of the key frame when in screenshare with 1 temporal // layer. @@ -833,13 +839,13 @@ int VP8EncoderImpl::Encode(const VideoFrame& frame, // the frame must be reencoded with the same parameters again because // target bitrate is exceeded and encoder state has been reset. while (num_tries == 0 || - (num_tries == 1 && + (num_tries == 1 && error == WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT)) { ++num_tries; // Note we must pass 0 for |flags| field in encode call below since they are // set above in |vpx_codec_control| function for each encoder/spatial layer. error = vpx_codec_encode(&encoders_[0], &raw_images_[0], timestamp_, - duration, 0, VPX_DL_REALTIME); + duration, 0, VPX_DL_REALTIME); // Reset specific intra frame thresholds, following the key frame. if (send_key_frame) { vpx_codec_control(&(encoders_[0]), VP8E_SET_MAX_INTRA_BITRATE_PCT, @@ -1233,8 +1239,8 @@ int VP8DecoderImpl::ReturnFrame(const vpx_image_t* img, img->planes[VPX_PLANE_V], img->stride[VPX_PLANE_V], buffer->MutableDataY(), buffer->StrideY(), buffer->MutableDataU(), buffer->StrideU(), - buffer->MutableDataV(), buffer->StrideV(), - img->d_w, img->d_h); + buffer->MutableDataV(), buffer->StrideV(), img->d_w, + img->d_h); VideoFrame decoded_image(buffer, timestamp, 0, kVideoRotation_0); decoded_image.set_ntp_time_ms(ntp_time_ms); diff --git a/modules/video_coding/codecs/vp8/vp8_impl.h b/modules/video_coding/codecs/vp8/vp8_impl.h index 069cca9de2..6964760cd5 100644 --- a/modules/video_coding/codecs/vp8/vp8_impl.h +++ b/modules/video_coding/codecs/vp8/vp8_impl.h @@ -18,10 +18,10 @@ // NOTE: This include order must remain to avoid compile errors, even though // it breaks the style guide. -#include "vpx/vpx_encoder.h" -#include "vpx/vpx_decoder.h" #include "vpx/vp8cx.h" #include "vpx/vp8dx.h" +#include "vpx/vpx_decoder.h" +#include "vpx/vpx_encoder.h" #include "api/video/video_frame.h" #include "common_video/include/i420_buffer_pool.h" @@ -105,6 +105,7 @@ class VP8EncoderImpl : public VP8Encoder { int number_of_cores_; uint32_t rc_max_intra_target_; std::vector> temporal_layers_; + std::vector> temporal_layers_checkers_; std::vector picture_id_; std::vector tl0_pic_idx_; std::vector key_frame_request_;