diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers.cc b/modules/video_coding/codecs/vp8/default_temporal_layers.cc index 0b593a58b3..d53f46994d 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers.cc +++ b/modules/video_coding/codecs/vp8/default_temporal_layers.cc @@ -58,6 +58,15 @@ TemporalLayers::FrameConfig::FrameConfig(TemporalLayers::BufferFlags last, first_reference(Vp8BufferReference::kNone), second_reference(Vp8BufferReference::kNone) {} +DefaultTemporalLayers::PendingFrame::PendingFrame() = default; +DefaultTemporalLayers::PendingFrame::PendingFrame( + bool expired, + uint8_t updated_buffers_mask, + const FrameConfig& frame_config) + : expired(expired), + updated_buffer_mask(updated_buffers_mask), + frame_config(frame_config) {} + namespace { static constexpr uint8_t kUninitializedPatternIndex = std::numeric_limits::max(); @@ -251,7 +260,10 @@ DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers) temporal_ids_(GetTemporalIds(num_layers_)), temporal_pattern_(GetTemporalPattern(num_layers_)), kf_buffers_(FindKfBuffers(temporal_pattern_)), - pattern_idx_(kUninitializedPatternIndex) { + pattern_idx_(kUninitializedPatternIndex), + checker_(TemporalLayersChecker::CreateTemporalLayersChecker( + TemporalLayersType::kFixedPattern, + number_of_temporal_layers)) { RTC_CHECK_GE(kMaxTemporalStreams, number_of_temporal_layers); RTC_CHECK_GE(number_of_temporal_layers, 0); RTC_CHECK_LE(number_of_temporal_layers, 4); @@ -348,7 +360,9 @@ TemporalLayers::FrameConfig DefaultTemporalLayers::UpdateLayerConfig( // Start of new pattern iteration, set up clear state by invalidating any // pending frames, so that we don't make an invalid reference to a buffer // containing data from a previous iteration. - pending_frames_.clear(); + for (auto& it : pending_frames_) { + it.second.expired = true; + } } // Last is always ok to reference as it contains the base layer. For other @@ -376,7 +390,15 @@ TemporalLayers::FrameConfig DefaultTemporalLayers::UpdateLayerConfig( } // Add frame to set of pending frames, awaiting completion. - pending_frames_[timestamp] = GetUpdatedBuffers(tl_config); + pending_frames_[timestamp] = + PendingFrame{false, GetUpdatedBuffers(tl_config), tl_config}; + + if (checker_) { + // Checker does not yet support encoder frame dropping, so validate flags + // here before they can be dropped. + // TODO(sprang): Update checker to support dropping. + RTC_DCHECK(checker_->CheckTemporalConfig(false, tl_config)); + } return tl_config; } @@ -437,61 +459,60 @@ void DefaultTemporalLayers::UpdateSearchOrder(FrameConfig* config) { } } -void DefaultTemporalLayers::PopulateCodecSpecific( - bool frame_is_keyframe, - const TemporalLayers::FrameConfig& tl_config, - CodecSpecificInfoVP8* vp8_info, - uint32_t timestamp) { +void DefaultTemporalLayers::OnEncodeDone(uint32_t rtp_timestamp, + size_t size_bytes, + bool is_keyframe, + int qp, + CodecSpecificInfoVP8* vp8_info) { RTC_DCHECK_GT(num_layers_, 0); + auto pending_frame = pending_frames_.find(rtp_timestamp); + RTC_DCHECK(pending_frame != pending_frames_.end()); + + if (size_bytes == 0) { + pending_frames_.erase(pending_frame); + return; + } + + PendingFrame& frame = pending_frame->second; + if (is_keyframe && checker_) { + // Signal key-frame so checker resets state. + RTC_DCHECK(checker_->CheckTemporalConfig(true, frame.frame_config)); + } + if (num_layers_ == 1) { vp8_info->temporalIdx = kNoTemporalIdx; vp8_info->layerSync = false; } else { - if (frame_is_keyframe) { + if (is_keyframe) { // Restart the temporal pattern on keyframes. pattern_idx_ = 0; vp8_info->temporalIdx = 0; vp8_info->layerSync = true; // Keyframes are always sync frames. - // Update frame count of all kf-only buffers, regardless of state of - // |pending_frames_|. - for (auto it : kf_buffers_) { - frames_since_buffer_refresh_[it] = 0; - } - auto pending_frames = pending_frames_.find(timestamp); - if (pending_frames != pending_frames_.end()) { - for (Vp8BufferReference buffer : kAllBuffers) { - if (kf_buffers_.find(buffer) == kf_buffers_.end()) { - // Key-frames update all buffers, this should be reflected if when - // updating state in FrameEncoded(). - pending_frames->second |= static_cast(buffer); - } + + for (Vp8BufferReference buffer : kAllBuffers) { + if (kf_buffers_.find(buffer) != kf_buffers_.end()) { + // Update frame count of all kf-only buffers, regardless of state of + // |pending_frames_|. + frames_since_buffer_refresh_[buffer] = 0; + } else { + // Key-frames update all buffers, this should be reflected when + // updating state in FrameEncoded(). + frame.updated_buffer_mask |= static_cast(buffer); } } } else { // Delta frame, update codec specifics with temporal id and sync flag. - vp8_info->temporalIdx = tl_config.packetizer_temporal_idx; - vp8_info->layerSync = tl_config.layer_sync; + vp8_info->temporalIdx = frame.frame_config.packetizer_temporal_idx; + vp8_info->layerSync = frame.frame_config.layer_sync; } } -} -void DefaultTemporalLayers::FrameEncoded(uint32_t rtp_timestamp, - size_t size, - int qp) { - auto pending_frame = pending_frames_.find(rtp_timestamp); - if (pending_frame == pending_frames_.end()) { - // Might happen if pipelined encoder delayed encoding until after pattern - // looped. - return; - } - if (size == 0) { - pending_frames_.erase(pending_frame); - return; - } - for (Vp8BufferReference buffer : kAllBuffers) { - if (pending_frame->second & static_cast(buffer)) { - frames_since_buffer_refresh_[buffer] = 0; + if (!frame.expired) { + for (Vp8BufferReference buffer : kAllBuffers) { + if (frame.updated_buffer_mask & static_cast(buffer)) { + frames_since_buffer_refresh_[buffer] = 0; + } } } } diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers.h b/modules/video_coding/codecs/vp8/default_temporal_layers.h index ab54621e95..bda43cd6d9 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers.h +++ b/modules/video_coding/codecs/vp8/default_temporal_layers.h @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -41,12 +42,11 @@ class DefaultTemporalLayers : public TemporalLayers { bool UpdateConfiguration(Vp8EncoderConfig* cfg) override; - void PopulateCodecSpecific(bool frame_is_keyframe, - const TemporalLayers::FrameConfig& tl_config, - CodecSpecificInfoVP8* vp8_info, - uint32_t timestamp) override; - - void FrameEncoded(uint32_t rtp_timestamp, size_t size, int qp) override; + void OnEncodeDone(uint32_t rtp_timestamp, + size_t size_bytes, + bool is_keyframe, + int qp, + CodecSpecificInfoVP8* vp8_info) override; private: static constexpr size_t kKeyframeBuffer = std::numeric_limits::max(); @@ -66,14 +66,30 @@ class DefaultTemporalLayers : public TemporalLayers { // Updated cumulative bitrates, per temporal layer. absl::optional> new_bitrates_bps_; - // Map from rtp timestamp to a bitmask of Vp8BufferReference indicating which - // buffers this frame should update. Reset on pattern loop. - std::map pending_frames_; + struct PendingFrame { + PendingFrame(); + PendingFrame(bool expired, + uint8_t updated_buffers_mask, + const FrameConfig& frame_config); + // Flag indicating if this frame has expired, ie it belongs to a previous + // iteration of the temporal pattern. + bool expired = false; + // Bitmask of Vp8BufferReference flags, indicating which buffers this frame + // updates. + uint8_t updated_buffer_mask = 0; + // The frame config return by UpdateLayerConfig() for this frame. + FrameConfig frame_config; + }; + // Map from rtp timestamp to pending frame status. Reset on pattern loop. + std::map pending_frames_; // One counter per Vp8BufferReference, indicating number of frames since last // refresh. For non-base-layer frames (ie golden, altref buffers), this is // reset when the pattern loops. std::map frames_since_buffer_refresh_; + + // Optional utility used to verify reference validity. + std::unique_ptr checker_; }; class DefaultTemporalLayersChecker : public TemporalLayersChecker { diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc b/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc index 6021c65bce..ad31a6ef4a 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc +++ b/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc @@ -116,8 +116,8 @@ TEST(TemporalLayersTest, 2Layers) { for (int i = 0; i < 16; ++i) { TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[i], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i; - tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp, + &vp8_info); EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config)); EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx); @@ -168,8 +168,8 @@ TEST(TemporalLayersTest, 3Layers) { for (int i = 0; i < 16; ++i) { TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[i], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i; - tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp, + &vp8_info); EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config)); EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx); @@ -209,8 +209,8 @@ TEST(TemporalLayersTest, Alternative3Layers) { for (int i = 0; i < 8; ++i) { TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[i], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i; - tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp, + &vp8_info); EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config)); EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx); @@ -239,28 +239,28 @@ TEST(TemporalLayersTest, SearchOrder) { // Start with a key-frame. tl_config flags can be ignored. uint32_t timestamp = 0; TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); - tl.PopulateCodecSpecific(true, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, + &vp8_info); // TL2 frame. First one only references TL0. Updates altref. tl_config = tl.UpdateLayerConfig(++timestamp); - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast); EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone); // TL1 frame. Can only reference TL0. Updated golden. tl_config = tl.UpdateLayerConfig(++timestamp); - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast); EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone); // TL2 frame. Can reference all three buffers. Golden was the last to be // updated, the next to last was altref. tl_config = tl.UpdateLayerConfig(++timestamp); - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kGolden); EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kAltref); } @@ -283,25 +283,25 @@ TEST(TemporalLayersTest, SearchOrderWithDrop) { // Start with a key-frame. tl_config flags can be ignored. uint32_t timestamp = 0; TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); - tl.PopulateCodecSpecific(true, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, + &vp8_info); // TL2 frame. First one only references TL0. Updates altref. tl_config = tl.UpdateLayerConfig(++timestamp); - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast); EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone); // Dropped TL1 frame. Can only reference TL0. Should have updated golden. tl_config = tl.UpdateLayerConfig(++timestamp); - tl.FrameEncoded(timestamp, 0, 0); + tl.OnEncodeDone(timestamp, 0, false, 0, nullptr); // TL2 frame. Can normally reference all three buffers, but golden has not // been populated this cycle. Altref was last to be updated, before that last. tl_config = tl.UpdateLayerConfig(++timestamp); - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kAltref); EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kLast); } @@ -345,8 +345,8 @@ TEST(TemporalLayersTest, 4Layers) { for (int i = 0; i < 16; ++i) { TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[i], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i; - tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp, + &vp8_info); EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config)); EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx); @@ -374,22 +374,22 @@ TEST(TemporalLayersTest, DoesNotReferenceDroppedFrames) { // Start with a keyframe. uint32_t timestamp = 0; TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); - tl.PopulateCodecSpecific(true, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, + &vp8_info); // Dropped TL2 frame. tl_config = tl.UpdateLayerConfig(++timestamp); - tl.FrameEncoded(timestamp, 0, 0); + tl.OnEncodeDone(timestamp, 0, false, 0, nullptr); // Dropped TL1 frame. tl_config = tl.UpdateLayerConfig(++timestamp); - tl.FrameEncoded(timestamp, 0, 0); + tl.OnEncodeDone(timestamp, 0, false, 0, nullptr); // TL2 frame. Can reference all three buffers, valid since golden and altref // both contain the last keyframe. tl_config = tl.UpdateLayerConfig(++timestamp); - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference); EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference); EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference); @@ -398,24 +398,24 @@ TEST(TemporalLayersTest, DoesNotReferenceDroppedFrames) { // TL0 base layer frame, updating and referencing last. tl_config = tl.UpdateLayerConfig(++timestamp); - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); // TL2 frame, updating altref. tl_config = tl.UpdateLayerConfig(++timestamp); - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); // TL1 frame, updating golden. tl_config = tl.UpdateLayerConfig(++timestamp); - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); // TL2 frame. Can still reference all buffer since they have been update this // cycle. tl_config = tl.UpdateLayerConfig(++timestamp); - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference); EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference); EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference); @@ -424,22 +424,22 @@ TEST(TemporalLayersTest, DoesNotReferenceDroppedFrames) { // TL0 base layer frame, updating and referencing last. tl_config = tl.UpdateLayerConfig(++timestamp); - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); // Dropped TL2 frame. tl_config = tl.UpdateLayerConfig(++timestamp); - tl.FrameEncoded(timestamp, 0, 0); + tl.OnEncodeDone(timestamp, 0, false, 0, nullptr); // Dropped TL1 frame. tl_config = tl.UpdateLayerConfig(++timestamp); - tl.FrameEncoded(timestamp, 0, 0); + tl.OnEncodeDone(timestamp, 0, false, 0, nullptr); // TL2 frame. This time golden and altref contain data from the previous cycle // and cannot be referenced. tl_config = tl.UpdateLayerConfig(++timestamp); - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference); EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference); EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference); @@ -461,25 +461,25 @@ TEST(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExist) { // Start with a keyframe. uint32_t timestamp = 0; TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); - tl.PopulateCodecSpecific(true, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, + &vp8_info); // Do a full cycle of the pattern. for (int i = 0; i < 7; ++i) { tl_config = tl.UpdateLayerConfig(++timestamp); - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); } // TL0 base layer frame, starting the cycle over. tl_config = tl.UpdateLayerConfig(++timestamp); - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); // TL2 frame. tl_config = tl.UpdateLayerConfig(++timestamp); - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); // Encoder has a hiccup and builds a queue, so frame encoding is delayed. // TL1 frame, updating golden. @@ -498,14 +498,14 @@ TEST(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExist) { // The previous four enqueued frames finally get encoded, and the updated // buffers are now OK to reference. // Enqueued TL1 frame ready. - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); // Enqueued TL2 frame. - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, ++timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(++timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); // Enqueued TL0 frame. - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, ++timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(++timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); // TL2 frame, all buffers are now in a known good state, OK to reference. tl_config = tl.UpdateLayerConfig(++timestamp + 1); @@ -531,25 +531,25 @@ TEST(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExistLongDelay) { // Start with a keyframe. uint32_t timestamp = 0; TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); - tl.PopulateCodecSpecific(true, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, + &vp8_info); // Do a full cycle of the pattern. for (int i = 0; i < 3; ++i) { tl_config = tl.UpdateLayerConfig(++timestamp); - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); } // TL0 base layer frame, starting the cycle over. tl_config = tl.UpdateLayerConfig(++timestamp); - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); // TL2 frame. tl_config = tl.UpdateLayerConfig(++timestamp); - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); // Encoder has a hiccup and builds a queue, so frame encoding is delayed. // Encoded, but delayed frames in TL 1, 2. @@ -563,11 +563,11 @@ TEST(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExistLongDelay) { tl_config = tl.UpdateLayerConfig(timestamp + 4); // TL1 frame from last cycle is ready. - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp + 1); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp + 1, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); // TL2 frame from last cycle is ready. - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp + 2); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp + 2, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); // TL2 frame, that should be referencing all buffers, but altref and golden // haven not been updated this cycle. (Don't be fooled by the late frames from @@ -613,8 +613,8 @@ TEST(TemporalLayersTest, KeyFrame) { TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[j], LibvpxVp8Encoder::EncodeFlags(tl_config)) << j; - tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + &vp8_info); EXPECT_TRUE(checker.CheckTemporalConfig(false, tl_config)); EXPECT_EQ(expected_temporal_idx[j], tl_config.packetizer_temporal_idx); EXPECT_EQ(expected_temporal_idx[j], tl_config.encoder_layer_id); @@ -623,8 +623,8 @@ TEST(TemporalLayersTest, KeyFrame) { } TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); - tl.PopulateCodecSpecific(true, tl_config, &vp8_info, timestamp); - tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp); + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, + &vp8_info); EXPECT_TRUE(vp8_info.layerSync) << "Key frame should be marked layer sync."; EXPECT_EQ(0, vp8_info.temporalIdx) << "Key frame should always be packetized as layer 0"; @@ -712,7 +712,8 @@ TEST_P(TemporalLayersReferenceTest, ValidFrameConfigs) { std::vector tl_configs(kMaxPatternLength); for (int i = 0; i < kMaxPatternLength; ++i) { TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp_); - tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_specifics, timestamp_); + tl.OnEncodeDone(timestamp_, kDefaultBytesPerFrame, i == 0, kDefaultQp, + &vp8_specifics); ++timestamp_; EXPECT_FALSE(tl_config.drop_frame); tl_configs.push_back(tl_config); diff --git a/modules/video_coding/codecs/vp8/include/vp8_temporal_layers.h b/modules/video_coding/codecs/vp8/include/vp8_temporal_layers.h index 1b668ebf34..b5dbc4ec2b 100644 --- a/modules/video_coding/codecs/vp8/include/vp8_temporal_layers.h +++ b/modules/video_coding/codecs/vp8/include/vp8_temporal_layers.h @@ -28,10 +28,10 @@ namespace webrtc { // - UpdateLayerConfig(timestampB) // - PopulateCodecSpecific(timestampA, ...) // - UpdateLayerConfig(timestampC) -// - FrameEncoded(timestampA, 1234, ...) -// - FrameEncoded(timestampB, 0, ...) -// - PopulateCodecSpecific(timestampC, ...) -// - FrameEncoded(timestampC, 1234, ...) +// - OnEncodeDone(timestampA, 1234, ...) +// - UpdateLayerConfig(timestampC) +// - OnEncodeDone(timestampB, 0, ...) +// - OnEncodeDone(timestampC, 1234, ...) // Note that UpdateLayerConfig() for a new frame can happen before // FrameEncoded() for a previous one, but calls themselves must be both // synchronized (e.g. run on a task queue) and in order (per type). @@ -142,14 +142,26 @@ class TemporalLayers { virtual ~TemporalLayers() = default; + // If this method returns true, the encoder is free to drop frames for + // instance in an effort to uphold encoding bitrate. + // If this return false, the encoder must not drop any frames unless: + // 1. Requested to do so via FrameConfig.drop_frame + // 2. The frame to be encoded is requested to be a keyframe + // 3. The encoded detected a large overshoot and decided to drop and then + // re-encode the image at a low bitrate. In this case the encoder should + // call OnEncodeDone() once with size = 0 to indicate drop, and then call + // OnEncodeDone() again when the frame has actually been encoded. virtual bool SupportsEncoderFrameDropping() const = 0; // New target bitrate, per temporal layer. virtual void OnRatesUpdated(const std::vector& bitrates_bps, int framerate_fps) = 0; - // Update the encoder configuration with target bitrates or other parameters. - // Returns true iff the configuration was actually modified. + // Called by the encoder before encoding a frame. |cfg| contains the current + // configuration. If the TemporalLayers instance wishes any part of that + // to be changed before the encode step, |cfg| should be changed and then + // return true. If false is returned, the encoder will proceed without + // updating the configuration. virtual bool UpdateConfiguration(Vp8EncoderConfig* cfg) = 0; // Returns the recommended VP8 encode flags needed, and moves the temporal @@ -157,30 +169,28 @@ class TemporalLayers { // The timestamp may be used as both a time and a unique identifier, and so // the caller must make sure no two frames use the same timestamp. // The timestamp uses a 90kHz RTP clock. - // After calling this method, the actual encoder should be called with the - // provided frame configuration, after which: - // * On success, call PopulateCodecSpecific() and then FrameEncoded(); - // * On failure/ frame drop: Call FrameEncoded() with size = 0. + // After calling this method, first call the actual encoder with the provided + // frame configuration, and then OnEncodeDone() below. virtual FrameConfig UpdateLayerConfig(uint32_t rtp_timestamp) = 0; - // Called after successful encoding of a frame. The rtp timestamp must match - // the one using in UpdateLayerConfig(). Some fields in |vp8_info| may have - // already been populated by the encoder, check before overwriting. - // |tl_config| is the frame config returned by UpdateLayerConfig() for this - // rtp_timestamp; - // If |is_keyframe| is true, the flags in |tl_config| will be ignored. - virtual void PopulateCodecSpecific( - bool is_keyframe, - const TemporalLayers::FrameConfig& tl_config, - CodecSpecificInfoVP8* vp8_info, - uint32_t rtp_timestamp) = 0; - - // Called after an encode event. If the frame was dropped, |size_bytes| must - // be set to 0. The rtp timestamp must match the one using in - // UpdateLayerConfig() - virtual void FrameEncoded(uint32_t rtp_timestamp, + // Called after the encode step is done. |rtp_timestamp| must match the + // parameter use in the UpdateLayerConfig() call. + // |is_keyframe| must be true iff the encoder decided to encode this frame as + // a keyframe. + // If the encoder decided to drop this frame, |size_bytes| must be set to 0, + // otherwise it should indicate the size in bytes of the encoded frame. + // If |size_bytes| > 0, and |vp8_info| is not null, the TemporalLayers + // instance my update |vp8_info| with codec specific data such as temporal id. + // Some fields of this struct may have already been populated by the encoder, + // check before overwriting. + // If |size_bytes| > 0, |qp| should indicate the frame-level QP this frame was + // encoded at. If the encoder does not support extracting this, |qp| should be + // set to 0. + virtual void OnEncodeDone(uint32_t rtp_timestamp, size_t size_bytes, - int qp) = 0; + bool is_keyframe, + int qp, + CodecSpecificInfoVP8* vp8_info) = 0; }; } // namespace webrtc diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc index 2b7250823c..6d07eb5464 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc @@ -821,31 +821,31 @@ int LibvpxVp8Encoder::Encode(const VideoFrame& frame, return WEBRTC_VIDEO_CODEC_ERROR; timestamp_ += duration; // Examines frame timestamps only. - error = GetEncodedPartitions(tl_configs, frame); + error = GetEncodedPartitions(frame); } return error; } -void LibvpxVp8Encoder::PopulateCodecSpecific( - CodecSpecificInfo* codec_specific, - const TemporalLayers::FrameConfig& tl_config, - const vpx_codec_cx_pkt_t& pkt, - int stream_idx, - uint32_t timestamp) { +void LibvpxVp8Encoder::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, + const vpx_codec_cx_pkt_t& pkt, + int stream_idx, + int encoder_idx, + uint32_t timestamp) { assert(codec_specific != NULL); codec_specific->codecType = kVideoCodecVP8; codec_specific->codec_name = ImplementationName(); CodecSpecificInfoVP8* vp8Info = &(codec_specific->codecSpecific.VP8); vp8Info->keyIdx = kNoKeyIdx; // TODO(hlundin) populate this vp8Info->nonReference = (pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE) != 0; - temporal_layers_[stream_idx]->PopulateCodecSpecific( - (pkt.data.frame.flags & VPX_FRAME_IS_KEY) != 0, tl_config, vp8Info, - timestamp); + + int qp = 0; + vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp); + temporal_layers_[stream_idx]->OnEncodeDone( + timestamp, encoded_images_[encoder_idx]._length, + (pkt.data.frame.flags & VPX_FRAME_IS_KEY) != 0, qp, vp8Info); } -int LibvpxVp8Encoder::GetEncodedPartitions( - const TemporalLayers::FrameConfig tl_configs[], - const VideoFrame& input_image) { +int LibvpxVp8Encoder::GetEncodedPartitions(const VideoFrame& input_image) { int stream_idx = static_cast(encoders_.size()) - 1; int result = WEBRTC_VIDEO_CODEC_OK; for (size_t encoder_idx = 0; encoder_idx < encoders_.size(); @@ -895,8 +895,8 @@ int LibvpxVp8Encoder::GetEncodedPartitions( is_keyframe = true; } encoded_images_[encoder_idx].SetSpatialIndex(stream_idx); - PopulateCodecSpecific(&codec_specific, tl_configs[stream_idx], *pkt, - stream_idx, input_image.timestamp()); + PopulateCodecSpecific(&codec_specific, *pkt, stream_idx, encoder_idx, + input_image.timestamp()); break; } } @@ -910,10 +910,6 @@ int LibvpxVp8Encoder::GetEncodedPartitions( : VideoContentType::UNSPECIFIED; encoded_images_[encoder_idx].timing_.flags = VideoSendTiming::kInvalid; - int qp = -1; - vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp); - temporal_layers_[stream_idx]->FrameEncoded( - input_image.timestamp(), encoded_images_[encoder_idx]._length, qp); if (send_stream_[stream_idx]) { if (encoded_images_[encoder_idx]._length > 0) { TRACE_COUNTER_ID1("webrtc", "EncodedFrameSize", encoder_idx, @@ -928,16 +924,16 @@ int LibvpxVp8Encoder::GetEncodedPartitions( encoded_images_[encoder_idx].qp_ = qp_128; encoded_complete_callback_->OnEncodedImage(encoded_images_[encoder_idx], &codec_specific, &frag_info); - } else if (codec_.mode == VideoCodecMode::kScreensharing) { + } else if (!temporal_layers_[stream_idx] + ->SupportsEncoderFrameDropping()) { result = WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT; + if (encoded_images_[encoder_idx]._length == 0) { + // Dropped frame that will be re-encoded. + temporal_layers_[stream_idx]->OnEncodeDone(input_image.timestamp(), 0, + false, 0, nullptr); + } } } - if (result != WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT) { - // Don't run checker on drop before reencode as that will incorrectly - // increase the pattern index twice. - RTC_DCHECK(temporal_layers_checkers_[stream_idx]->CheckTemporalConfig( - is_keyframe, tl_configs[stream_idx])); - } } return result; } diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h index 8ee359e5b4..98caf501d5 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h @@ -70,13 +70,12 @@ class LibvpxVp8Encoder : public VP8Encoder { int InitAndSetControlSettings(); void PopulateCodecSpecific(CodecSpecificInfo* codec_specific, - const TemporalLayers::FrameConfig& tl_config, const vpx_codec_cx_pkt& pkt, int stream_idx, + int encoder_idx, uint32_t timestamp); - int GetEncodedPartitions(const TemporalLayers::FrameConfig tl_configs[], - const VideoFrame& input_image); + int GetEncodedPartitions(const VideoFrame& input_image); // Set the stream state for stream |stream_idx|. void SetStreamState(bool send_stream, int stream_idx); diff --git a/modules/video_coding/codecs/vp8/screenshare_layers.cc b/modules/video_coding/codecs/vp8/screenshare_layers.cc index 12b74fa0d5..76c4382a21 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers.cc +++ b/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -21,12 +21,13 @@ #include "system_wrappers/include/metrics.h" namespace webrtc { - +namespace { static const int kOneSecond90Khz = 90000; static const int kMinTimeBetweenSyncs = kOneSecond90Khz * 2; static const int kMaxTimeBetweenSyncs = kOneSecond90Khz * 4; static const int kQpDeltaThresholdForSync = 8; static const int kMinBitrateKbpsForQpBoost = 500; +} // namespace const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5; const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0; @@ -37,8 +38,7 @@ constexpr int ScreenshareLayers::kMaxNumTemporalLayers; // been exceeded. This prevents needless keyframe requests. const int ScreenshareLayers::kMaxFrameIntervalMs = 2750; -ScreenshareLayers::ScreenshareLayers(int num_temporal_layers, - Clock* clock) +ScreenshareLayers::ScreenshareLayers(int num_temporal_layers, Clock* clock) : clock_(clock), number_of_temporal_layers_( std::min(kMaxNumTemporalLayers, num_temporal_layers)), @@ -51,7 +51,10 @@ ScreenshareLayers::ScreenshareLayers(int num_temporal_layers, max_qp_(-1), max_debt_bytes_(0), encode_framerate_(1000.0f, 1000.0f), // 1 second window, second scale. - bitrate_updated_(false) { + bitrate_updated_(false), + checker_(TemporalLayersChecker::CreateTemporalLayersChecker( + TemporalLayersType::kBitrateDynamic, + num_temporal_layers)) { RTC_CHECK_GT(number_of_temporal_layers_, 0); RTC_CHECK_LE(number_of_temporal_layers_, kMaxNumTemporalLayers); } @@ -67,11 +70,18 @@ bool ScreenshareLayers::SupportsEncoderFrameDropping() const { TemporalLayers::FrameConfig ScreenshareLayers::UpdateLayerConfig( uint32_t timestamp) { + auto it = pending_frame_configs_.find(timestamp); + if (it != pending_frame_configs_.end()) { + // Drop and re-encode, reuse the previous config. + return it->second; + } + if (number_of_temporal_layers_ <= 1) { // No flags needed for 1 layer screenshare. // TODO(pbos): Consider updating only last, and not all buffers. TemporalLayers::FrameConfig tl_config( kReferenceAndUpdate, kReferenceAndUpdate, kReferenceAndUpdate); + pending_frame_configs_[timestamp] = tl_config; return tl_config; } @@ -201,6 +211,7 @@ TemporalLayers::FrameConfig ScreenshareLayers::UpdateLayerConfig( } tl_config.layer_sync = layer_state == TemporalLayerState::kTl1Sync; + pending_frame_configs_[timestamp] = tl_config; return tl_config; } @@ -244,20 +255,57 @@ void ScreenshareLayers::OnRatesUpdated( layers_[1].target_rate_kbps_ = tl1_kbps; } -void ScreenshareLayers::FrameEncoded(uint32_t timestamp, size_t size, int qp) { - if (size > 0) - encode_framerate_.Update(1, clock_->TimeInMilliseconds()); - - if (number_of_temporal_layers_ == 1) - return; - - RTC_DCHECK_NE(-1, active_layer_); - if (size == 0) { +void ScreenshareLayers::OnEncodeDone(uint32_t rtp_timestamp, + size_t size_bytes, + bool is_keyframe, + int qp, + CodecSpecificInfoVP8* vp8_info) { + if (size_bytes == 0) { layers_[active_layer_].state = TemporalLayer::State::kDropped; ++stats_.num_overshoots_; return; } + absl::optional frame_config; + auto it = pending_frame_configs_.find(rtp_timestamp); + if (it != pending_frame_configs_.end()) { + frame_config = it->second; + pending_frame_configs_.erase(it); + + if (checker_) { + RTC_DCHECK(checker_->CheckTemporalConfig(is_keyframe, *frame_config)); + } + } + + if (number_of_temporal_layers_ == 1) { + vp8_info->temporalIdx = kNoTemporalIdx; + vp8_info->layerSync = false; + } else { + int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(rtp_timestamp); + if (frame_config) { + vp8_info->temporalIdx = frame_config->packetizer_temporal_idx; + vp8_info->layerSync = frame_config->layer_sync; + } else { + // Frame requested to be dropped, but was not. Fall back to base-layer. + vp8_info->temporalIdx = 0; + vp8_info->layerSync = false; + } + if (is_keyframe) { + vp8_info->temporalIdx = 0; + last_sync_timestamp_ = unwrapped_timestamp; + vp8_info->layerSync = true; + layers_[0].state = TemporalLayer::State::kKeyFrame; + layers_[1].state = TemporalLayer::State::kKeyFrame; + active_layer_ = 1; + } + } + + encode_framerate_.Update(1, clock_->TimeInMilliseconds()); + + if (number_of_temporal_layers_ == 1) + return; + + RTC_DCHECK_NE(-1, active_layer_); if (layers_[active_layer_].state == TemporalLayer::State::kDropped) { layers_[active_layer_].state = TemporalLayer::State::kQualityBoost; } @@ -266,42 +314,19 @@ void ScreenshareLayers::FrameEncoded(uint32_t timestamp, size_t size, int qp) { layers_[active_layer_].last_qp = qp; if (active_layer_ == 0) { - layers_[0].debt_bytes_ += size; - layers_[1].debt_bytes_ += size; + layers_[0].debt_bytes_ += size_bytes; + layers_[1].debt_bytes_ += size_bytes; ++stats_.num_tl0_frames_; stats_.tl0_target_bitrate_sum_ += layers_[0].target_rate_kbps_; stats_.tl0_qp_sum_ += qp; } else if (active_layer_ == 1) { - layers_[1].debt_bytes_ += size; + layers_[1].debt_bytes_ += size_bytes; ++stats_.num_tl1_frames_; stats_.tl1_target_bitrate_sum_ += layers_[1].target_rate_kbps_; stats_.tl1_qp_sum_ += qp; } } -void ScreenshareLayers::PopulateCodecSpecific( - bool frame_is_keyframe, - const TemporalLayers::FrameConfig& tl_config, - CodecSpecificInfoVP8* vp8_info, - uint32_t timestamp) { - if (number_of_temporal_layers_ == 1) { - vp8_info->temporalIdx = kNoTemporalIdx; - vp8_info->layerSync = false; - } else { - int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp); - vp8_info->temporalIdx = tl_config.packetizer_temporal_idx; - vp8_info->layerSync = tl_config.layer_sync; - if (frame_is_keyframe) { - vp8_info->temporalIdx = 0; - last_sync_timestamp_ = unwrapped_timestamp; - vp8_info->layerSync = true; - layers_[0].state = TemporalLayer::State::kKeyFrame; - layers_[1].state = TemporalLayer::State::kKeyFrame; - active_layer_ = 1; - } - } -} - bool ScreenshareLayers::TimeToSync(int64_t timestamp) const { RTC_DCHECK_EQ(1, active_layer_); RTC_DCHECK_NE(-1, layers_[0].last_qp); diff --git a/modules/video_coding/codecs/vp8/screenshare_layers.h b/modules/video_coding/codecs/vp8/screenshare_layers.h index cd12ac464e..6193be3600 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers.h +++ b/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -9,8 +9,11 @@ #ifndef MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_ #define MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_ +#include +#include #include +#include "modules/video_coding/codecs/vp8/include/temporal_layers_checker.h" #include "modules/video_coding/codecs/vp8/include/vp8_temporal_layers.h" #include "modules/video_coding/utility/frame_dropper.h" #include "rtc_base/rate_statistics.h" @@ -46,12 +49,11 @@ class ScreenshareLayers : public TemporalLayers { // Returns true iff the configuration was actually modified. bool UpdateConfiguration(Vp8EncoderConfig* cfg) override; - void PopulateCodecSpecific(bool base_layer_sync, - const TemporalLayers::FrameConfig& tl_config, - CodecSpecificInfoVP8* vp8_info, - uint32_t rtp_timestamp) override; - - void FrameEncoded(uint32_t rtp_timestamp, size_t size, int qp) override; + void OnEncodeDone(uint32_t rtp_timestamp, + size_t size_bytes, + bool is_keyframe, + int qp, + CodecSpecificInfoVP8* vp8_info) override; private: enum class TemporalLayerState : int { kDrop, kTl0, kTl1, kTl1Sync }; @@ -72,6 +74,8 @@ class ScreenshareLayers : public TemporalLayers { int max_qp_; uint32_t max_debt_bytes_; + std::map pending_frame_configs_; + // Configured max framerate. absl::optional target_framerate_; // Incoming framerate from capturer. @@ -118,6 +122,9 @@ class ScreenshareLayers : public TemporalLayers { int64_t tl0_target_bitrate_sum_ = 0; int64_t tl1_target_bitrate_sum_ = 0; } stats_; + + // Optional utility used to verify reference validity. + std::unique_ptr checker_; }; } // namespace webrtc diff --git a/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc b/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc index bf40894d04..4ae7ed5e7e 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc +++ b/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc @@ -67,7 +67,8 @@ class ScreenshareLayerTest : public ::testing::Test { int EncodeFrame(bool base_sync) { int flags = ConfigureFrame(base_sync); if (flags != -1) - layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp); + layers_->OnEncodeDone(timestamp_, frame_size_, base_sync, kDefaultQp, + &vp8_info_); return flags; } @@ -82,8 +83,6 @@ class ScreenshareLayerTest : public ::testing::Test { } config_updated_ = layers_->UpdateConfiguration(&cfg_); int flags = LibvpxVp8Encoder::EncodeFlags(tl_config_); - layers_->PopulateCodecSpecific(key_frame, tl_config_, &vp8_info_, - timestamp_); EXPECT_NE(-1, frame_size_); return flags; } @@ -147,10 +146,11 @@ class ScreenshareLayerTest : public ::testing::Test { 1 + (sync.value_or(false) ? kMaxSyncPeriodSeconds : 1) * kFrameRate; for (int i = 0; i < kMaxFramesToSkip; ++i) { flags = ConfigureFrame(false); - timestamp_ += kTimestampDelta5Fps; - if (vp8_info_.temporalIdx != layer || - (sync && *sync != vp8_info_.layerSync)) { - layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp); + if (tl_config_.packetizer_temporal_idx != layer || + (sync && *sync != tl_config_.layer_sync)) { + layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, + &vp8_info_); + timestamp_ += kTimestampDelta5Fps; } else { // Found frame from sought after layer. return flags; @@ -211,13 +211,14 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterTimeout) { for (int i = 0; i < kNumFrames; ++i) { tl_config_ = UpdateLayerConfig(timestamp_); config_updated_ = layers_->UpdateConfiguration(&cfg_); - layers_->PopulateCodecSpecific(false, tl_config_, &vp8_info_, timestamp_); // Simulate TL1 being at least 8 qp steps better. - if (vp8_info_.temporalIdx == 0) { - layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp); + if (tl_config_.packetizer_temporal_idx == 0) { + layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, + &vp8_info_); } else { - layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp - 8); + layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp - 8, + &vp8_info_); } if (vp8_info_.temporalIdx == 1 && vp8_info_.layerSync) @@ -240,10 +241,12 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) { ConfigureFrame(false); // Simulate TL1 being at least 8 qp steps better. - if (vp8_info_.temporalIdx == 0) { - layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp); + if (tl_config_.packetizer_temporal_idx == 0) { + layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, + &vp8_info_); } else { - layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp - 8); + layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp - 8, + &vp8_info_); } if (vp8_info_.temporalIdx == 1 && vp8_info_.layerSync) @@ -257,12 +260,12 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) { bool bumped_tl0_quality = false; for (int i = 0; i < 3; ++i) { int flags = ConfigureFrame(false); + layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp - 8, + &vp8_info_); if (vp8_info_.temporalIdx == 0) { // Bump TL0 to same quality as TL1. - layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp - 8); bumped_tl0_quality = true; } else { - layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp - 8); if (bumped_tl0_quality) { EXPECT_TRUE(vp8_info_.layerSync); EXPECT_EQ(kTl1SyncFlags, flags); @@ -380,7 +383,7 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) { SkipUntilTl(0); // Size 0 indicates dropped frame. - layers_->FrameEncoded(timestamp_, 0, kDefaultQp); + layers_->OnEncodeDone(timestamp_, 0, false, 0, &vp8_info_); // Re-encode frame (so don't advance timestamp). int flags = EncodeFrame(false); @@ -392,18 +395,18 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) { SkipUntilTl(0); EXPECT_TRUE(config_updated_); EXPECT_LT(cfg_.rc_max_quantizer, static_cast(kDefaultQp)); - layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp); + layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, &vp8_info_); timestamp_ += kTimestampDelta5Fps; // ...then back to standard setup. SkipUntilTl(0); - layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp); + layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, &vp8_info_); timestamp_ += kTimestampDelta5Fps; EXPECT_EQ(cfg_.rc_max_quantizer, static_cast(kDefaultQp)); // Next drop in TL1. SkipUntilTl(1); - layers_->FrameEncoded(timestamp_, 0, kDefaultQp); + layers_->OnEncodeDone(timestamp_, 0, false, 0, &vp8_info_); // Re-encode frame (so don't advance timestamp). flags = EncodeFrame(false); @@ -415,13 +418,13 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) { SkipUntilTl(1); EXPECT_TRUE(config_updated_); EXPECT_LT(cfg_.rc_max_quantizer, static_cast(kDefaultQp)); - layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp); + layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, &vp8_info_); timestamp_ += kTimestampDelta5Fps; // ...and back to normal. SkipUntilTl(1); EXPECT_EQ(cfg_.rc_max_quantizer, static_cast(kDefaultQp)); - layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp); + layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, &vp8_info_); timestamp_ += kTimestampDelta5Fps; } @@ -436,7 +439,8 @@ TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) { EXPECT_EQ(kTl0Flags, LibvpxVp8Encoder::EncodeFlags(UpdateLayerConfig(kStartTimestamp))); - layers_->FrameEncoded(timestamp_, kLargeFrameSizeBytes, kDefaultQp); + layers_->OnEncodeDone(kStartTimestamp, kLargeFrameSizeBytes, false, + kDefaultQp, &vp8_info_); const uint32_t kTwoSecondsLater = kStartTimestamp + (ScreenshareLayers::kMaxFrameIntervalMs * 90); @@ -478,20 +482,22 @@ TEST_F(ScreenshareLayerTest, UpdatesHistograms) { if (timestamp >= kTimestampDelta5Fps * 5 && !overshoot && flags != -1) { // Simulate one overshoot. - layers_->FrameEncoded(timestamp_, 0, 0); + layers_->OnEncodeDone(timestamp, 0, false, 0, nullptr); overshoot = true; } if (flags == kTl0Flags) { if (timestamp >= kTimestampDelta5Fps * 20 && !trigger_drop) { // Simulate a too large frame, to cause frame drop. - layers_->FrameEncoded(timestamp_, frame_size_ * 10, kTl0Qp); + layers_->OnEncodeDone(timestamp, frame_size_ * 10, false, kTl0Qp, + &vp8_info_); trigger_drop = true; } else { - layers_->FrameEncoded(timestamp_, frame_size_, kTl0Qp); + layers_->OnEncodeDone(timestamp, frame_size_, false, kTl0Qp, + &vp8_info_); } } else if (flags == kTl1Flags || flags == kTl1SyncFlags) { - layers_->FrameEncoded(timestamp_, frame_size_, kTl1Qp); + layers_->OnEncodeDone(timestamp, frame_size_, false, kTl1Qp, &vp8_info_); } else if (flags == -1) { dropped_frame = true; } else { @@ -557,7 +563,8 @@ TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) { ++num_discarded_frames; } else { size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8; - layers_->FrameEncoded(timestamp_, frame_size_bytes, kDefaultQp); + layers_->OnEncodeDone(timestamp, frame_size_bytes, false, kDefaultQp, + &vp8_info_); } timestamp += kFrameIntervalsMs * 90; clock_.AdvanceTimeMilliseconds(kFrameIntervalsMs); @@ -573,7 +580,8 @@ TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) { ++num_discarded_frames; } else { size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8; - layers_->FrameEncoded(timestamp_, frame_size_bytes, kDefaultQp); + layers_->OnEncodeDone(timestamp, frame_size_bytes, false, kDefaultQp, + &vp8_info_); } timestamp += kFrameIntervalsMs * 90 / 2; clock_.AdvanceTimeMilliseconds(kFrameIntervalsMs / 2); @@ -590,16 +598,17 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAtOvershootDrop) { // Move ahead until we have a sync frame in TL1. EXPECT_EQ(kTl1SyncFlags, SkipUntilTlAndSync(1, true)); - ASSERT_TRUE(vp8_info_.layerSync); + ASSERT_TRUE(tl_config_.layer_sync); // Simulate overshoot of this frame. - layers_->FrameEncoded(timestamp_, 0, -1); + layers_->OnEncodeDone(timestamp_, 0, false, 0, nullptr); config_updated_ = layers_->UpdateConfiguration(&cfg_); EXPECT_EQ(kTl1SyncFlags, LibvpxVp8Encoder::EncodeFlags(tl_config_)); CodecSpecificInfoVP8 new_vp8_info; - layers_->PopulateCodecSpecific(false, tl_config_, &new_vp8_info, timestamp_); + layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, + &new_vp8_info); EXPECT_TRUE(new_vp8_info.layerSync); } @@ -610,7 +619,7 @@ TEST_F(ScreenshareLayerTest, DropOnTooShortFrameInterval) { // Add a large gap, so there's plenty of room in the rate tracker. timestamp_ += kTimestampDelta5Fps * 3; EXPECT_FALSE(UpdateLayerConfig(timestamp_).drop_frame); - layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp); + layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, &vp8_info_); // Frame interval below 90% if desired time is not allowed, try inserting // frame just before this limit. @@ -666,14 +675,13 @@ TEST_F(ScreenshareLayerTest, MaxQpRestoredAfterDoubleDrop) { // Move ahead until we have a sync frame in TL1. EXPECT_EQ(kTl1SyncFlags, SkipUntilTlAndSync(1, true)); - ASSERT_TRUE(vp8_info_.layerSync); + ASSERT_TRUE(tl_config_.layer_sync); // Simulate overshoot of this frame. - layers_->FrameEncoded(timestamp_, 0, -1); + layers_->OnEncodeDone(timestamp_, 0, false, -1, nullptr); // Simulate re-encoded frame. - layers_->PopulateCodecSpecific(false, tl_config_, &vp8_info_, timestamp_); - layers_->FrameEncoded(timestamp_, 1, max_qp_); + layers_->OnEncodeDone(timestamp_, 1, false, max_qp_, &vp8_info_); // Next frame, expect boosted quality. // Slightly alter bitrate between each frame. @@ -686,11 +694,10 @@ TEST_F(ScreenshareLayerTest, MaxQpRestoredAfterDoubleDrop) { uint32_t adjusted_qp = cfg_.rc_max_quantizer; // Simulate overshoot of this frame. - layers_->FrameEncoded(timestamp_, 0, -1); + layers_->OnEncodeDone(timestamp_, 0, false, -1, nullptr); // Simulate re-encoded frame. - layers_->PopulateCodecSpecific(false, tl_config_, &vp8_info_, timestamp_); - layers_->FrameEncoded(timestamp_, frame_size_, max_qp_); + layers_->OnEncodeDone(timestamp_, frame_size_, false, max_qp_, &vp8_info_); // A third frame, expect boosted quality. layers_->OnRatesUpdated(kDefault2TlBitratesBps, kFrameRate); @@ -700,13 +707,11 @@ TEST_F(ScreenshareLayerTest, MaxQpRestoredAfterDoubleDrop) { EXPECT_EQ(adjusted_qp, cfg_.rc_max_quantizer); // Frame encoded. - layers_->PopulateCodecSpecific(false, tl_config_, &vp8_info_, timestamp_); - layers_->FrameEncoded(timestamp_, frame_size_, max_qp_); + layers_->OnEncodeDone(timestamp_, frame_size_, false, max_qp_, &vp8_info_); // A fourth frame, max qp should be restored. layers_->OnRatesUpdated(kDefault2TlBitratesBpsAlt, kFrameRate); EXPECT_EQ(kTl1Flags, SkipUntilTlAndSync(1, false)); - EXPECT_TRUE(config_updated_); EXPECT_EQ(cfg_.rc_max_quantizer, max_qp_); } diff --git a/modules/video_coding/codecs/vp8/temporal_layers_checker.cc b/modules/video_coding/codecs/vp8/temporal_layers_checker.cc index b77e8bc219..c0a736a4ed 100644 --- a/modules/video_coding/codecs/vp8/temporal_layers_checker.cc +++ b/modules/video_coding/codecs/vp8/temporal_layers_checker.cc @@ -70,7 +70,8 @@ bool TemporalLayersChecker::CheckAndUpdateBufferState( bool TemporalLayersChecker::CheckTemporalConfig( bool frame_is_keyframe, const TemporalLayers::FrameConfig& frame_config) { - if (frame_config.drop_frame) { + if (frame_config.drop_frame || + frame_config.packetizer_temporal_idx == kNoTemporalIdx) { return true; } ++sequence_number_; diff --git a/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc b/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc index 05d881e407..7f54932a81 100644 --- a/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc +++ b/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc @@ -39,11 +39,8 @@ class MockTemporalLayers : public TemporalLayers { MOCK_METHOD1(UpdateLayerConfig, TemporalLayers::FrameConfig(uint32_t)); MOCK_METHOD2(OnRatesUpdated, void(const std::vector&, int)); MOCK_METHOD1(UpdateConfiguration, bool(Vp8EncoderConfig*)); - MOCK_METHOD4(PopulateCodecSpecific, - void(bool, - const TemporalLayers::FrameConfig&, - CodecSpecificInfoVP8*, - uint32_t)); + MOCK_METHOD5(OnEncodeDone, + void(uint32_t, size_t, bool, int, CodecSpecificInfoVP8*)); MOCK_METHOD3(FrameEncoded, void(uint32_t, size_t, int)); MOCK_CONST_METHOD0(Tl0PicIdx, uint8_t()); MOCK_CONST_METHOD1(GetTemporalLayerId, diff --git a/test/fake_vp8_encoder.cc b/test/fake_vp8_encoder.cc index 07f9d8f666..4038837798 100644 --- a/test/fake_vp8_encoder.cc +++ b/test/fake_vp8_encoder.cc @@ -78,20 +78,19 @@ void FakeVP8Encoder::SetupTemporalLayers(const VideoCodec& codec) { } } -void FakeVP8Encoder::PopulateCodecSpecific( - CodecSpecificInfo* codec_specific, - const TemporalLayers::FrameConfig& tl_config, - FrameType frame_type, - int stream_idx, - uint32_t timestamp) { +void FakeVP8Encoder::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, + size_t size_bytes, + FrameType frame_type, + int stream_idx, + uint32_t timestamp) { RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); codec_specific->codecType = kVideoCodecVP8; codec_specific->codec_name = ImplementationName(); CodecSpecificInfoVP8* vp8Info = &(codec_specific->codecSpecific.VP8); vp8Info->keyIdx = kNoKeyIdx; vp8Info->nonReference = false; - temporal_layers_[stream_idx]->PopulateCodecSpecific( - frame_type == kVideoFrameKey, tl_config, vp8Info, timestamp); + temporal_layers_[stream_idx]->OnEncodeDone( + timestamp, size_bytes, frame_type == kVideoFrameKey, -1, vp8Info); } EncodedImageCallback::Result FakeVP8Encoder::OnEncodedImage( @@ -101,14 +100,10 @@ EncodedImageCallback::Result FakeVP8Encoder::OnEncodedImage( RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); uint8_t stream_idx = encoded_image.SpatialIndex().value_or(0); CodecSpecificInfo overrided_specific_info; - TemporalLayers::FrameConfig tl_config = - temporal_layers_[stream_idx]->UpdateLayerConfig( - encoded_image.Timestamp()); - PopulateCodecSpecific(&overrided_specific_info, tl_config, + temporal_layers_[stream_idx]->UpdateLayerConfig(encoded_image.Timestamp()); + PopulateCodecSpecific(&overrided_specific_info, encoded_image._length, encoded_image._frameType, stream_idx, encoded_image.Timestamp()); - temporal_layers_[stream_idx]->FrameEncoded(encoded_image.Timestamp(), - encoded_image._length, -1); return callback_->OnEncodedImage(encoded_image, &overrided_specific_info, fragments); diff --git a/test/fake_vp8_encoder.h b/test/fake_vp8_encoder.h index f21f9f50aa..7518e3a2db 100644 --- a/test/fake_vp8_encoder.h +++ b/test/fake_vp8_encoder.h @@ -46,7 +46,7 @@ class FakeVP8Encoder : public FakeEncoder, public EncodedImageCallback { private: void SetupTemporalLayers(const VideoCodec& codec); void PopulateCodecSpecific(CodecSpecificInfo* codec_specific, - const TemporalLayers::FrameConfig& tl_config, + size_t size_bytes, FrameType frame_type, int stream_idx, uint32_t timestamp);