admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactoring of VP8 TemporalLayers interface.

Browse Source 
This refactoring merged PopulateCodecSpecific and FrameEncoded into a
single callback method. It also removes the FrameConfig parameter and
instead relies on the temporal layer to remember it internally.

Bug: webrtc:9012
Change-Id: I489b76821b534398ad452643f1322f411d3455b1
Reviewed-on: https://webrtc-review.googlesource.com/95681
Reviewed-by: Sebastian Jansson <srte@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#24957}
This commit is contained in:
Erik Språng 2018-10-03 11:05:16 +02:00 committed by Commit Bot
parent 7988589e48
commit 59021ba4e1
13 changed files with 362 additions and 289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
modules/video_coding/codecs/vp8/default_temporal_layers.cc
View File

@ -58,6 +58,15 @@ TemporalLayers::FrameConfig::FrameConfig(TemporalLayers::BufferFlags last,
first_reference(Vp8BufferReference::kNone),
second_reference(Vp8BufferReference::kNone) {}
DefaultTemporalLayers::PendingFrame::PendingFrame() = default;
DefaultTemporalLayers::PendingFrame::PendingFrame(
bool expired,
uint8_t updated_buffers_mask,
const FrameConfig& frame_config)
: expired(expired),
updated_buffer_mask(updated_buffers_mask),
frame_config(frame_config) {}
namespace {
static constexpr uint8_t kUninitializedPatternIndex =
std::numeric_limits<uint8_t>::max();
@ -251,7 +260,10 @@ DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers)
temporal_ids_(GetTemporalIds(num_layers_)),
temporal_pattern_(GetTemporalPattern(num_layers_)),
kf_buffers_(FindKfBuffers(temporal_pattern_)),
pattern_idx_(kUninitializedPatternIndex) {
pattern_idx_(kUninitializedPatternIndex),
checker_(TemporalLayersChecker::CreateTemporalLayersChecker(
TemporalLayersType::kFixedPattern,
number_of_temporal_layers)) {
RTC_CHECK_GE(kMaxTemporalStreams, number_of_temporal_layers);
RTC_CHECK_GE(number_of_temporal_layers, 0);
RTC_CHECK_LE(number_of_temporal_layers, 4);
@ -348,7 +360,9 @@ TemporalLayers::FrameConfig DefaultTemporalLayers::UpdateLayerConfig(
// Start of new pattern iteration, set up clear state by invalidating any
// pending frames, so that we don't make an invalid reference to a buffer
// containing data from a previous iteration.
pending_frames_.clear();
for (auto& it : pending_frames_) {
it.second.expired = true;
}
}
// Last is always ok to reference as it contains the base layer. For other
@ -376,7 +390,15 @@ TemporalLayers::FrameConfig DefaultTemporalLayers::UpdateLayerConfig(
}
// Add frame to set of pending frames, awaiting completion.
pending_frames_[timestamp] = GetUpdatedBuffers(tl_config);
pending_frames_[timestamp] =
PendingFrame{false, GetUpdatedBuffers(tl_config), tl_config};
if (checker_) {
// Checker does not yet support encoder frame dropping, so validate flags
// here before they can be dropped.
// TODO(sprang): Update checker to support dropping.
RTC_DCHECK(checker_->CheckTemporalConfig(false, tl_config));
}
return tl_config;
}
@ -437,61 +459,60 @@ void DefaultTemporalLayers::UpdateSearchOrder(FrameConfig* config) {
}
}
void DefaultTemporalLayers::PopulateCodecSpecific(
bool frame_is_keyframe,
const TemporalLayers::FrameConfig& tl_config,
CodecSpecificInfoVP8* vp8_info,
uint32_t timestamp) {
void DefaultTemporalLayers::OnEncodeDone(uint32_t rtp_timestamp,
size_t size_bytes,
bool is_keyframe,
int qp,
CodecSpecificInfoVP8* vp8_info) {
RTC_DCHECK_GT(num_layers_, 0);
auto pending_frame = pending_frames_.find(rtp_timestamp);
RTC_DCHECK(pending_frame != pending_frames_.end());
if (size_bytes == 0) {
pending_frames_.erase(pending_frame);
return;
}
PendingFrame& frame = pending_frame->second;
if (is_keyframe && checker_) {
// Signal key-frame so checker resets state.
RTC_DCHECK(checker_->CheckTemporalConfig(true, frame.frame_config));
}
if (num_layers_ == 1) {
vp8_info->temporalIdx = kNoTemporalIdx;
vp8_info->layerSync = false;
} else {
if (frame_is_keyframe) {
if (is_keyframe) {
// Restart the temporal pattern on keyframes.
pattern_idx_ = 0;
vp8_info->temporalIdx = 0;
vp8_info->layerSync = true; // Keyframes are always sync frames.
// Update frame count of all kf-only buffers, regardless of state of
// |pending_frames_|.
for (auto it : kf_buffers_) {
frames_since_buffer_refresh_[it] = 0;
}
auto pending_frames = pending_frames_.find(timestamp);
if (pending_frames != pending_frames_.end()) {
for (Vp8BufferReference buffer : kAllBuffers) {
if (kf_buffers_.find(buffer) == kf_buffers_.end()) {
// Key-frames update all buffers, this should be reflected if when
// updating state in FrameEncoded().
pending_frames->second |= static_cast<uint8_t>(buffer);
}
for (Vp8BufferReference buffer : kAllBuffers) {
if (kf_buffers_.find(buffer) != kf_buffers_.end()) {
// Update frame count of all kf-only buffers, regardless of state of
// |pending_frames_|.
frames_since_buffer_refresh_[buffer] = 0;
} else {
// Key-frames update all buffers, this should be reflected when
// updating state in FrameEncoded().
frame.updated_buffer_mask |= static_cast<uint8_t>(buffer);
}
}
} else {
// Delta frame, update codec specifics with temporal id and sync flag.
vp8_info->temporalIdx = tl_config.packetizer_temporal_idx;
vp8_info->layerSync = tl_config.layer_sync;
vp8_info->temporalIdx = frame.frame_config.packetizer_temporal_idx;
vp8_info->layerSync = frame.frame_config.layer_sync;
}
}
}
void DefaultTemporalLayers::FrameEncoded(uint32_t rtp_timestamp,
size_t size,
int qp) {
auto pending_frame = pending_frames_.find(rtp_timestamp);
if (pending_frame == pending_frames_.end()) {
// Might happen if pipelined encoder delayed encoding until after pattern
// looped.
return;
}
if (size == 0) {
pending_frames_.erase(pending_frame);
return;
}
for (Vp8BufferReference buffer : kAllBuffers) {
if (pending_frame->second & static_cast<uint8_t>(buffer)) {
frames_since_buffer_refresh_[buffer] = 0;
if (!frame.expired) {
for (Vp8BufferReference buffer : kAllBuffers) {
if (frame.updated_buffer_mask & static_cast<uint8_t>(buffer)) {
frames_since_buffer_refresh_[buffer] = 0;
}
}
}
}

34
modules/video_coding/codecs/vp8/default_temporal_layers.h
View File

@ -14,6 +14,7 @@
#include <limits>
#include <map>
#include <memory>
#include <set>
#include <vector>
@ -41,12 +42,11 @@ class DefaultTemporalLayers : public TemporalLayers {
bool UpdateConfiguration(Vp8EncoderConfig* cfg) override;
void PopulateCodecSpecific(bool frame_is_keyframe,
const TemporalLayers::FrameConfig& tl_config,
CodecSpecificInfoVP8* vp8_info,
uint32_t timestamp) override;
void FrameEncoded(uint32_t rtp_timestamp, size_t size, int qp) override;
void OnEncodeDone(uint32_t rtp_timestamp,
size_t size_bytes,
bool is_keyframe,
int qp,
CodecSpecificInfoVP8* vp8_info) override;
private:
static constexpr size_t kKeyframeBuffer = std::numeric_limits<size_t>::max();
@ -66,14 +66,30 @@ class DefaultTemporalLayers : public TemporalLayers {
// Updated cumulative bitrates, per temporal layer.
absl::optional<std::vector<uint32_t>> new_bitrates_bps_;
// Map from rtp timestamp to a bitmask of Vp8BufferReference indicating which
// buffers this frame should update. Reset on pattern loop.
std::map<uint32_t, uint8_t> pending_frames_;
struct PendingFrame {
PendingFrame();
PendingFrame(bool expired,
uint8_t updated_buffers_mask,
const FrameConfig& frame_config);
// Flag indicating if this frame has expired, ie it belongs to a previous
// iteration of the temporal pattern.
bool expired = false;
// Bitmask of Vp8BufferReference flags, indicating which buffers this frame
// updates.
uint8_t updated_buffer_mask = 0;
// The frame config return by UpdateLayerConfig() for this frame.
FrameConfig frame_config;
};
// Map from rtp timestamp to pending frame status. Reset on pattern loop.
std::map<uint32_t, PendingFrame> pending_frames_;
// One counter per Vp8BufferReference, indicating number of frames since last
// refresh. For non-base-layer frames (ie golden, altref buffers), this is
// reset when the pattern loops.
std::map<Vp8BufferReference, size_t> frames_since_buffer_refresh_;
// Optional utility used to verify reference validity.
std::unique_ptr<TemporalLayersChecker> checker_;
};
class DefaultTemporalLayersChecker : public TemporalLayersChecker {

149
modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc
View File

@ -116,8 +116,8 @@ TEST(TemporalLayersTest, 2Layers) {
for (int i = 0; i < 16; ++i) {
TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
EXPECT_EQ(expected_flags[i], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i;
tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp,
&vp8_info);
EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config));
EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx);
EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
@ -168,8 +168,8 @@ TEST(TemporalLayersTest, 3Layers) {
for (int i = 0; i < 16; ++i) {
TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
EXPECT_EQ(expected_flags[i], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i;
tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp,
&vp8_info);
EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config));
EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx);
EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
@ -209,8 +209,8 @@ TEST(TemporalLayersTest, Alternative3Layers) {
for (int i = 0; i < 8; ++i) {
TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
EXPECT_EQ(expected_flags[i], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i;
tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp,
&vp8_info);
EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config));
EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx);
EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
@ -239,28 +239,28 @@ TEST(TemporalLayersTest, SearchOrder) {
// Start with a key-frame. tl_config flags can be ignored.
uint32_t timestamp = 0;
TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
tl.PopulateCodecSpecific(true, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
&vp8_info);
// TL2 frame. First one only references TL0. Updates altref.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast);
EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone);
// TL1 frame. Can only reference TL0. Updated golden.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast);
EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone);
// TL2 frame. Can reference all three buffers. Golden was the last to be
// updated, the next to last was altref.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kGolden);
EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kAltref);
}
@ -283,25 +283,25 @@ TEST(TemporalLayersTest, SearchOrderWithDrop) {
// Start with a key-frame. tl_config flags can be ignored.
uint32_t timestamp = 0;
TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
tl.PopulateCodecSpecific(true, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
&vp8_info);
// TL2 frame. First one only references TL0. Updates altref.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast);
EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone);
// Dropped TL1 frame. Can only reference TL0. Should have updated golden.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.FrameEncoded(timestamp, 0, 0);
tl.OnEncodeDone(timestamp, 0, false, 0, nullptr);
// TL2 frame. Can normally reference all three buffers, but golden has not
// been populated this cycle. Altref was last to be updated, before that last.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kAltref);
EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kLast);
}
@ -345,8 +345,8 @@ TEST(TemporalLayersTest, 4Layers) {
for (int i = 0; i < 16; ++i) {
TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
EXPECT_EQ(expected_flags[i], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i;
tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp,
&vp8_info);
EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config));
EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx);
EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
@ -374,22 +374,22 @@ TEST(TemporalLayersTest, DoesNotReferenceDroppedFrames) {
// Start with a keyframe.
uint32_t timestamp = 0;
TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
tl.PopulateCodecSpecific(true, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
&vp8_info);
// Dropped TL2 frame.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.FrameEncoded(timestamp, 0, 0);
tl.OnEncodeDone(timestamp, 0, false, 0, nullptr);
// Dropped TL1 frame.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.FrameEncoded(timestamp, 0, 0);
tl.OnEncodeDone(timestamp, 0, false, 0, nullptr);
// TL2 frame. Can reference all three buffers, valid since golden and altref
// both contain the last keyframe.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference);
EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference);
@ -398,24 +398,24 @@ TEST(TemporalLayersTest, DoesNotReferenceDroppedFrames) {
// TL0 base layer frame, updating and referencing last.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
// TL2 frame, updating altref.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
// TL1 frame, updating golden.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
// TL2 frame. Can still reference all buffer since they have been update this
// cycle.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference);
EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference);
@ -424,22 +424,22 @@ TEST(TemporalLayersTest, DoesNotReferenceDroppedFrames) {
// TL0 base layer frame, updating and referencing last.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
// Dropped TL2 frame.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.FrameEncoded(timestamp, 0, 0);
tl.OnEncodeDone(timestamp, 0, false, 0, nullptr);
// Dropped TL1 frame.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.FrameEncoded(timestamp, 0, 0);
tl.OnEncodeDone(timestamp, 0, false, 0, nullptr);
// TL2 frame. This time golden and altref contain data from the previous cycle
// and cannot be referenced.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference);
EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
@ -461,25 +461,25 @@ TEST(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExist) {
// Start with a keyframe.
uint32_t timestamp = 0;
TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
tl.PopulateCodecSpecific(true, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
&vp8_info);
// Do a full cycle of the pattern.
for (int i = 0; i < 7; ++i) {
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
}
// TL0 base layer frame, starting the cycle over.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
// TL2 frame.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
// Encoder has a hiccup and builds a queue, so frame encoding is delayed.
// TL1 frame, updating golden.
@ -498,14 +498,14 @@ TEST(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExist) {
// The previous four enqueued frames finally get encoded, and the updated
// buffers are now OK to reference.
// Enqueued TL1 frame ready.
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
// Enqueued TL2 frame.
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, ++timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(++timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
// Enqueued TL0 frame.
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, ++timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(++timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
// TL2 frame, all buffers are now in a known good state, OK to reference.
tl_config = tl.UpdateLayerConfig(++timestamp + 1);
@ -531,25 +531,25 @@ TEST(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExistLongDelay) {
// Start with a keyframe.
uint32_t timestamp = 0;
TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
tl.PopulateCodecSpecific(true, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
&vp8_info);
// Do a full cycle of the pattern.
for (int i = 0; i < 3; ++i) {
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
}
// TL0 base layer frame, starting the cycle over.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
// TL2 frame.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
// Encoder has a hiccup and builds a queue, so frame encoding is delayed.
// Encoded, but delayed frames in TL 1, 2.
@ -563,11 +563,11 @@ TEST(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExistLongDelay) {
tl_config = tl.UpdateLayerConfig(timestamp + 4);
// TL1 frame from last cycle is ready.
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp + 1);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp + 1, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
// TL2 frame from last cycle is ready.
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp + 2);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp + 2, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
// TL2 frame, that should be referencing all buffers, but altref and golden
// haven not been updated this cycle. (Don't be fooled by the late frames from
@ -613,8 +613,8 @@ TEST(TemporalLayersTest, KeyFrame) {
TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
EXPECT_EQ(expected_flags[j], LibvpxVp8Encoder::EncodeFlags(tl_config))
<< j;
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
&vp8_info);
EXPECT_TRUE(checker.CheckTemporalConfig(false, tl_config));
EXPECT_EQ(expected_temporal_idx[j], tl_config.packetizer_temporal_idx);
EXPECT_EQ(expected_temporal_idx[j], tl_config.encoder_layer_id);
@ -623,8 +623,8 @@ TEST(TemporalLayersTest, KeyFrame) {
}
TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
tl.PopulateCodecSpecific(true, tl_config, &vp8_info, timestamp);
tl.FrameEncoded(timestamp, kDefaultBytesPerFrame, kDefaultQp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
&vp8_info);
EXPECT_TRUE(vp8_info.layerSync) << "Key frame should be marked layer sync.";
EXPECT_EQ(0, vp8_info.temporalIdx)
<< "Key frame should always be packetized as layer 0";
@ -712,7 +712,8 @@ TEST_P(TemporalLayersReferenceTest, ValidFrameConfigs) {
std::vector<TemporalLayers::FrameConfig> tl_configs(kMaxPatternLength);
for (int i = 0; i < kMaxPatternLength; ++i) {
TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp_);
tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_specifics, timestamp_);
tl.OnEncodeDone(timestamp_, kDefaultBytesPerFrame, i == 0, kDefaultQp,
&vp8_specifics);
++timestamp_;
EXPECT_FALSE(tl_config.drop_frame);
tl_configs.push_back(tl_config);

64
modules/video_coding/codecs/vp8/include/vp8_temporal_layers.h
View File

@ -28,10 +28,10 @@ namespace webrtc {
// - UpdateLayerConfig(timestampB)
// - PopulateCodecSpecific(timestampA, ...)
// - UpdateLayerConfig(timestampC)
// - FrameEncoded(timestampA, 1234, ...)
// - FrameEncoded(timestampB, 0, ...)
// - PopulateCodecSpecific(timestampC, ...)
// - FrameEncoded(timestampC, 1234, ...)
// - OnEncodeDone(timestampA, 1234, ...)
// - UpdateLayerConfig(timestampC)
// - OnEncodeDone(timestampB, 0, ...)
// - OnEncodeDone(timestampC, 1234, ...)
// Note that UpdateLayerConfig() for a new frame can happen before
// FrameEncoded() for a previous one, but calls themselves must be both
// synchronized (e.g. run on a task queue) and in order (per type).
@ -142,14 +142,26 @@ class TemporalLayers {
virtual ~TemporalLayers() = default;
// If this method returns true, the encoder is free to drop frames for
// instance in an effort to uphold encoding bitrate.
// If this return false, the encoder must not drop any frames unless:
// 1. Requested to do so via FrameConfig.drop_frame
// 2. The frame to be encoded is requested to be a keyframe
// 3. The encoded detected a large overshoot and decided to drop and then
// re-encode the image at a low bitrate. In this case the encoder should
// call OnEncodeDone() once with size = 0 to indicate drop, and then call
// OnEncodeDone() again when the frame has actually been encoded.
virtual bool SupportsEncoderFrameDropping() const = 0;
// New target bitrate, per temporal layer.
virtual void OnRatesUpdated(const std::vector<uint32_t>& bitrates_bps,
int framerate_fps) = 0;
// Update the encoder configuration with target bitrates or other parameters.
// Returns true iff the configuration was actually modified.
// Called by the encoder before encoding a frame. |cfg| contains the current
// configuration. If the TemporalLayers instance wishes any part of that
// to be changed before the encode step, |cfg| should be changed and then
// return true. If false is returned, the encoder will proceed without
// updating the configuration.
virtual bool UpdateConfiguration(Vp8EncoderConfig* cfg) = 0;
// Returns the recommended VP8 encode flags needed, and moves the temporal
@ -157,30 +169,28 @@ class TemporalLayers {
// The timestamp may be used as both a time and a unique identifier, and so
// the caller must make sure no two frames use the same timestamp.
// The timestamp uses a 90kHz RTP clock.
// After calling this method, the actual encoder should be called with the
// provided frame configuration, after which:
// * On success, call PopulateCodecSpecific() and then FrameEncoded();
// * On failure/ frame drop: Call FrameEncoded() with size = 0.
// After calling this method, first call the actual encoder with the provided
// frame configuration, and then OnEncodeDone() below.
virtual FrameConfig UpdateLayerConfig(uint32_t rtp_timestamp) = 0;
// Called after successful encoding of a frame. The rtp timestamp must match
// the one using in UpdateLayerConfig(). Some fields in |vp8_info| may have
// already been populated by the encoder, check before overwriting.
// |tl_config| is the frame config returned by UpdateLayerConfig() for this
// rtp_timestamp;
// If |is_keyframe| is true, the flags in |tl_config| will be ignored.
virtual void PopulateCodecSpecific(
bool is_keyframe,
const TemporalLayers::FrameConfig& tl_config,
CodecSpecificInfoVP8* vp8_info,
uint32_t rtp_timestamp) = 0;
// Called after an encode event. If the frame was dropped, |size_bytes| must
// be set to 0. The rtp timestamp must match the one using in
// UpdateLayerConfig()
virtual void FrameEncoded(uint32_t rtp_timestamp,
// Called after the encode step is done. |rtp_timestamp| must match the
// parameter use in the UpdateLayerConfig() call.
// |is_keyframe| must be true iff the encoder decided to encode this frame as
// a keyframe.
// If the encoder decided to drop this frame, |size_bytes| must be set to 0,
// otherwise it should indicate the size in bytes of the encoded frame.
// If |size_bytes| > 0, and |vp8_info| is not null, the TemporalLayers
// instance my update |vp8_info| with codec specific data such as temporal id.
// Some fields of this struct may have already been populated by the encoder,
// check before overwriting.
// If |size_bytes| > 0, |qp| should indicate the frame-level QP this frame was
// encoded at. If the encoder does not support extracting this, |qp| should be
// set to 0.
virtual void OnEncodeDone(uint32_t rtp_timestamp,
size_t size_bytes,
int qp) = 0;
bool is_keyframe,
int qp,
CodecSpecificInfoVP8* vp8_info) = 0;
};
} // namespace webrtc

48
modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc
View File

@ -821,31 +821,31 @@ int LibvpxVp8Encoder::Encode(const VideoFrame& frame,
return WEBRTC_VIDEO_CODEC_ERROR;
timestamp_ += duration;
// Examines frame timestamps only.
error = GetEncodedPartitions(tl_configs, frame);
error = GetEncodedPartitions(frame);
}
return error;
}
void LibvpxVp8Encoder::PopulateCodecSpecific(
CodecSpecificInfo* codec_specific,
const TemporalLayers::FrameConfig& tl_config,
const vpx_codec_cx_pkt_t& pkt,
int stream_idx,
uint32_t timestamp) {
void LibvpxVp8Encoder::PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
const vpx_codec_cx_pkt_t& pkt,
int stream_idx,
int encoder_idx,
uint32_t timestamp) {
assert(codec_specific != NULL);
codec_specific->codecType = kVideoCodecVP8;
codec_specific->codec_name = ImplementationName();
CodecSpecificInfoVP8* vp8Info = &(codec_specific->codecSpecific.VP8);
vp8Info->keyIdx = kNoKeyIdx; // TODO(hlundin) populate this
vp8Info->nonReference = (pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE) != 0;
temporal_layers_[stream_idx]->PopulateCodecSpecific(
(pkt.data.frame.flags & VPX_FRAME_IS_KEY) != 0, tl_config, vp8Info,
timestamp);
int qp = 0;
vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp);
temporal_layers_[stream_idx]->OnEncodeDone(
timestamp, encoded_images_[encoder_idx]._length,
(pkt.data.frame.flags & VPX_FRAME_IS_KEY) != 0, qp, vp8Info);
}
int LibvpxVp8Encoder::GetEncodedPartitions(
const TemporalLayers::FrameConfig tl_configs[],
const VideoFrame& input_image) {
int LibvpxVp8Encoder::GetEncodedPartitions(const VideoFrame& input_image) {
int stream_idx = static_cast<int>(encoders_.size()) - 1;
int result = WEBRTC_VIDEO_CODEC_OK;
for (size_t encoder_idx = 0; encoder_idx < encoders_.size();
@ -895,8 +895,8 @@ int LibvpxVp8Encoder::GetEncodedPartitions(
is_keyframe = true;
}
encoded_images_[encoder_idx].SetSpatialIndex(stream_idx);
PopulateCodecSpecific(&codec_specific, tl_configs[stream_idx], *pkt,
stream_idx, input_image.timestamp());
PopulateCodecSpecific(&codec_specific, *pkt, stream_idx, encoder_idx,
input_image.timestamp());
break;
}
}
@ -910,10 +910,6 @@ int LibvpxVp8Encoder::GetEncodedPartitions(
: VideoContentType::UNSPECIFIED;
encoded_images_[encoder_idx].timing_.flags = VideoSendTiming::kInvalid;
int qp = -1;
vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp);
temporal_layers_[stream_idx]->FrameEncoded(
input_image.timestamp(), encoded_images_[encoder_idx]._length, qp);
if (send_stream_[stream_idx]) {
if (encoded_images_[encoder_idx]._length > 0) {
TRACE_COUNTER_ID1("webrtc", "EncodedFrameSize", encoder_idx,
@ -928,16 +924,16 @@ int LibvpxVp8Encoder::GetEncodedPartitions(
encoded_images_[encoder_idx].qp_ = qp_128;
encoded_complete_callback_->OnEncodedImage(encoded_images_[encoder_idx],
&codec_specific, &frag_info);
} else if (codec_.mode == VideoCodecMode::kScreensharing) {
} else if (!temporal_layers_[stream_idx]
->SupportsEncoderFrameDropping()) {
result = WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT;
if (encoded_images_[encoder_idx]._length == 0) {
// Dropped frame that will be re-encoded.
temporal_layers_[stream_idx]->OnEncodeDone(input_image.timestamp(), 0,
false, 0, nullptr);
}
}
}
if (result != WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT) {
// Don't run checker on drop before reencode as that will incorrectly
// increase the pattern index twice.
RTC_DCHECK(temporal_layers_checkers_[stream_idx]->CheckTemporalConfig(
is_keyframe, tl_configs[stream_idx]));
}
}
return result;
}

5
modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h
View File

@ -70,13 +70,12 @@ class LibvpxVp8Encoder : public VP8Encoder {
int InitAndSetControlSettings();
void PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
const TemporalLayers::FrameConfig& tl_config,
const vpx_codec_cx_pkt& pkt,
int stream_idx,
int encoder_idx,
uint32_t timestamp);
int GetEncodedPartitions(const TemporalLayers::FrameConfig tl_configs[],
const VideoFrame& input_image);
int GetEncodedPartitions(const VideoFrame& input_image);
// Set the stream state for stream |stream_idx|.
void SetStreamState(bool send_stream, int stream_idx);

103
modules/video_coding/codecs/vp8/screenshare_layers.cc
View File

@ -21,12 +21,13 @@
#include "system_wrappers/include/metrics.h"
namespace webrtc {
namespace {
static const int kOneSecond90Khz = 90000;
static const int kMinTimeBetweenSyncs = kOneSecond90Khz * 2;
static const int kMaxTimeBetweenSyncs = kOneSecond90Khz * 4;
static const int kQpDeltaThresholdForSync = 8;
static const int kMinBitrateKbpsForQpBoost = 500;
} // namespace
const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5;
const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0;
@ -37,8 +38,7 @@ constexpr int ScreenshareLayers::kMaxNumTemporalLayers;
// been exceeded. This prevents needless keyframe requests.
const int ScreenshareLayers::kMaxFrameIntervalMs = 2750;
ScreenshareLayers::ScreenshareLayers(int num_temporal_layers,
Clock* clock)
ScreenshareLayers::ScreenshareLayers(int num_temporal_layers, Clock* clock)
: clock_(clock),
number_of_temporal_layers_(
std::min(kMaxNumTemporalLayers, num_temporal_layers)),
@ -51,7 +51,10 @@ ScreenshareLayers::ScreenshareLayers(int num_temporal_layers,
max_qp_(-1),
max_debt_bytes_(0),
encode_framerate_(1000.0f, 1000.0f), // 1 second window, second scale.
bitrate_updated_(false) {
bitrate_updated_(false),
checker_(TemporalLayersChecker::CreateTemporalLayersChecker(
TemporalLayersType::kBitrateDynamic,
num_temporal_layers)) {
RTC_CHECK_GT(number_of_temporal_layers_, 0);
RTC_CHECK_LE(number_of_temporal_layers_, kMaxNumTemporalLayers);
}
@ -67,11 +70,18 @@ bool ScreenshareLayers::SupportsEncoderFrameDropping() const {
TemporalLayers::FrameConfig ScreenshareLayers::UpdateLayerConfig(
uint32_t timestamp) {
auto it = pending_frame_configs_.find(timestamp);
if (it != pending_frame_configs_.end()) {
// Drop and re-encode, reuse the previous config.
return it->second;
}
if (number_of_temporal_layers_ <= 1) {
// No flags needed for 1 layer screenshare.
// TODO(pbos): Consider updating only last, and not all buffers.
TemporalLayers::FrameConfig tl_config(
kReferenceAndUpdate, kReferenceAndUpdate, kReferenceAndUpdate);
pending_frame_configs_[timestamp] = tl_config;
return tl_config;
}
@ -201,6 +211,7 @@ TemporalLayers::FrameConfig ScreenshareLayers::UpdateLayerConfig(
}
tl_config.layer_sync = layer_state == TemporalLayerState::kTl1Sync;
pending_frame_configs_[timestamp] = tl_config;
return tl_config;
}
@ -244,20 +255,57 @@ void ScreenshareLayers::OnRatesUpdated(
layers_[1].target_rate_kbps_ = tl1_kbps;
}
void ScreenshareLayers::FrameEncoded(uint32_t timestamp, size_t size, int qp) {
if (size > 0)
encode_framerate_.Update(1, clock_->TimeInMilliseconds());
if (number_of_temporal_layers_ == 1)
return;
RTC_DCHECK_NE(-1, active_layer_);
if (size == 0) {
void ScreenshareLayers::OnEncodeDone(uint32_t rtp_timestamp,
size_t size_bytes,
bool is_keyframe,
int qp,
CodecSpecificInfoVP8* vp8_info) {
if (size_bytes == 0) {
layers_[active_layer_].state = TemporalLayer::State::kDropped;
++stats_.num_overshoots_;
return;
}
absl::optional<FrameConfig> frame_config;
auto it = pending_frame_configs_.find(rtp_timestamp);
if (it != pending_frame_configs_.end()) {
frame_config = it->second;
pending_frame_configs_.erase(it);
if (checker_) {
RTC_DCHECK(checker_->CheckTemporalConfig(is_keyframe, *frame_config));
}
}
if (number_of_temporal_layers_ == 1) {
vp8_info->temporalIdx = kNoTemporalIdx;
vp8_info->layerSync = false;
} else {
int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(rtp_timestamp);
if (frame_config) {
vp8_info->temporalIdx = frame_config->packetizer_temporal_idx;
vp8_info->layerSync = frame_config->layer_sync;
} else {
// Frame requested to be dropped, but was not. Fall back to base-layer.
vp8_info->temporalIdx = 0;
vp8_info->layerSync = false;
}
if (is_keyframe) {
vp8_info->temporalIdx = 0;
last_sync_timestamp_ = unwrapped_timestamp;
vp8_info->layerSync = true;
layers_[0].state = TemporalLayer::State::kKeyFrame;
layers_[1].state = TemporalLayer::State::kKeyFrame;
active_layer_ = 1;
}
}
encode_framerate_.Update(1, clock_->TimeInMilliseconds());
if (number_of_temporal_layers_ == 1)
return;
RTC_DCHECK_NE(-1, active_layer_);
if (layers_[active_layer_].state == TemporalLayer::State::kDropped) {
layers_[active_layer_].state = TemporalLayer::State::kQualityBoost;
}
@ -266,42 +314,19 @@ void ScreenshareLayers::FrameEncoded(uint32_t timestamp, size_t size, int qp) {
layers_[active_layer_].last_qp = qp;
if (active_layer_ == 0) {
layers_[0].debt_bytes_ += size;
layers_[1].debt_bytes_ += size;
layers_[0].debt_bytes_ += size_bytes;
layers_[1].debt_bytes_ += size_bytes;
++stats_.num_tl0_frames_;
stats_.tl0_target_bitrate_sum_ += layers_[0].target_rate_kbps_;
stats_.tl0_qp_sum_ += qp;
} else if (active_layer_ == 1) {
layers_[1].debt_bytes_ += size;
layers_[1].debt_bytes_ += size_bytes;
++stats_.num_tl1_frames_;
stats_.tl1_target_bitrate_sum_ += layers_[1].target_rate_kbps_;
stats_.tl1_qp_sum_ += qp;
}
}
void ScreenshareLayers::PopulateCodecSpecific(
bool frame_is_keyframe,
const TemporalLayers::FrameConfig& tl_config,
CodecSpecificInfoVP8* vp8_info,
uint32_t timestamp) {
if (number_of_temporal_layers_ == 1) {
vp8_info->temporalIdx = kNoTemporalIdx;
vp8_info->layerSync = false;
} else {
int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
vp8_info->temporalIdx = tl_config.packetizer_temporal_idx;
vp8_info->layerSync = tl_config.layer_sync;
if (frame_is_keyframe) {
vp8_info->temporalIdx = 0;
last_sync_timestamp_ = unwrapped_timestamp;
vp8_info->layerSync = true;
layers_[0].state = TemporalLayer::State::kKeyFrame;
layers_[1].state = TemporalLayer::State::kKeyFrame;
active_layer_ = 1;
}
}
}
bool ScreenshareLayers::TimeToSync(int64_t timestamp) const {
RTC_DCHECK_EQ(1, active_layer_);
RTC_DCHECK_NE(-1, layers_[0].last_qp);

19
modules/video_coding/codecs/vp8/screenshare_layers.h
View File

@ -9,8 +9,11 @@
#ifndef MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_
#define MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_
#include <map>
#include <memory>
#include <vector>
#include "modules/video_coding/codecs/vp8/include/temporal_layers_checker.h"
#include "modules/video_coding/codecs/vp8/include/vp8_temporal_layers.h"
#include "modules/video_coding/utility/frame_dropper.h"
#include "rtc_base/rate_statistics.h"
@ -46,12 +49,11 @@ class ScreenshareLayers : public TemporalLayers {
// Returns true iff the configuration was actually modified.
bool UpdateConfiguration(Vp8EncoderConfig* cfg) override;
void PopulateCodecSpecific(bool base_layer_sync,
const TemporalLayers::FrameConfig& tl_config,
CodecSpecificInfoVP8* vp8_info,
uint32_t rtp_timestamp) override;
void FrameEncoded(uint32_t rtp_timestamp, size_t size, int qp) override;
void OnEncodeDone(uint32_t rtp_timestamp,
size_t size_bytes,
bool is_keyframe,
int qp,
CodecSpecificInfoVP8* vp8_info) override;
private:
enum class TemporalLayerState : int { kDrop, kTl0, kTl1, kTl1Sync };
@ -72,6 +74,8 @@ class ScreenshareLayers : public TemporalLayers {
int max_qp_;
uint32_t max_debt_bytes_;
std::map<uint32_t, TemporalLayers::FrameConfig> pending_frame_configs_;
// Configured max framerate.
absl::optional<uint32_t> target_framerate_;
// Incoming framerate from capturer.
@ -118,6 +122,9 @@ class ScreenshareLayers : public TemporalLayers {
int64_t tl0_target_bitrate_sum_ = 0;
int64_t tl1_target_bitrate_sum_ = 0;
} stats_;
// Optional utility used to verify reference validity.
std::unique_ptr<TemporalLayersChecker> checker_;
};
} // namespace webrtc

91
modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc
View File

@ -67,7 +67,8 @@ class ScreenshareLayerTest : public ::testing::Test {
int EncodeFrame(bool base_sync) {
int flags = ConfigureFrame(base_sync);
if (flags != -1)
layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp);
layers_->OnEncodeDone(timestamp_, frame_size_, base_sync, kDefaultQp,
&vp8_info_);
return flags;
}
@ -82,8 +83,6 @@ class ScreenshareLayerTest : public ::testing::Test {
}
config_updated_ = layers_->UpdateConfiguration(&cfg_);
int flags = LibvpxVp8Encoder::EncodeFlags(tl_config_);
layers_->PopulateCodecSpecific(key_frame, tl_config_, &vp8_info_,
timestamp_);
EXPECT_NE(-1, frame_size_);
return flags;
}
@ -147,10 +146,11 @@ class ScreenshareLayerTest : public ::testing::Test {
1 + (sync.value_or(false) ? kMaxSyncPeriodSeconds : 1) * kFrameRate;
for (int i = 0; i < kMaxFramesToSkip; ++i) {
flags = ConfigureFrame(false);
timestamp_ += kTimestampDelta5Fps;
if (vp8_info_.temporalIdx != layer ||
(sync && *sync != vp8_info_.layerSync)) {
layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp);
if (tl_config_.packetizer_temporal_idx != layer ||
(sync && *sync != tl_config_.layer_sync)) {
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp,
&vp8_info_);
timestamp_ += kTimestampDelta5Fps;
} else {
// Found frame from sought after layer.
return flags;
@ -211,13 +211,14 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterTimeout) {
for (int i = 0; i < kNumFrames; ++i) {
tl_config_ = UpdateLayerConfig(timestamp_);
config_updated_ = layers_->UpdateConfiguration(&cfg_);
layers_->PopulateCodecSpecific(false, tl_config_, &vp8_info_, timestamp_);
// Simulate TL1 being at least 8 qp steps better.
if (vp8_info_.temporalIdx == 0) {
layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp);
if (tl_config_.packetizer_temporal_idx == 0) {
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp,
&vp8_info_);
} else {
layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp - 8);
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp - 8,
&vp8_info_);
}
if (vp8_info_.temporalIdx == 1 && vp8_info_.layerSync)
@ -240,10 +241,12 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) {
ConfigureFrame(false);
// Simulate TL1 being at least 8 qp steps better.
if (vp8_info_.temporalIdx == 0) {
layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp);
if (tl_config_.packetizer_temporal_idx == 0) {
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp,
&vp8_info_);
} else {
layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp - 8);
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp - 8,
&vp8_info_);
}
if (vp8_info_.temporalIdx == 1 && vp8_info_.layerSync)
@ -257,12 +260,12 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) {
bool bumped_tl0_quality = false;
for (int i = 0; i < 3; ++i) {
int flags = ConfigureFrame(false);
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp - 8,
&vp8_info_);
if (vp8_info_.temporalIdx == 0) {
// Bump TL0 to same quality as TL1.
layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp - 8);
bumped_tl0_quality = true;
} else {
layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp - 8);
if (bumped_tl0_quality) {
EXPECT_TRUE(vp8_info_.layerSync);
EXPECT_EQ(kTl1SyncFlags, flags);
@ -380,7 +383,7 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) {
SkipUntilTl(0);
// Size 0 indicates dropped frame.
layers_->FrameEncoded(timestamp_, 0, kDefaultQp);
layers_->OnEncodeDone(timestamp_, 0, false, 0, &vp8_info_);
// Re-encode frame (so don't advance timestamp).
int flags = EncodeFrame(false);
@ -392,18 +395,18 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) {
SkipUntilTl(0);
EXPECT_TRUE(config_updated_);
EXPECT_LT(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp);
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, &vp8_info_);
timestamp_ += kTimestampDelta5Fps;
// ...then back to standard setup.
SkipUntilTl(0);
layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp);
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, &vp8_info_);
timestamp_ += kTimestampDelta5Fps;
EXPECT_EQ(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
// Next drop in TL1.
SkipUntilTl(1);
layers_->FrameEncoded(timestamp_, 0, kDefaultQp);
layers_->OnEncodeDone(timestamp_, 0, false, 0, &vp8_info_);
// Re-encode frame (so don't advance timestamp).
flags = EncodeFrame(false);
@ -415,13 +418,13 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) {
SkipUntilTl(1);
EXPECT_TRUE(config_updated_);
EXPECT_LT(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp);
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, &vp8_info_);
timestamp_ += kTimestampDelta5Fps;
// ...and back to normal.
SkipUntilTl(1);
EXPECT_EQ(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp);
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, &vp8_info_);
timestamp_ += kTimestampDelta5Fps;
}
@ -436,7 +439,8 @@ TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) {
EXPECT_EQ(kTl0Flags,
LibvpxVp8Encoder::EncodeFlags(UpdateLayerConfig(kStartTimestamp)));
layers_->FrameEncoded(timestamp_, kLargeFrameSizeBytes, kDefaultQp);
layers_->OnEncodeDone(kStartTimestamp, kLargeFrameSizeBytes, false,
kDefaultQp, &vp8_info_);
const uint32_t kTwoSecondsLater =
kStartTimestamp + (ScreenshareLayers::kMaxFrameIntervalMs * 90);
@ -478,20 +482,22 @@ TEST_F(ScreenshareLayerTest, UpdatesHistograms) {
if (timestamp >= kTimestampDelta5Fps * 5 && !overshoot && flags != -1) {
// Simulate one overshoot.
layers_->FrameEncoded(timestamp_, 0, 0);
layers_->OnEncodeDone(timestamp, 0, false, 0, nullptr);
overshoot = true;
}
if (flags == kTl0Flags) {
if (timestamp >= kTimestampDelta5Fps * 20 && !trigger_drop) {
// Simulate a too large frame, to cause frame drop.
layers_->FrameEncoded(timestamp_, frame_size_ * 10, kTl0Qp);
layers_->OnEncodeDone(timestamp, frame_size_ * 10, false, kTl0Qp,
&vp8_info_);
trigger_drop = true;
} else {
layers_->FrameEncoded(timestamp_, frame_size_, kTl0Qp);
layers_->OnEncodeDone(timestamp, frame_size_, false, kTl0Qp,
&vp8_info_);
}
} else if (flags == kTl1Flags || flags == kTl1SyncFlags) {
layers_->FrameEncoded(timestamp_, frame_size_, kTl1Qp);
layers_->OnEncodeDone(timestamp, frame_size_, false, kTl1Qp, &vp8_info_);
} else if (flags == -1) {
dropped_frame = true;
} else {
@ -557,7 +563,8 @@ TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) {
++num_discarded_frames;
} else {
size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8;
layers_->FrameEncoded(timestamp_, frame_size_bytes, kDefaultQp);
layers_->OnEncodeDone(timestamp, frame_size_bytes, false, kDefaultQp,
&vp8_info_);
}
timestamp += kFrameIntervalsMs * 90;
clock_.AdvanceTimeMilliseconds(kFrameIntervalsMs);
@ -573,7 +580,8 @@ TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) {
++num_discarded_frames;
} else {
size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8;
layers_->FrameEncoded(timestamp_, frame_size_bytes, kDefaultQp);
layers_->OnEncodeDone(timestamp, frame_size_bytes, false, kDefaultQp,
&vp8_info_);
}
timestamp += kFrameIntervalsMs * 90 / 2;
clock_.AdvanceTimeMilliseconds(kFrameIntervalsMs / 2);
@ -590,16 +598,17 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAtOvershootDrop) {
// Move ahead until we have a sync frame in TL1.
EXPECT_EQ(kTl1SyncFlags, SkipUntilTlAndSync(1, true));
ASSERT_TRUE(vp8_info_.layerSync);
ASSERT_TRUE(tl_config_.layer_sync);
// Simulate overshoot of this frame.
layers_->FrameEncoded(timestamp_, 0, -1);
layers_->OnEncodeDone(timestamp_, 0, false, 0, nullptr);
config_updated_ = layers_->UpdateConfiguration(&cfg_);
EXPECT_EQ(kTl1SyncFlags, LibvpxVp8Encoder::EncodeFlags(tl_config_));
CodecSpecificInfoVP8 new_vp8_info;
layers_->PopulateCodecSpecific(false, tl_config_, &new_vp8_info, timestamp_);
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp,
&new_vp8_info);
EXPECT_TRUE(new_vp8_info.layerSync);
}
@ -610,7 +619,7 @@ TEST_F(ScreenshareLayerTest, DropOnTooShortFrameInterval) {
// Add a large gap, so there's plenty of room in the rate tracker.
timestamp_ += kTimestampDelta5Fps * 3;
EXPECT_FALSE(UpdateLayerConfig(timestamp_).drop_frame);
layers_->FrameEncoded(timestamp_, frame_size_, kDefaultQp);
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, &vp8_info_);
// Frame interval below 90% if desired time is not allowed, try inserting
// frame just before this limit.
@ -666,14 +675,13 @@ TEST_F(ScreenshareLayerTest, MaxQpRestoredAfterDoubleDrop) {
// Move ahead until we have a sync frame in TL1.
EXPECT_EQ(kTl1SyncFlags, SkipUntilTlAndSync(1, true));
ASSERT_TRUE(vp8_info_.layerSync);
ASSERT_TRUE(tl_config_.layer_sync);
// Simulate overshoot of this frame.
layers_->FrameEncoded(timestamp_, 0, -1);
layers_->OnEncodeDone(timestamp_, 0, false, -1, nullptr);
// Simulate re-encoded frame.
layers_->PopulateCodecSpecific(false, tl_config_, &vp8_info_, timestamp_);
layers_->FrameEncoded(timestamp_, 1, max_qp_);
layers_->OnEncodeDone(timestamp_, 1, false, max_qp_, &vp8_info_);
// Next frame, expect boosted quality.
// Slightly alter bitrate between each frame.
@ -686,11 +694,10 @@ TEST_F(ScreenshareLayerTest, MaxQpRestoredAfterDoubleDrop) {
uint32_t adjusted_qp = cfg_.rc_max_quantizer;
// Simulate overshoot of this frame.
layers_->FrameEncoded(timestamp_, 0, -1);
layers_->OnEncodeDone(timestamp_, 0, false, -1, nullptr);
// Simulate re-encoded frame.
layers_->PopulateCodecSpecific(false, tl_config_, &vp8_info_, timestamp_);
layers_->FrameEncoded(timestamp_, frame_size_, max_qp_);
layers_->OnEncodeDone(timestamp_, frame_size_, false, max_qp_, &vp8_info_);
// A third frame, expect boosted quality.
layers_->OnRatesUpdated(kDefault2TlBitratesBps, kFrameRate);
@ -700,13 +707,11 @@ TEST_F(ScreenshareLayerTest, MaxQpRestoredAfterDoubleDrop) {
EXPECT_EQ(adjusted_qp, cfg_.rc_max_quantizer);
// Frame encoded.
layers_->PopulateCodecSpecific(false, tl_config_, &vp8_info_, timestamp_);
layers_->FrameEncoded(timestamp_, frame_size_, max_qp_);
layers_->OnEncodeDone(timestamp_, frame_size_, false, max_qp_, &vp8_info_);
// A fourth frame, max qp should be restored.
layers_->OnRatesUpdated(kDefault2TlBitratesBpsAlt, kFrameRate);
EXPECT_EQ(kTl1Flags, SkipUntilTlAndSync(1, false));
EXPECT_TRUE(config_updated_);
EXPECT_EQ(cfg_.rc_max_quantizer, max_qp_);
}

3
modules/video_coding/codecs/vp8/temporal_layers_checker.cc
View File

@ -70,7 +70,8 @@ bool TemporalLayersChecker::CheckAndUpdateBufferState(
bool TemporalLayersChecker::CheckTemporalConfig(
bool frame_is_keyframe,
const TemporalLayers::FrameConfig& frame_config) {
if (frame_config.drop_frame) {
if (frame_config.drop_frame ||
frame_config.packetizer_temporal_idx == kNoTemporalIdx) {
return true;
}
++sequence_number_;

7
modules/video_coding/utility/simulcast_rate_allocator_unittest.cc
View File

@ -39,11 +39,8 @@ class MockTemporalLayers : public TemporalLayers {
MOCK_METHOD1(UpdateLayerConfig, TemporalLayers::FrameConfig(uint32_t));
MOCK_METHOD2(OnRatesUpdated, void(const std::vector<uint32_t>&, int));
MOCK_METHOD1(UpdateConfiguration, bool(Vp8EncoderConfig*));
MOCK_METHOD4(PopulateCodecSpecific,
void(bool,
const TemporalLayers::FrameConfig&,
CodecSpecificInfoVP8*,
uint32_t));
MOCK_METHOD5(OnEncodeDone,
void(uint32_t, size_t, bool, int, CodecSpecificInfoVP8*));
MOCK_METHOD3(FrameEncoded, void(uint32_t, size_t, int));
MOCK_CONST_METHOD0(Tl0PicIdx, uint8_t());
MOCK_CONST_METHOD1(GetTemporalLayerId,

23
test/fake_vp8_encoder.cc
View File

@ -78,20 +78,19 @@ void FakeVP8Encoder::SetupTemporalLayers(const VideoCodec& codec) {
}
}
void FakeVP8Encoder::PopulateCodecSpecific(
CodecSpecificInfo* codec_specific,
const TemporalLayers::FrameConfig& tl_config,
FrameType frame_type,
int stream_idx,
uint32_t timestamp) {
void FakeVP8Encoder::PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
size_t size_bytes,
FrameType frame_type,
int stream_idx,
uint32_t timestamp) {
RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
codec_specific->codecType = kVideoCodecVP8;
codec_specific->codec_name = ImplementationName();
CodecSpecificInfoVP8* vp8Info = &(codec_specific->codecSpecific.VP8);
vp8Info->keyIdx = kNoKeyIdx;
vp8Info->nonReference = false;
temporal_layers_[stream_idx]->PopulateCodecSpecific(
frame_type == kVideoFrameKey, tl_config, vp8Info, timestamp);
temporal_layers_[stream_idx]->OnEncodeDone(
timestamp, size_bytes, frame_type == kVideoFrameKey, -1, vp8Info);
}
EncodedImageCallback::Result FakeVP8Encoder::OnEncodedImage(
@ -101,14 +100,10 @@ EncodedImageCallback::Result FakeVP8Encoder::OnEncodedImage(
RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
uint8_t stream_idx = encoded_image.SpatialIndex().value_or(0);
CodecSpecificInfo overrided_specific_info;
TemporalLayers::FrameConfig tl_config =
temporal_layers_[stream_idx]->UpdateLayerConfig(
encoded_image.Timestamp());
PopulateCodecSpecific(&overrided_specific_info, tl_config,
temporal_layers_[stream_idx]->UpdateLayerConfig(encoded_image.Timestamp());
PopulateCodecSpecific(&overrided_specific_info, encoded_image._length,
encoded_image._frameType, stream_idx,
encoded_image.Timestamp());
temporal_layers_[stream_idx]->FrameEncoded(encoded_image.Timestamp(),
encoded_image._length, -1);
return callback_->OnEncodedImage(encoded_image, &overrided_specific_info,
fragments);

2
test/fake_vp8_encoder.h
View File

@ -46,7 +46,7 @@ class FakeVP8Encoder : public FakeEncoder, public EncodedImageCallback {
private:
void SetupTemporalLayers(const VideoCodec& codec);
void PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
const TemporalLayers::FrameConfig& tl_config,
size_t size_bytes,
FrameType frame_type,
int stream_idx,
uint32_t timestamp);