admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove VideoLayerFrameId::spatial_layer, use EncodedImage::SpatialIndex instead.

Browse Source 
Next step is to replace VideoLayerFrameId with int64_t.

Bug: webrtc:12206
Change-Id: I414f491e383acf7f8efd97f7bf93dc55a5194fbf
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/206804
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Sam Zackrisson <saza@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#33245}
This commit is contained in:
philipel 2021-02-11 15:25:08 +01:00 committed by Commit Bot
parent cae4656d4a
commit f109193fba
11 changed files with 88 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
api/video/encoded_frame.h
View File

@ -23,12 +23,11 @@ namespace video_coding {
struct VideoLayerFrameId {
// TODO(philipel): The default ctor is currently used internaly, but have a
// look if we can remove it.
VideoLayerFrameId() : picture_id(-1), spatial_layer(0) {}
VideoLayerFrameId(int64_t picture_id, uint8_t spatial_layer)
: picture_id(picture_id), spatial_layer(spatial_layer) {}
VideoLayerFrameId() : picture_id(-1) {}
explicit VideoLayerFrameId(int64_t picture_id) : picture_id(picture_id) {}
bool operator==(const VideoLayerFrameId& rhs) const {
return picture_id == rhs.picture_id && spatial_layer == rhs.spatial_layer;
return picture_id == rhs.picture_id;
}
bool operator!=(const VideoLayerFrameId& rhs) const {
@ -36,8 +35,6 @@ struct VideoLayerFrameId {
}
bool operator<(const VideoLayerFrameId& rhs) const {
if (picture_id == rhs.picture_id)
return spatial_layer < rhs.spatial_layer;
return picture_id < rhs.picture_id;
}
@ -46,7 +43,6 @@ struct VideoLayerFrameId {
bool operator>=(const VideoLayerFrameId& rhs) const { return rhs <= *this; }
int64_t picture_id;
uint8_t spatial_layer;
};
// TODO(philipel): Remove webrtc::VCMEncodedFrame inheritance.

39
modules/video_coding/frame_buffer2.cc
View File

@ -402,26 +402,20 @@ int64_t FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) {
!last_continuous_frame_ ? -1 : last_continuous_frame_->picture_id;
if (!ValidReferences(*frame)) {
RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
<< id.picture_id << ":"
<< static_cast<int>(id.spatial_layer)
<< ") has invalid frame references, dropping frame.";
RTC_LOG(LS_WARNING) << "Frame " << id.picture_id
<< " has invalid frame references, dropping frame.";
return last_continuous_picture_id;
}
if (frames_.size() >= kMaxFramesBuffered) {
if (frame->is_keyframe()) {
RTC_LOG(LS_WARNING) << "Inserting keyframe (picture_id:spatial_id) ("
<< id.picture_id << ":"
<< static_cast<int>(id.spatial_layer)
<< ") but buffer is full, clearing"
RTC_LOG(LS_WARNING) << "Inserting keyframe " << id.picture_id
<< " but buffer is full, clearing"
" buffer and inserting the frame.";
ClearFramesAndHistory();
} else {
RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
<< id.picture_id << ":"
<< static_cast<int>(id.spatial_layer)
<< ") could not be inserted due to the frame "
RTC_LOG(LS_WARNING) << "Frame " << id.picture_id
<< " could not be inserted due to the frame "
"buffer being full, dropping frame.";
return last_continuous_picture_id;
}
@ -443,13 +437,10 @@ int64_t FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) {
ClearFramesAndHistory();
last_continuous_picture_id = -1;
} else {
RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
<< id.picture_id << ":"
<< static_cast<int>(id.spatial_layer)
<< ") inserted after frame ("
<< last_decoded_frame->picture_id << ":"
<< static_cast<int>(last_decoded_frame->spatial_layer)
<< ") was handed off for decoding, dropping frame.";
RTC_LOG(LS_WARNING) << "Frame " << id.picture_id
<< " inserted after frame "
<< last_decoded_frame->picture_id
<< " was handed off for decoding, dropping frame.";
return last_continuous_picture_id;
}
}
@ -579,7 +570,7 @@ bool FrameBuffer::UpdateFrameInfoWithIncomingFrame(const EncodedFrame& frame,
// Find all dependencies that have not yet been fulfilled.
for (size_t i = 0; i < frame.num_references; ++i) {
VideoLayerFrameId ref_key(frame.references[i], frame.id.spatial_layer);
VideoLayerFrameId ref_key(frame.references[i]);
// Does |frame| depend on a frame earlier than the last decoded one?
if (last_decoded_frame && ref_key <= *last_decoded_frame) {
// Was that frame decoded? If not, this |frame| will never become
@ -588,10 +579,9 @@ bool FrameBuffer::UpdateFrameInfoWithIncomingFrame(const EncodedFrame& frame,
int64_t now_ms = clock_->TimeInMilliseconds();
if (last_log_non_decoded_ms_ + kLogNonDecodedIntervalMs < now_ms) {
RTC_LOG(LS_WARNING)
<< "Frame with (picture_id:spatial_id) (" << id.picture_id << ":"
<< static_cast<int>(id.spatial_layer)
<< ") depends on a non-decoded frame more previous than"
" the last decoded frame, dropping frame.";
<< "Frame " << id.picture_id
<< " depends on a non-decoded frame more previous than the last "
"decoded frame, dropping frame.";
last_log_non_decoded_ms_ = now_ms;
}
return false;
@ -683,7 +673,6 @@ EncodedFrame* FrameBuffer::CombineAndDeleteFrames(
// Spatial index of combined frame is set equal to spatial index of its top
// spatial layer.
first_frame->SetSpatialIndex(last_frame->SpatialIndex().value_or(0));
first_frame->id.spatial_layer = last_frame->id.spatial_layer;
first_frame->video_timing_mutable()->network2_timestamp_ms =
last_frame->video_timing().network2_timestamp_ms;

1
modules/video_coding/rtp_frame_id_only_ref_finder.cc
View File

@ -20,6 +20,7 @@ namespace video_coding {
RtpFrameReferenceFinder::ReturnVector RtpFrameIdOnlyRefFinder::ManageFrame(
std::unique_ptr<RtpFrameObject> frame,
int frame_id) {
frame->SetSpatialIndex(0);
frame->id.picture_id = unwrapper_.Unwrap(frame_id & (kFrameIdLength - 1));
frame->num_references =
frame->frame_type() == VideoFrameType::kVideoFrameKey ? 0 : 1;

2
modules/video_coding/rtp_frame_reference_finder_unittest.cc
View File

@ -73,7 +73,7 @@ class TestRtpFrameReferenceFinder : public ::testing::Test,
void OnCompleteFrame(std::unique_ptr<EncodedFrame> frame) override {
int64_t pid = frame->id.picture_id;
uint16_t sidx = frame->id.spatial_layer;
uint16_t sidx = *frame->SpatialIndex();
auto frame_it = frames_from_callback_.find(std::make_pair(pid, sidx));
if (frame_it != frames_from_callback_.end()) {
ADD_FAILURE() << "Already received frame with (pid:sidx): (" << pid << ":"

1
modules/video_coding/rtp_seq_num_only_ref_finder.cc
View File

@ -96,6 +96,7 @@ RtpSeqNumOnlyRefFinder::ManageFrameInternal(RtpFrameObject* frame) {
}
UpdateLastPictureIdWithPadding(frame->id.picture_id);
frame->SetSpatialIndex(0);
frame->id.picture_id = rtp_seq_num_unwrapper_.Unwrap(frame->id.picture_id);
return kHandOff;
}

1
modules/video_coding/rtp_vp8_ref_finder.cc
View File

@ -49,6 +49,7 @@ RtpVp8RefFinder::FrameDecision RtpVp8RefFinder::ManageFrameInternal(
if (codec_header.temporalIdx >= kMaxTemporalLayers)
return kDrop;
frame->SetSpatialIndex(0);
frame->id.picture_id = codec_header.pictureId & 0x7FFF;
if (last_picture_id_ == -1)

73
modules/video_coding/utility/decoded_frames_history.cc
View File

@ -18,11 +18,8 @@
namespace webrtc {
namespace video_coding {
DecodedFramesHistory::LayerHistory::LayerHistory() = default;
DecodedFramesHistory::LayerHistory::~LayerHistory() = default;
DecodedFramesHistory::DecodedFramesHistory(size_t window_size)
: window_size_(window_size) {}
: buffer_(window_size) {}
DecodedFramesHistory::~DecodedFramesHistory() = default;
@ -30,73 +27,53 @@ void DecodedFramesHistory::InsertDecoded(const VideoLayerFrameId& frameid,
uint32_t timestamp) {
last_decoded_frame_ = frameid;
last_decoded_frame_timestamp_ = timestamp;
if (static_cast<int>(layers_.size()) < frameid.spatial_layer + 1) {
size_t old_size = layers_.size();
layers_.resize(frameid.spatial_layer + 1);
for (size_t i = old_size; i < layers_.size(); ++i)
layers_[i].buffer.resize(window_size_);
layers_[frameid.spatial_layer].last_picture_id = frameid.picture_id;
layers_[frameid.spatial_layer]
.buffer[PictureIdToIndex(frameid.picture_id)] = true;
return;
}
int new_index = PictureIdToIndex(frameid.picture_id);
LayerHistory& history = layers_[frameid.spatial_layer];
RTC_DCHECK(history.last_picture_id < frameid.picture_id);
RTC_DCHECK(last_picture_id_ < frameid.picture_id);
// Clears expired values from the cyclic buffer.
if (history.last_picture_id) {
int64_t id_jump = frameid.picture_id - *history.last_picture_id;
int last_index = PictureIdToIndex(*history.last_picture_id);
// Clears expired values from the cyclic buffer_.
if (last_picture_id_) {
int64_t id_jump = frameid.picture_id - *last_picture_id_;
int last_index = PictureIdToIndex(*last_picture_id_);
if (id_jump >= window_size_) {
std::fill(history.buffer.begin(), history.buffer.end(), false);
if (id_jump >= static_cast<int64_t>(buffer_.size())) {
std::fill(buffer_.begin(), buffer_.end(), false);
} else if (new_index > last_index) {
std::fill(history.buffer.begin() + last_index + 1,
history.buffer.begin() + new_index, false);
std::fill(buffer_.begin() + last_index + 1, buffer_.begin() + new_index,
false);
} else {
std::fill(history.buffer.begin() + last_index + 1, history.buffer.end(),
false);
std::fill(history.buffer.begin(), history.buffer.begin() + new_index,
false);
std::fill(buffer_.begin() + last_index + 1, buffer_.end(), false);
std::fill(buffer_.begin(), buffer_.begin() + new_index, false);
}
}
history.buffer[new_index] = true;
history.last_picture_id = frameid.picture_id;
buffer_[new_index] = true;
last_picture_id_ = frameid.picture_id;
}
bool DecodedFramesHistory::WasDecoded(const VideoLayerFrameId& frameid) {
// Unseen before spatial layer.
if (static_cast<int>(layers_.size()) < frameid.spatial_layer + 1)
if (!last_picture_id_)
return false;
LayerHistory& history = layers_[frameid.spatial_layer];
if (!history.last_picture_id)
return false;
// Reference to the picture_id out of the stored history should happen.
if (frameid.picture_id <= *history.last_picture_id - window_size_) {
RTC_LOG(LS_WARNING) << "Referencing a frame out of the history window. "
// Reference to the picture_id out of the stored should happen.
if (frameid.picture_id <=
*last_picture_id_ - static_cast<int64_t>(buffer_.size())) {
RTC_LOG(LS_WARNING) << "Referencing a frame out of the window. "
"Assuming it was undecoded to avoid artifacts.";
return false;
}
if (frameid.picture_id > history.last_picture_id)
if (frameid.picture_id > last_picture_id_)
return false;
return history.buffer[PictureIdToIndex(frameid.picture_id)];
return buffer_[PictureIdToIndex(frameid.picture_id)];
}
void DecodedFramesHistory::Clear() {
layers_.clear();
last_decoded_frame_timestamp_.reset();
last_decoded_frame_.reset();
std::fill(buffer_.begin(), buffer_.end(), false);
last_picture_id_.reset();
}
absl::optional<VideoLayerFrameId>
@ -109,8 +86,8 @@ absl::optional<uint32_t> DecodedFramesHistory::GetLastDecodedFrameTimestamp() {
}
int DecodedFramesHistory::PictureIdToIndex(int64_t frame_id) const {
int m = frame_id % window_size_;
return m >= 0 ? m : m + window_size_;
int m = frame_id % buffer_.size();
return m >= 0 ? m : m + buffer_.size();
}
} // namespace video_coding

12
modules/video_coding/utility/decoded_frames_history.h
View File

@ -39,18 +39,10 @@ class DecodedFramesHistory {
absl::optional<uint32_t> GetLastDecodedFrameTimestamp();
private:
struct LayerHistory {
LayerHistory();
~LayerHistory();
// Cyclic bitset buffer. Stores last known |window_size| bits.
std::vector<bool> buffer;
absl::optional<int64_t> last_picture_id;
};
int PictureIdToIndex(int64_t frame_id) const;
const int window_size_;
std::vector<LayerHistory> layers_;
std::vector<bool> buffer_;
absl::optional<int64_t> last_picture_id_;
absl::optional<VideoLayerFrameId> last_decoded_frame_;
absl::optional<uint32_t> last_decoded_frame_timestamp_;
};

108
modules/video_coding/utility/decoded_frames_history_unittest.cc
View File

@ -20,125 +20,93 @@ constexpr int kHistorySize = 1 << 13;
TEST(DecodedFramesHistory, RequestOnEmptyHistory) {
DecodedFramesHistory history(kHistorySize);
EXPECT_EQ(history.WasDecoded({1234, 0}), false);
EXPECT_EQ(history.WasDecoded(VideoLayerFrameId{1234}), false);
}
TEST(DecodedFramesHistory, FindsLastDecodedFrame) {
DecodedFramesHistory history(kHistorySize);
history.InsertDecoded({1234, 0}, 0);
EXPECT_EQ(history.WasDecoded({1234, 0}), true);
history.InsertDecoded(VideoLayerFrameId{1234}, 0);
EXPECT_EQ(history.WasDecoded(VideoLayerFrameId{1234}), true);
}
TEST(DecodedFramesHistory, FindsPreviousFrame) {
DecodedFramesHistory history(kHistorySize);
history.InsertDecoded({1234, 0}, 0);
history.InsertDecoded({1235, 0}, 0);
EXPECT_EQ(history.WasDecoded({1234, 0}), true);
history.InsertDecoded(VideoLayerFrameId{1234}, 0);
history.InsertDecoded(VideoLayerFrameId{1235}, 0);
EXPECT_EQ(history.WasDecoded(VideoLayerFrameId{1234}), true);
}
TEST(DecodedFramesHistory, ReportsMissingFrame) {
DecodedFramesHistory history(kHistorySize);
history.InsertDecoded({1234, 0}, 0);
history.InsertDecoded({1236, 0}, 0);
EXPECT_EQ(history.WasDecoded({1235, 0}), false);
history.InsertDecoded(VideoLayerFrameId{1234}, 0);
history.InsertDecoded(VideoLayerFrameId{1236}, 0);
EXPECT_EQ(history.WasDecoded(VideoLayerFrameId{1235}), false);
}
TEST(DecodedFramesHistory, ClearsHistory) {
DecodedFramesHistory history(kHistorySize);
history.InsertDecoded({1234, 0}, 0);
history.InsertDecoded(VideoLayerFrameId{1234}, 0);
history.Clear();
EXPECT_EQ(history.WasDecoded({1234, 0}), false);
EXPECT_EQ(history.WasDecoded(VideoLayerFrameId{1234}), false);
EXPECT_EQ(history.GetLastDecodedFrameId(), absl::nullopt);
EXPECT_EQ(history.GetLastDecodedFrameTimestamp(), absl::nullopt);
}
TEST(DecodedFramesHistory, HandlesMultipleLayers) {
DecodedFramesHistory history(kHistorySize);
history.InsertDecoded({1234, 0}, 0);
history.InsertDecoded({1234, 1}, 0);
history.InsertDecoded({1235, 0}, 0);
history.InsertDecoded({1236, 0}, 0);
history.InsertDecoded({1236, 1}, 0);
EXPECT_EQ(history.WasDecoded({1235, 0}), true);
EXPECT_EQ(history.WasDecoded({1235, 1}), false);
}
TEST(DecodedFramesHistory, HandlesNewLayer) {
DecodedFramesHistory history(kHistorySize);
history.InsertDecoded({1234, 0}, 0);
history.InsertDecoded({1234, 1}, 0);
history.InsertDecoded({1235, 0}, 0);
history.InsertDecoded({1235, 1}, 0);
history.InsertDecoded({1236, 0}, 0);
history.InsertDecoded({1236, 1}, 0);
EXPECT_EQ(history.WasDecoded({1234, 2}), false);
}
TEST(DecodedFramesHistory, HandlesSkippedLayer) {
DecodedFramesHistory history(kHistorySize);
history.InsertDecoded({1234, 0}, 0);
history.InsertDecoded({1234, 2}, 0);
history.InsertDecoded({1235, 0}, 0);
history.InsertDecoded({1235, 1}, 0);
EXPECT_EQ(history.WasDecoded({1234, 1}), false);
EXPECT_EQ(history.WasDecoded({1235, 1}), true);
}
TEST(DecodedFramesHistory, HandlesBigJumpInPictureId) {
DecodedFramesHistory history(kHistorySize);
history.InsertDecoded({1234, 0}, 0);
history.InsertDecoded({1235, 0}, 0);
history.InsertDecoded({1236, 0}, 0);
history.InsertDecoded({1236 + kHistorySize / 2, 0}, 0);
EXPECT_EQ(history.WasDecoded({1234, 0}), true);
EXPECT_EQ(history.WasDecoded({1237, 0}), false);
history.InsertDecoded(VideoLayerFrameId{1234}, 0);
history.InsertDecoded(VideoLayerFrameId{1235}, 0);
history.InsertDecoded(VideoLayerFrameId{1236}, 0);
history.InsertDecoded(VideoLayerFrameId{1236 + kHistorySize / 2}, 0);
EXPECT_EQ(history.WasDecoded(VideoLayerFrameId{1234}), true);
EXPECT_EQ(history.WasDecoded(VideoLayerFrameId{1237}), false);
}
TEST(DecodedFramesHistory, ForgetsTooOldHistory) {
DecodedFramesHistory history(kHistorySize);
history.InsertDecoded({1234, 0}, 0);
history.InsertDecoded({1235, 0}, 0);
history.InsertDecoded({1236, 0}, 0);
history.InsertDecoded({1236 + kHistorySize * 2, 0}, 0);
EXPECT_EQ(history.WasDecoded({1234, 0}), false);
EXPECT_EQ(history.WasDecoded({1237, 0}), false);
history.InsertDecoded(VideoLayerFrameId{1234}, 0);
history.InsertDecoded(VideoLayerFrameId{1235}, 0);
history.InsertDecoded(VideoLayerFrameId{1236}, 0);
history.InsertDecoded(VideoLayerFrameId{1236 + kHistorySize * 2}, 0);
EXPECT_EQ(history.WasDecoded(VideoLayerFrameId{1234}), false);
EXPECT_EQ(history.WasDecoded(VideoLayerFrameId{1237}), false);
}
TEST(DecodedFramesHistory, ReturnsLastDecodedFrameId) {
DecodedFramesHistory history(kHistorySize);
EXPECT_EQ(history.GetLastDecodedFrameId(), absl::nullopt);
history.InsertDecoded({1234, 0}, 0);
EXPECT_EQ(history.GetLastDecodedFrameId(), VideoLayerFrameId(1234, 0));
history.InsertDecoded({1235, 0}, 0);
EXPECT_EQ(history.GetLastDecodedFrameId(), VideoLayerFrameId(1235, 0));
history.InsertDecoded(VideoLayerFrameId{1234}, 0);
EXPECT_EQ(history.GetLastDecodedFrameId(), VideoLayerFrameId(1234));
history.InsertDecoded(VideoLayerFrameId{1235}, 0);
EXPECT_EQ(history.GetLastDecodedFrameId(), VideoLayerFrameId(1235));
}
TEST(DecodedFramesHistory, ReturnsLastDecodedFrameTimestamp) {
DecodedFramesHistory history(kHistorySize);
EXPECT_EQ(history.GetLastDecodedFrameTimestamp(), absl::nullopt);
history.InsertDecoded({1234, 0}, 12345);
history.InsertDecoded(VideoLayerFrameId{1234}, 12345);
EXPECT_EQ(history.GetLastDecodedFrameTimestamp(), 12345u);
history.InsertDecoded({1235, 0}, 12366);
history.InsertDecoded(VideoLayerFrameId{1235}, 12366);
EXPECT_EQ(history.GetLastDecodedFrameTimestamp(), 12366u);
}
TEST(DecodedFramesHistory, NegativePictureIds) {
DecodedFramesHistory history(kHistorySize);
history.InsertDecoded({-1234, 0}, 12345);
history.InsertDecoded({-1233, 0}, 12366);
history.InsertDecoded(VideoLayerFrameId{-1234}, 12345);
history.InsertDecoded(VideoLayerFrameId{-1233}, 12366);
EXPECT_EQ(history.GetLastDecodedFrameId()->picture_id, -1233);
history.InsertDecoded({-1, 0}, 12377);
history.InsertDecoded({0, 0}, 12388);
history.InsertDecoded(VideoLayerFrameId{-1}, 12377);
history.InsertDecoded(VideoLayerFrameId{0}, 12388);
EXPECT_EQ(history.GetLastDecodedFrameId()->picture_id, 0);
history.InsertDecoded({1, 0}, 12399);
history.InsertDecoded(VideoLayerFrameId{1}, 12399);
EXPECT_EQ(history.GetLastDecodedFrameId()->picture_id, 1);
EXPECT_EQ(history.WasDecoded({-1234, 0}), true);
EXPECT_EQ(history.WasDecoded({-1, 0}), true);
EXPECT_EQ(history.WasDecoded({0, 0}), true);
EXPECT_EQ(history.WasDecoded({1, 0}), true);
EXPECT_EQ(history.WasDecoded(VideoLayerFrameId{-1234}), true);
EXPECT_EQ(history.WasDecoded(VideoLayerFrameId{-1}), true);
EXPECT_EQ(history.WasDecoded(VideoLayerFrameId{0}), true);
EXPECT_EQ(history.WasDecoded(VideoLayerFrameId{1}), true);
}
} // namespace

2
test/fuzzers/frame_buffer2_fuzzer.cc
View File

@ -78,7 +78,7 @@ void FuzzOneInput(const uint8_t* data, size_t size) {
if (reader.GetNum<uint8_t>() % 2) {
std::unique_ptr<FuzzyFrameObject> frame(new FuzzyFrameObject());
frame->id.picture_id = reader.GetNum<int64_t>();
frame->id.spatial_layer = reader.GetNum<uint8_t>() % 5;
frame->SetSpatialIndex(reader.GetNum<uint8_t>() % 5);
frame->SetTimestamp(reader.GetNum<uint32_t>());
frame->num_references = reader.GetNum<uint8_t>() %
video_coding::EncodedFrame::kMaxFrameReferences;

2
video/video_stream_decoder_impl.cc
View File

@ -64,7 +64,7 @@ void VideoStreamDecoderImpl::OnFrame(
RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
uint64_t continuous_pid = frame_buffer_.InsertFrame(std::move(frame));
video_coding::VideoLayerFrameId continuous_id(continuous_pid, 0);
video_coding::VideoLayerFrameId continuous_id(continuous_pid);
if (last_continuous_id_ < continuous_id) {
last_continuous_id_ = continuous_id;
callbacks_->OnContinuousUntil(last_continuous_id_);