Replace timestamp map with a deque in generic decoder
* Add test to Generic decoder unittests to ensure drop behaviour is covered. * Use simulated time in the generic decoder unittests. Bug: webrtc:14324 Change-Id: I10b28b45c434f92d5344683fb9ca6676efe0e08c Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/270662 Reviewed-by: Erik Språng <sprang@webrtc.org> Commit-Queue: Evan Shrubsole <eshr@webrtc.org> Cr-Commit-Position: refs/heads/main@{#37710}
This commit is contained in:
Evan Shrubsole
WebRTC LUCI CQ
parent
f8542b8c35
commit
1c51ec4d74
@ -196,8 +196,6 @@ rtc_library("video_coding") {
|
||||
"rtp_vp8_ref_finder.h",
|
||||
"rtp_vp9_ref_finder.cc",
|
||||
"rtp_vp9_ref_finder.h",
|
||||
"timestamp_map.cc",
|
||||
"timestamp_map.h",
|
||||
"video_codec_initializer.cc",
|
||||
"video_receiver2.cc",
|
||||
"video_receiver2.h",
|
||||
@ -275,6 +273,7 @@ rtc_library("video_coding") {
|
||||
"timing:timing_module",
|
||||
]
|
||||
absl_deps = [
|
||||
"//third_party/abseil-cpp/absl/algorithm:container",
|
||||
"//third_party/abseil-cpp/absl/base:core_headers",
|
||||
"//third_party/abseil-cpp/absl/container:inlined_vector",
|
||||
"//third_party/abseil-cpp/absl/types:optional",
|
||||
@ -1105,7 +1104,6 @@ if (rtc_include_tests) {
|
||||
"session_info_unittest.cc",
|
||||
"test/stream_generator.cc",
|
||||
"test/stream_generator.h",
|
||||
"timestamp_map_unittest.cc",
|
||||
"utility/bandwidth_quality_scaler_unittest.cc",
|
||||
"utility/decoded_frames_history_unittest.cc",
|
||||
"utility/frame_dropper_unittest.cc",
|
||||
@ -1154,6 +1152,7 @@ if (rtc_include_tests) {
|
||||
"../../api:mock_fec_controller_override",
|
||||
"../../api:mock_video_decoder",
|
||||
"../../api:mock_video_encoder",
|
||||
"../../api:rtp_packet_info",
|
||||
"../../api:scoped_refptr",
|
||||
"../../api:simulcast_test_fixture_api",
|
||||
"../../api:videocodec_test_fixture_api",
|
||||
|
||||
@ -14,9 +14,13 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <iterator>
|
||||
#include <utility>
|
||||
|
||||
#include "absl/algorithm/container.h"
|
||||
#include "absl/types/optional.h"
|
||||
#include "api/video/video_timing.h"
|
||||
#include "modules/include/module_common_types_public.h"
|
||||
#include "modules/video_coding/include/video_error_codes.h"
|
||||
#include "rtc_base/checks.h"
|
||||
#include "rtc_base/logging.h"
|
||||
@ -25,11 +29,17 @@
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr size_t kDecoderFrameMemoryLength = 10;
|
||||
|
||||
}
|
||||
|
||||
VCMDecodedFrameCallback::VCMDecodedFrameCallback(
|
||||
VCMTiming* timing,
|
||||
Clock* clock,
|
||||
const FieldTrialsView& field_trials)
|
||||
: _clock(clock), _timing(timing), _timestampMap(kDecoderFrameMemoryLength) {
|
||||
: _clock(clock), _timing(timing) {
|
||||
ntp_offset_ =
|
||||
_clock->CurrentNtpInMilliseconds() - _clock->TimeInMilliseconds();
|
||||
}
|
||||
@ -66,6 +76,26 @@ int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
std::pair<absl::optional<FrameInfo>, size_t>
|
||||
VCMDecodedFrameCallback::FindFrameInfo(uint32_t rtp_timestamp) {
|
||||
absl::optional<FrameInfo> frame_info;
|
||||
|
||||
auto it = absl::c_find_if(frame_infos_, [rtp_timestamp](const auto& entry) {
|
||||
return entry.rtp_timestamp == rtp_timestamp ||
|
||||
IsNewerTimestamp(entry.rtp_timestamp, rtp_timestamp);
|
||||
});
|
||||
size_t dropped_frames = std::distance(frame_infos_.begin(), it);
|
||||
|
||||
if (it != frame_infos_.end() && it->rtp_timestamp == rtp_timestamp) {
|
||||
// Frame was found and should also be removed from the queue.
|
||||
frame_info = std::move(*it);
|
||||
++it;
|
||||
}
|
||||
|
||||
frame_infos_.erase(frame_infos_.begin(), it);
|
||||
return std::make_pair(std::move(frame_info), dropped_frames);
|
||||
}
|
||||
|
||||
void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
|
||||
absl::optional<int32_t> decode_time_ms,
|
||||
absl::optional<uint8_t> qp) {
|
||||
@ -74,36 +104,29 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
|
||||
"timestamp", decodedImage.timestamp());
|
||||
// TODO(holmer): We should improve this so that we can handle multiple
|
||||
// callbacks from one call to Decode().
|
||||
absl::optional<FrameInformation> frameInfo;
|
||||
absl::optional<FrameInfo> frame_info;
|
||||
int timestamp_map_size = 0;
|
||||
int dropped_frames = 0;
|
||||
{
|
||||
MutexLock lock(&lock_);
|
||||
int initial_timestamp_map_size = _timestampMap.Size();
|
||||
frameInfo = _timestampMap.Pop(decodedImage.timestamp());
|
||||
timestamp_map_size = _timestampMap.Size();
|
||||
// _timestampMap.Pop() erases all frame upto the specified timestamp and
|
||||
// return the frame info for this timestamp if it exists. Thus, the
|
||||
// difference in the _timestampMap size before and after Pop() will show
|
||||
// internally dropped frames.
|
||||
dropped_frames =
|
||||
initial_timestamp_map_size - timestamp_map_size - (frameInfo ? 1 : 0);
|
||||
std::tie(frame_info, dropped_frames) =
|
||||
FindFrameInfo(decodedImage.timestamp());
|
||||
timestamp_map_size = frame_infos_.size();
|
||||
}
|
||||
|
||||
if (dropped_frames > 0) {
|
||||
_receiveCallback->OnDroppedFrames(dropped_frames);
|
||||
}
|
||||
|
||||
if (!frameInfo) {
|
||||
if (!frame_info) {
|
||||
RTC_LOG(LS_WARNING) << "Too many frames backed up in the decoder, dropping "
|
||||
"frame with timestamp "
|
||||
<< decodedImage.timestamp();
|
||||
return;
|
||||
}
|
||||
|
||||
decodedImage.set_ntp_time_ms(frameInfo->ntp_time_ms);
|
||||
decodedImage.set_packet_infos(frameInfo->packet_infos);
|
||||
decodedImage.set_rotation(frameInfo->rotation);
|
||||
decodedImage.set_ntp_time_ms(frame_info->ntp_time_ms);
|
||||
decodedImage.set_packet_infos(frame_info->packet_infos);
|
||||
decodedImage.set_rotation(frame_info->rotation);
|
||||
VideoFrame::RenderParameters render_parameters = _timing->RenderParameters();
|
||||
if (render_parameters.max_composition_delay_in_frames) {
|
||||
// Subtract frames that are in flight.
|
||||
@ -113,70 +136,70 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
|
||||
}
|
||||
decodedImage.set_render_parameters(render_parameters);
|
||||
|
||||
RTC_DCHECK(frameInfo->decode_start);
|
||||
RTC_DCHECK(frame_info->decode_start);
|
||||
const Timestamp now = _clock->CurrentTime();
|
||||
const TimeDelta decode_time = decode_time_ms
|
||||
? TimeDelta::Millis(*decode_time_ms)
|
||||
: now - *frameInfo->decode_start;
|
||||
: now - *frame_info->decode_start;
|
||||
_timing->StopDecodeTimer(decode_time, now);
|
||||
decodedImage.set_processing_time(
|
||||
{*frameInfo->decode_start, *frameInfo->decode_start + decode_time});
|
||||
{*frame_info->decode_start, *frame_info->decode_start + decode_time});
|
||||
|
||||
// Report timing information.
|
||||
TimingFrameInfo timing_frame_info;
|
||||
if (frameInfo->timing.flags != VideoSendTiming::kInvalid) {
|
||||
if (frame_info->timing.flags != VideoSendTiming::kInvalid) {
|
||||
int64_t capture_time_ms = decodedImage.ntp_time_ms() - ntp_offset_;
|
||||
// Convert remote timestamps to local time from ntp timestamps.
|
||||
frameInfo->timing.encode_start_ms -= ntp_offset_;
|
||||
frameInfo->timing.encode_finish_ms -= ntp_offset_;
|
||||
frameInfo->timing.packetization_finish_ms -= ntp_offset_;
|
||||
frameInfo->timing.pacer_exit_ms -= ntp_offset_;
|
||||
frameInfo->timing.network_timestamp_ms -= ntp_offset_;
|
||||
frameInfo->timing.network2_timestamp_ms -= ntp_offset_;
|
||||
frame_info->timing.encode_start_ms -= ntp_offset_;
|
||||
frame_info->timing.encode_finish_ms -= ntp_offset_;
|
||||
frame_info->timing.packetization_finish_ms -= ntp_offset_;
|
||||
frame_info->timing.pacer_exit_ms -= ntp_offset_;
|
||||
frame_info->timing.network_timestamp_ms -= ntp_offset_;
|
||||
frame_info->timing.network2_timestamp_ms -= ntp_offset_;
|
||||
|
||||
int64_t sender_delta_ms = 0;
|
||||
if (decodedImage.ntp_time_ms() < 0) {
|
||||
// Sender clock is not estimated yet. Make sure that sender times are all
|
||||
// negative to indicate that. Yet they still should be relatively correct.
|
||||
sender_delta_ms =
|
||||
std::max({capture_time_ms, frameInfo->timing.encode_start_ms,
|
||||
frameInfo->timing.encode_finish_ms,
|
||||
frameInfo->timing.packetization_finish_ms,
|
||||
frameInfo->timing.pacer_exit_ms,
|
||||
frameInfo->timing.network_timestamp_ms,
|
||||
frameInfo->timing.network2_timestamp_ms}) +
|
||||
std::max({capture_time_ms, frame_info->timing.encode_start_ms,
|
||||
frame_info->timing.encode_finish_ms,
|
||||
frame_info->timing.packetization_finish_ms,
|
||||
frame_info->timing.pacer_exit_ms,
|
||||
frame_info->timing.network_timestamp_ms,
|
||||
frame_info->timing.network2_timestamp_ms}) +
|
||||
1;
|
||||
}
|
||||
|
||||
timing_frame_info.capture_time_ms = capture_time_ms - sender_delta_ms;
|
||||
timing_frame_info.encode_start_ms =
|
||||
frameInfo->timing.encode_start_ms - sender_delta_ms;
|
||||
frame_info->timing.encode_start_ms - sender_delta_ms;
|
||||
timing_frame_info.encode_finish_ms =
|
||||
frameInfo->timing.encode_finish_ms - sender_delta_ms;
|
||||
frame_info->timing.encode_finish_ms - sender_delta_ms;
|
||||
timing_frame_info.packetization_finish_ms =
|
||||
frameInfo->timing.packetization_finish_ms - sender_delta_ms;
|
||||
frame_info->timing.packetization_finish_ms - sender_delta_ms;
|
||||
timing_frame_info.pacer_exit_ms =
|
||||
frameInfo->timing.pacer_exit_ms - sender_delta_ms;
|
||||
frame_info->timing.pacer_exit_ms - sender_delta_ms;
|
||||
timing_frame_info.network_timestamp_ms =
|
||||
frameInfo->timing.network_timestamp_ms - sender_delta_ms;
|
||||
frame_info->timing.network_timestamp_ms - sender_delta_ms;
|
||||
timing_frame_info.network2_timestamp_ms =
|
||||
frameInfo->timing.network2_timestamp_ms - sender_delta_ms;
|
||||
frame_info->timing.network2_timestamp_ms - sender_delta_ms;
|
||||
}
|
||||
|
||||
timing_frame_info.flags = frameInfo->timing.flags;
|
||||
timing_frame_info.decode_start_ms = frameInfo->decode_start->ms();
|
||||
timing_frame_info.flags = frame_info->timing.flags;
|
||||
timing_frame_info.decode_start_ms = frame_info->decode_start->ms();
|
||||
timing_frame_info.decode_finish_ms = now.ms();
|
||||
timing_frame_info.render_time_ms =
|
||||
frameInfo->render_time ? frameInfo->render_time->ms() : -1;
|
||||
frame_info->render_time ? frame_info->render_time->ms() : -1;
|
||||
timing_frame_info.rtp_timestamp = decodedImage.timestamp();
|
||||
timing_frame_info.receive_start_ms = frameInfo->timing.receive_start_ms;
|
||||
timing_frame_info.receive_finish_ms = frameInfo->timing.receive_finish_ms;
|
||||
timing_frame_info.receive_start_ms = frame_info->timing.receive_start_ms;
|
||||
timing_frame_info.receive_finish_ms = frame_info->timing.receive_finish_ms;
|
||||
_timing->SetTimingFrameInfo(timing_frame_info);
|
||||
|
||||
decodedImage.set_timestamp_us(
|
||||
frameInfo->render_time ? frameInfo->render_time->us() : -1);
|
||||
frame_info->render_time ? frame_info->render_time->us() : -1);
|
||||
_receiveCallback->FrameToRender(decodedImage, qp, decode_time,
|
||||
frameInfo->content_type);
|
||||
frame_info->content_type);
|
||||
}
|
||||
|
||||
void VCMDecodedFrameCallback::OnDecoderImplementationName(
|
||||
@ -184,15 +207,17 @@ void VCMDecodedFrameCallback::OnDecoderImplementationName(
|
||||
_receiveCallback->OnDecoderImplementationName(implementation_name);
|
||||
}
|
||||
|
||||
void VCMDecodedFrameCallback::Map(uint32_t timestamp,
|
||||
const FrameInformation& frameInfo) {
|
||||
void VCMDecodedFrameCallback::Map(FrameInfo frameInfo) {
|
||||
int dropped_frames = 0;
|
||||
{
|
||||
MutexLock lock(&lock_);
|
||||
int initial_size = _timestampMap.Size();
|
||||
_timestampMap.Add(timestamp, frameInfo);
|
||||
int initial_size = frame_infos_.size();
|
||||
if (initial_size == kDecoderFrameMemoryLength) {
|
||||
frame_infos_.pop_front();
|
||||
dropped_frames = 1;
|
||||
}
|
||||
frame_infos_.push_back(std::move(frameInfo));
|
||||
// If no frame is dropped, the new size should be `initial_size` + 1
|
||||
dropped_frames = (initial_size + 1) - _timestampMap.Size();
|
||||
}
|
||||
if (dropped_frames > 0) {
|
||||
_receiveCallback->OnDroppedFrames(dropped_frames);
|
||||
@ -203,8 +228,8 @@ void VCMDecodedFrameCallback::ClearTimestampMap() {
|
||||
int dropped_frames = 0;
|
||||
{
|
||||
MutexLock lock(&lock_);
|
||||
dropped_frames = _timestampMap.Size();
|
||||
_timestampMap.Clear();
|
||||
dropped_frames = frame_infos_.size();
|
||||
frame_infos_.clear();
|
||||
}
|
||||
if (dropped_frames > 0) {
|
||||
_receiveCallback->OnDroppedFrames(dropped_frames);
|
||||
@ -238,7 +263,8 @@ bool VCMGenericDecoder::Configure(const VideoDecoder::Settings& settings) {
|
||||
int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) {
|
||||
TRACE_EVENT1("webrtc", "VCMGenericDecoder::Decode", "timestamp",
|
||||
frame.Timestamp());
|
||||
FrameInformation frame_info;
|
||||
FrameInfo frame_info;
|
||||
frame_info.rtp_timestamp = frame.Timestamp();
|
||||
frame_info.decode_start = now;
|
||||
frame_info.render_time =
|
||||
frame.RenderTimeMs() >= 0
|
||||
@ -258,7 +284,7 @@ int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) {
|
||||
} else {
|
||||
frame_info.content_type = _last_keyframe_content_type;
|
||||
}
|
||||
_callback->Map(frame.Timestamp(), frame_info);
|
||||
_callback->Map(std::move(frame_info));
|
||||
|
||||
int32_t ret = decoder_->Decode(frame.EncodedImage(), frame.MissingFrame(),
|
||||
frame.RenderTimeMs());
|
||||
|
||||
@ -11,14 +11,15 @@
|
||||
#ifndef MODULES_VIDEO_CODING_GENERIC_DECODER_H_
|
||||
#define MODULES_VIDEO_CODING_GENERIC_DECODER_H_
|
||||
|
||||
#include <cstdint>
|
||||
#include <deque>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
#include "api/field_trials_view.h"
|
||||
#include "api/sequence_checker.h"
|
||||
#include "api/video_codecs/video_decoder.h"
|
||||
#include "modules/video_coding/encoded_frame.h"
|
||||
#include "modules/video_coding/include/video_codec_interface.h"
|
||||
#include "modules/video_coding/timestamp_map.h"
|
||||
#include "modules/video_coding/timing/timing.h"
|
||||
#include "rtc_base/synchronization/mutex.h"
|
||||
|
||||
@ -26,7 +27,26 @@ namespace webrtc {
|
||||
|
||||
class VCMReceiveCallback;
|
||||
|
||||
enum { kDecoderFrameMemoryLength = 10 };
|
||||
struct FrameInfo {
|
||||
FrameInfo() = default;
|
||||
FrameInfo(const FrameInfo&) = delete;
|
||||
FrameInfo& operator=(const FrameInfo&) = delete;
|
||||
FrameInfo(FrameInfo&&) = default;
|
||||
FrameInfo& operator=(FrameInfo&&) = default;
|
||||
|
||||
uint32_t rtp_timestamp;
|
||||
// This is likely not optional, but some inputs seem to sometimes be negative.
|
||||
// TODO(bugs.webrtc.org/13756): See if this can be replaced with Timestamp
|
||||
// once all inputs to this field use Timestamp instead of an integer.
|
||||
absl::optional<Timestamp> render_time;
|
||||
absl::optional<Timestamp> decode_start;
|
||||
VideoRotation rotation;
|
||||
VideoContentType content_type;
|
||||
EncodedImage::Timing timing;
|
||||
int64_t ntp_time_ms;
|
||||
RtpPacketInfos packet_infos;
|
||||
// ColorSpace is not stored here, as it might be modified by decoders.
|
||||
};
|
||||
|
||||
class VCMDecodedFrameCallback : public DecodedImageCallback {
|
||||
public:
|
||||
@ -45,12 +65,14 @@ class VCMDecodedFrameCallback : public DecodedImageCallback {
|
||||
|
||||
void OnDecoderImplementationName(const char* implementation_name);
|
||||
|
||||
void Map(uint32_t timestamp, const FrameInformation& frameInfo);
|
||||
void Map(FrameInfo frameInfo);
|
||||
void ClearTimestampMap();
|
||||
|
||||
private:
|
||||
std::pair<absl::optional<FrameInfo>, size_t> FindFrameInfo(
|
||||
uint32_t rtp_timestamp) RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_);
|
||||
|
||||
SequenceChecker construction_thread_;
|
||||
// Protect `_timestampMap`.
|
||||
Clock* const _clock;
|
||||
// This callback must be set before the decoder thread starts running
|
||||
// and must only be unset when external threads (e.g decoder thread)
|
||||
@ -60,7 +82,7 @@ class VCMDecodedFrameCallback : public DecodedImageCallback {
|
||||
VCMReceiveCallback* _receiveCallback = nullptr;
|
||||
VCMTiming* _timing;
|
||||
Mutex lock_;
|
||||
TimestampMap _timestampMap RTC_GUARDED_BY(lock_);
|
||||
std::deque<FrameInfo> frame_infos_ RTC_GUARDED_BY(lock_);
|
||||
int64_t ntp_offset_;
|
||||
};
|
||||
|
||||
|
||||
@ -10,67 +10,65 @@
|
||||
|
||||
#include "modules/video_coding/generic_decoder.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/types/optional.h"
|
||||
#include "api/task_queue/default_task_queue_factory.h"
|
||||
#include "api/array_view.h"
|
||||
#include "api/rtp_packet_infos.h"
|
||||
#include "api/video_codecs/video_decoder.h"
|
||||
#include "common_video/test/utilities.h"
|
||||
#include "modules/video_coding/timing/timing.h"
|
||||
#include "rtc_base/event.h"
|
||||
#include "rtc_base/synchronization/mutex.h"
|
||||
#include "system_wrappers/include/clock.h"
|
||||
#include "test/fake_decoder.h"
|
||||
#include "test/gmock.h"
|
||||
#include "test/gtest.h"
|
||||
#include "test/scoped_key_value_config.h"
|
||||
#include "test/time_controller/simulated_time_controller.h"
|
||||
|
||||
namespace webrtc {
|
||||
namespace video_coding {
|
||||
|
||||
class ReceiveCallback : public VCMReceiveCallback {
|
||||
public:
|
||||
int32_t FrameToRender(VideoFrame& videoFrame, // NOLINT
|
||||
int32_t FrameToRender(VideoFrame& frame,
|
||||
absl::optional<uint8_t> qp,
|
||||
TimeDelta decode_time,
|
||||
VideoContentType content_type) override {
|
||||
{
|
||||
MutexLock lock(&lock_);
|
||||
last_frame_ = videoFrame;
|
||||
}
|
||||
received_frame_event_.Set();
|
||||
frames_.push_back(frame);
|
||||
return 0;
|
||||
}
|
||||
|
||||
absl::optional<VideoFrame> GetLastFrame() {
|
||||
MutexLock lock(&lock_);
|
||||
return last_frame_;
|
||||
absl::optional<VideoFrame> PopLastFrame() {
|
||||
if (frames_.empty())
|
||||
return absl::nullopt;
|
||||
auto ret = frames_.front();
|
||||
frames_.pop_back();
|
||||
return ret;
|
||||
}
|
||||
|
||||
absl::optional<VideoFrame> WaitForFrame(int64_t wait_ms) {
|
||||
if (received_frame_event_.Wait(wait_ms)) {
|
||||
MutexLock lock(&lock_);
|
||||
return last_frame_;
|
||||
} else {
|
||||
return absl::nullopt;
|
||||
}
|
||||
rtc::ArrayView<const VideoFrame> GetAllFrames() const { return frames_; }
|
||||
|
||||
void OnDroppedFrames(uint32_t frames_dropped) {
|
||||
frames_dropped_ += frames_dropped;
|
||||
}
|
||||
|
||||
uint32_t frames_dropped() const { return frames_dropped_; }
|
||||
|
||||
private:
|
||||
Mutex lock_;
|
||||
rtc::Event received_frame_event_;
|
||||
absl::optional<VideoFrame> last_frame_ RTC_GUARDED_BY(lock_);
|
||||
std::vector<VideoFrame> frames_;
|
||||
uint32_t frames_dropped_ = 0;
|
||||
};
|
||||
|
||||
class GenericDecoderTest : public ::testing::Test {
|
||||
protected:
|
||||
GenericDecoderTest()
|
||||
: clock_(0),
|
||||
timing_(&clock_, field_trials_),
|
||||
task_queue_factory_(CreateDefaultTaskQueueFactory()),
|
||||
decoder_(task_queue_factory_.get()),
|
||||
vcm_callback_(&timing_, &clock_, field_trials_),
|
||||
: time_controller_(Timestamp::Zero()),
|
||||
clock_(time_controller_.GetClock()),
|
||||
timing_(time_controller_.GetClock(), field_trials_),
|
||||
decoder_(time_controller_.GetTaskQueueFactory()),
|
||||
vcm_callback_(&timing_, time_controller_.GetClock(), field_trials_),
|
||||
generic_decoder_(&decoder_) {}
|
||||
|
||||
void SetUp() override {
|
||||
@ -83,10 +81,10 @@ class GenericDecoderTest : public ::testing::Test {
|
||||
generic_decoder_.Configure(settings);
|
||||
}
|
||||
|
||||
GlobalSimulatedTimeController time_controller_;
|
||||
Clock* const clock_;
|
||||
test::ScopedKeyValueConfig field_trials_;
|
||||
SimulatedClock clock_;
|
||||
VCMTiming timing_;
|
||||
std::unique_ptr<TaskQueueFactory> task_queue_factory_;
|
||||
webrtc::test::FakeDecoder decoder_;
|
||||
VCMDecodedFrameCallback vcm_callback_;
|
||||
VCMGenericDecoder generic_decoder_;
|
||||
@ -97,12 +95,32 @@ TEST_F(GenericDecoderTest, PassesPacketInfos) {
|
||||
RtpPacketInfos packet_infos = CreatePacketInfos(3);
|
||||
VCMEncodedFrame encoded_frame;
|
||||
encoded_frame.SetPacketInfos(packet_infos);
|
||||
generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
|
||||
absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(10);
|
||||
generic_decoder_.Decode(encoded_frame, clock_->CurrentTime());
|
||||
time_controller_.AdvanceTime(TimeDelta::Millis(10));
|
||||
absl::optional<VideoFrame> decoded_frame = user_callback_.PopLastFrame();
|
||||
ASSERT_TRUE(decoded_frame.has_value());
|
||||
EXPECT_EQ(decoded_frame->packet_infos().size(), 3U);
|
||||
}
|
||||
|
||||
TEST_F(GenericDecoderTest, FrameDroppedIfTooManyFramesInFlight) {
|
||||
constexpr int kMaxFramesInFlight = 10;
|
||||
decoder_.SetDelayedDecoding(10);
|
||||
for (int i = 0; i < kMaxFramesInFlight + 1; ++i) {
|
||||
VCMEncodedFrame encoded_frame;
|
||||
encoded_frame.SetTimestamp(90000 * i);
|
||||
generic_decoder_.Decode(encoded_frame, clock_->CurrentTime());
|
||||
}
|
||||
|
||||
time_controller_.AdvanceTime(TimeDelta::Millis(10));
|
||||
|
||||
auto frames = user_callback_.GetAllFrames();
|
||||
ASSERT_EQ(10U, frames.size());
|
||||
// Expect that the first frame was dropped since all decodes released at the
|
||||
// same time and the oldest frame info is the first one dropped.
|
||||
EXPECT_EQ(frames[0].timestamp(), 90000u);
|
||||
EXPECT_EQ(1u, user_callback_.frames_dropped());
|
||||
}
|
||||
|
||||
TEST_F(GenericDecoderTest, PassesPacketInfosForDelayedDecoders) {
|
||||
RtpPacketInfos packet_infos = CreatePacketInfos(3);
|
||||
decoder_.SetDelayedDecoding(100);
|
||||
@ -111,18 +129,20 @@ TEST_F(GenericDecoderTest, PassesPacketInfosForDelayedDecoders) {
|
||||
// Ensure the original frame is destroyed before the decoding is completed.
|
||||
VCMEncodedFrame encoded_frame;
|
||||
encoded_frame.SetPacketInfos(packet_infos);
|
||||
generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
|
||||
generic_decoder_.Decode(encoded_frame, clock_->CurrentTime());
|
||||
}
|
||||
|
||||
absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(200);
|
||||
time_controller_.AdvanceTime(TimeDelta::Millis(200));
|
||||
absl::optional<VideoFrame> decoded_frame = user_callback_.PopLastFrame();
|
||||
ASSERT_TRUE(decoded_frame.has_value());
|
||||
EXPECT_EQ(decoded_frame->packet_infos().size(), 3U);
|
||||
}
|
||||
|
||||
TEST_F(GenericDecoderTest, MaxCompositionDelayNotSetByDefault) {
|
||||
VCMEncodedFrame encoded_frame;
|
||||
generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
|
||||
absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(10);
|
||||
generic_decoder_.Decode(encoded_frame, clock_->CurrentTime());
|
||||
time_controller_.AdvanceTime(TimeDelta::Millis(10));
|
||||
absl::optional<VideoFrame> decoded_frame = user_callback_.PopLastFrame();
|
||||
ASSERT_TRUE(decoded_frame.has_value());
|
||||
EXPECT_THAT(
|
||||
decoded_frame->render_parameters().max_composition_delay_in_frames,
|
||||
@ -136,8 +156,9 @@ TEST_F(GenericDecoderTest, MaxCompositionDelayActivatedByPlayoutDelay) {
|
||||
constexpr int kMaxCompositionDelayInFrames = 3; // ~50 ms at 60 fps.
|
||||
timing_.SetMaxCompositionDelayInFrames(
|
||||
absl::make_optional(kMaxCompositionDelayInFrames));
|
||||
generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
|
||||
absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(10);
|
||||
generic_decoder_.Decode(encoded_frame, clock_->CurrentTime());
|
||||
time_controller_.AdvanceTime(TimeDelta::Millis(10));
|
||||
absl::optional<VideoFrame> decoded_frame = user_callback_.PopLastFrame();
|
||||
ASSERT_TRUE(decoded_frame.has_value());
|
||||
EXPECT_THAT(
|
||||
decoded_frame->render_parameters().max_composition_delay_in_frames,
|
||||
@ -146,8 +167,9 @@ TEST_F(GenericDecoderTest, MaxCompositionDelayActivatedByPlayoutDelay) {
|
||||
|
||||
TEST_F(GenericDecoderTest, IsLowLatencyStreamFalseByDefault) {
|
||||
VCMEncodedFrame encoded_frame;
|
||||
generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
|
||||
absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(10);
|
||||
generic_decoder_.Decode(encoded_frame, clock_->CurrentTime());
|
||||
time_controller_.AdvanceTime(TimeDelta::Millis(10));
|
||||
absl::optional<VideoFrame> decoded_frame = user_callback_.PopLastFrame();
|
||||
ASSERT_TRUE(decoded_frame.has_value());
|
||||
EXPECT_FALSE(decoded_frame->render_parameters().use_low_latency_rendering);
|
||||
}
|
||||
@ -157,8 +179,9 @@ TEST_F(GenericDecoderTest, IsLowLatencyStreamActivatedByPlayoutDelay) {
|
||||
const VideoPlayoutDelay kPlayoutDelay = {0, 50};
|
||||
timing_.set_min_playout_delay(TimeDelta::Millis(kPlayoutDelay.min_ms));
|
||||
timing_.set_max_playout_delay(TimeDelta::Millis(kPlayoutDelay.max_ms));
|
||||
generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
|
||||
absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(10);
|
||||
generic_decoder_.Decode(encoded_frame, clock_->CurrentTime());
|
||||
time_controller_.AdvanceTime(TimeDelta::Millis(10));
|
||||
absl::optional<VideoFrame> decoded_frame = user_callback_.PopLastFrame();
|
||||
ASSERT_TRUE(decoded_frame.has_value());
|
||||
EXPECT_TRUE(decoded_frame->render_parameters().use_low_latency_rendering);
|
||||
}
|
||||
|
||||
@ -1,79 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "modules/video_coding/timestamp_map.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "modules/include/module_common_types_public.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
TimestampMap::TimestampMap(size_t capacity)
|
||||
: ring_buffer_(new TimestampDataTuple[capacity]),
|
||||
capacity_(capacity),
|
||||
next_add_idx_(0),
|
||||
next_pop_idx_(0) {}
|
||||
|
||||
TimestampMap::~TimestampMap() {}
|
||||
|
||||
void TimestampMap::Add(uint32_t timestamp, const FrameInformation& data) {
|
||||
ring_buffer_[next_add_idx_].timestamp = timestamp;
|
||||
ring_buffer_[next_add_idx_].data = data;
|
||||
next_add_idx_ = (next_add_idx_ + 1) % capacity_;
|
||||
|
||||
if (next_add_idx_ == next_pop_idx_) {
|
||||
// Circular list full; forget oldest entry.
|
||||
next_pop_idx_ = (next_pop_idx_ + 1) % capacity_;
|
||||
}
|
||||
}
|
||||
|
||||
absl::optional<FrameInformation> TimestampMap::Pop(uint32_t timestamp) {
|
||||
while (!IsEmpty()) {
|
||||
if (ring_buffer_[next_pop_idx_].timestamp == timestamp) {
|
||||
// Found start time for this timestamp.
|
||||
const FrameInformation& data = ring_buffer_[next_pop_idx_].data;
|
||||
ring_buffer_[next_pop_idx_].timestamp = 0;
|
||||
next_pop_idx_ = (next_pop_idx_ + 1) % capacity_;
|
||||
return data;
|
||||
} else if (IsNewerTimestamp(ring_buffer_[next_pop_idx_].timestamp,
|
||||
timestamp)) {
|
||||
// The timestamp we are looking for is not in the list.
|
||||
return absl::nullopt;
|
||||
}
|
||||
|
||||
// Not in this position, check next (and forget this position).
|
||||
next_pop_idx_ = (next_pop_idx_ + 1) % capacity_;
|
||||
}
|
||||
|
||||
// Could not find matching timestamp in list.
|
||||
return absl::nullopt;
|
||||
}
|
||||
|
||||
bool TimestampMap::IsEmpty() const {
|
||||
return (next_add_idx_ == next_pop_idx_);
|
||||
}
|
||||
|
||||
size_t TimestampMap::Size() const {
|
||||
// The maximum number of elements in the list is `capacity_` - 1. The list is
|
||||
// empty if the add and pop indices are equal.
|
||||
return next_add_idx_ >= next_pop_idx_
|
||||
? next_add_idx_ - next_pop_idx_
|
||||
: next_add_idx_ + capacity_ - next_pop_idx_;
|
||||
}
|
||||
|
||||
void TimestampMap::Clear() {
|
||||
while (!IsEmpty()) {
|
||||
ring_buffer_[next_pop_idx_].timestamp = 0;
|
||||
next_pop_idx_ = (next_pop_idx_ + 1) % capacity_;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
@ -1,64 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef MODULES_VIDEO_CODING_TIMESTAMP_MAP_H_
|
||||
#define MODULES_VIDEO_CODING_TIMESTAMP_MAP_H_
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "absl/types/optional.h"
|
||||
#include "api/rtp_packet_infos.h"
|
||||
#include "api/units/timestamp.h"
|
||||
#include "api/video/encoded_image.h"
|
||||
#include "api/video/video_content_type.h"
|
||||
#include "api/video/video_rotation.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
struct FrameInformation {
|
||||
// This is likely not optional, but some inputs seem to sometimes be negative.
|
||||
// TODO(bugs.webrtc.org/13756): See if this can be replaced with Timestamp
|
||||
// once all inputs to this field use Timestamp instead of an integer.
|
||||
absl::optional<Timestamp> render_time;
|
||||
absl::optional<Timestamp> decode_start;
|
||||
VideoRotation rotation;
|
||||
VideoContentType content_type;
|
||||
EncodedImage::Timing timing;
|
||||
int64_t ntp_time_ms;
|
||||
RtpPacketInfos packet_infos;
|
||||
// ColorSpace is not stored here, as it might be modified by decoders.
|
||||
};
|
||||
|
||||
class TimestampMap {
|
||||
public:
|
||||
explicit TimestampMap(size_t capacity);
|
||||
~TimestampMap();
|
||||
|
||||
void Add(uint32_t timestamp, const FrameInformation& data);
|
||||
absl::optional<FrameInformation> Pop(uint32_t timestamp);
|
||||
size_t Size() const;
|
||||
void Clear();
|
||||
|
||||
private:
|
||||
struct TimestampDataTuple {
|
||||
uint32_t timestamp;
|
||||
FrameInformation data;
|
||||
};
|
||||
bool IsEmpty() const;
|
||||
|
||||
std::unique_ptr<TimestampDataTuple[]> ring_buffer_;
|
||||
const size_t capacity_;
|
||||
size_t next_add_idx_;
|
||||
size_t next_pop_idx_;
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_VIDEO_CODING_TIMESTAMP_MAP_H_
|
||||
@ -1,121 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "modules/video_coding/timestamp_map.h"
|
||||
|
||||
#include "test/gmock.h"
|
||||
#include "test/gtest.h"
|
||||
|
||||
namespace webrtc {
|
||||
namespace video_coding {
|
||||
namespace {
|
||||
constexpr int kTimestampMapSize = 6;
|
||||
constexpr int kTimestamp1 = 1;
|
||||
constexpr int kTimestamp2 = 2;
|
||||
constexpr int kNoExistingTimestamp3 = 3;
|
||||
constexpr int kTimestamp4 = 4;
|
||||
constexpr int kTimestamp5 = 5;
|
||||
constexpr int kTimestamp6 = 6;
|
||||
constexpr int kTimestamp7 = 7;
|
||||
constexpr Timestamp kRenderTime1 = Timestamp::Seconds(1);
|
||||
constexpr Timestamp kRenderTime2 = Timestamp::Seconds(2);
|
||||
constexpr Timestamp kRenderTime4 = Timestamp::Seconds(4);
|
||||
constexpr Timestamp kRenderTime5 = Timestamp::Seconds(5);
|
||||
constexpr Timestamp kRenderTime6 = Timestamp::Seconds(6);
|
||||
constexpr Timestamp kRenderTime7 = Timestamp::Seconds(7);
|
||||
} // namespace
|
||||
|
||||
class VcmTimestampMapTest : public ::testing::Test {
|
||||
protected:
|
||||
VcmTimestampMapTest() : _timestampMap(kTimestampMapSize) {}
|
||||
|
||||
void SetUp() override {
|
||||
_timestampMap.Add(kTimestamp1, FrameInformation({kRenderTime1}));
|
||||
_timestampMap.Add(kTimestamp2, FrameInformation({kRenderTime2}));
|
||||
_timestampMap.Add(kTimestamp4, FrameInformation({kRenderTime4}));
|
||||
}
|
||||
|
||||
TimestampMap _timestampMap;
|
||||
};
|
||||
|
||||
TEST_F(VcmTimestampMapTest, PopExistingFrameInfo) {
|
||||
EXPECT_EQ(_timestampMap.Size(), 3u);
|
||||
auto frameInfo = _timestampMap.Pop(kTimestamp1);
|
||||
ASSERT_TRUE(frameInfo);
|
||||
EXPECT_EQ(frameInfo->render_time, kRenderTime1);
|
||||
frameInfo = _timestampMap.Pop(kTimestamp2);
|
||||
ASSERT_TRUE(frameInfo);
|
||||
EXPECT_EQ(frameInfo->render_time, kRenderTime2);
|
||||
frameInfo = _timestampMap.Pop(kTimestamp4);
|
||||
ASSERT_TRUE(frameInfo);
|
||||
EXPECT_EQ(frameInfo->render_time, kRenderTime4);
|
||||
}
|
||||
|
||||
TEST_F(VcmTimestampMapTest, PopNonexistingClearsOlderFrameInfos) {
|
||||
auto frameInfo = _timestampMap.Pop(kNoExistingTimestamp3);
|
||||
EXPECT_FALSE(frameInfo);
|
||||
EXPECT_EQ(_timestampMap.Size(), 1u);
|
||||
}
|
||||
|
||||
TEST_F(VcmTimestampMapTest, SizeIsIncrementedWhenAddingNewFrameInfo) {
|
||||
EXPECT_EQ(_timestampMap.Size(), 3u);
|
||||
_timestampMap.Add(kTimestamp5, FrameInformation({kRenderTime5}));
|
||||
EXPECT_EQ(_timestampMap.Size(), 4u);
|
||||
_timestampMap.Add(kTimestamp6, FrameInformation({kRenderTime6}));
|
||||
EXPECT_EQ(_timestampMap.Size(), 5u);
|
||||
}
|
||||
|
||||
TEST_F(VcmTimestampMapTest, SizeIsDecreasedWhenPoppingFrameInfo) {
|
||||
EXPECT_EQ(_timestampMap.Size(), 3u);
|
||||
EXPECT_TRUE(_timestampMap.Pop(kTimestamp1));
|
||||
EXPECT_EQ(_timestampMap.Size(), 2u);
|
||||
EXPECT_TRUE(_timestampMap.Pop(kTimestamp2));
|
||||
EXPECT_EQ(_timestampMap.Size(), 1u);
|
||||
EXPECT_FALSE(_timestampMap.Pop(kNoExistingTimestamp3));
|
||||
EXPECT_EQ(_timestampMap.Size(), 1u);
|
||||
EXPECT_TRUE(_timestampMap.Pop(kTimestamp4));
|
||||
EXPECT_EQ(_timestampMap.Size(), 0u);
|
||||
}
|
||||
|
||||
TEST_F(VcmTimestampMapTest, ClearEmptiesMap) {
|
||||
EXPECT_EQ(_timestampMap.Size(), 3u);
|
||||
_timestampMap.Clear();
|
||||
EXPECT_EQ(_timestampMap.Size(), 0u);
|
||||
// Clear empty map does nothing.
|
||||
_timestampMap.Clear();
|
||||
EXPECT_EQ(_timestampMap.Size(), 0u);
|
||||
}
|
||||
|
||||
TEST_F(VcmTimestampMapTest, PopLastAddedClearsMap) {
|
||||
EXPECT_EQ(_timestampMap.Size(), 3u);
|
||||
EXPECT_TRUE(_timestampMap.Pop(kTimestamp4));
|
||||
EXPECT_EQ(_timestampMap.Size(), 0u);
|
||||
}
|
||||
|
||||
TEST_F(VcmTimestampMapTest, LastAddedIsDiscardedIfMapGetsFull) {
|
||||
EXPECT_EQ(_timestampMap.Size(), 3u);
|
||||
_timestampMap.Add(kTimestamp5, FrameInformation({kRenderTime5}));
|
||||
EXPECT_EQ(_timestampMap.Size(), 4u);
|
||||
_timestampMap.Add(kTimestamp6, FrameInformation({kRenderTime6}));
|
||||
EXPECT_EQ(_timestampMap.Size(), 5u);
|
||||
_timestampMap.Add(kTimestamp7, FrameInformation({kRenderTime7}));
|
||||
// Size is not incremented since the oldest element is discarded.
|
||||
EXPECT_EQ(_timestampMap.Size(), 5u);
|
||||
EXPECT_FALSE(_timestampMap.Pop(kTimestamp1));
|
||||
EXPECT_TRUE(_timestampMap.Pop(kTimestamp2));
|
||||
EXPECT_TRUE(_timestampMap.Pop(kTimestamp4));
|
||||
EXPECT_TRUE(_timestampMap.Pop(kTimestamp5));
|
||||
EXPECT_TRUE(_timestampMap.Pop(kTimestamp6));
|
||||
EXPECT_TRUE(_timestampMap.Pop(kTimestamp7));
|
||||
EXPECT_EQ(_timestampMap.Size(), 0u);
|
||||
}
|
||||
|
||||
} // namespace video_coding
|
||||
} // namespace webrtc
|
||||
Loading…
x
Reference in New Issue
Block a user