diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 6b370f5541..fa56f6a35e 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -196,8 +196,6 @@ rtc_library("video_coding") { "rtp_vp8_ref_finder.h", "rtp_vp9_ref_finder.cc", "rtp_vp9_ref_finder.h", - "timestamp_map.cc", - "timestamp_map.h", "video_codec_initializer.cc", "video_receiver2.cc", "video_receiver2.h", @@ -275,6 +273,7 @@ rtc_library("video_coding") { "timing:timing_module", ] absl_deps = [ + "//third_party/abseil-cpp/absl/algorithm:container", "//third_party/abseil-cpp/absl/base:core_headers", "//third_party/abseil-cpp/absl/container:inlined_vector", "//third_party/abseil-cpp/absl/types:optional", @@ -1105,7 +1104,6 @@ if (rtc_include_tests) { "session_info_unittest.cc", "test/stream_generator.cc", "test/stream_generator.h", - "timestamp_map_unittest.cc", "utility/bandwidth_quality_scaler_unittest.cc", "utility/decoded_frames_history_unittest.cc", "utility/frame_dropper_unittest.cc", @@ -1154,6 +1152,7 @@ if (rtc_include_tests) { "../../api:mock_fec_controller_override", "../../api:mock_video_decoder", "../../api:mock_video_encoder", + "../../api:rtp_packet_info", "../../api:scoped_refptr", "../../api:simulcast_test_fixture_api", "../../api:videocodec_test_fixture_api", diff --git a/modules/video_coding/generic_decoder.cc b/modules/video_coding/generic_decoder.cc index dd8f52aef0..dac8f2cd43 100644 --- a/modules/video_coding/generic_decoder.cc +++ b/modules/video_coding/generic_decoder.cc @@ -14,9 +14,13 @@ #include #include +#include +#include +#include "absl/algorithm/container.h" #include "absl/types/optional.h" #include "api/video/video_timing.h" +#include "modules/include/module_common_types_public.h" #include "modules/video_coding/include/video_error_codes.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" @@ -25,11 +29,17 @@ namespace webrtc { +namespace { + +constexpr size_t kDecoderFrameMemoryLength = 10; + +} + VCMDecodedFrameCallback::VCMDecodedFrameCallback( VCMTiming* timing, Clock* clock, const FieldTrialsView& field_trials) - : _clock(clock), _timing(timing), _timestampMap(kDecoderFrameMemoryLength) { + : _clock(clock), _timing(timing) { ntp_offset_ = _clock->CurrentNtpInMilliseconds() - _clock->TimeInMilliseconds(); } @@ -66,6 +76,26 @@ int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, return WEBRTC_VIDEO_CODEC_OK; } +std::pair, size_t> +VCMDecodedFrameCallback::FindFrameInfo(uint32_t rtp_timestamp) { + absl::optional frame_info; + + auto it = absl::c_find_if(frame_infos_, [rtp_timestamp](const auto& entry) { + return entry.rtp_timestamp == rtp_timestamp || + IsNewerTimestamp(entry.rtp_timestamp, rtp_timestamp); + }); + size_t dropped_frames = std::distance(frame_infos_.begin(), it); + + if (it != frame_infos_.end() && it->rtp_timestamp == rtp_timestamp) { + // Frame was found and should also be removed from the queue. + frame_info = std::move(*it); + ++it; + } + + frame_infos_.erase(frame_infos_.begin(), it); + return std::make_pair(std::move(frame_info), dropped_frames); +} + void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, absl::optional decode_time_ms, absl::optional qp) { @@ -74,36 +104,29 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, "timestamp", decodedImage.timestamp()); // TODO(holmer): We should improve this so that we can handle multiple // callbacks from one call to Decode(). - absl::optional frameInfo; + absl::optional frame_info; int timestamp_map_size = 0; int dropped_frames = 0; { MutexLock lock(&lock_); - int initial_timestamp_map_size = _timestampMap.Size(); - frameInfo = _timestampMap.Pop(decodedImage.timestamp()); - timestamp_map_size = _timestampMap.Size(); - // _timestampMap.Pop() erases all frame upto the specified timestamp and - // return the frame info for this timestamp if it exists. Thus, the - // difference in the _timestampMap size before and after Pop() will show - // internally dropped frames. - dropped_frames = - initial_timestamp_map_size - timestamp_map_size - (frameInfo ? 1 : 0); + std::tie(frame_info, dropped_frames) = + FindFrameInfo(decodedImage.timestamp()); + timestamp_map_size = frame_infos_.size(); } - if (dropped_frames > 0) { _receiveCallback->OnDroppedFrames(dropped_frames); } - if (!frameInfo) { + if (!frame_info) { RTC_LOG(LS_WARNING) << "Too many frames backed up in the decoder, dropping " "frame with timestamp " << decodedImage.timestamp(); return; } - decodedImage.set_ntp_time_ms(frameInfo->ntp_time_ms); - decodedImage.set_packet_infos(frameInfo->packet_infos); - decodedImage.set_rotation(frameInfo->rotation); + decodedImage.set_ntp_time_ms(frame_info->ntp_time_ms); + decodedImage.set_packet_infos(frame_info->packet_infos); + decodedImage.set_rotation(frame_info->rotation); VideoFrame::RenderParameters render_parameters = _timing->RenderParameters(); if (render_parameters.max_composition_delay_in_frames) { // Subtract frames that are in flight. @@ -113,70 +136,70 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, } decodedImage.set_render_parameters(render_parameters); - RTC_DCHECK(frameInfo->decode_start); + RTC_DCHECK(frame_info->decode_start); const Timestamp now = _clock->CurrentTime(); const TimeDelta decode_time = decode_time_ms ? TimeDelta::Millis(*decode_time_ms) - : now - *frameInfo->decode_start; + : now - *frame_info->decode_start; _timing->StopDecodeTimer(decode_time, now); decodedImage.set_processing_time( - {*frameInfo->decode_start, *frameInfo->decode_start + decode_time}); + {*frame_info->decode_start, *frame_info->decode_start + decode_time}); // Report timing information. TimingFrameInfo timing_frame_info; - if (frameInfo->timing.flags != VideoSendTiming::kInvalid) { + if (frame_info->timing.flags != VideoSendTiming::kInvalid) { int64_t capture_time_ms = decodedImage.ntp_time_ms() - ntp_offset_; // Convert remote timestamps to local time from ntp timestamps. - frameInfo->timing.encode_start_ms -= ntp_offset_; - frameInfo->timing.encode_finish_ms -= ntp_offset_; - frameInfo->timing.packetization_finish_ms -= ntp_offset_; - frameInfo->timing.pacer_exit_ms -= ntp_offset_; - frameInfo->timing.network_timestamp_ms -= ntp_offset_; - frameInfo->timing.network2_timestamp_ms -= ntp_offset_; + frame_info->timing.encode_start_ms -= ntp_offset_; + frame_info->timing.encode_finish_ms -= ntp_offset_; + frame_info->timing.packetization_finish_ms -= ntp_offset_; + frame_info->timing.pacer_exit_ms -= ntp_offset_; + frame_info->timing.network_timestamp_ms -= ntp_offset_; + frame_info->timing.network2_timestamp_ms -= ntp_offset_; int64_t sender_delta_ms = 0; if (decodedImage.ntp_time_ms() < 0) { // Sender clock is not estimated yet. Make sure that sender times are all // negative to indicate that. Yet they still should be relatively correct. sender_delta_ms = - std::max({capture_time_ms, frameInfo->timing.encode_start_ms, - frameInfo->timing.encode_finish_ms, - frameInfo->timing.packetization_finish_ms, - frameInfo->timing.pacer_exit_ms, - frameInfo->timing.network_timestamp_ms, - frameInfo->timing.network2_timestamp_ms}) + + std::max({capture_time_ms, frame_info->timing.encode_start_ms, + frame_info->timing.encode_finish_ms, + frame_info->timing.packetization_finish_ms, + frame_info->timing.pacer_exit_ms, + frame_info->timing.network_timestamp_ms, + frame_info->timing.network2_timestamp_ms}) + 1; } timing_frame_info.capture_time_ms = capture_time_ms - sender_delta_ms; timing_frame_info.encode_start_ms = - frameInfo->timing.encode_start_ms - sender_delta_ms; + frame_info->timing.encode_start_ms - sender_delta_ms; timing_frame_info.encode_finish_ms = - frameInfo->timing.encode_finish_ms - sender_delta_ms; + frame_info->timing.encode_finish_ms - sender_delta_ms; timing_frame_info.packetization_finish_ms = - frameInfo->timing.packetization_finish_ms - sender_delta_ms; + frame_info->timing.packetization_finish_ms - sender_delta_ms; timing_frame_info.pacer_exit_ms = - frameInfo->timing.pacer_exit_ms - sender_delta_ms; + frame_info->timing.pacer_exit_ms - sender_delta_ms; timing_frame_info.network_timestamp_ms = - frameInfo->timing.network_timestamp_ms - sender_delta_ms; + frame_info->timing.network_timestamp_ms - sender_delta_ms; timing_frame_info.network2_timestamp_ms = - frameInfo->timing.network2_timestamp_ms - sender_delta_ms; + frame_info->timing.network2_timestamp_ms - sender_delta_ms; } - timing_frame_info.flags = frameInfo->timing.flags; - timing_frame_info.decode_start_ms = frameInfo->decode_start->ms(); + timing_frame_info.flags = frame_info->timing.flags; + timing_frame_info.decode_start_ms = frame_info->decode_start->ms(); timing_frame_info.decode_finish_ms = now.ms(); timing_frame_info.render_time_ms = - frameInfo->render_time ? frameInfo->render_time->ms() : -1; + frame_info->render_time ? frame_info->render_time->ms() : -1; timing_frame_info.rtp_timestamp = decodedImage.timestamp(); - timing_frame_info.receive_start_ms = frameInfo->timing.receive_start_ms; - timing_frame_info.receive_finish_ms = frameInfo->timing.receive_finish_ms; + timing_frame_info.receive_start_ms = frame_info->timing.receive_start_ms; + timing_frame_info.receive_finish_ms = frame_info->timing.receive_finish_ms; _timing->SetTimingFrameInfo(timing_frame_info); decodedImage.set_timestamp_us( - frameInfo->render_time ? frameInfo->render_time->us() : -1); + frame_info->render_time ? frame_info->render_time->us() : -1); _receiveCallback->FrameToRender(decodedImage, qp, decode_time, - frameInfo->content_type); + frame_info->content_type); } void VCMDecodedFrameCallback::OnDecoderImplementationName( @@ -184,15 +207,17 @@ void VCMDecodedFrameCallback::OnDecoderImplementationName( _receiveCallback->OnDecoderImplementationName(implementation_name); } -void VCMDecodedFrameCallback::Map(uint32_t timestamp, - const FrameInformation& frameInfo) { +void VCMDecodedFrameCallback::Map(FrameInfo frameInfo) { int dropped_frames = 0; { MutexLock lock(&lock_); - int initial_size = _timestampMap.Size(); - _timestampMap.Add(timestamp, frameInfo); + int initial_size = frame_infos_.size(); + if (initial_size == kDecoderFrameMemoryLength) { + frame_infos_.pop_front(); + dropped_frames = 1; + } + frame_infos_.push_back(std::move(frameInfo)); // If no frame is dropped, the new size should be `initial_size` + 1 - dropped_frames = (initial_size + 1) - _timestampMap.Size(); } if (dropped_frames > 0) { _receiveCallback->OnDroppedFrames(dropped_frames); @@ -203,8 +228,8 @@ void VCMDecodedFrameCallback::ClearTimestampMap() { int dropped_frames = 0; { MutexLock lock(&lock_); - dropped_frames = _timestampMap.Size(); - _timestampMap.Clear(); + dropped_frames = frame_infos_.size(); + frame_infos_.clear(); } if (dropped_frames > 0) { _receiveCallback->OnDroppedFrames(dropped_frames); @@ -238,7 +263,8 @@ bool VCMGenericDecoder::Configure(const VideoDecoder::Settings& settings) { int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) { TRACE_EVENT1("webrtc", "VCMGenericDecoder::Decode", "timestamp", frame.Timestamp()); - FrameInformation frame_info; + FrameInfo frame_info; + frame_info.rtp_timestamp = frame.Timestamp(); frame_info.decode_start = now; frame_info.render_time = frame.RenderTimeMs() >= 0 @@ -258,7 +284,7 @@ int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) { } else { frame_info.content_type = _last_keyframe_content_type; } - _callback->Map(frame.Timestamp(), frame_info); + _callback->Map(std::move(frame_info)); int32_t ret = decoder_->Decode(frame.EncodedImage(), frame.MissingFrame(), frame.RenderTimeMs()); diff --git a/modules/video_coding/generic_decoder.h b/modules/video_coding/generic_decoder.h index 70c79dace2..d7e1850abb 100644 --- a/modules/video_coding/generic_decoder.h +++ b/modules/video_coding/generic_decoder.h @@ -11,14 +11,15 @@ #ifndef MODULES_VIDEO_CODING_GENERIC_DECODER_H_ #define MODULES_VIDEO_CODING_GENERIC_DECODER_H_ +#include +#include #include +#include #include "api/field_trials_view.h" #include "api/sequence_checker.h" #include "api/video_codecs/video_decoder.h" #include "modules/video_coding/encoded_frame.h" -#include "modules/video_coding/include/video_codec_interface.h" -#include "modules/video_coding/timestamp_map.h" #include "modules/video_coding/timing/timing.h" #include "rtc_base/synchronization/mutex.h" @@ -26,7 +27,26 @@ namespace webrtc { class VCMReceiveCallback; -enum { kDecoderFrameMemoryLength = 10 }; +struct FrameInfo { + FrameInfo() = default; + FrameInfo(const FrameInfo&) = delete; + FrameInfo& operator=(const FrameInfo&) = delete; + FrameInfo(FrameInfo&&) = default; + FrameInfo& operator=(FrameInfo&&) = default; + + uint32_t rtp_timestamp; + // This is likely not optional, but some inputs seem to sometimes be negative. + // TODO(bugs.webrtc.org/13756): See if this can be replaced with Timestamp + // once all inputs to this field use Timestamp instead of an integer. + absl::optional render_time; + absl::optional decode_start; + VideoRotation rotation; + VideoContentType content_type; + EncodedImage::Timing timing; + int64_t ntp_time_ms; + RtpPacketInfos packet_infos; + // ColorSpace is not stored here, as it might be modified by decoders. +}; class VCMDecodedFrameCallback : public DecodedImageCallback { public: @@ -45,12 +65,14 @@ class VCMDecodedFrameCallback : public DecodedImageCallback { void OnDecoderImplementationName(const char* implementation_name); - void Map(uint32_t timestamp, const FrameInformation& frameInfo); + void Map(FrameInfo frameInfo); void ClearTimestampMap(); private: + std::pair, size_t> FindFrameInfo( + uint32_t rtp_timestamp) RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_); + SequenceChecker construction_thread_; - // Protect `_timestampMap`. Clock* const _clock; // This callback must be set before the decoder thread starts running // and must only be unset when external threads (e.g decoder thread) @@ -60,7 +82,7 @@ class VCMDecodedFrameCallback : public DecodedImageCallback { VCMReceiveCallback* _receiveCallback = nullptr; VCMTiming* _timing; Mutex lock_; - TimestampMap _timestampMap RTC_GUARDED_BY(lock_); + std::deque frame_infos_ RTC_GUARDED_BY(lock_); int64_t ntp_offset_; }; diff --git a/modules/video_coding/generic_decoder_unittest.cc b/modules/video_coding/generic_decoder_unittest.cc index 811c296a67..68bc307e65 100644 --- a/modules/video_coding/generic_decoder_unittest.cc +++ b/modules/video_coding/generic_decoder_unittest.cc @@ -10,67 +10,65 @@ #include "modules/video_coding/generic_decoder.h" +#include #include #include #include "absl/types/optional.h" -#include "api/task_queue/default_task_queue_factory.h" +#include "api/array_view.h" +#include "api/rtp_packet_infos.h" #include "api/video_codecs/video_decoder.h" #include "common_video/test/utilities.h" #include "modules/video_coding/timing/timing.h" -#include "rtc_base/event.h" -#include "rtc_base/synchronization/mutex.h" #include "system_wrappers/include/clock.h" #include "test/fake_decoder.h" #include "test/gmock.h" #include "test/gtest.h" #include "test/scoped_key_value_config.h" +#include "test/time_controller/simulated_time_controller.h" namespace webrtc { namespace video_coding { class ReceiveCallback : public VCMReceiveCallback { public: - int32_t FrameToRender(VideoFrame& videoFrame, // NOLINT + int32_t FrameToRender(VideoFrame& frame, absl::optional qp, TimeDelta decode_time, VideoContentType content_type) override { - { - MutexLock lock(&lock_); - last_frame_ = videoFrame; - } - received_frame_event_.Set(); + frames_.push_back(frame); return 0; } - absl::optional GetLastFrame() { - MutexLock lock(&lock_); - return last_frame_; + absl::optional PopLastFrame() { + if (frames_.empty()) + return absl::nullopt; + auto ret = frames_.front(); + frames_.pop_back(); + return ret; } - absl::optional WaitForFrame(int64_t wait_ms) { - if (received_frame_event_.Wait(wait_ms)) { - MutexLock lock(&lock_); - return last_frame_; - } else { - return absl::nullopt; - } + rtc::ArrayView GetAllFrames() const { return frames_; } + + void OnDroppedFrames(uint32_t frames_dropped) { + frames_dropped_ += frames_dropped; } + uint32_t frames_dropped() const { return frames_dropped_; } + private: - Mutex lock_; - rtc::Event received_frame_event_; - absl::optional last_frame_ RTC_GUARDED_BY(lock_); + std::vector frames_; + uint32_t frames_dropped_ = 0; }; class GenericDecoderTest : public ::testing::Test { protected: GenericDecoderTest() - : clock_(0), - timing_(&clock_, field_trials_), - task_queue_factory_(CreateDefaultTaskQueueFactory()), - decoder_(task_queue_factory_.get()), - vcm_callback_(&timing_, &clock_, field_trials_), + : time_controller_(Timestamp::Zero()), + clock_(time_controller_.GetClock()), + timing_(time_controller_.GetClock(), field_trials_), + decoder_(time_controller_.GetTaskQueueFactory()), + vcm_callback_(&timing_, time_controller_.GetClock(), field_trials_), generic_decoder_(&decoder_) {} void SetUp() override { @@ -83,10 +81,10 @@ class GenericDecoderTest : public ::testing::Test { generic_decoder_.Configure(settings); } + GlobalSimulatedTimeController time_controller_; + Clock* const clock_; test::ScopedKeyValueConfig field_trials_; - SimulatedClock clock_; VCMTiming timing_; - std::unique_ptr task_queue_factory_; webrtc::test::FakeDecoder decoder_; VCMDecodedFrameCallback vcm_callback_; VCMGenericDecoder generic_decoder_; @@ -97,12 +95,32 @@ TEST_F(GenericDecoderTest, PassesPacketInfos) { RtpPacketInfos packet_infos = CreatePacketInfos(3); VCMEncodedFrame encoded_frame; encoded_frame.SetPacketInfos(packet_infos); - generic_decoder_.Decode(encoded_frame, clock_.CurrentTime()); - absl::optional decoded_frame = user_callback_.WaitForFrame(10); + generic_decoder_.Decode(encoded_frame, clock_->CurrentTime()); + time_controller_.AdvanceTime(TimeDelta::Millis(10)); + absl::optional decoded_frame = user_callback_.PopLastFrame(); ASSERT_TRUE(decoded_frame.has_value()); EXPECT_EQ(decoded_frame->packet_infos().size(), 3U); } +TEST_F(GenericDecoderTest, FrameDroppedIfTooManyFramesInFlight) { + constexpr int kMaxFramesInFlight = 10; + decoder_.SetDelayedDecoding(10); + for (int i = 0; i < kMaxFramesInFlight + 1; ++i) { + VCMEncodedFrame encoded_frame; + encoded_frame.SetTimestamp(90000 * i); + generic_decoder_.Decode(encoded_frame, clock_->CurrentTime()); + } + + time_controller_.AdvanceTime(TimeDelta::Millis(10)); + + auto frames = user_callback_.GetAllFrames(); + ASSERT_EQ(10U, frames.size()); + // Expect that the first frame was dropped since all decodes released at the + // same time and the oldest frame info is the first one dropped. + EXPECT_EQ(frames[0].timestamp(), 90000u); + EXPECT_EQ(1u, user_callback_.frames_dropped()); +} + TEST_F(GenericDecoderTest, PassesPacketInfosForDelayedDecoders) { RtpPacketInfos packet_infos = CreatePacketInfos(3); decoder_.SetDelayedDecoding(100); @@ -111,18 +129,20 @@ TEST_F(GenericDecoderTest, PassesPacketInfosForDelayedDecoders) { // Ensure the original frame is destroyed before the decoding is completed. VCMEncodedFrame encoded_frame; encoded_frame.SetPacketInfos(packet_infos); - generic_decoder_.Decode(encoded_frame, clock_.CurrentTime()); + generic_decoder_.Decode(encoded_frame, clock_->CurrentTime()); } - absl::optional decoded_frame = user_callback_.WaitForFrame(200); + time_controller_.AdvanceTime(TimeDelta::Millis(200)); + absl::optional decoded_frame = user_callback_.PopLastFrame(); ASSERT_TRUE(decoded_frame.has_value()); EXPECT_EQ(decoded_frame->packet_infos().size(), 3U); } TEST_F(GenericDecoderTest, MaxCompositionDelayNotSetByDefault) { VCMEncodedFrame encoded_frame; - generic_decoder_.Decode(encoded_frame, clock_.CurrentTime()); - absl::optional decoded_frame = user_callback_.WaitForFrame(10); + generic_decoder_.Decode(encoded_frame, clock_->CurrentTime()); + time_controller_.AdvanceTime(TimeDelta::Millis(10)); + absl::optional decoded_frame = user_callback_.PopLastFrame(); ASSERT_TRUE(decoded_frame.has_value()); EXPECT_THAT( decoded_frame->render_parameters().max_composition_delay_in_frames, @@ -136,8 +156,9 @@ TEST_F(GenericDecoderTest, MaxCompositionDelayActivatedByPlayoutDelay) { constexpr int kMaxCompositionDelayInFrames = 3; // ~50 ms at 60 fps. timing_.SetMaxCompositionDelayInFrames( absl::make_optional(kMaxCompositionDelayInFrames)); - generic_decoder_.Decode(encoded_frame, clock_.CurrentTime()); - absl::optional decoded_frame = user_callback_.WaitForFrame(10); + generic_decoder_.Decode(encoded_frame, clock_->CurrentTime()); + time_controller_.AdvanceTime(TimeDelta::Millis(10)); + absl::optional decoded_frame = user_callback_.PopLastFrame(); ASSERT_TRUE(decoded_frame.has_value()); EXPECT_THAT( decoded_frame->render_parameters().max_composition_delay_in_frames, @@ -146,8 +167,9 @@ TEST_F(GenericDecoderTest, MaxCompositionDelayActivatedByPlayoutDelay) { TEST_F(GenericDecoderTest, IsLowLatencyStreamFalseByDefault) { VCMEncodedFrame encoded_frame; - generic_decoder_.Decode(encoded_frame, clock_.CurrentTime()); - absl::optional decoded_frame = user_callback_.WaitForFrame(10); + generic_decoder_.Decode(encoded_frame, clock_->CurrentTime()); + time_controller_.AdvanceTime(TimeDelta::Millis(10)); + absl::optional decoded_frame = user_callback_.PopLastFrame(); ASSERT_TRUE(decoded_frame.has_value()); EXPECT_FALSE(decoded_frame->render_parameters().use_low_latency_rendering); } @@ -157,8 +179,9 @@ TEST_F(GenericDecoderTest, IsLowLatencyStreamActivatedByPlayoutDelay) { const VideoPlayoutDelay kPlayoutDelay = {0, 50}; timing_.set_min_playout_delay(TimeDelta::Millis(kPlayoutDelay.min_ms)); timing_.set_max_playout_delay(TimeDelta::Millis(kPlayoutDelay.max_ms)); - generic_decoder_.Decode(encoded_frame, clock_.CurrentTime()); - absl::optional decoded_frame = user_callback_.WaitForFrame(10); + generic_decoder_.Decode(encoded_frame, clock_->CurrentTime()); + time_controller_.AdvanceTime(TimeDelta::Millis(10)); + absl::optional decoded_frame = user_callback_.PopLastFrame(); ASSERT_TRUE(decoded_frame.has_value()); EXPECT_TRUE(decoded_frame->render_parameters().use_low_latency_rendering); } diff --git a/modules/video_coding/timestamp_map.cc b/modules/video_coding/timestamp_map.cc deleted file mode 100644 index ef5b4b2550..0000000000 --- a/modules/video_coding/timestamp_map.cc +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_coding/timestamp_map.h" - -#include - -#include "modules/include/module_common_types_public.h" - -namespace webrtc { - -TimestampMap::TimestampMap(size_t capacity) - : ring_buffer_(new TimestampDataTuple[capacity]), - capacity_(capacity), - next_add_idx_(0), - next_pop_idx_(0) {} - -TimestampMap::~TimestampMap() {} - -void TimestampMap::Add(uint32_t timestamp, const FrameInformation& data) { - ring_buffer_[next_add_idx_].timestamp = timestamp; - ring_buffer_[next_add_idx_].data = data; - next_add_idx_ = (next_add_idx_ + 1) % capacity_; - - if (next_add_idx_ == next_pop_idx_) { - // Circular list full; forget oldest entry. - next_pop_idx_ = (next_pop_idx_ + 1) % capacity_; - } -} - -absl::optional TimestampMap::Pop(uint32_t timestamp) { - while (!IsEmpty()) { - if (ring_buffer_[next_pop_idx_].timestamp == timestamp) { - // Found start time for this timestamp. - const FrameInformation& data = ring_buffer_[next_pop_idx_].data; - ring_buffer_[next_pop_idx_].timestamp = 0; - next_pop_idx_ = (next_pop_idx_ + 1) % capacity_; - return data; - } else if (IsNewerTimestamp(ring_buffer_[next_pop_idx_].timestamp, - timestamp)) { - // The timestamp we are looking for is not in the list. - return absl::nullopt; - } - - // Not in this position, check next (and forget this position). - next_pop_idx_ = (next_pop_idx_ + 1) % capacity_; - } - - // Could not find matching timestamp in list. - return absl::nullopt; -} - -bool TimestampMap::IsEmpty() const { - return (next_add_idx_ == next_pop_idx_); -} - -size_t TimestampMap::Size() const { - // The maximum number of elements in the list is `capacity_` - 1. The list is - // empty if the add and pop indices are equal. - return next_add_idx_ >= next_pop_idx_ - ? next_add_idx_ - next_pop_idx_ - : next_add_idx_ + capacity_ - next_pop_idx_; -} - -void TimestampMap::Clear() { - while (!IsEmpty()) { - ring_buffer_[next_pop_idx_].timestamp = 0; - next_pop_idx_ = (next_pop_idx_ + 1) % capacity_; - } -} - -} // namespace webrtc diff --git a/modules/video_coding/timestamp_map.h b/modules/video_coding/timestamp_map.h deleted file mode 100644 index 5ff75b10e3..0000000000 --- a/modules/video_coding/timestamp_map.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_VIDEO_CODING_TIMESTAMP_MAP_H_ -#define MODULES_VIDEO_CODING_TIMESTAMP_MAP_H_ - -#include - -#include "absl/types/optional.h" -#include "api/rtp_packet_infos.h" -#include "api/units/timestamp.h" -#include "api/video/encoded_image.h" -#include "api/video/video_content_type.h" -#include "api/video/video_rotation.h" - -namespace webrtc { - -struct FrameInformation { - // This is likely not optional, but some inputs seem to sometimes be negative. - // TODO(bugs.webrtc.org/13756): See if this can be replaced with Timestamp - // once all inputs to this field use Timestamp instead of an integer. - absl::optional render_time; - absl::optional decode_start; - VideoRotation rotation; - VideoContentType content_type; - EncodedImage::Timing timing; - int64_t ntp_time_ms; - RtpPacketInfos packet_infos; - // ColorSpace is not stored here, as it might be modified by decoders. -}; - -class TimestampMap { - public: - explicit TimestampMap(size_t capacity); - ~TimestampMap(); - - void Add(uint32_t timestamp, const FrameInformation& data); - absl::optional Pop(uint32_t timestamp); - size_t Size() const; - void Clear(); - - private: - struct TimestampDataTuple { - uint32_t timestamp; - FrameInformation data; - }; - bool IsEmpty() const; - - std::unique_ptr ring_buffer_; - const size_t capacity_; - size_t next_add_idx_; - size_t next_pop_idx_; -}; - -} // namespace webrtc - -#endif // MODULES_VIDEO_CODING_TIMESTAMP_MAP_H_ diff --git a/modules/video_coding/timestamp_map_unittest.cc b/modules/video_coding/timestamp_map_unittest.cc deleted file mode 100644 index 4fc8897dbe..0000000000 --- a/modules/video_coding/timestamp_map_unittest.cc +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_coding/timestamp_map.h" - -#include "test/gmock.h" -#include "test/gtest.h" - -namespace webrtc { -namespace video_coding { -namespace { -constexpr int kTimestampMapSize = 6; -constexpr int kTimestamp1 = 1; -constexpr int kTimestamp2 = 2; -constexpr int kNoExistingTimestamp3 = 3; -constexpr int kTimestamp4 = 4; -constexpr int kTimestamp5 = 5; -constexpr int kTimestamp6 = 6; -constexpr int kTimestamp7 = 7; -constexpr Timestamp kRenderTime1 = Timestamp::Seconds(1); -constexpr Timestamp kRenderTime2 = Timestamp::Seconds(2); -constexpr Timestamp kRenderTime4 = Timestamp::Seconds(4); -constexpr Timestamp kRenderTime5 = Timestamp::Seconds(5); -constexpr Timestamp kRenderTime6 = Timestamp::Seconds(6); -constexpr Timestamp kRenderTime7 = Timestamp::Seconds(7); -} // namespace - -class VcmTimestampMapTest : public ::testing::Test { - protected: - VcmTimestampMapTest() : _timestampMap(kTimestampMapSize) {} - - void SetUp() override { - _timestampMap.Add(kTimestamp1, FrameInformation({kRenderTime1})); - _timestampMap.Add(kTimestamp2, FrameInformation({kRenderTime2})); - _timestampMap.Add(kTimestamp4, FrameInformation({kRenderTime4})); - } - - TimestampMap _timestampMap; -}; - -TEST_F(VcmTimestampMapTest, PopExistingFrameInfo) { - EXPECT_EQ(_timestampMap.Size(), 3u); - auto frameInfo = _timestampMap.Pop(kTimestamp1); - ASSERT_TRUE(frameInfo); - EXPECT_EQ(frameInfo->render_time, kRenderTime1); - frameInfo = _timestampMap.Pop(kTimestamp2); - ASSERT_TRUE(frameInfo); - EXPECT_EQ(frameInfo->render_time, kRenderTime2); - frameInfo = _timestampMap.Pop(kTimestamp4); - ASSERT_TRUE(frameInfo); - EXPECT_EQ(frameInfo->render_time, kRenderTime4); -} - -TEST_F(VcmTimestampMapTest, PopNonexistingClearsOlderFrameInfos) { - auto frameInfo = _timestampMap.Pop(kNoExistingTimestamp3); - EXPECT_FALSE(frameInfo); - EXPECT_EQ(_timestampMap.Size(), 1u); -} - -TEST_F(VcmTimestampMapTest, SizeIsIncrementedWhenAddingNewFrameInfo) { - EXPECT_EQ(_timestampMap.Size(), 3u); - _timestampMap.Add(kTimestamp5, FrameInformation({kRenderTime5})); - EXPECT_EQ(_timestampMap.Size(), 4u); - _timestampMap.Add(kTimestamp6, FrameInformation({kRenderTime6})); - EXPECT_EQ(_timestampMap.Size(), 5u); -} - -TEST_F(VcmTimestampMapTest, SizeIsDecreasedWhenPoppingFrameInfo) { - EXPECT_EQ(_timestampMap.Size(), 3u); - EXPECT_TRUE(_timestampMap.Pop(kTimestamp1)); - EXPECT_EQ(_timestampMap.Size(), 2u); - EXPECT_TRUE(_timestampMap.Pop(kTimestamp2)); - EXPECT_EQ(_timestampMap.Size(), 1u); - EXPECT_FALSE(_timestampMap.Pop(kNoExistingTimestamp3)); - EXPECT_EQ(_timestampMap.Size(), 1u); - EXPECT_TRUE(_timestampMap.Pop(kTimestamp4)); - EXPECT_EQ(_timestampMap.Size(), 0u); -} - -TEST_F(VcmTimestampMapTest, ClearEmptiesMap) { - EXPECT_EQ(_timestampMap.Size(), 3u); - _timestampMap.Clear(); - EXPECT_EQ(_timestampMap.Size(), 0u); - // Clear empty map does nothing. - _timestampMap.Clear(); - EXPECT_EQ(_timestampMap.Size(), 0u); -} - -TEST_F(VcmTimestampMapTest, PopLastAddedClearsMap) { - EXPECT_EQ(_timestampMap.Size(), 3u); - EXPECT_TRUE(_timestampMap.Pop(kTimestamp4)); - EXPECT_EQ(_timestampMap.Size(), 0u); -} - -TEST_F(VcmTimestampMapTest, LastAddedIsDiscardedIfMapGetsFull) { - EXPECT_EQ(_timestampMap.Size(), 3u); - _timestampMap.Add(kTimestamp5, FrameInformation({kRenderTime5})); - EXPECT_EQ(_timestampMap.Size(), 4u); - _timestampMap.Add(kTimestamp6, FrameInformation({kRenderTime6})); - EXPECT_EQ(_timestampMap.Size(), 5u); - _timestampMap.Add(kTimestamp7, FrameInformation({kRenderTime7})); - // Size is not incremented since the oldest element is discarded. - EXPECT_EQ(_timestampMap.Size(), 5u); - EXPECT_FALSE(_timestampMap.Pop(kTimestamp1)); - EXPECT_TRUE(_timestampMap.Pop(kTimestamp2)); - EXPECT_TRUE(_timestampMap.Pop(kTimestamp4)); - EXPECT_TRUE(_timestampMap.Pop(kTimestamp5)); - EXPECT_TRUE(_timestampMap.Pop(kTimestamp6)); - EXPECT_TRUE(_timestampMap.Pop(kTimestamp7)); - EXPECT_EQ(_timestampMap.Size(), 0u); -} - -} // namespace video_coding -} // namespace webrtc