diff --git a/webrtc/modules/video_coding/frame_buffer2.cc b/webrtc/modules/video_coding/frame_buffer2.cc index 53b30c9240..dab7832546 100644 --- a/webrtc/modules/video_coding/frame_buffer2.cc +++ b/webrtc/modules/video_coding/frame_buffer2.cc @@ -42,13 +42,15 @@ bool FrameBuffer::FrameComp::operator()(const FrameKey& f1, FrameBuffer::FrameBuffer(Clock* clock, VCMJitterEstimator* jitter_estimator, - const VCMTiming* timing) + VCMTiming* timing) : clock_(clock), frame_inserted_event_(false, false), jitter_estimator_(jitter_estimator), timing_(timing), + inter_frame_delay_(clock_->TimeInMilliseconds()), newest_picture_id_(-1), - stopped_(false) {} + stopped_(false), + protection_mode_(kProtectionNack) {} std::unique_ptr FrameBuffer::NextFrame(int64_t max_wait_time_ms) { int64_t latest_return_time = clock_->TimeInMilliseconds() + max_wait_time_ms; @@ -56,7 +58,7 @@ std::unique_ptr FrameBuffer::NextFrame(int64_t max_wait_time_ms) { int64_t wait_ms = max_wait_time_ms; while (true) { std::map, FrameComp>::iterator - next_frame; + next_frame_it; { rtc::CritScope lock(&crit_); frame_inserted_event_.Reset(); @@ -65,14 +67,18 @@ std::unique_ptr FrameBuffer::NextFrame(int64_t max_wait_time_ms) { now = clock_->TimeInMilliseconds(); wait_ms = max_wait_time_ms; - next_frame = frames_.end(); + next_frame_it = frames_.end(); for (auto frame_it = frames_.begin(); frame_it != frames_.end(); ++frame_it) { const FrameObject& frame = *frame_it->second; if (IsContinuous(frame)) { - next_frame = frame_it; - int64_t render_time = timing_->RenderTimeMs(frame.timestamp, now); + next_frame_it = frame_it; + int64_t render_time = + next_frame_it->second->RenderTime() == -1 + ? timing_->RenderTimeMs(frame.timestamp, now) + : next_frame_it->second->RenderTime(); wait_ms = timing_->MaxWaitingTime(render_time, now); + frame_it->second->SetRenderTime(render_time); // This will cause the frame buffer to prefer high framerate rather // than high resolution in the case of the decoder not decoding fast @@ -85,19 +91,30 @@ std::unique_ptr FrameBuffer::NextFrame(int64_t max_wait_time_ms) { } } - // If the timout occures, return. Otherwise a new frame has been inserted - // and the best frame to decode next will be selected again. wait_ms = std::min(wait_ms, latest_return_time - now); wait_ms = std::max(wait_ms, 0); + // If the timeout occurs, return. Otherwise a new frame has been inserted + // and the best frame to decode next will be selected again. if (!frame_inserted_event_.Wait(wait_ms)) { rtc::CritScope lock(&crit_); - if (next_frame != frames_.end()) { - // TODO(philipel): update jitter estimator with correct values. - jitter_estimator_->UpdateEstimate(100, 100); + if (next_frame_it != frames_.end()) { + int64_t received_timestamp = next_frame_it->second->ReceivedTime(); + uint32_t timestamp = next_frame_it->second->Timestamp(); - decoded_frames_.insert(next_frame->first); - std::unique_ptr frame = std::move(next_frame->second); - frames_.erase(frames_.begin(), ++next_frame); + int64_t frame_delay; + if (inter_frame_delay_.CalculateDelay(timestamp, &frame_delay, + received_timestamp)) { + jitter_estimator_->UpdateEstimate(frame_delay, + next_frame_it->second->size); + } + float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0; + timing_->SetJitterDelay(jitter_estimator_->GetJitterEstimate(rtt_mult)); + timing_->UpdateCurrentDelay(next_frame_it->second->RenderTime(), + clock_->TimeInMilliseconds()); + + decoded_frames_.insert(next_frame_it->first); + std::unique_ptr frame = std::move(next_frame_it->second); + frames_.erase(frames_.begin(), ++next_frame_it); return frame; } else { return std::unique_ptr(); @@ -106,6 +123,11 @@ std::unique_ptr FrameBuffer::NextFrame(int64_t max_wait_time_ms) { } } +void FrameBuffer::SetProtectionMode(VCMVideoProtection mode) { + rtc::CritScope lock(&crit_); + protection_mode_ = mode; +} + void FrameBuffer::Start() { rtc::CritScope lock(&crit_); stopped_ = false; @@ -119,6 +141,7 @@ void FrameBuffer::Stop() { void FrameBuffer::InsertFrame(std::unique_ptr frame) { rtc::CritScope lock(&crit_); + // If |newest_picture_id_| is -1 then this is the first frame we received. if (newest_picture_id_ == -1) newest_picture_id_ = frame->picture_id; @@ -129,7 +152,7 @@ void FrameBuffer::InsertFrame(std::unique_ptr frame) { while (decoded_frames_.size() > kMaxNumHistoryFrames) decoded_frames_.erase(decoded_frames_.begin()); - // Remove frames that are too old, |kMaxNumHistoryFrames|. + // Remove frames that are too old. uint16_t old_picture_id = Subtract<1 << 16>(newest_picture_id_, kMaxFrameAge); auto old_decoded_it = decoded_frames_.lower_bound(FrameKey(old_picture_id, 0)); diff --git a/webrtc/modules/video_coding/frame_buffer2.h b/webrtc/modules/video_coding/frame_buffer2.h index 3201f627ce..0af2bf9523 100644 --- a/webrtc/modules/video_coding/frame_buffer2.h +++ b/webrtc/modules/video_coding/frame_buffer2.h @@ -21,6 +21,8 @@ #include "webrtc/base/criticalsection.h" #include "webrtc/base/event.h" #include "webrtc/base/thread_annotations.h" +#include "webrtc/modules/video_coding/include/video_coding_defines.h" +#include "webrtc/modules/video_coding/inter_frame_delay.h" namespace webrtc { @@ -36,7 +38,7 @@ class FrameBuffer { public: FrameBuffer(Clock* clock, VCMJitterEstimator* jitter_estimator, - const VCMTiming* timing); + VCMTiming* timing); // Insert a frame into the frame buffer. void InsertFrame(std::unique_ptr frame); @@ -46,6 +48,12 @@ class FrameBuffer { // unique ptr if there is no available frame for decoding. std::unique_ptr NextFrame(int64_t max_wait_time_ms); + // Tells the FrameBuffer which protection mode that is in use. Affects + // the frame timing. + // TODO(philipel): Remove this when new timing calculations has been + // implemented. + void SetProtectionMode(VCMVideoProtection mode); + // Start the frame buffer, has no effect if the frame buffer is started. // The frame buffer is started upon construction. void Start(); @@ -78,10 +86,12 @@ class FrameBuffer { rtc::CriticalSection crit_; Clock* const clock_; rtc::Event frame_inserted_event_; - VCMJitterEstimator* const jitter_estimator_; - const VCMTiming* const timing_; + VCMJitterEstimator* const jitter_estimator_ GUARDED_BY(crit_); + VCMTiming* const timing_ GUARDED_BY(crit_); + VCMInterFrameDelay inter_frame_delay_ GUARDED_BY(crit_); int newest_picture_id_ GUARDED_BY(crit_); bool stopped_ GUARDED_BY(crit_); + VCMVideoProtection protection_mode_ GUARDED_BY(crit_); RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(FrameBuffer); }; diff --git a/webrtc/modules/video_coding/frame_buffer2_unittest.cc b/webrtc/modules/video_coding/frame_buffer2_unittest.cc index 5c81d0284d..674ae29bfc 100644 --- a/webrtc/modules/video_coding/frame_buffer2_unittest.cc +++ b/webrtc/modules/video_coding/frame_buffer2_unittest.cc @@ -70,6 +70,7 @@ class VCMJitterEstimatorMock : public VCMJitterEstimator { void(int64_t frameDelayMs, uint32_t frameSizeBytes, bool incompleteFrame)); + MOCK_METHOD1(GetJitterEstimate, int(double rttMultiplier)); }; class FrameObjectFake : public FrameObject { @@ -178,7 +179,7 @@ class TestFrameBuffer2 : public ::testing::Test { SimulatedClock clock_; VCMTimingFake timing_; - VCMJitterEstimatorMock jitter_estimator_; + ::testing::NiceMock jitter_estimator_; FrameBuffer buffer_; std::vector> frames_; Random rand_; @@ -336,5 +337,19 @@ TEST_F(TestFrameBuffer2, InsertLateFrame) { CheckNoFrame(2); } +TEST_F(TestFrameBuffer2, ProtectionMode) { + uint16_t pid = Rand(); + uint32_t ts = Rand(); + + EXPECT_CALL(jitter_estimator_, GetJitterEstimate(1.0)); + InsertFrame(pid, 0, ts, false); + ExtractFrame(); + + buffer_.SetProtectionMode(kProtectionNackFEC); + EXPECT_CALL(jitter_estimator_, GetJitterEstimate(0.0)); + InsertFrame(pid + 1, 0, ts, false); + ExtractFrame(); +} + } // namespace video_coding } // namespace webrtc diff --git a/webrtc/modules/video_coding/jitter_estimator.h b/webrtc/modules/video_coding/jitter_estimator.h index a7b4b3e3df..7ef050f032 100644 --- a/webrtc/modules/video_coding/jitter_estimator.h +++ b/webrtc/modules/video_coding/jitter_estimator.h @@ -50,7 +50,7 @@ class VCMJitterEstimator { // - rttMultiplier : RTT param multiplier (when applicable). // // Return value : Jitter estimate in milliseconds - int GetJitterEstimate(double rttMultiplier); + virtual int GetJitterEstimate(double rttMultiplier); // Updates the nack counter. void FrameNacked();