diff --git a/webrtc/modules/video_coding/main/source/decoding_state.cc b/webrtc/modules/video_coding/main/source/decoding_state.cc index 9d74c94cd2..1196b8866e 100644 --- a/webrtc/modules/video_coding/main/source/decoding_state.cc +++ b/webrtc/modules/video_coding/main/source/decoding_state.cc @@ -82,21 +82,11 @@ void VCMDecodingState::CopyFrom(const VCMDecodingState& state) { in_initial_state_ = state.in_initial_state_; } -bool VCMDecodingState::UpdateEmptyFrame(const VCMFrameBuffer* frame) { - bool empty_packet = frame->GetHighSeqNum() == frame->GetLowSeqNum(); - if (in_initial_state_ && empty_packet) { - // Drop empty packets as long as we are in the initial state. - return true; - } - if ((empty_packet && ContinuousSeqNum(frame->GetHighSeqNum())) || - ContinuousFrame(frame)) { - // Continuous empty packets or continuous frames can be dropped if we - // advance the sequence number. - sequence_num_ = frame->GetHighSeqNum(); +void VCMDecodingState::UpdateEmptyFrame(const VCMFrameBuffer* frame) { + if (ContinuousFrame(frame)) { time_stamp_ = frame->TimeStamp(); - return true; + sequence_num_ = frame->GetHighSeqNum(); } - return false; } void VCMDecodingState::UpdateOldPacket(const VCMPacket* packet) { @@ -149,14 +139,11 @@ bool VCMDecodingState::ContinuousFrame(const VCMFrameBuffer* frame) const { // Return true when in initial state. // Note that when a method is not applicable it will return false. assert(frame != NULL); - // A key frame is always considered continuous as it doesn't refer to any - // frames and therefore won't introduce any errors even if prior frames are - // missing. - if (frame->FrameType() == kVideoFrameKey) - return true; - // When in the initial state we always require a key frame to start decoding. - if (in_initial_state_) + if (in_initial_state_) { + // Always start with a key frame. + if (frame->FrameType() == kVideoFrameKey) return true; return false; + } if (!ContinuousLayer(frame->TemporalId(), frame->Tl0PicId())) { // Base layers are not continuous or temporal layers are inactive. diff --git a/webrtc/modules/video_coding/main/source/decoding_state.h b/webrtc/modules/video_coding/main/source/decoding_state.h index 1fdc1fb23a..b579cba5da 100644 --- a/webrtc/modules/video_coding/main/source/decoding_state.h +++ b/webrtc/modules/video_coding/main/source/decoding_state.h @@ -32,7 +32,7 @@ class VCMDecodingState { bool ContinuousFrame(const VCMFrameBuffer* frame) const; void SetState(const VCMFrameBuffer* frame); void CopyFrom(const VCMDecodingState& state); - bool UpdateEmptyFrame(const VCMFrameBuffer* frame); + void UpdateEmptyFrame(const VCMFrameBuffer* frame); // Update the sequence number if the timestamp matches current state and the // sequence number is higher than the current one. This accounts for packets // arriving late. diff --git a/webrtc/modules/video_coding/main/source/jitter_buffer.cc b/webrtc/modules/video_coding/main/source/jitter_buffer.cc index c0ea9de594..0c0306a83c 100644 --- a/webrtc/modules/video_coding/main/source/jitter_buffer.cc +++ b/webrtc/modules/video_coding/main/source/jitter_buffer.cc @@ -73,82 +73,6 @@ bool HasNonEmptyState(VCMFrameBuffer* frame) { return frame->GetState() != kStateEmpty; } -void FrameList::InsertFrame(VCMFrameBuffer* frame) { - reverse_iterator rit = std::find_if( - rbegin(), rend(), FrameSmallerTimestamp(frame->TimeStamp())); - insert(rit.base(), frame); -} - -VCMFrameBuffer* FrameList::FindFrame(uint32_t timestamp) const { - FrameList::const_iterator it = std::find_if(begin(), end(), - FrameEqualTimestamp(timestamp)); - if (it == end()) - return NULL; - return *it; -} - -VCMFrameBuffer* FrameList::PopFrame(uint32_t timestamp) { - FrameList::iterator it = std::find_if(begin(), end(), - FrameEqualTimestamp(timestamp)); - if (it == end()) - return NULL; - VCMFrameBuffer* frame = *it; - erase(it); - return frame; -} - -int FrameList::RecycleFramesUntilKeyFrame(FrameList::iterator* key_frame_it) { - int drop_count = 0; - *key_frame_it = begin(); - while (!empty()) { - // Throw at least one frame. - WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding, -1, - "Recycling: type=%s, low seqnum=%u", - (**key_frame_it)->FrameType() == kVideoFrameKey ? - "key" : "delta", (**key_frame_it)->GetLowSeqNum()); - if ((**key_frame_it)->GetState() != kStateDecoding) { - (**key_frame_it)->SetState(kStateFree); - } - *key_frame_it = erase(*key_frame_it); - ++drop_count; - if (*key_frame_it != end() && - (**key_frame_it)->FrameType() == kVideoFrameKey) { - return drop_count; - } - } - return drop_count; -} - -int FrameList::CleanUpOldOrEmptyFrames(VCMDecodingState* decoding_state) { - int drop_count = 0; - while (!empty()) { - VCMFrameBuffer* oldest_frame = front(); - bool remove_frame = false; - if (oldest_frame->GetState() == kStateEmpty && size() > 1) { - // This frame is empty, try to update the last decoded state and drop it - // if successful. - remove_frame = decoding_state->UpdateEmptyFrame(oldest_frame); - } else { - remove_frame = decoding_state->IsOldFrame(oldest_frame); - } - if (!remove_frame) { - break; - } - if (front()->GetState() != kStateDecoding) { - front()->SetState(kStateFree); - } - ++drop_count; - TRACE_EVENT_INSTANT1("webrtc", "JB::OldOrEmptyFrameDropped", "timestamp", - oldest_frame->TimeStamp()); - erase(begin()); - } - if (empty()) { - TRACE_EVENT_INSTANT1("webrtc", "JB::FrameListEmptied", - "type", "CleanUpOldOrEmptyFrames"); - } - return drop_count; -} - VCMJitterBuffer::VCMJitterBuffer(Clock* clock, EventFactory* event_factory, int vcm_id, @@ -164,8 +88,7 @@ VCMJitterBuffer::VCMJitterBuffer(Clock* clock, packet_event_(event_factory->CreateEvent()), max_number_of_frames_(kStartNumberOfFrames), frame_buffers_(), - decodable_frames_(), - incomplete_frames_(), + frame_list_(), last_decoded_state_(), first_packet_since_reset_(true), num_not_decodable_packets_(0), @@ -249,18 +172,15 @@ void VCMJitterBuffer::CopyFrom(const VCMJitterBuffer& rhs) { frame_buffers_[i] = NULL; } } - decodable_frames_.clear(); - incomplete_frames_.clear(); - int i = 0; - for (FrameList::const_iterator it = rhs.decodable_frames_.begin(); - it != rhs.decodable_frames_.end(); ++it, ++i) { - frame_buffers_[i] = new VCMFrameBuffer(**it); - decodable_frames_.push_back(frame_buffers_[i]); - } - for (FrameList::const_iterator it = rhs.incomplete_frames_.begin(); - it != rhs.incomplete_frames_.end(); ++it, ++i) { - frame_buffers_[i] = new VCMFrameBuffer(**it); - incomplete_frames_.push_back(frame_buffers_[i]); + frame_list_.clear(); + for (int i = 0; i < max_number_of_frames_; i++) { + frame_buffers_[i] = new VCMFrameBuffer(*(rhs.frame_buffers_[i])); + if (frame_buffers_[i]->Length() > 0) { + FrameList::reverse_iterator rit = std::find_if( + frame_list_.rbegin(), frame_list_.rend(), + FrameSmallerTimestamp(frame_buffers_[i]->TimeStamp())); + frame_list_.insert(rit.base(), frame_buffers_[i]); + } } rhs.crit_sect_->Leave(); crit_sect_->Leave(); @@ -301,8 +221,7 @@ void VCMJitterBuffer::Stop() { crit_sect_->Enter(); running_ = false; last_decoded_state_.Reset(); - decodable_frames_.clear(); - incomplete_frames_.clear(); + frame_list_.clear(); TRACE_EVENT_INSTANT1("webrtc", "JB::FrameListEmptied", "type", "Stop"); for (int i = 0; i < kMaxNumberOfFrames; i++) { if (frame_buffers_[i] != NULL) { @@ -327,8 +246,7 @@ bool VCMJitterBuffer::Running() const { void VCMJitterBuffer::Flush() { CriticalSectionScoped cs(crit_sect_); // Erase all frames from the sorted list and set their state to free. - decodable_frames_.clear(); - incomplete_frames_.clear(); + frame_list_.clear(); TRACE_EVENT_INSTANT2("webrtc", "JB::FrameListEmptied", "type", "Flush", "frames", max_number_of_frames_); for (int i = 0; i < max_number_of_frames_; i++) { @@ -443,13 +361,32 @@ bool VCMJitterBuffer::CompleteSequenceWithNextFrame() { CriticalSectionScoped cs(crit_sect_); // Finding oldest frame ready for decoder, check sequence number and size CleanUpOldOrEmptyFrames(); - if (!decodable_frames_.empty()) + + if (frame_list_.empty()) return true; - if (incomplete_frames_.size() <= 1) { + + VCMFrameBuffer* oldest_frame = frame_list_.front(); + if (frame_list_.size() <= 1 && + oldest_frame->GetState() != kStateComplete) { // Frame not ready to be decoded. return true; } - return false; + if (oldest_frame->GetState() != kStateComplete) { + return false; + } + + // See if we have lost a frame before this one. + if (last_decoded_state_.in_initial_state()) { + // Following start, reset or flush -> check for key frame. + if (oldest_frame->FrameType() != kVideoFrameKey) { + return false; + } + } else if (oldest_frame->GetLowSeqNum() == -1) { + return false; + } else if (!last_decoded_state_.ContinuousFrame(oldest_frame)) { + return false; + } + return true; } // Returns immediately or a |max_wait_time_ms| ms event hang waiting for a @@ -459,12 +396,13 @@ bool VCMJitterBuffer::NextCompleteTimestamp( TRACE_EVENT0("webrtc", "JB::NextCompleteTimestamp"); crit_sect_->Enter(); if (!running_) { - crit_sect_->Leave(); return false; } CleanUpOldOrEmptyFrames(); - if (decodable_frames_.empty()) { + FrameList::iterator it = FindOldestCompleteContinuousFrame( + frame_list_.begin(), &last_decoded_state_); + if (it == frame_list_.end()) { const int64_t end_wait_time_ms = clock_->TimeInMilliseconds() + max_wait_time_ms; int64_t wait_time_ms = max_wait_time_ms; @@ -474,7 +412,7 @@ bool VCMJitterBuffer::NextCompleteTimestamp( frame_event_->Wait(static_cast(wait_time_ms)); crit_sect_->Enter(); if (ret == kEventSignaled) { - // Are we shutting down the jitter buffer? + // Are we closing down the Jitter buffer? if (!running_) { crit_sect_->Leave(); return false; @@ -482,7 +420,9 @@ bool VCMJitterBuffer::NextCompleteTimestamp( // Finding oldest frame ready for decoder, but check // sequence number and size CleanUpOldOrEmptyFrames(); - if (decodable_frames_.empty()) { + it = FindOldestCompleteContinuousFrame( + frame_list_.begin(), &last_decoded_state_); + if (it == frame_list_.end()) { wait_time_ms = end_wait_time_ms - clock_->TimeInMilliseconds(); } else { break; @@ -493,19 +433,31 @@ bool VCMJitterBuffer::NextCompleteTimestamp( } // Inside |crit_sect_|. } else { - // We already have a frame, reset the event. + // We already have a frame reset the event. frame_event_->Reset(); } - if (decodable_frames_.empty()) { - crit_sect_->Leave(); - return false; + + if (!decode_with_errors_ && it == frame_list_.end()) { + // We're still missing a complete continuous frame. + // Look for a complete key frame if we're not decoding with errors. + it = find_if(frame_list_.begin(), frame_list_.end(), + CompleteKeyFrameCriteria()); } - *timestamp = decodable_frames_.front()->TimeStamp(); + + if (it == frame_list_.end()) { + crit_sect_->Leave(); + return false; + } + + VCMFrameBuffer* oldest_frame = *it; + + *timestamp = oldest_frame->TimeStamp(); crit_sect_->Leave(); return true; } -bool VCMJitterBuffer::NextMaybeIncompleteTimestamp(uint32_t* timestamp) { +bool VCMJitterBuffer::NextMaybeIncompleteTimestamp( + uint32_t* timestamp) { TRACE_EVENT0("webrtc", "JB::NextMaybeIncompleteTimestamp"); CriticalSectionScoped cs(crit_sect_); if (!running_) { @@ -518,17 +470,17 @@ bool VCMJitterBuffer::NextMaybeIncompleteTimestamp(uint32_t* timestamp) { CleanUpOldOrEmptyFrames(); - VCMFrameBuffer* oldest_frame = NextFrame(); - if (!oldest_frame) { + if (frame_list_.empty()) { return false; } - if (decodable_frames_.empty() && incomplete_frames_.size() <= 1 && - oldest_frame->GetState() == kStateIncomplete) { - // If we have only one frame in the buffer, release it only if it is - // complete. + + VCMFrameBuffer* oldest_frame = frame_list_.front(); + // If we have only one frame in the buffer, release it only if it is complete. + if (frame_list_.size() <= 1 && oldest_frame->GetState() != kStateComplete) { return false; } - // Always start with a complete key frame. + + // Always start with a key frame. if (last_decoded_state_.in_initial_state() && oldest_frame->FrameType() != kVideoFrameKey) { return false; @@ -546,13 +498,18 @@ VCMEncodedFrame* VCMJitterBuffer::ExtractAndSetDecode(uint32_t timestamp) { return NULL; } // Extract the frame with the desired timestamp. - VCMFrameBuffer* frame = decodable_frames_.PopFrame(timestamp); - if (!frame) { - frame = incomplete_frames_.PopFrame(timestamp); - if (!frame) - return NULL; + FrameList::iterator it = std::find_if( + frame_list_.begin(), + frame_list_.end(), + FrameEqualTimestamp(timestamp)); + + if (it == frame_list_.end()) { + return NULL; } - if (!NextFrame()) { + // We got the frame. + VCMFrameBuffer* frame = *it; + frame_list_.erase(it); + if (frame_list_.empty()) { TRACE_EVENT_INSTANT1("webrtc", "JB::FrameListEmptied", "type", "ExtractAndSetDecode"); } @@ -628,14 +585,16 @@ VCMFrameBufferEnum VCMJitterBuffer::GetFrame(const VCMPacket& packet, } num_consecutive_old_packets_ = 0; - *frame = incomplete_frames_.FindFrame(packet.timestamp); - if (*frame) { - return kNoError; - } - *frame = decodable_frames_.FindFrame(packet.timestamp); - if (*frame) { + FrameList::iterator it = std::find_if( + frame_list_.begin(), + frame_list_.end(), + FrameEqualTimestamp(packet.timestamp)); + + if (it != frame_list_.end()) { + *frame = *it; return kNoError; } + // No match, return empty frame. *frame = GetEmptyFrame(); if (*frame != NULL) { @@ -681,23 +640,6 @@ VCMFrameBufferEnum VCMJitterBuffer::InsertPacket(const VCMPacket& packet, // reset the delay estimate. inter_frame_delay_.Reset(clock_->TimeInMilliseconds()); } - if (last_decoded_state_.IsOldPacket(&packet)) { - // This packet belongs to an old, already decoded frame, we want to update - // the last decoded sequence number. - last_decoded_state_.UpdateOldPacket(&packet); - frame->SetState(kStateFree); - TRACE_EVENT_INSTANT1("webrtc", "JB::DropLateFrame", - "timestamp", frame->TimeStamp()); - drop_count_++; - // Flush() if this happens consistently. - num_consecutive_old_frames_++; - if (num_consecutive_old_frames_ > kMaxConsecutiveOldFrames) { - Flush(); - return kFlushIndicator; - } - return kNoError; - } - num_consecutive_old_frames_ = 0; // Empty packets may bias the jitter estimate (lacking size component), // therefore don't let empty packet trigger the following updates: @@ -717,7 +659,7 @@ VCMFrameBufferEnum VCMJitterBuffer::InsertPacket(const VCMPacket& packet, } } - VCMFrameBufferStateEnum previous_state = frame->GetState(); + VCMFrameBufferStateEnum state = frame->GetState(); // Insert packet. // Check for first packet. High sequence number will be -1 if neither an empty // packet nor a media packet has been inserted. @@ -732,6 +674,17 @@ VCMFrameBufferEnum VCMJitterBuffer::InsertPacket(const VCMPacket& packet, ret = buffer_return; if (buffer_return > 0) { incoming_bit_count_ += packet.sizeBytes << 3; + + // Insert each frame once on the arrival of the first packet + // belonging to that frame (media or empty). + if (state == kStateEmpty && first) { + ret = kFirstPacket; + FrameList::reverse_iterator rit = std::find_if( + frame_list_.rbegin(), + frame_list_.rend(), + FrameSmallerTimestamp(frame->TimeStamp())); + frame_list_.insert(rit.base(), frame); + } if (first_packet_since_reset_) { latest_received_sequence_number_ = packet.seqNum; first_packet_since_reset_ = false; @@ -752,8 +705,9 @@ VCMFrameBufferEnum VCMJitterBuffer::InsertPacket(const VCMPacket& packet, case kTimeStampError: case kSizeError: { if (frame != NULL) { + // Will be released when it gets old. frame->Reset(); - frame->SetState(kStateFree); + frame->SetState(kStateEmpty); } break; } @@ -761,36 +715,17 @@ VCMFrameBufferEnum VCMJitterBuffer::InsertPacket(const VCMPacket& packet, // Don't let the first packet be overridden by a complete session. ret = kCompleteSession; // Only update return value for a JB flush indicator. - UpdateFrameState(frame); + if (UpdateFrameState(frame) == kFlushIndicator) + ret = kFlushIndicator; *retransmitted = (frame->GetNackCount() > 0); - if (IsContinuous(*frame) && previous_state != kStateComplete) { - if (!first) { - incomplete_frames_.PopFrame(packet.timestamp); - } - decodable_frames_.InsertFrame(frame); - FindAndInsertContinuousFrames(*frame); - // Signal that we have a decodable frame. - frame_event_->Set(); - } else if (first) { - incomplete_frames_.InsertFrame(frame); - } // Signal that we have a received packet. packet_event_->Set(); break; } case kDecodableSession: case kIncomplete: { - // No point in storing empty continuous frames. - if (frame->GetState() == kStateEmpty && - last_decoded_state_.UpdateEmptyFrame(frame)) { - frame->SetState(kStateFree); - ret = kNoError; - } else if (first) { - ret = kFirstPacket; - incomplete_frames_.InsertFrame(frame); - // Signal that we have received a packet. - packet_event_->Set(); - } + // Signal that we have a received packet. + packet_event_->Set(); break; } case kNoError: @@ -807,66 +742,6 @@ VCMFrameBufferEnum VCMJitterBuffer::InsertPacket(const VCMPacket& packet, return ret; } -bool VCMJitterBuffer::IsContinuousInState(const VCMFrameBuffer& frame, - const VCMDecodingState& decoding_state) const { - // Is this frame complete or decodable and continuous? - if ((frame.GetState() == kStateComplete || - (decode_with_errors_ && frame.GetState() == kStateDecodable)) && - decoding_state.ContinuousFrame(&frame)) { - return true; - } else { - return false; - } -} - -bool VCMJitterBuffer::IsContinuous(const VCMFrameBuffer& frame) const { - if (IsContinuousInState(frame, last_decoded_state_)) { - return true; - } - VCMDecodingState decoding_state; - decoding_state.CopyFrom(last_decoded_state_); - for (FrameList::const_iterator it = decodable_frames_.begin(); - it != decodable_frames_.end(); ++it) { - VCMFrameBuffer* decodable_frame = *it; - if (IsNewerTimestamp(decodable_frame->TimeStamp(), frame.TimeStamp())) { - break; - } - decoding_state.SetState(decodable_frame); - if (IsContinuousInState(frame, decoding_state)) { - return true; - } - } - return false; -} - -void VCMJitterBuffer::FindAndInsertContinuousFrames( - const VCMFrameBuffer& new_frame) { - VCMDecodingState decoding_state; - decoding_state.CopyFrom(last_decoded_state_); - decoding_state.SetState(&new_frame); - // When temporal layers are available, we search for a complete or decodable - // frame until we hit one of the following: - // 1. Continuous base or sync layer. - // 2. The end of the list was reached. - for (FrameList::iterator it = incomplete_frames_.begin(); - it != incomplete_frames_.end();) { - VCMFrameBuffer* frame = *it; - if (IsNewerTimestamp(new_frame.TimeStamp(), frame->TimeStamp())) { - ++it; - continue; - } - if (IsContinuousInState(*frame, decoding_state)) { - decodable_frames_.InsertFrame(frame); - it = incomplete_frames_.erase(it); - decoding_state.SetState(frame); - } else if (frame->TemporalId() <= 0) { - break; - } else { - ++it; - } - } -} - void VCMJitterBuffer::SetMaxJitterEstimate(bool enable) { CriticalSectionScoped cs(crit_sect_); jitter_estimate_.SetMaxJitterEstimate(enable); @@ -934,14 +809,16 @@ VCMNackMode VCMJitterBuffer::nack_mode() const { } int VCMJitterBuffer::NonContinuousOrIncompleteDuration() { - if (incomplete_frames_.empty()) { + if (frame_list_.empty()) { return 0; } - uint32_t start_timestamp = incomplete_frames_.front()->TimeStamp(); - if (!decodable_frames_.empty()) { - start_timestamp = decodable_frames_.back()->TimeStamp(); - } - return incomplete_frames_.back()->TimeStamp() - start_timestamp; + FrameList::iterator start_it; + FrameList::iterator end_it; + RenderBuffer(&start_it, &end_it); + if (end_it == frame_list_.end()) + end_it = frame_list_.begin(); + return frame_list_.back()->TimeStamp() - + (*end_it)->TimeStamp(); } uint16_t VCMJitterBuffer::EstimatedLowSequenceNumber( @@ -964,19 +841,13 @@ uint16_t* VCMJitterBuffer::GetNackList(uint16_t* nack_list_size, return NULL; } if (last_decoded_state_.in_initial_state()) { - const bool first_frame_is_key = NextFrame() && - NextFrame()->FrameType() == kVideoFrameKey && - NextFrame()->HaveFirstPacket(); + bool first_frame_is_key = !frame_list_.empty() && + frame_list_.front()->FrameType() == kVideoFrameKey && + frame_list_.front()->HaveFirstPacket(); if (!first_frame_is_key) { + const bool have_non_empty_frame = frame_list_.end() != find_if( + frame_list_.begin(), frame_list_.end(), HasNonEmptyState); LOG_F(LS_INFO) << "First frame is not key; Recycling."; - bool have_non_empty_frame = decodable_frames_.end() != find_if( - decodable_frames_.begin(), decodable_frames_.end(), - HasNonEmptyState); - if (!have_non_empty_frame) { - have_non_empty_frame = incomplete_frames_.end() != find_if( - incomplete_frames_.begin(), incomplete_frames_.end(), - HasNonEmptyState); - } bool found_key_frame = RecycleFramesUntilKeyFrame(); if (!found_key_frame) { *request_key_frame = have_non_empty_frame; @@ -999,9 +870,10 @@ uint16_t* VCMJitterBuffer::GetNackList(uint16_t* nack_list_size, LOG_F(LS_INFO) << "Too long non-decodable duration: " << non_continuous_incomplete_duration << " > " << 90 * max_incomplete_time_ms_; - FrameList::reverse_iterator rit = find_if(incomplete_frames_.rbegin(), - incomplete_frames_.rend(), KeyFrameCriteria()); - if (rit == incomplete_frames_.rend()) { + FrameList::reverse_iterator rit = find_if(frame_list_.rbegin(), + frame_list_.rend(), + KeyFrameCriteria()); + if (rit == frame_list_.rend()) { // Request a key frame if we don't have one already. *request_key_frame = true; *nack_list_size = 0; @@ -1025,14 +897,6 @@ uint16_t* VCMJitterBuffer::GetNackList(uint16_t* nack_list_size, return &nack_seq_nums_[0]; } -VCMFrameBuffer* VCMJitterBuffer::NextFrame() const { - if (!decodable_frames_.empty()) - return decodable_frames_.front(); - if (!incomplete_frames_.empty()) - return incomplete_frames_.front(); - return NULL; -} - bool VCMJitterBuffer::UpdateNackList(uint16_t sequence_number) { if (nack_mode_ == kNoNack) { return true; @@ -1121,17 +985,73 @@ int64_t VCMJitterBuffer::LastDecodedTimestamp() const { return last_decoded_state_.time_stamp(); } +FrameList::iterator VCMJitterBuffer::FindLastContinuousAndComplete( + FrameList::iterator start_it) { + // Search for a complete and continuous sequence (starting from the last + // decoded state or current frame if in initial state). + VCMDecodingState previous_state; + previous_state.SetState(*start_it); + FrameList::iterator previous_it = start_it; + ++start_it; + while (start_it != frame_list_.end()) { + start_it = FindOldestCompleteContinuousFrame(start_it, &previous_state); + if (start_it == frame_list_.end()) + break; + previous_state.SetState(*start_it); + previous_it = start_it; + ++start_it; + } + // Desired frame is the previous one. + return previous_it; +} + +void VCMJitterBuffer::RenderBuffer(FrameList::iterator* start_it, + FrameList::iterator* end_it) { + *start_it = FindOldestCompleteContinuousFrame( + frame_list_.begin(), &last_decoded_state_); + if (!decode_with_errors_ && *start_it == frame_list_.end()) { + // No complete continuous frame found. + // Look for a complete key frame if we're not decoding with errors. + *start_it = find_if(frame_list_.begin(), frame_list_.end(), + CompleteKeyFrameCriteria()); + } + if (*start_it == frame_list_.end()) { + *end_it = *start_it; + } else { + *end_it = *start_it; + // Look for the last complete key frame and use that as the end of the + // render buffer it's later than the last complete continuous frame. + FrameList::reverse_iterator rend(*end_it); + FrameList::reverse_iterator rit = find_if(frame_list_.rbegin(), + rend, + CompleteKeyFrameCriteria()); + if (rit != rend) { + // A key frame was found. The reverse iterator base points to the + // frame after it, so subtracting 1. + *end_it = rit.base(); + --*end_it; + } + *end_it = FindLastContinuousAndComplete(*end_it); + } +} + void VCMJitterBuffer::RenderBufferSize(uint32_t* timestamp_start, uint32_t* timestamp_end) { CriticalSectionScoped cs(crit_sect_); CleanUpOldOrEmptyFrames(); *timestamp_start = 0; *timestamp_end = 0; - if (decodable_frames_.empty()) { + if (frame_list_.empty()) { return; } - *timestamp_start = decodable_frames_.front()->TimeStamp(); - *timestamp_end = decodable_frames_.back()->TimeStamp(); + FrameList::iterator start_it; + FrameList::iterator end_it; + RenderBuffer(&start_it, &end_it); + if (start_it == frame_list_.end()) { + return; + } + *timestamp_start = (*start_it)->TimeStamp(); + *timestamp_end = (*end_it)->TimeStamp(); } // Set the frame state to free and remove it from the sorted @@ -1173,42 +1093,48 @@ VCMFrameBuffer* VCMJitterBuffer::GetEmptyFrame() { // Recycle oldest frames up to a key frame, used if jitter buffer is completely // full. bool VCMJitterBuffer::RecycleFramesUntilKeyFrame() { - // First release incomplete frames, and only release decodable frames if there - // are no incomplete ones. - FrameList::iterator key_frame_it; - bool key_frame_found = false; - int dropped_frames = 0; - dropped_frames += incomplete_frames_.RecycleFramesUntilKeyFrame( - &key_frame_it); - key_frame_found = key_frame_it != incomplete_frames_.end(); - if (dropped_frames == 0) { - dropped_frames += decodable_frames_.RecycleFramesUntilKeyFrame( - &key_frame_it); - key_frame_found = key_frame_it != decodable_frames_.end(); - if (!key_frame_found) { - TRACE_EVENT_INSTANT1("webrtc", "JB::FrameListEmptied", "type", - "RecycleFramesUntilKeyFrame"); + // Remove up to oldest key frame + while (!frame_list_.empty()) { + // Throw at least one frame. + drop_count_++; + FrameList::iterator it = frame_list_.begin(); + WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding, + VCMId(vcm_id_, receiver_id_), + "Jitter buffer drop count:%d, low_seq %d, frame type: %s", + drop_count_, (*it)->GetLowSeqNum(), + (*it)->FrameType() == kVideoFrameKey ? "key" : "delta"); + TRACE_EVENT_INSTANT0("webrtc", "JB::RecycleFramesUntilKeyFrame"); + ReleaseFrameIfNotDecoding(*it); + it = frame_list_.erase(it); + if (it != frame_list_.end() && (*it)->FrameType() == kVideoFrameKey) { + // Reset last decoded state to make sure the next frame decoded is a key + // frame, and start NACKing from here. + // Note that the estimated low sequence number is correct for VP8 + // streams because only the first packet of a key frame is marked. + last_decoded_state_.Reset(); + DropPacketsFromNackList(EstimatedLowSequenceNumber(**it)); + return true; } } - drop_count_ += dropped_frames; - WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding, - VCMId(vcm_id_, receiver_id_), - "Jitter buffer drop count:%u", drop_count_); - TRACE_EVENT_INSTANT0("webrtc", "JB::RecycleFramesUntilKeyFrame"); - if (key_frame_found) { - // Reset last decoded state to make sure the next frame decoded is a key - // frame, and start NACKing from here. - last_decoded_state_.Reset(); - DropPacketsFromNackList(EstimatedLowSequenceNumber(**key_frame_it)); - } else if (decodable_frames_.empty()) { - last_decoded_state_.Reset(); // TODO(mikhal): No sync. - missing_sequence_numbers_.clear(); + if (frame_list_.empty()) { + TRACE_EVENT_INSTANT1("webrtc", "JB::FrameListEmptied", + "type", "RecycleFramesUntilKeyFrame"); } - return key_frame_found; + last_decoded_state_.Reset(); // TODO(mikhal): No sync. + missing_sequence_numbers_.clear(); + return false; } // Must be called under the critical section |crit_sect_|. -void VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) { +VCMFrameBufferEnum VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) { + if (frame == NULL) { + WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding, + VCMId(vcm_id_, receiver_id_), "JB(0x%x) FB(0x%x): " + "UpdateFrameState NULL frame pointer", this, frame); + return kNoError; + } + + int length = frame->Length(); if (master_) { // Only trace the primary jitter buffer to make it possible to parse // and plot the trace file. @@ -1216,17 +1142,43 @@ void VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) { VCMId(vcm_id_, receiver_id_), "JB(0x%x) FB(0x%x): Complete frame added to jitter buffer," " size:%d type %d", - this, frame, frame->Length(), frame->FrameType()); + this, frame, length, frame->FrameType()); } bool frame_counted = false; - if (!frame->GetCountedFrame()) { + if (length != 0 && !frame->GetCountedFrame()) { // Ignore ACK frames. incoming_frame_count_++; frame->SetCountedFrame(true); frame_counted = true; } + // Check if we should drop the frame. A complete frame can arrive too late. + if (last_decoded_state_.IsOldFrame(frame)) { + // Frame is older than the latest decoded frame, drop it. Will be + // released by CleanUpOldFrames later. + TRACE_EVENT_INSTANT1("webrtc", "JB::DropLateFrame", + "timestamp", frame->TimeStamp()); + frame->Reset(); + frame->SetState(kStateEmpty); + WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, + VCMId(vcm_id_, receiver_id_), + "JB(0x%x) FB(0x%x): Dropping old frame in Jitter buffer", + this, frame); + drop_count_++; + WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding, + VCMId(vcm_id_, receiver_id_), + "Jitter buffer drop count: %d, consecutive drops: %u", + drop_count_, num_consecutive_old_frames_); + // Flush() if this happens consistently. + num_consecutive_old_frames_++; + if (num_consecutive_old_frames_ > kMaxConsecutiveOldFrames) { + Flush(); + return kFlushIndicator; + } + return kNoError; + } + num_consecutive_old_frames_ = 0; frame->SetState(kStateComplete); if (frame->FrameType() == kVideoFrameKey) { TRACE_EVENT_INSTANT2("webrtc", "JB::AddKeyFrame", @@ -1262,15 +1214,83 @@ void VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) { assert(false); } } + const FrameList::iterator it = FindOldestCompleteContinuousFrame( + frame_list_.begin(), &last_decoded_state_); + VCMFrameBuffer* old_frame = NULL; + if (it != frame_list_.end()) { + old_frame = *it; + } + + // Only signal if this is the oldest frame. + // Not necessarily the case due to packet reordering or NACK. + if (!WaitForRetransmissions() || (old_frame != NULL && old_frame == frame)) { + frame_event_->Set(); + } + return kNoError; +} + +// Find oldest complete frame used for getting next frame to decode +// Must be called under critical section +FrameList::iterator VCMJitterBuffer::FindOldestCompleteContinuousFrame( + FrameList::iterator start_it, + const VCMDecodingState* decoding_state) { + // If we have more than one frame done since last time, pick oldest. + VCMFrameBuffer* oldest_frame = NULL; + + // When temporal layers are available, we search for a complete or decodable + // frame until we hit one of the following: + // 1. Continuous base or sync layer. + // 2. The end of the list was reached. + for (; start_it != frame_list_.end(); ++start_it) { + oldest_frame = *start_it; + VCMFrameBufferStateEnum state = oldest_frame->GetState(); + // Is this frame complete or decodable and continuous? + if ((state == kStateComplete || + (decode_with_errors_ && state == kStateDecodable)) && + decoding_state->ContinuousFrame(oldest_frame)) { + break; + } else { + int temporal_id = oldest_frame->TemporalId(); + oldest_frame = NULL; + if (temporal_id <= 0) { + // When temporal layers are disabled or we have hit a base layer + // we break (regardless of continuity and completeness). + break; + } + } + } + + if (oldest_frame == NULL) { + // No complete frame no point to continue. + return frame_list_.end(); + } + + // We have a complete continuous frame. + return start_it; } // Must be called under the critical section |crit_sect_|. void VCMJitterBuffer::CleanUpOldOrEmptyFrames() { - drop_count_ += - decodable_frames_.CleanUpOldOrEmptyFrames(&last_decoded_state_); - drop_count_ += - incomplete_frames_.CleanUpOldOrEmptyFrames(&last_decoded_state_); - TRACE_COUNTER1("webrtc", "JBDroppedLateFrames", drop_count_); + while (frame_list_.size() > 0) { + VCMFrameBuffer* oldest_frame = frame_list_.front(); + if (oldest_frame->GetState() == kStateEmpty && frame_list_.size() > 1) { + // This frame is empty, mark it as decoded, thereby making it old. + last_decoded_state_.UpdateEmptyFrame(oldest_frame); + } + if (last_decoded_state_.IsOldFrame(oldest_frame)) { + ReleaseFrameIfNotDecoding(frame_list_.front()); + TRACE_EVENT_INSTANT1("webrtc", "JB::OldFrameDropped", + "timestamp", oldest_frame->TimeStamp()); + TRACE_COUNTER1("webrtc", "JBDroppedLateFrames", drop_count_); + frame_list_.erase(frame_list_.begin()); + } else { + break; + } + } + if (frame_list_.empty()) { + TRACE_EVENT_INSTANT1("webrtc", "JB::FrameListEmptied", + "type", "CleanUpOldOrEmptyFrames"); + } if (!last_decoded_state_.in_initial_state()) { DropPacketsFromNackList(last_decoded_state_.sequence_num()); } diff --git a/webrtc/modules/video_coding/main/source/jitter_buffer.h b/webrtc/modules/video_coding/main/source/jitter_buffer.h index 16a73e2a4c..05f2939dbb 100644 --- a/webrtc/modules/video_coding/main/source/jitter_buffer.h +++ b/webrtc/modules/video_coding/main/source/jitter_buffer.h @@ -32,6 +32,8 @@ enum VCMNackMode { kNoNack }; +typedef std::list FrameList; + // forward declarations class Clock; class EventFactory; @@ -47,15 +49,6 @@ struct VCMJitterSample { int64_t latest_packet_time; }; -class FrameList : public std::list { - public: - void InsertFrame(VCMFrameBuffer* frame); - VCMFrameBuffer* FindFrame(uint32_t timestamp) const; - VCMFrameBuffer* PopFrame(uint32_t timestamp); - int RecycleFramesUntilKeyFrame(FrameList::iterator* key_frame_it); - int CleanUpOldOrEmptyFrames(VCMDecodingState* decoding_state); -}; - class VCMJitterBuffer { public: VCMJitterBuffer(Clock* clock, @@ -127,8 +120,6 @@ class VCMJitterBuffer { bool* retransmitted) const; // Inserts a packet into a frame returned from GetFrame(). - // If the return value is <= 0, |frame| is invalidated and the pointer must - // be dropped after this function returns. VCMFrameBufferEnum InsertPacket(const VCMPacket& packet, bool* retransmitted); @@ -184,18 +175,6 @@ class VCMJitterBuffer { // existing frames if no free frames are available. Returns an error code if // failing, or kNoError on success. VCMFrameBufferEnum GetFrame(const VCMPacket& packet, VCMFrameBuffer** frame); - // Returns true if |frame| is continuous in |decoding_state|, not taking - // decodable frames into account. - bool IsContinuousInState(const VCMFrameBuffer& frame, - const VCMDecodingState& decoding_state) const; - // Returns true if |frame| is continuous in the |last_decoded_state_|, taking - // all decodable frames into account. - bool IsContinuous(const VCMFrameBuffer& frame) const; - // Looks for frames in |incomplete_frames_| which are continuous in - // |last_decoded_state_| taking all decodable frames into account. Starts - // the search from |new_frame|. - void FindAndInsertContinuousFrames(const VCMFrameBuffer& new_frame); - VCMFrameBuffer* NextFrame() const; // Returns true if the NACK list was updated to cover sequence numbers up to // |sequence_number|. If false a key frame is needed to get into a state where // we can continue decoding. @@ -223,8 +202,19 @@ class VCMJitterBuffer { bool RecycleFramesUntilKeyFrame(); // Sets the state of |frame| to complete if it's not too old to be decoded. - // Also updates the frame statistics. - void UpdateFrameState(VCMFrameBuffer* frame); + // Also updates the frame statistics. Signals the |frame_event| if this is + // the next frame to be decoded. + VCMFrameBufferEnum UpdateFrameState(VCMFrameBuffer* frame); + + // Finds the oldest complete frame, used for getting next frame to decode. + // Can return a decodable, incomplete frame when enabled. + FrameList::iterator FindOldestCompleteContinuousFrame( + FrameList::iterator start_it, + const VCMDecodingState* decoding_state); + FrameList::iterator FindLastContinuousAndComplete( + FrameList::iterator start_it); + void RenderBuffer(FrameList::iterator* start_it, + FrameList::iterator* end_it); // Cleans the frame list in the JB from old/empty frames. // Should only be called prior to actual use. @@ -273,8 +263,7 @@ class VCMJitterBuffer { int max_number_of_frames_; // Array of pointers to the frames in jitter buffer. VCMFrameBuffer* frame_buffers_[kMaxNumberOfFrames]; - FrameList decodable_frames_; - FrameList incomplete_frames_; + FrameList frame_list_; VCMDecodingState last_decoded_state_; bool first_packet_since_reset_; diff --git a/webrtc/modules/video_coding/main/source/jitter_buffer_unittest.cc b/webrtc/modules/video_coding/main/source/jitter_buffer_unittest.cc index 32bb390997..5521093854 100644 --- a/webrtc/modules/video_coding/main/source/jitter_buffer_unittest.cc +++ b/webrtc/modules/video_coding/main/source/jitter_buffer_unittest.cc @@ -69,10 +69,10 @@ class TestBasicJitterBuffer : public ::testing::Test { VCMEncodedFrame* frame = jitter_buffer_->ExtractAndSetDecode(timestamp); return frame; } - void CheckOutFrame(VCMEncodedFrame* frame_out, + int CheckOutFrame(VCMEncodedFrame* frame_out, unsigned int size, bool startCode) { - ASSERT_TRUE(frame_out); + EXPECT_FALSE(frame_out == NULL); const uint8_t* outData = frame_out->Buffer(); unsigned int i = 0; @@ -104,6 +104,7 @@ class TestBasicJitterBuffer : public ::testing::Test { } } } + return 0; } uint16_t seq_num_; @@ -187,9 +188,7 @@ class TestRunningJitterBuffer : public ::testing::Test { void DropFrame(int num_packets) { stream_generator_->GenerateFrame(kVideoFrameDelta, num_packets, 0, - clock_->TimeInMilliseconds()); - for (int i = 0; i < num_packets; ++i) - stream_generator_->DropLastPacket(); + clock_->TimeInMilliseconds()); clock_->AdvanceTimeMilliseconds(kDefaultFramePeriodMs); } @@ -262,7 +261,7 @@ TEST_F(TestBasicJitterBuffer, SinglePacketFrame) { EXPECT_EQ(kCompleteSession, jitter_buffer_->InsertPacket(*packet_, &retransmitted)); VCMEncodedFrame* frame_out = DecodeCompleteFrame(); - CheckOutFrame(frame_out, size_, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, size_, false)); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); } @@ -287,7 +286,7 @@ TEST_F(TestBasicJitterBuffer, DualPacketFrame) { &retransmitted)); frame_out = DecodeCompleteFrame(); - CheckOutFrame(frame_out, 2 * size_, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, 2 * size_, false)); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); } @@ -330,7 +329,7 @@ TEST_F(TestBasicJitterBuffer, 100PacketKeyFrame) { frame_out = DecodeCompleteFrame(); - CheckOutFrame(frame_out, 100 * size_, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, 100 * size_, false)); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); } @@ -384,7 +383,7 @@ TEST_F(TestBasicJitterBuffer, 100PacketDeltaFrame) { frame_out = DecodeCompleteFrame(); - CheckOutFrame(frame_out, 100 * size_, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, 100 * size_, false)); EXPECT_EQ(kVideoFrameDelta, frame_out->FrameType()); } @@ -429,7 +428,7 @@ TEST_F(TestBasicJitterBuffer, PacketReorderingReverseOrder) { frame_out = DecodeCompleteFrame();; - CheckOutFrame(frame_out, 100 * size_, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, 100 * size_, false)); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); } @@ -485,7 +484,7 @@ TEST_F(TestBasicJitterBuffer, FrameReordering2Frames2PacketsEach) { frame_out = DecodeCompleteFrame(); - CheckOutFrame(frame_out, 2 * size_, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, 2 * size_, false)); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); @@ -493,7 +492,7 @@ TEST_F(TestBasicJitterBuffer, FrameReordering2Frames2PacketsEach) { frame_out = DecodeCompleteFrame(); - CheckOutFrame(frame_out, 2 * size_, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, 2 * size_, false)); EXPECT_EQ(kVideoFrameDelta, frame_out->FrameType()); } @@ -528,7 +527,7 @@ TEST_F(TestBasicJitterBuffer, DuplicatePackets) { frame_out = DecodeCompleteFrame(); - CheckOutFrame(frame_out, 2 * size_, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, 2 * size_, false)); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); } @@ -560,7 +559,7 @@ TEST_F(TestBasicJitterBuffer, H264InsertStartCode) { frame_out = DecodeCompleteFrame(); - CheckOutFrame(frame_out, size_ * 2 + 4 * 2, true); + EXPECT_EQ(0, CheckOutFrame(frame_out, size_ * 2 + 4 * 2, true)); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); } @@ -632,7 +631,7 @@ TEST_F(TestBasicJitterBuffer, PacketLoss) { // One of the packets has been discarded by the jitter buffer. // Last frame can't be extracted yet. if (i < 10) { - CheckOutFrame(frame_out, size_, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, size_, false)); if (i == 0) { EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); @@ -717,7 +716,7 @@ TEST_F(TestBasicJitterBuffer, DeltaFrame100PacketsWithSeqNumWrap) { frame_out = DecodeCompleteFrame(); - CheckOutFrame(frame_out, 100 * size_, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, 100 * size_, false)); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); } @@ -766,7 +765,7 @@ TEST_F(TestBasicJitterBuffer, PacketReorderingReverseWithNegSeqNumWrap) { &retransmitted)); frame_out = DecodeCompleteFrame(); - CheckOutFrame(frame_out, 100 * size_, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, 100 * size_, false)); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); } @@ -790,7 +789,7 @@ TEST_F(TestBasicJitterBuffer, TestInsertOldFrame) { VCMEncodedFrame* frame_out = DecodeCompleteFrame(); EXPECT_EQ(3000u, frame_out->TimeStamp()); - CheckOutFrame(frame_out, size_, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, size_, false)); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); @@ -829,7 +828,7 @@ TEST_F(TestBasicJitterBuffer, TestInsertOldFrameWithSeqNumWrap) { VCMEncodedFrame* frame_out = DecodeCompleteFrame(); EXPECT_EQ(timestamp_, frame_out->TimeStamp()); - CheckOutFrame(frame_out, size_, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, size_, false)); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); @@ -880,7 +879,7 @@ TEST_F(TestBasicJitterBuffer, TimestampWrap) { frame_out = DecodeCompleteFrame(); - CheckOutFrame(frame_out, 2 * size_, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, 2 * size_, false)); jitter_buffer_->ReleaseFrame(frame_out); @@ -909,7 +908,7 @@ TEST_F(TestBasicJitterBuffer, TimestampWrap) { frame_out = DecodeCompleteFrame(); - CheckOutFrame(frame_out, 2 * size_, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, 2 * size_, false)); EXPECT_EQ(kVideoFrameDelta, frame_out->FrameType()); } @@ -946,14 +945,14 @@ TEST_F(TestBasicJitterBuffer, 2FrameWithTimestampWrap) { VCMEncodedFrame* frame_out = DecodeCompleteFrame(); EXPECT_EQ(0xffffff00, frame_out->TimeStamp()); - CheckOutFrame(frame_out, size_, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, size_, false)); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); VCMEncodedFrame* frame_out2 = DecodeCompleteFrame(); EXPECT_EQ(2700u, frame_out2->TimeStamp()); - CheckOutFrame(frame_out2, size_, false); + EXPECT_EQ(0, CheckOutFrame(frame_out2, size_, false)); EXPECT_EQ(kVideoFrameDelta, frame_out2->FrameType()); } @@ -991,14 +990,14 @@ TEST_F(TestBasicJitterBuffer, Insert2FramesReOrderedWithTimestampWrap) { VCMEncodedFrame* frame_out = DecodeCompleteFrame(); EXPECT_EQ(0xffffff00, frame_out->TimeStamp()); - CheckOutFrame(frame_out, size_, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, size_, false)); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); VCMEncodedFrame* frame_out2 = DecodeCompleteFrame(); EXPECT_EQ(2700u, frame_out2->TimeStamp()); - CheckOutFrame(frame_out2, size_, false); + EXPECT_EQ(0, CheckOutFrame(frame_out2, size_, false)); EXPECT_EQ(kVideoFrameDelta, frame_out2->FrameType()); } @@ -1091,7 +1090,7 @@ TEST_F(TestBasicJitterBuffer, ExceedNumOfFrameWithSeqNumWrap) { VCMEncodedFrame* frame_out = DecodeCompleteFrame(); EXPECT_EQ(first_key_frame_timestamp, frame_out->TimeStamp()); - CheckOutFrame(frame_out, size_, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, size_, false)); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); } @@ -1112,8 +1111,8 @@ TEST_F(TestBasicJitterBuffer, EmptyLastFrame) { packet_->timestamp = timestamp_; packet_->frameType = kFrameEmpty; - EXPECT_EQ(kNoError, jitter_buffer_->InsertPacket(*packet_, - &retransmitted)); + EXPECT_EQ(kFirstPacket, jitter_buffer_->InsertPacket(*packet_, + &retransmitted)); VCMEncodedFrame* testFrame = DecodeIncompleteFrame(); // Timestamp should never be the last TS inserted. if (testFrame != NULL) { @@ -1185,7 +1184,7 @@ TEST_F(TestBasicJitterBuffer, H264IncompleteNalu) { // We can decode everything from a NALU until a packet has been lost. // Thus we can decode the first packet of the first NALU and the second NALU // which consists of one packet. - CheckOutFrame(frame_out, packet_->sizeBytes * 2, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, packet_->sizeBytes * 2, false)); jitter_buffer_->ReleaseFrame(frame_out); // Test reordered start frame + 1 lost. @@ -1252,7 +1251,7 @@ TEST_F(TestBasicJitterBuffer, H264IncompleteNalu) { // is the last. frame_out = DecodeIncompleteFrame(); // Only last NALU is complete. - CheckOutFrame(frame_out, insertedLength, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, insertedLength, false)); jitter_buffer_->ReleaseFrame(frame_out); @@ -1303,7 +1302,7 @@ TEST_F(TestBasicJitterBuffer, H264IncompleteNalu) { frame_out = DecodeCompleteFrame(); // Only last NALU is complete - CheckOutFrame(frame_out, packet_->sizeBytes, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, packet_->sizeBytes, false)); } TEST_F(TestBasicJitterBuffer, NextFrameWhenIncomplete) { @@ -1345,7 +1344,7 @@ TEST_F(TestBasicJitterBuffer, NextFrameWhenIncomplete) { frame_out = DecodeIncompleteFrame(); - CheckOutFrame(frame_out, packet_->sizeBytes, false); + EXPECT_EQ(0, CheckOutFrame(frame_out, packet_->sizeBytes, false)); } TEST_F(TestRunningJitterBuffer, Full) { @@ -1367,19 +1366,16 @@ TEST_F(TestRunningJitterBuffer, Full) { TEST_F(TestRunningJitterBuffer, EmptyPackets) { // Make sure a frame can get complete even though empty packets are missing. stream_generator_->GenerateFrame(kVideoFrameKey, 3, 3, - clock_->TimeInMilliseconds()); + clock_->TimeInMilliseconds()); bool request_key_frame = false; - // Insert empty packet. - EXPECT_EQ(kNoError, InsertPacketAndPop(4)); + EXPECT_EQ(kFirstPacket, InsertPacketAndPop(4)); EXPECT_FALSE(request_key_frame); - // Insert 3 media packets. - EXPECT_EQ(kFirstPacket, InsertPacketAndPop(0)); + EXPECT_EQ(kIncomplete, InsertPacketAndPop(4)); EXPECT_FALSE(request_key_frame); EXPECT_EQ(kIncomplete, InsertPacketAndPop(0)); EXPECT_FALSE(request_key_frame); - EXPECT_EQ(kCompleteSession, InsertPacketAndPop(0)); + EXPECT_EQ(kIncomplete, InsertPacketAndPop(0)); EXPECT_FALSE(request_key_frame); - // Insert empty packet. EXPECT_EQ(kCompleteSession, InsertPacketAndPop(0)); EXPECT_FALSE(request_key_frame); } @@ -1486,22 +1482,6 @@ TEST_F(TestRunningJitterBuffer, KeyDeltaKeyDelta) { } } -TEST_F(TestRunningJitterBuffer, TwoPacketsNonContinuous) { - InsertFrame(kVideoFrameKey); - EXPECT_TRUE(DecodeCompleteFrame()); - stream_generator_->GenerateFrame(kVideoFrameDelta, 1, 0, - clock_->TimeInMilliseconds()); - clock_->AdvanceTimeMilliseconds(kDefaultFramePeriodMs); - stream_generator_->GenerateFrame(kVideoFrameDelta, 2, 0, - clock_->TimeInMilliseconds()); - EXPECT_EQ(kFirstPacket, InsertPacketAndPop(1)); - EXPECT_EQ(kCompleteSession, InsertPacketAndPop(1)); - EXPECT_FALSE(DecodeCompleteFrame()); - EXPECT_EQ(kCompleteSession, InsertPacketAndPop(0)); - EXPECT_TRUE(DecodeCompleteFrame()); - EXPECT_TRUE(DecodeCompleteFrame()); -} - TEST_F(TestJitterBufferNack, EmptyPackets) { // Make sure empty packets doesn't clog the jitter buffer. jitter_buffer_->SetNackMode(kNack, media_optimization::kLowRttNackMs, -1); @@ -1519,7 +1499,7 @@ TEST_F(TestJitterBufferNack, NackTooOldPackets) { // old packet. DropFrame(1); // Insert a frame which should trigger a recycle until the next key frame. - EXPECT_EQ(kFlushIndicator, InsertFrames(oldest_packet_to_nack_ + 1, + EXPECT_EQ(kFlushIndicator, InsertFrames(oldest_packet_to_nack_, kVideoFrameDelta)); EXPECT_FALSE(DecodeCompleteFrame()); @@ -1575,7 +1555,7 @@ TEST_F(TestJitterBufferNack, NackListFull) { EXPECT_TRUE(DecodeCompleteFrame()); // Generate and drop |kNackHistoryLength| packets to fill the NACK list. - DropFrame(max_nack_list_size_ + 1); + DropFrame(max_nack_list_size_); // Insert a frame which should trigger a recycle until the next key frame. EXPECT_EQ(kFlushIndicator, InsertFrame(kVideoFrameDelta)); EXPECT_FALSE(DecodeCompleteFrame()); @@ -1632,33 +1612,6 @@ TEST_F(TestJitterBufferNack, NackListBuiltBeforeFirstDecode) { EXPECT_TRUE(list != NULL); } -TEST_F(TestJitterBufferNack, VerifyRetransmittedFlag) { - stream_generator_->Init(0, 0, clock_->TimeInMilliseconds()); - stream_generator_->GenerateFrame(kVideoFrameKey, 3, 0, - clock_->TimeInMilliseconds()); - VCMPacket packet; - stream_generator_->PopPacket(&packet, 0); - bool retransmitted = false; - EXPECT_EQ(kFirstPacket, jitter_buffer_->InsertPacket(packet, &retransmitted)); - EXPECT_FALSE(retransmitted); - // Drop second packet. - stream_generator_->PopPacket(&packet, 1); - EXPECT_EQ(kIncomplete, jitter_buffer_->InsertPacket(packet, &retransmitted)); - EXPECT_FALSE(retransmitted); - EXPECT_FALSE(DecodeCompleteFrame()); - uint16_t nack_list_size = 0; - bool extended = false; - uint16_t* list = jitter_buffer_->GetNackList(&nack_list_size, &extended); - EXPECT_EQ(1, nack_list_size); - ASSERT_TRUE(list != NULL); - stream_generator_->PopPacket(&packet, 0); - EXPECT_EQ(packet.seqNum, list[0]); - EXPECT_EQ(kCompleteSession, jitter_buffer_->InsertPacket(packet, - &retransmitted)); - EXPECT_TRUE(retransmitted); - EXPECT_TRUE(DecodeCompleteFrame()); -} - TEST_F(TestJitterBufferNack, UseNackToRecoverFirstKeyFrame) { stream_generator_->Init(0, 0, clock_->TimeInMilliseconds()); stream_generator_->GenerateFrame(kVideoFrameKey, 3, 0, diff --git a/webrtc/modules/video_coding/main/source/stream_generator.cc b/webrtc/modules/video_coding/main/source/stream_generator.cc index 652b014061..a2da3e50c8 100644 --- a/webrtc/modules/video_coding/main/source/stream_generator.cc +++ b/webrtc/modules/video_coding/main/source/stream_generator.cc @@ -44,6 +44,10 @@ void StreamGenerator::GenerateFrame(FrameType type, int num_empty_packets, int64_t current_time) { timestamp_ = 90 * (current_time - start_time_); + // Move the sequence number counter if all packets from the previous frame + // wasn't collected. + sequence_number_ += packets_.size(); + packets_.clear(); for (int i = 0; i < num_media_packets; ++i) { const int packet_size = (kFrameSize + num_media_packets / 2) / num_media_packets; @@ -119,10 +123,6 @@ bool StreamGenerator::NextPacket(VCMPacket* packet) { return true; } -void StreamGenerator::DropLastPacket() { - packets_.pop_back(); -} - uint16_t StreamGenerator::NextSequenceNumber() const { if (packets_.empty()) return sequence_number_; diff --git a/webrtc/modules/video_coding/main/source/stream_generator.h b/webrtc/modules/video_coding/main/source/stream_generator.h index dd6deadaad..24eeccc9d7 100644 --- a/webrtc/modules/video_coding/main/source/stream_generator.h +++ b/webrtc/modules/video_coding/main/source/stream_generator.h @@ -46,7 +46,6 @@ class StreamGenerator { FrameType type); bool PopPacket(VCMPacket* packet, int index); - void DropLastPacket(); bool GetPacket(VCMPacket* packet, int index);