From cdc943e2d583816af4a98d3a6a449b6caf3ce9f4 Mon Sep 17 00:00:00 2001 From: "mikhal@google.com" Date: Fri, 1 Jul 2011 18:15:11 +0000 Subject: [PATCH] VCM: 1. Updating handling of empty packets. 2. Updating JB test. 3. Removing un-used code. Review URL: http://webrtc-codereview.appspot.com/59001 git-svn-id: http://webrtc.googlecode.com/svn/trunk@142 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../video_coding/main/source/frame_buffer.cc | 21 +- .../video_coding/main/source/jitter_buffer.cc | 66 ++--- .../main/source/jitter_buffer_common.h | 10 +- modules/video_coding/main/source/receiver.cc | 4 +- .../video_coding/main/source/session_info.cc | 227 ++++++++---------- .../video_coding/main/source/session_info.h | 6 +- .../main/test/jitter_buffer_test.cc | 118 ++++++++- 7 files changed, 272 insertions(+), 180 deletions(-) diff --git a/modules/video_coding/main/source/frame_buffer.cc b/modules/video_coding/main/source/frame_buffer.cc index bd8be3ef7d..df7b37b4c1 100644 --- a/modules/video_coding/main/source/frame_buffer.cc +++ b/modules/video_coding/main/source/frame_buffer.cc @@ -113,8 +113,9 @@ VCMFrameBuffer::InsertPacket(const VCMPacket& packet, WebRtc_Word64 timeInMs) } // sanity checks - if (_size + packet.sizeBytes + (packet.insertStartCode?kH264StartCodeLengthBytes:0) > - kMaxJBFrameSizeBytes) + if (_size + packet.sizeBytes + + (packet.insertStartCode ? kH264StartCodeLengthBytes : 0 ) + > kMaxJBFrameSizeBytes) { return kSizeError; } @@ -122,7 +123,8 @@ VCMFrameBuffer::InsertPacket(const VCMPacket& packet, WebRtc_Word64 timeInMs) { return kSizeError; } - if(!_sessionInfo.HaveStartSeqNumber()) + if ((packet.frameType != kFrameEmpty) && + (!_sessionInfo.HaveStartSeqNumber())) { _sessionInfo.SetStartSeqNumber(packet.seqNum); } @@ -133,13 +135,13 @@ VCMFrameBuffer::InsertPacket(const VCMPacket& packet, WebRtc_Word64 timeInMs) if (kStateEmpty == _state) { - // This is the first packet (empty and/or data) inserted into this frame. + // First packet (empty and/or media) inserted into this frame. // store some info and set some initial values. _timeStamp = packet.timestamp; _codec = packet.codec; - // for the first media packet if (packet.frameType != kFrameEmpty) { + // first media packet SetState(kStateIncomplete); } } @@ -148,9 +150,12 @@ VCMFrameBuffer::InsertPacket(const VCMPacket& packet, WebRtc_Word64 timeInMs) (packet.insertStartCode ? kH264StartCodeLengthBytes : 0); if (requiredSizeBytes >= _size) { - const WebRtc_UWord32 increments = requiredSizeBytes / kBufferIncStepSizeBytes + - (requiredSizeBytes % kBufferIncStepSizeBytes > 0); - const WebRtc_UWord32 newSize = _size + increments * kBufferIncStepSizeBytes; + const WebRtc_UWord32 increments = requiredSizeBytes / + kBufferIncStepSizeBytes + + (requiredSizeBytes % + kBufferIncStepSizeBytes > 0); + const WebRtc_UWord32 newSize = _size + + increments * kBufferIncStepSizeBytes; if (newSize > kMaxJBFrameSizeBytes) { return kSizeError; diff --git a/modules/video_coding/main/source/jitter_buffer.cc b/modules/video_coding/main/source/jitter_buffer.cc index 746432d7e6..d4d7ab9aac 100644 --- a/modules/video_coding/main/source/jitter_buffer.cc +++ b/modules/video_coding/main/source/jitter_buffer.cc @@ -666,7 +666,7 @@ VCMJitterBuffer::CheckForCompleteFrame(VCMFrameListItem* oldestFrameItem) // Verify that we have received the first packet of the next frame. // This is the only way we can be sure we're not missing the last packet. if (nextFrame != NULL && nextFrame->GetLowSeqNum() == - static_cast(oldestFrame->GetHighSeqNum()+1)) // Sequence number is only 16 bit + static_cast(oldestFrame->GetHighSeqNum() + 1)) { _missingMarkerBits = true; bool completeSession = oldestFrame->ForceSetHaveLastPacket(); @@ -800,7 +800,8 @@ VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS) _critSect.Leave(); return NULL; } - const WebRtc_Word64 endWaitTimeMs = VCMTickTime::MillisecondTimestamp() + maxWaitTimeMS; + const WebRtc_Word64 endWaitTimeMs = VCMTickTime::MillisecondTimestamp() + + maxWaitTimeMS; WebRtc_Word64 waitTimeMs = maxWaitTimeMS; while (waitTimeMs > 0) { @@ -849,7 +850,7 @@ VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS) if (oldestFrame == NULL) { - // Even after signalling we're still missing a complete _continuous_ frame + // Even after signaling we're still missing a complete _continuous_ frame _critSect.Leave(); return NULL; } @@ -1269,7 +1270,8 @@ VCMJitterBuffer::GetNackList(WebRtc_UWord16& nackSize,bool& listExtended) // Assume called internally with critsect WebRtc_Word32 -VCMJitterBuffer::GetLowHighSequenceNumbers(WebRtc_Word32& lowSeqNum, WebRtc_Word32& highSeqNum) const +VCMJitterBuffer::GetLowHighSequenceNumbers(WebRtc_Word32& lowSeqNum, + WebRtc_Word32& highSeqNum) const { int i = 0; int seqNum = -1; @@ -1277,7 +1279,7 @@ VCMJitterBuffer::GetLowHighSequenceNumbers(WebRtc_Word32& lowSeqNum, WebRtc_Word highSeqNum = -1; lowSeqNum = _lastDecodedSeqNum; - // find highest seqnumbers + // find highest seq numbers for (i = 0; i < _maxNumberOfFrames; ++i) { seqNum = _frameBuffers[i]->GetHighSeqNum(); @@ -1342,11 +1344,13 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended) //This happens if we lose the first packet, nothing is popped if (highSeqNum == -1) { - nackSize = 0;// we have not received any packets yet + // we have not received any packets yet + nackSize = 0; } else { - nackSize = 0xffff; // signal that we want a key frame request to be sent + // signal that we want a key frame request to be sent + nackSize = 0xffff; } return NULL; } @@ -1368,9 +1372,10 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended) if (numberOfSeqNum > kNackHistoryLength) { // Nack list is too big, flush and try to restart. - WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), - "Nack list too large, try to find a key frame and restart from seq: %d." - " Lowest seq in jb %d", highSeqNum,lowSeqNum); + WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding, + VCMId(_vcmId, _receiverId), + "Nack list too large, try to find a key frame and restart " + "from seq: %d. Lowest seq in jb %d", highSeqNum,lowSeqNum); // This nack size will trigger a key request... bool foundIFrame = false; @@ -1463,8 +1468,8 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended) (kStateDecoding != state)) { // Reaching thus far means we are going to update the nack list - // When in hybrid mode, we also need to check empty frames, so as not - // to add empty packets to the nack list + // When in hybrid mode, we also need to check empty frames, so as + // not to add empty packets to the nack list if (_nackMode == kNackHybrid) { // build external rttScore based on RTT value @@ -1534,7 +1539,7 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended) listExtended = true; } - for(WebRtc_UWord32 j = 0; j < nackSize; j++) + for (WebRtc_UWord32 j = 0; j < nackSize; j++) { // Check if the list has been extended since it was last created. I.e, // new items have been added @@ -1543,7 +1548,7 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended) WebRtc_UWord32 k = 0; for (k = j; k < _NACKSeqNumLength; k++) { - // Found the item in the last list. I.e, no new items found yet. + // Found the item in the last list, i.e, no new items found yet. if (_NACKSeqNum[k] == (WebRtc_UWord16)_NACKSeqNumInternal[j]) { break; @@ -1727,7 +1732,8 @@ void VCMJitterBuffer::UpdateOldJitterSample(const VCMPacket& packet) { if (_waitingForCompletion.timestamp != packet.timestamp && - LatestTimestamp(_waitingForCompletion.timestamp, packet.timestamp) == packet.timestamp) + LatestTimestamp(_waitingForCompletion.timestamp, packet.timestamp) == + packet.timestamp) { // This is a newer frame than the one waiting for completion. _waitingForCompletion.frameSize = packet.sizeBytes; @@ -1798,7 +1804,10 @@ VCMJitterBuffer::RecycleFramesUntilKeyFrame() { // Throw at least one frame. _dropCount++; - WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), "Jitter buffer drop count:%d, lowSeq %d", _dropCount, oldestFrame->GetLowSeqNum()); + WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding, + VCMId(_vcmId, _receiverId), + "Jitter buffer drop count:%d, lowSeq %d", _dropCount, + oldestFrame->GetLowSeqNum()); _frameBuffersTSOrder.Erase(oldestFrameListItem); RecycleFrame(oldestFrame); @@ -1810,12 +1819,15 @@ VCMJitterBuffer::RecycleFramesUntilKeyFrame() if(oldestFrame != NULL) { - foundIFrame = foundIFrame || (oldestFrame->FrameType() != kVideoFrameDelta); + foundIFrame = foundIFrame || + (oldestFrame->FrameType() != kVideoFrameDelta); if (foundIFrame) { // fake the last played out to match the start of this key frame - _lastDecodedSeqNum = (WebRtc_UWord16)((WebRtc_UWord16)(oldestFrame->GetLowSeqNum()) - 1); - _lastDecodedTimeStamp = (WebRtc_UWord32)(oldestFrame->TimeStamp() - 1); + _lastDecodedSeqNum = (WebRtc_UWord16)((WebRtc_UWord16) + (oldestFrame->GetLowSeqNum()) - 1); + _lastDecodedTimeStamp = (WebRtc_UWord32) + (oldestFrame->TimeStamp() - 1); break; } } @@ -1843,7 +1855,8 @@ VCMJitterBuffer::CleanUpOldFrames() // Release the frame if it's older than the last decoded frame. if (_lastDecodedTimeStamp > -1 && - LatestTimestamp(static_cast(_lastDecodedTimeStamp), frameTimeStamp) + LatestTimestamp(static_cast(_lastDecodedTimeStamp), + frameTimeStamp) == static_cast(_lastDecodedTimeStamp)) { const WebRtc_Word32 frameLowSeqNum = oldestFrame->GetLowSeqNum(); @@ -1855,7 +1868,8 @@ VCMJitterBuffer::CleanUpOldFrames() { // Could happen when sending filler data. // Filler packet (size = 0) belonging to last decoded frame. - // Frame: | packet | packet | packet M=1 | filler data (size = 0) | filler data (size = 0)| ... + // Frame: | packet | packet | packet M=1 | + // filler data (size = 0) | filler data (size = 0)| ... // This frame follows the last decoded frame _lastDecodedSeqNum = frameHighSeqNum; @@ -1966,7 +1980,8 @@ VCMJitterBuffer::VerifyAndSetPreviousFrameLost(VCMFrameBuffer& frame) // First frame frame.SetPreviousFrameLoss(); } - else if (frame.GetLowSeqNum() != ((WebRtc_UWord16)_lastDecodedSeqNum + (WebRtc_UWord16)1)) + else if (frame.GetLowSeqNum() != ((WebRtc_UWord16)_lastDecodedSeqNum + + (WebRtc_UWord16)1)) { // Frame loss frame.SetPreviousFrameLoss(); @@ -1992,12 +2007,7 @@ VCMJitterBuffer::WaitForNack() { return false; } - // RTT low, we can afford the wait - else if (_rttMs <= kLowRttNackMs) - { - return true; - } - // interim values - hybrid mode + // Either NACK only or hybrid return true; } diff --git a/modules/video_coding/main/source/jitter_buffer_common.h b/modules/video_coding/main/source/jitter_buffer_common.h index 6f7bcad6f2..035a3c1178 100644 --- a/modules/video_coding/main/source/jitter_buffer_common.h +++ b/modules/video_coding/main/source/jitter_buffer_common.h @@ -55,11 +55,11 @@ enum { kH264StartCodeLengthBytes = 4}; // Used to indicate if a received packet contain a complete NALU (or equivalent) enum VCMNaluCompleteness { - kNaluUnset=0, //Packet has not been filled. - kNaluComplete=1, //Packet can be decoded as is. - kNaluStart, // Packet contain beginning of NALU - kNaluIncomplete, //Packet is not beginning or end of NALU - kNaluEnd // Packet is the end of a NALU + kNaluUnset = 0, //Packet has not been filled. + kNaluComplete = 1, //Packet can be decoded as is. + kNaluStart, // Packet contain beginning of NALU + kNaluIncomplete, //Packet is not beginning or end of NALU + kNaluEnd, // Packet is the end of a NALU }; } // namespace webrtc diff --git a/modules/video_coding/main/source/receiver.cc b/modules/video_coding/main/source/receiver.cc index 113d878e57..676bab88a2 100644 --- a/modules/video_coding/main/source/receiver.cc +++ b/modules/video_coding/main/source/receiver.cc @@ -113,7 +113,7 @@ VCMReceiver::InsertPacket(const VCMPacket& packet, WebRtc_Word64 renderTimeMs = _timing.RenderTimeMs(packet.timestamp, nowMs); - if(renderTimeMs < 0) + if (renderTimeMs < 0) { // Render time error. Assume that this is due to some change in // the incoming video stream and reset the JB and the timing. @@ -163,7 +163,7 @@ VCMReceiver::InsertPacket(const VCMPacket& packet, } // Insert packet into jitter buffer - // both data and empty packets + // both media and empty packets const VCMFrameBufferEnum ret = _jitterBuffer.InsertPacket(buffer, packet); if (ret < 0) diff --git a/modules/video_coding/main/source/session_info.cc b/modules/video_coding/main/source/session_info.cc index 2624da8387..7145b29026 100644 --- a/modules/video_coding/main/source/session_info.cc +++ b/modules/video_coding/main/source/session_info.cc @@ -27,7 +27,8 @@ VCMSessionInfo::VCMSessionInfo(): _highSeqNum(-1), _highestPacketIndex(0), _emptySeqNumLow(-1), - _emptySeqNumHigh(-1) + _emptySeqNumHigh(-1), + _markerSeqNum(-1) { memset(_packetSizeBytes, 0, sizeof(_packetSizeBytes)); memset(_naluCompleteness, kNaluUnset, sizeof(_naluCompleteness)); @@ -47,6 +48,10 @@ VCMSessionInfo::GetLowSeqNum() const WebRtc_Word32 VCMSessionInfo::GetHighSeqNum() const { + if (_emptySeqNumHigh != -1) + { + return _emptySeqNumHigh; + } return _highSeqNum; } @@ -64,6 +69,7 @@ VCMSessionInfo::Reset() _previousFrameLoss = false; _sessionNACK = false; _highestPacketIndex = 0; + _markerSeqNum = -1; memset(_packetSizeBytes, 0, sizeof(_packetSizeBytes)); memset(_naluCompleteness, kNaluUnset, sizeof(_naluCompleteness)); memset(_ORwithPrevByte, 0, sizeof(_ORwithPrevByte)); @@ -89,7 +95,7 @@ VCMSessionInfo::SetStartSeqNumber(WebRtc_UWord16 seqNumber) bool VCMSessionInfo::HaveStartSeqNumber() { - if(_lowSeqNum == -1 || _highSeqNum == -1) + if (_lowSeqNum == -1 || _highSeqNum == -1) { return false; } @@ -118,7 +124,7 @@ VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, else { packetSize = packet.sizeBytes + - (packet.insertStartCode?kH264StartCodeLengthBytes:0); + (packet.insertStartCode ? kH264StartCodeLengthBytes : 0); } _packetSizeBytes[packetIndex] += packetSize; @@ -146,7 +152,8 @@ VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, _ORwithPrevByte[packetIndex] = true; if (packet.dataPtr != NULL) { - memcpy((void*)(ptrStartOfLayer + offset), packet.dataPtr, packetSize); + memcpy((void*)(ptrStartOfLayer + offset), packet.dataPtr, + packetSize); } returnLength = packetSize; } @@ -155,14 +162,14 @@ VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, _ORwithPrevByte[packetIndex] = false; if (packet.dataPtr != NULL) { - const unsigned char startCode[] = {0, 0, 0, 1}; + const unsigned char startCode[] = {0, 0, 0, 1}; if(packet.insertStartCode) { memcpy((void*)(ptrStartOfLayer + offset), startCode, kH264StartCodeLengthBytes); } memcpy((void*)(ptrStartOfLayer + offset - + (packet.insertStartCode?kH264StartCodeLengthBytes:0)), + + (packet.insertStartCode ? kH264StartCodeLengthBytes : 0)), packet.dataPtr, packet.sizeBytes); } @@ -172,16 +179,14 @@ VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, if (packet.isFirstPacket) { _haveFirstPacket = true; - //initializing FEC sequence numbers - _emptySeqNumHigh = -1; - _emptySeqNumLow = -1; } if (packet.markerBit) { _markerBit = true; + _markerSeqNum = packet.seqNum; } // Store information about if the packet is decodable as is or not. - _naluCompleteness[packetIndex]=packet.completeNALU; + _naluCompleteness[packetIndex] = packet.completeNALU; UpdateCompleteSession(); @@ -193,9 +198,10 @@ VCMSessionInfo::UpdateCompleteSession() { if (_haveFirstPacket && _markerBit) { - // do we have all packets in this session? + // Do we have all the packets in this session? bool completeSession = true; - for (int i = 0; i<= _highestPacketIndex; ++i) + + for (int i = 0; i <= _highestPacketIndex; ++i) { if (_naluCompleteness[i] == kNaluUnset) { @@ -213,13 +219,12 @@ bool VCMSessionInfo::IsSessionComplete() } // Find the start and end index of packetIndex packet. -// startIndex -1 if start not found endIndex=-1 if end index not found +// startIndex -1 if start not found endIndex = -1 if end index not found void VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex, WebRtc_Word32& startIndex, WebRtc_Word32& endIndex) { - if (_naluCompleteness[packetIndex] == kNaluStart || _naluCompleteness[packetIndex] == kNaluComplete) { @@ -230,10 +235,11 @@ VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex, for (startIndex = packetIndex - 1; startIndex >= 0; --startIndex) { - if( (_naluCompleteness[startIndex] == kNaluComplete && - _packetSizeBytes[startIndex] > 0) || - // Found previous NALU. - (_naluCompleteness[startIndex] == kNaluEnd && startIndex>0)) + if ((_naluCompleteness[startIndex] == kNaluComplete && + _packetSizeBytes[startIndex] > 0) || + // Found previous NALU. + (_naluCompleteness[startIndex] == kNaluEnd && + startIndex > 0)) { startIndex++; break; @@ -246,31 +252,35 @@ VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex, } } - if(_naluCompleteness[packetIndex] == kNaluEnd || + if (_naluCompleteness[packetIndex] == kNaluEnd || _naluCompleteness[packetIndex] == kNaluComplete) { - endIndex=packetIndex; + endIndex = packetIndex; } else { // Find the next NALU - for (endIndex=packetIndex+1;endIndex<=_highestPacketIndex;++endIndex) + for (endIndex = packetIndex + 1; endIndex <= _highestPacketIndex; + ++endIndex) { - if ((_naluCompleteness[endIndex]==kNaluComplete && - _packetSizeBytes[endIndex]>0) || - _naluCompleteness[endIndex]==kNaluStart) // Found next NALU. + if ((_naluCompleteness[endIndex] == kNaluComplete && + _packetSizeBytes[endIndex] > 0) || + // Found next NALU. + _naluCompleteness[endIndex] == kNaluStart) { endIndex--; break; } - if ( _naluCompleteness[endIndex]==kNaluEnd) + if ( _naluCompleteness[endIndex] == kNaluEnd) { // This is where the NALU end. break; } } if (endIndex > _highestPacketIndex) + { endIndex = -1; + } } } @@ -293,7 +303,7 @@ VCMSessionInfo::DeletePackets(WebRtc_UWord8* ptrStartOfLayer, { // Get the offset we want to move to. int destOffset = 0; - for(int j = 0;j < startIndex;j++) + for (int j = 0;j < startIndex;j++) { destOffset += _packetSizeBytes[j]; } @@ -318,15 +328,10 @@ VCMSessionInfo::DeletePackets(WebRtc_UWord8* ptrStartOfLayer, WebRtc_UWord32 VCMSessionInfo::MakeSessionDecodable(WebRtc_UWord8* ptrStartOfLayer) { - if(_lowSeqNum < 0) // No packets in this session + if (_lowSeqNum < 0) // No packets in this session { return 0; } - else if (_lowSeqNum == _emptySeqNumLow) - { - // no data packets in this session - return 0; - } WebRtc_Word32 startIndex = 0; WebRtc_Word32 endIndex = 0; @@ -336,7 +341,7 @@ VCMSessionInfo::MakeSessionDecodable(WebRtc_UWord8* ptrStartOfLayer) { if (_naluCompleteness[packetIndex] == kNaluUnset) // Found a lost packet { - FindNaluBorder(packetIndex,startIndex,endIndex); + FindNaluBorder(packetIndex, startIndex, endIndex); if (startIndex == -1) { startIndex = 0; @@ -346,34 +351,37 @@ VCMSessionInfo::MakeSessionDecodable(WebRtc_UWord8* ptrStartOfLayer) endIndex = _highestPacketIndex; } - returnLength += DeletePackets(ptrStartOfLayer,packetIndex,endIndex); + returnLength += DeletePackets(ptrStartOfLayer, + packetIndex, endIndex); packetIndex = endIndex; }// end lost packet } - //Make sure the first packet is decodable (Either complete nalu or start of NALU) + // Make sure the first packet is decodable (Either complete nalu or start + // of NALU) if (_packetSizeBytes[0] > 0) { switch (_naluCompleteness[0]) { - case kNaluComplete: //Packet can be decoded as is. + case kNaluComplete: // Packet can be decoded as is. break; - case kNaluStart: // Packet contain beginning of NALU- No need to do anything. + case kNaluStart: + // Packet contain beginning of NALU- No need to do anything. break; case kNaluIncomplete: //Packet is not beginning or end of NALU - //Need to find the end of this fua NALU and delete all packets. + // Need to find the end of this NALU and delete all packets. FindNaluBorder(0,startIndex,endIndex); - if(endIndex == -1) // No end found. Delete + if (endIndex == -1) // No end found. Delete { endIndex = _highestPacketIndex; } - //Delete this NALU. - returnLength += DeletePackets(ptrStartOfLayer,0,endIndex); + // Delete this NALU. + returnLength += DeletePackets(ptrStartOfLayer, 0, endIndex); break; case kNaluEnd: // Packet is the end of a NALU - //Delete this NALU - returnLength += DeletePackets(ptrStartOfLayer,0,0); + // Delete this NALU + returnLength += DeletePackets(ptrStartOfLayer, 0, 0); break; default: assert(false); @@ -441,7 +449,7 @@ VCMSessionInfo::ZeroOutSeqNumHybrid(WebRtc_Word32* list, } if (_lowSeqNum == -1) { - // no packets in this frame + // no media packets in this frame return 0; } @@ -478,8 +486,17 @@ VCMSessionInfo::ZeroOutSeqNumHybrid(WebRtc_Word32* list, int i = 0; // Score place holder - based on RTT and partition (when available). const float nackScoreTh = 0.25f; - WebRtc_Word32 highMediaPacket = _emptySeqNumLow > _lowSeqNum ? - _emptySeqNumLow - 1: _highSeqNum; + + WebRtc_Word32 highMediaPacket; + if (_markerSeqNum != -1) + { + highMediaPacket = _markerSeqNum; + } + else + { + highMediaPacket = _emptySeqNumLow - 1 > _highSeqNum ? + _emptySeqNumLow - 1: _highSeqNum; + } while (list[index] <= highMediaPacket && index < numberOfSeqNum) { @@ -569,66 +586,12 @@ VCMSessionInfo::UpdatePacketSize(WebRtc_Word32 packetIndex, _packetSizeBytes[packetIndex] = length; } -void -VCMSessionInfo::PrependPacketIndices(WebRtc_Word32 numberOfPacketIndices) -{ - // sanity - if ((numberOfPacketIndices + - GetHighestPacketIndex() >= kMaxPacketsInJitterBuffer) - || numberOfPacketIndices < 0) - { - // not allowed - assert(!"SessionInfo::PrependPacketIndexes Error: invalid packetIndex"); - return; - } - // Works if we have new packets before packetIndex = 0 - int numOfPacketsToMove = GetHighestPacketIndex()+1; - memmove(&_packetSizeBytes[numberOfPacketIndices], &_packetSizeBytes[0], - (numOfPacketsToMove)*sizeof(WebRtc_UWord16)); - memset(&_packetSizeBytes[0], 0, numberOfPacketIndices*sizeof(WebRtc_UWord16)); - - _highestPacketIndex += (WebRtc_UWord16)numberOfPacketIndices; -} - -void -VCMSessionInfo::ClearPacketSize(WebRtc_Word32 packetIndex) -{ - // sanity - if (packetIndex >= kMaxPacketsInJitterBuffer || packetIndex < 0) - { - // not allowed - assert(!"SessionInfo::ClearPacketSize Error: invalid packetIndex"); - return; - } - _packetSizeBytes[packetIndex] = 0; -} - -WebRtc_UWord32 -VCMSessionInfo::GetPacketSize(WebRtc_Word32 packetIndex) -{ - // sanity - if (packetIndex >= kMaxPacketsInJitterBuffer || packetIndex < 0) - { - //not allowed - assert(!"SessionInfo::GetPacketSize Error: invalid packetIndex"); - return 0; - } - return _packetSizeBytes[packetIndex]; -} - WebRtc_Word64 VCMSessionInfo::InsertPacket(const VCMPacket& packet, WebRtc_UWord8* ptrStartOfLayer) { // not allowed assert(!packet.insertStartCode || !packet.bits); - - if (packet.frameType == kFrameEmpty) - { - // update seq number as an empty packet - // empty packets will be counted twice: both empty and standard packets. - InformOfEmptyPacket(packet.seqNum); - } // Check if this is first packet (only valid for some codecs) if (packet.isFirstPacket) { @@ -637,12 +600,18 @@ VCMSessionInfo::InsertPacket(const VCMPacket& packet, } else if (_frameType == kFrameEmpty && packet.frameType != kFrameEmpty) { - // in case an empty packet came in first, update the frame type + // Update the frame type with the first media packet _frameType = packet.frameType; } + if (packet.frameType == kFrameEmpty) + { + // update seq number as an empty packet + InformOfEmptyPacket(packet.seqNum); + return 0; + } - // Check sequence number and update highest and lowest sequence numbers received. - // Move data if this seq num is lower than previously lowest. + // Check sequence number and update highest and lowest sequence numbers + // received. Move data if this seq num is lower than previously lowest. if (packet.seqNum > _highSeqNum) { @@ -672,19 +641,22 @@ VCMSessionInfo::InsertPacket(const VCMPacket& packet, + (WebRtc_UWord16)1; } else { - // This packet's seq num is lower than previously lowest seq num, but no wrap - // We need to move the data in all arrays indexed by packetIndex and insert the new - // packet's info + // This packet's seq num is lower than previously lowest seq num, + // but no wrap We need to move the data in all arrays indexed by + // packetIndex and insert the new packet's info // How many packets should we leave room for (positions to shift)? // Example - this seq num is 3 lower than previously lowest seq num // Before: |--prev packet with lowest seq num--|--|...| - // After: |--new lowest seq num--|--|--|--prev packet with lowest seq num--|--|...| + // After: |--new lowest seq num--|--|--|--prev packet with + // lowest seq num--|--|...| - WebRtc_UWord16 positionsToShift = (WebRtc_UWord16)_lowSeqNum - packet.seqNum; + WebRtc_UWord16 positionsToShift = (WebRtc_UWord16)_lowSeqNum - + packet.seqNum; WebRtc_UWord16 numOfPacketsToMove = _highestPacketIndex + 1; // sanity, do we have room for the shift? - if ((positionsToShift + numOfPacketsToMove) > kMaxPacketsInJitterBuffer) + if ((positionsToShift + numOfPacketsToMove) > + kMaxPacketsInJitterBuffer) { return -1; } @@ -696,13 +668,17 @@ VCMSessionInfo::InsertPacket(const VCMPacket& packet, // Shift _packetSizeBytes array memmove(&_packetSizeBytes[positionsToShift], - &_packetSizeBytes[0], numOfPacketsToMove*sizeof(WebRtc_UWord32)); - memset(&_packetSizeBytes[0], 0, positionsToShift*sizeof(WebRtc_UWord32)); + &_packetSizeBytes[0], + numOfPacketsToMove * sizeof(WebRtc_UWord32)); + memset(&_packetSizeBytes[0], 0, + positionsToShift * sizeof(WebRtc_UWord32)); - //Shift _naluCompleteness + // Shift _naluCompleteness memmove(&_naluCompleteness[positionsToShift], - &_naluCompleteness[0], numOfPacketsToMove*sizeof(WebRtc_UWord8)); - memset(&_naluCompleteness[0], kNaluUnset, positionsToShift*sizeof(WebRtc_UWord8)); + &_naluCompleteness[0], + numOfPacketsToMove * sizeof(WebRtc_UWord8)); + memset(&_naluCompleteness[0], kNaluUnset, + positionsToShift * sizeof(WebRtc_UWord8)); _highestPacketIndex += positionsToShift; _lowSeqNum = packet.seqNum; @@ -723,7 +699,7 @@ VCMSessionInfo::InsertPacket(const VCMPacket& packet, // Check for duplicate packets if (_packetSizeBytes[packetIndex] != 0) { - // We have already received a packet with this sequence number, ignore it. + // We have already received a packet with this seq number, ignore it. return -2; } @@ -743,7 +719,7 @@ VCMSessionInfo::InformOfEmptyPacket(const WebRtc_UWord16 seqNum) // and low sequence numbers and may assume that the packets in between are // empty packets belonging to the same frame (timestamp). - if (_emptySeqNumLow == -1 && _emptySeqNumHigh == -1) + if (_emptySeqNumLow == -1 && _emptySeqNumHigh == -1) { _emptySeqNumLow = seqNum; _emptySeqNumHigh = seqNum; @@ -795,23 +771,27 @@ VCMSessionInfo::PrepareForDecode(WebRtc_UWord8* ptrStartOfLayer, { if (currentPacketOffset > 0) { - WebRtc_UWord8* ptrFirstByte = ptrStartOfLayer + currentPacketOffset; + WebRtc_UWord8* ptrFirstByte = ptrStartOfLayer + + currentPacketOffset; if (_packetSizeBytes[i-1] == 0 || previousLost) { - // It is be better to throw away this packet if we are missing the - // previous packet. + // It is be better to throw away this packet if we are + // missing the previous packet. memset(ptrFirstByte, 0, _packetSizeBytes[i]); previousLost = true; } else if (_packetSizeBytes[i] > 0) // Ignore if empty packet { // Glue with previous byte - // Move everything from [this packet start + 1, end of buffer] one byte to the left + // Move everything from [this packet start + 1, + // end of buffer] one byte to the left WebRtc_UWord8* ptrPrevByte = ptrFirstByte - 1; *ptrPrevByte = (*ptrPrevByte) | (*ptrFirstByte); - WebRtc_UWord32 lengthToEnd = length - (currentPacketOffset + 1); - memmove((void*)ptrFirstByte, (void*)(ptrFirstByte + 1), lengthToEnd); + WebRtc_UWord32 lengthToEnd = length - + (currentPacketOffset + 1); + memmove((void*)ptrFirstByte, (void*)(ptrFirstByte + 1), + lengthToEnd); _packetSizeBytes[i]--; length--; previousLost = false; @@ -827,7 +807,8 @@ VCMSessionInfo::PrepareForDecode(WebRtc_UWord8* ptrStartOfLayer, else if (_packetSizeBytes[i] == 0 && codec == kVideoCodecH263) { WebRtc_UWord8* ptrFirstByte = ptrStartOfLayer + currentPacketOffset; - memmove(ptrFirstByte + 10, ptrFirstByte, length - currentPacketOffset); + memmove(ptrFirstByte + 10, ptrFirstByte, + length - currentPacketOffset); memset(ptrFirstByte, 0, 10); _packetSizeBytes[i] = 10; length += _packetSizeBytes[i]; diff --git a/modules/video_coding/main/source/session_info.h b/modules/video_coding/main/source/session_info.h index adbcf43e8f..ff71defe5a 100644 --- a/modules/video_coding/main/source/session_info.h +++ b/modules/video_coding/main/source/session_info.h @@ -47,16 +47,14 @@ public: webrtc::FrameType FrameType() const { return _frameType; } virtual WebRtc_Word32 GetHighestPacketIndex(); - virtual WebRtc_UWord32 GetPacketSize(WebRtc_Word32 packetIndex); - virtual void ClearPacketSize(WebRtc_Word32 packetIndex); virtual void UpdatePacketSize(WebRtc_Word32 packetIndex, WebRtc_UWord32 length); - virtual void PrependPacketIndices(WebRtc_Word32 numberOfPacketIndexes); void SetStartSeqNumber(WebRtc_UWord16 seqNumber); bool HaveStartSeqNumber(); WebRtc_Word32 GetLowSeqNum() const; + // returns highest seqNum, media or empty WebRtc_Word32 GetHighSeqNum() const; WebRtc_UWord32 PrepareForDecode(WebRtc_UWord8* ptrStartOfLayer, VideoCodecType codec); @@ -94,6 +92,8 @@ protected: WebRtc_UWord8 _naluCompleteness[kMaxPacketsInJitterBuffer]; WebRtc_Word32 _emptySeqNumLow; WebRtc_Word32 _emptySeqNumHigh; + // Store the sequence number that marks the last media packet + WebRtc_Word32 _markerSeqNum; bool _ORwithPrevByte[kMaxPacketsInJitterBuffer]; }; diff --git a/modules/video_coding/main/test/jitter_buffer_test.cc b/modules/video_coding/main/test/jitter_buffer_test.cc index effc5b100c..87d708efcf 100644 --- a/modules/video_coding/main/test/jitter_buffer_test.cc +++ b/modules/video_coding/main/test/jitter_buffer_test.cc @@ -1786,7 +1786,8 @@ int JitterBufferTest(CmdArgs& args) //printf("DONE fill JB - number of delta frames > max number of frames\n"); // - // TEST fill JB with more than max number of frame (50 delta frames + 51 key frames) with wrap in seqNum + // TEST fill JB with more than max number of frame (50 delta frames + + // 51 key frames) with wrap in seqNum // // -------------------------------------------------------------- // | 65485 | 65486 | 65487 | .... | 65535 | 0 | 1 | 2 | .....| 50 | @@ -1829,7 +1830,8 @@ int JitterBufferTest(CmdArgs& args) TEST(kFirstPacket == jb.InsertPacket(frameIn, packet)); // Get packet notification, should be first inserted frame - TEST(timeStampStart == jb.GetNextTimeStamp(10, incomingFrameType, renderTimeMs)); + TEST(timeStampStart == jb.GetNextTimeStamp(10, incomingFrameType, + renderTimeMs)); // check incoming frame type TEST(incomingFrameType == kVideoFrameDelta); @@ -1849,13 +1851,15 @@ int JitterBufferTest(CmdArgs& args) // Now, no free frame - frames will be recycled until first key frame frameIn = jb.GetFrame(packet); - TEST(frameIn != 0 && frameIn && ptrLastDeltaFrame); // ptr to last inserted delta frame should be returned + // ptr to last inserted delta frame should be returned + TEST(frameIn != 0 && frameIn && ptrLastDeltaFrame); // Insert frame TEST(kFirstPacket == jb.InsertPacket(frameIn, packet)); // First inserted key frame should be oldest in buffer - TEST(timeStampFirstKey == jb.GetNextTimeStamp(10, incomingFrameType, renderTimeMs)); + TEST(timeStampFirstKey == jb.GetNextTimeStamp(10, incomingFrameType, + renderTimeMs)); // check incoming frame type TEST(incomingFrameType == kVideoFrameKey); @@ -1874,7 +1878,93 @@ int JitterBufferTest(CmdArgs& args) jb.Flush(); - //printf("DONE fill JB - nr of delta + key frames (w/ wrap in seqNum) > max nr of frames\n"); + // printf("DONE fill JB - nr of delta + key frames (w/ wrap in seqNum) > + // max nr of frames\n"); + + // Test handling empty packets + // first insert 2 empty packets + jb.ReleaseFrame(frameIn); + timeStamp = 33 * 90; + seqNum = 5; + packet.isFirstPacket = false; + packet.markerBit = false; + packet.seqNum = seqNum; + packet.timestamp = timeStamp; + packet.frameType = kFrameEmpty; + frameIn = jb.GetFrame(packet); + TEST(frameIn); + TEST(kFirstPacket == jb.InsertPacket(frameIn, packet)); + + seqNum = 6; + packet.isFirstPacket = false; + packet.markerBit = false; + packet.seqNum = seqNum; + packet.timestamp = timeStamp; + packet.frameType = kFrameEmpty; + TEST(kFirstPacket == jb.InsertPacket(frameIn, packet)); + // now insert the first data packet + seqNum = 1; + packet.isFirstPacket = true; + packet.markerBit = false; + packet.seqNum = seqNum; + packet.timestamp = timeStamp; + packet.frameType = kVideoFrameDelta; + TEST(kFirstPacket == jb.InsertPacket(frameIn, packet)); + // insert an additional data packet + seqNum = 2; + packet.isFirstPacket = false; + packet.markerBit = false; + packet.seqNum = seqNum; + packet.timestamp = timeStamp; + packet.frameType = kVideoFrameDelta; + TEST(kIncomplete == jb.InsertPacket(frameIn, packet)); + + // insert the last packet and verify frame completness + // (even though packet 4 (empty) is missing) + seqNum = 3; + packet.isFirstPacket = false; + packet.markerBit = true; + packet.seqNum = seqNum; + packet.timestamp = timeStamp; + packet.frameType = kVideoFrameDelta; + TEST(kCompleteSession == jb.InsertPacket(frameIn, packet)); + jb.Flush(); + + // testing that empty packets do not clog the jitter buffer + // Set hybrid mode + jb.SetNackMode(kNackHybrid); + TEST(jb.GetNackMode() == kNackHybrid); + + int maxSize = 100; + seqNum = 3; + VCMEncodedFrame* testFrame; + for (int i = 0; i < maxSize + 10; i++) + { + timeStamp += 33 * 90; + packet.isFirstPacket = false; + packet.markerBit = false; + packet.seqNum = seqNum; + packet.timestamp = timeStamp; + packet.frameType = kFrameEmpty; + testFrame = jb.GetFrame(packet); + TEST(frameIn != 0); + TEST(kFirstPacket == jb.InsertPacket(testFrame, packet)); + } + // verify insertion of a data packet (old empty frames will be flushed) + timeStamp += 33 * 90; + packet.isFirstPacket = true; + packet.markerBit = false; + packet.seqNum = seqNum; + packet.timestamp = timeStamp; + packet.frameType = kFrameEmpty; + testFrame = jb.GetFrame(packet); + TEST(frameIn != 0); + + jb.SetNackMode(kNoNack); + + + // printf(DONE testing inserting empty packets to the JB) + // H.264 tests //Test incomplete NALU frames @@ -1977,7 +2067,8 @@ int JitterBufferTest(CmdArgs& args) packet.completeNALU=kNaluStart; packet.markerBit=false; TEST(kIncomplete == jb.InsertPacket(frameIn, packet)); - insertedLength+=packet.sizeBytes; // This packet should be decoded since it's the beginning of a NAL + // This packet should be decoded since it's the beginning of a NAL + insertedLength+=packet.sizeBytes; seqNum+=2; packet.seqNum=seqNum; @@ -1987,10 +2078,13 @@ int JitterBufferTest(CmdArgs& args) packet.completeNALU=kNaluEnd; packet.markerBit=true; TEST(kIncomplete == jb.InsertPacket(frameIn, packet)); - insertedLength+=0; // This packet should not be decoded because it is an incomplete NAL if it is the last + // This packet should not be decoded because it is an incomplete NAL if it + // is the last + insertedLength+=0; frameOut = jb.GetFrameForDecoding(); - CheckOutFrame(frameOut, insertedLength, false); // Only last NALU is complete + // Only last NALU is complete + CheckOutFrame(frameOut, insertedLength, false); jb.ReleaseFrame(frameOut); @@ -2006,7 +2100,9 @@ int JitterBufferTest(CmdArgs& args) emptypacket.markerBit=true; TEST(frameIn=jb.GetFrame(emptypacket)); TEST(kFirstPacket == jb.InsertPacket(frameIn, emptypacket)); - insertedLength+=0; // This packet should not be decoded because it is an incomplete NAL if it is the last + // This packet should not be decoded because it is an incomplete NAL if it + // is the last + insertedLength+=0; TEST(-1 == jb.GetNextTimeStamp(10, incomingFrameType, renderTimeMs)); TEST(NULL==jb.GetFrameForDecoding()); @@ -2036,8 +2132,8 @@ int JitterBufferTest(CmdArgs& args) // get the frame frameOut = jb.GetCompleteFrameForDecoding(10); - CheckOutFrame(frameOut, packet.sizeBytes, false); // Only last NALU is complete - + // Only last NALU is complete + CheckOutFrame(frameOut, packet.sizeBytes, false); jb.Flush(); // Three reordered H263 packets with bits.