diff --git a/src/Limelight-internal.h b/src/Limelight-internal.h index ef06a07..0fc6181 100644 --- a/src/Limelight-internal.h +++ b/src/Limelight-internal.h @@ -97,6 +97,7 @@ void destroyVideoDepacketizer(void); void queueRtpPacket(PRTPV_QUEUE_ENTRY queueEntry); void stopVideoDepacketizer(void); void requestDecoderRefresh(void); +void notifyFrameLost(int frameNumber); void initializeVideoStream(void); void destroyVideoStream(void); diff --git a/src/RtpVideoQueue.c b/src/RtpVideoQueue.c index 96c16ce..d0db57b 100644 --- a/src/RtpVideoQueue.c +++ b/src/RtpVideoQueue.c @@ -84,9 +84,10 @@ static void removeEntryFromList(PRTPV_QUEUE_LIST list, PRTPV_QUEUE_ENTRY entry) } // newEntry is contained within the packet buffer so we free the whole entry by freeing entry->packet -static bool queuePacket(PRTP_VIDEO_QUEUE queue, PRTPV_QUEUE_ENTRY newEntry, PRTP_PACKET packet, int length, bool isParity) { +static bool queuePacket(PRTP_VIDEO_QUEUE queue, PRTPV_QUEUE_ENTRY newEntry, PRTP_PACKET packet, int length, bool isParity, bool isFecRecovery) { PRTPV_QUEUE_ENTRY entry; + LC_ASSERT(!(isFecRecovery && isParity)); LC_ASSERT(!isBefore16(packet->sequenceNumber, queue->nextContiguousSequenceNumber)); // If the packet is in order, we can take the fast path and avoid having @@ -99,14 +100,30 @@ static bool queuePacket(PRTP_VIDEO_QUEUE queue, PRTPV_QUEUE_ENTRY newEntry, PRTP // Check for duplicates entry = queue->pendingFecBlockList.head; while (entry != NULL) { - if (entry->packet->sequenceNumber == packet->sequenceNumber) { + if (packet->sequenceNumber == entry->packet->sequenceNumber) { return false; } + else if (!isFecRecovery && isBefore16(packet->sequenceNumber, entry->packet->sequenceNumber)) { + // This packet was received after a higher sequence number packet, so note that we + // received an out of order packet to disable our fast RFI recovery logic. + queue->lastOosSequenceNumber = packet->sequenceNumber; + if (!queue->receivedOosData) { + Limelog("Leaving fast RFI mode after OOS video data (%u < %u)\n", + packet->sequenceNumber, entry->packet->sequenceNumber); + queue->receivedOosData = true; + } + } entry = entry->next; } } + // This is just a fancy way of determining if 32767 packets have passed since our last OOS data + if (queue->receivedOosData && isBefore16(queue->bufferHighestSequenceNumber, queue->lastOosSequenceNumber)) { + Limelog("Entering fast RFI mode after sequenced video data\n"); + queue->receivedOosData = false; + } + newEntry->packet = packet; newEntry->length = length; newEntry->isParity = isParity; @@ -132,19 +149,35 @@ static bool queuePacket(PRTP_VIDEO_QUEUE queue, PRTPV_QUEUE_ENTRY newEntry, PRTP // Returns 0 if the frame is completely constructed static int reconstructFrame(PRTP_VIDEO_QUEUE queue) { unsigned int totalPackets = U16(queue->bufferHighestSequenceNumber - queue->bufferLowestSequenceNumber) + 1; + unsigned int neededPackets = queue->bufferDataPackets; int ret; #ifdef FEC_VALIDATION_MODE // We'll need an extra packet to run in FEC validation mode, because we will // be "dropping" one below and recovering it using parity. However, some frames // are so large that FEC is disabled entirely, so don't wait for parity on those. - if (queue->pendingFecBlockList.count < queue->bufferDataPackets + (queue->fecPercentage ? 1 : 0)) { -#else - if (queue->pendingFecBlockList.count < queue->bufferDataPackets) { + neededPackets += (queue->fecPercentage ? 1 : 0); #endif + + if (queue->pendingFecBlockList.count < neededPackets) { + // If we've never seen OOS data and we've seen parity packets but not enough to recover the frame without + // additional data (which is unlikely to come, given that we've never seen OOS data from this host), we + // will presume the frame is lost and tell the host immediately. + if (!queue->reportedLostFrame && + !queue->receivedOosData && + queue->receivedParityPackets != 0 && + neededPackets - queue->pendingFecBlockList.count > U16(queue->bufferHighestSequenceNumber - queue->receivedParityHighestSequenceNumber)) { + notifyFrameLost(queue->currentFrameNumber); + queue->reportedLostFrame = true; + } + // Not enough data to recover yet return -1; } + + // If we make it here and reported a lost frame, we lied to the host. This can happen if we happen to get + // unlucky and this particular frame happens to be the one with OOS data, but it should almost never happen. + LC_ASSERT(!queue->reportedLostFrame); #ifdef FEC_VALIDATION_MODE // If FEC is disabled or unsupported for this frame, we must bail early here. @@ -343,7 +376,7 @@ cleanup_packets: // it may be a legitimate part of the H.264 bytestream. LC_ASSERT(isBefore16(rtpPacket->sequenceNumber, queue->bufferFirstParitySequenceNumber)); - queuePacket(queue, queueEntry, rtpPacket, StreamConfig.packetSize + dataOffset, false); + queuePacket(queue, queueEntry, rtpPacket, StreamConfig.packetSize + dataOffset, false, true); } else if (packets[i] != NULL) { free(packets[i]); } @@ -487,7 +520,7 @@ int RtpvAddPacket(PRTP_VIDEO_QUEUE queue, PRTP_PACKET packet, int length, PRTPV_ queue->currentFrameNumber, queue->multiFecCurrentBlockNumber+1, queue->multiFecLastBlockNumber+1, queue->receivedBufferDataPackets, - queue->pendingFecBlockList.count - queue->receivedBufferDataPackets, + queue->receivedParityPackets, queue->pendingFecBlockList.count, queue->bufferDataPackets); @@ -499,6 +532,9 @@ int RtpvAddPacket(PRTP_VIDEO_QUEUE queue, PRTP_PACKET packet, int length, PRTPV_ purgeListEntries(&queue->pendingFecBlockList); purgeListEntries(&queue->completedFecBlockList); + // Notify the host of the loss of this frame + notifyFrameLost(queue->currentFrameNumber); + queue->currentFrameNumber++; queue->multiFecCurrentBlockNumber = 0; return RTPF_RET_REJECTED; @@ -507,7 +543,7 @@ int RtpvAddPacket(PRTP_VIDEO_QUEUE queue, PRTP_PACKET packet, int length, PRTPV_ else { Limelog("Unrecoverable frame %d: %d+%d=%d received < %d needed\n", queue->currentFrameNumber, queue->receivedBufferDataPackets, - queue->pendingFecBlockList.count - queue->receivedBufferDataPackets, + queue->receivedParityPackets, queue->pendingFecBlockList.count, queue->bufferDataPackets); } @@ -526,6 +562,9 @@ int RtpvAddPacket(PRTP_VIDEO_QUEUE queue, PRTP_PACKET packet, int length, PRTPV_ purgeListEntries(&queue->pendingFecBlockList); purgeListEntries(&queue->completedFecBlockList); + // Notify the host of the loss of this frame + notifyFrameLost(queue->currentFrameNumber); + // We dropped a block of this frame, so we must skip to the next one. queue->currentFrameNumber = nvPacket->frameIndex + 1; queue->multiFecCurrentBlockNumber = 0; @@ -550,6 +589,9 @@ int RtpvAddPacket(PRTP_VIDEO_QUEUE queue, PRTP_PACKET packet, int length, PRTPV_ queue->bufferLowestSequenceNumber = U16(packet->sequenceNumber - fecIndex); queue->nextContiguousSequenceNumber = queue->bufferLowestSequenceNumber; queue->receivedBufferDataPackets = 0; + queue->receivedParityPackets = 0; + queue->receivedParityHighestSequenceNumber = 0; + queue->reportedLostFrame = false; queue->bufferDataPackets = (nvPacket->fecInfo & 0xFFC00000) >> 22; queue->fecPercentage = (nvPacket->fecInfo & 0xFF0) >> 4; queue->bufferParityPackets = (queue->bufferDataPackets * queue->fecPercentage + 99) / 100; @@ -576,13 +618,19 @@ int RtpvAddPacket(PRTP_VIDEO_QUEUE queue, PRTP_PACKET packet, int length, PRTPV_ LC_ASSERT(((nvPacket->multiFecBlocks >> 6) & 0x3) == queue->multiFecLastBlockNumber); LC_ASSERT((nvPacket->flags & FLAG_EOF) || length - dataOffset == StreamConfig.packetSize); - if (!queuePacket(queue, packetEntry, packet, length, !isBefore16(packet->sequenceNumber, queue->bufferFirstParitySequenceNumber))) { + if (!queuePacket(queue, packetEntry, packet, length, !isBefore16(packet->sequenceNumber, queue->bufferFirstParitySequenceNumber), false)) { return RTPF_RET_REJECTED; } else { if (isBefore16(packet->sequenceNumber, queue->bufferFirstParitySequenceNumber)) { queue->receivedBufferDataPackets++; } + else { + queue->receivedParityPackets++; + if (queue->receivedParityPackets == 1 || isBefore16(queue->receivedParityHighestSequenceNumber, packet->sequenceNumber)) { + queue->receivedParityHighestSequenceNumber = packet->sequenceNumber; + } + } // Try to submit this frame. If we haven't received enough packets, // this will fail and we'll keep waiting. diff --git a/src/RtpVideoQueue.h b/src/RtpVideoQueue.h index f6bb963..61bc638 100644 --- a/src/RtpVideoQueue.h +++ b/src/RtpVideoQueue.h @@ -29,14 +29,20 @@ typedef struct _RTP_VIDEO_QUEUE { uint32_t bufferDataPackets; uint32_t bufferParityPackets; uint32_t receivedBufferDataPackets; + uint32_t receivedParityPackets; + uint32_t receivedParityHighestSequenceNumber; uint32_t fecPercentage; uint32_t nextContiguousSequenceNumber; + bool reportedLostFrame; uint32_t currentFrameNumber; bool multiFecCapable; uint8_t multiFecCurrentBlockNumber; uint8_t multiFecLastBlockNumber; + + uint16_t lastOosSequenceNumber; + bool receivedOosData; } RTP_VIDEO_QUEUE, *PRTP_VIDEO_QUEUE; #define RTPF_RET_QUEUED 0 diff --git a/src/VideoDepacketizer.c b/src/VideoDepacketizer.c index 0cefd2d..06067a3 100644 --- a/src/VideoDepacketizer.c +++ b/src/VideoDepacketizer.c @@ -884,6 +884,29 @@ static void processRtpPayload(PNV_VIDEO_PACKET videoPacket, int length, } } +// Called by the video RTP FEC queue to notify us of a lost frame +// if it determines the frame to be unrecoverable. This lets us +// avoid having to wait until the next received frame to determine +// that we lost a frame and submit an RFI request. +void notifyFrameLost(int frameNumber) { + // This may only be called at frame boundaries + LC_ASSERT(!decodingFrame); + + // If RFI is enabled, we will notify the host PC now + if (isReferenceFrameInvalidationEnabled()) { + Limelog("Sending predictive RFI request for probable loss of frame %d\n", frameNumber); + + // Advance the frame number since we won't be expecting this one anymore + nextFrameNumber = frameNumber + 1; + + // Drop any existing frame state (shouldn't have any) and set the RFI wait flag + dropFrameState(); + + // Notify the host that we lost this one + connectionDetectedFrameLoss(startFrameNumber, frameNumber); + } +} + // Add an RTP Packet to the queue void queueRtpPacket(PRTPV_QUEUE_ENTRY queueEntryPtr) { int dataOffset;