jyapayne/moonlight-common-c
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement predictive RFI to recover more quickly from frame loss

Browse source 
With predictive RFI, we can recover from a lost frame as soon as the next frame
by predicting whether we will have enough FEC data to be be able to recover a
frame based on the sequence numbers of the received packets.
This commit is contained in:
Cameron Gutman 2022-10-05 22:04:22 -05:00
commit 5e1be51b84
4 changed files with 87 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/Limelight-internal.h
View file

@ -97,6 +97,7 @@ void destroyVideoDepacketizer(void);
void queueRtpPacket(PRTPV_QUEUE_ENTRY queueEntry);
void stopVideoDepacketizer(void);
void requestDecoderRefresh(void);
void notifyFrameLost(int frameNumber);
void initializeVideoStream(void);
void destroyVideoStream(void);

66
src/RtpVideoQueue.c
View file

@ -84,9 +84,10 @@ static void removeEntryFromList(PRTPV_QUEUE_LIST list, PRTPV_QUEUE_ENTRY entry)
}
// newEntry is contained within the packet buffer so we free the whole entry by freeing entry->packet
static bool queuePacket(PRTP_VIDEO_QUEUE queue, PRTPV_QUEUE_ENTRY newEntry, PRTP_PACKET packet, int length, bool isParity) {
static bool queuePacket(PRTP_VIDEO_QUEUE queue, PRTPV_QUEUE_ENTRY newEntry, PRTP_PACKET packet, int length, bool isParity, bool isFecRecovery) {
PRTPV_QUEUE_ENTRY entry;
LC_ASSERT(!(isFecRecovery && isParity));
LC_ASSERT(!isBefore16(packet->sequenceNumber, queue->nextContiguousSequenceNumber));
// If the packet is in order, we can take the fast path and avoid having
@ -99,14 +100,30 @@ static bool queuePacket(PRTP_VIDEO_QUEUE queue, PRTPV_QUEUE_ENTRY newEntry, PRTP
// Check for duplicates
entry = queue->pendingFecBlockList.head;
while (entry != NULL) {
if (entry->packet->sequenceNumber == packet->sequenceNumber) {
if (packet->sequenceNumber == entry->packet->sequenceNumber) {
return false;
}
else if (!isFecRecovery && isBefore16(packet->sequenceNumber, entry->packet->sequenceNumber)) {
// This packet was received after a higher sequence number packet, so note that we
// received an out of order packet to disable our fast RFI recovery logic.
queue->lastOosSequenceNumber = packet->sequenceNumber;
if (!queue->receivedOosData) {
Limelog("Leaving fast RFI mode after OOS video data (%u < %u)\n",
packet->sequenceNumber, entry->packet->sequenceNumber);
queue->receivedOosData = true;
}
}
entry = entry->next;
}
}
// This is just a fancy way of determining if 32767 packets have passed since our last OOS data
if (queue->receivedOosData && isBefore16(queue->bufferHighestSequenceNumber, queue->lastOosSequenceNumber)) {
Limelog("Entering fast RFI mode after sequenced video data\n");
queue->receivedOosData = false;
}
newEntry->packet = packet;
newEntry->length = length;
newEntry->isParity = isParity;
@ -132,19 +149,35 @@ static bool queuePacket(PRTP_VIDEO_QUEUE queue, PRTPV_QUEUE_ENTRY newEntry, PRTP
// Returns 0 if the frame is completely constructed
static int reconstructFrame(PRTP_VIDEO_QUEUE queue) {
unsigned int totalPackets = U16(queue->bufferHighestSequenceNumber - queue->bufferLowestSequenceNumber) + 1;
unsigned int neededPackets = queue->bufferDataPackets;
int ret;
#ifdef FEC_VALIDATION_MODE
// We'll need an extra packet to run in FEC validation mode, because we will
// be "dropping" one below and recovering it using parity. However, some frames
// are so large that FEC is disabled entirely, so don't wait for parity on those.
if (queue->pendingFecBlockList.count < queue->bufferDataPackets + (queue->fecPercentage ? 1 : 0)) {
#else
if (queue->pendingFecBlockList.count < queue->bufferDataPackets) {
neededPackets += (queue->fecPercentage ? 1 : 0);
#endif
if (queue->pendingFecBlockList.count < neededPackets) {
// If we've never seen OOS data and we've seen parity packets but not enough to recover the frame without
// additional data (which is unlikely to come, given that we've never seen OOS data from this host), we
// will presume the frame is lost and tell the host immediately.
if (!queue->reportedLostFrame &&
!queue->receivedOosData &&
queue->receivedParityPackets != 0 &&
neededPackets - queue->pendingFecBlockList.count > U16(queue->bufferHighestSequenceNumber - queue->receivedParityHighestSequenceNumber)) {
notifyFrameLost(queue->currentFrameNumber);
queue->reportedLostFrame = true;
}
// Not enough data to recover yet
return -1;
}
// If we make it here and reported a lost frame, we lied to the host. This can happen if we happen to get
// unlucky and this particular frame happens to be the one with OOS data, but it should almost never happen.
LC_ASSERT(!queue->reportedLostFrame);
#ifdef FEC_VALIDATION_MODE
// If FEC is disabled or unsupported for this frame, we must bail early here.
@ -343,7 +376,7 @@ cleanup_packets:
// it may be a legitimate part of the H.264 bytestream.
LC_ASSERT(isBefore16(rtpPacket->sequenceNumber, queue->bufferFirstParitySequenceNumber));
queuePacket(queue, queueEntry, rtpPacket, StreamConfig.packetSize + dataOffset, false);
queuePacket(queue, queueEntry, rtpPacket, StreamConfig.packetSize + dataOffset, false, true);
} else if (packets[i] != NULL) {
free(packets[i]);
}
@ -487,7 +520,7 @@ int RtpvAddPacket(PRTP_VIDEO_QUEUE queue, PRTP_PACKET packet, int length, PRTPV_
queue->currentFrameNumber, queue->multiFecCurrentBlockNumber+1,
queue->multiFecLastBlockNumber+1,
queue->receivedBufferDataPackets,
queue->pendingFecBlockList.count - queue->receivedBufferDataPackets,
queue->receivedParityPackets,
queue->pendingFecBlockList.count,
queue->bufferDataPackets);
@ -499,6 +532,9 @@ int RtpvAddPacket(PRTP_VIDEO_QUEUE queue, PRTP_PACKET packet, int length, PRTPV_
purgeListEntries(&queue->pendingFecBlockList);
purgeListEntries(&queue->completedFecBlockList);
// Notify the host of the loss of this frame
notifyFrameLost(queue->currentFrameNumber);
queue->currentFrameNumber++;
queue->multiFecCurrentBlockNumber = 0;
return RTPF_RET_REJECTED;
@ -507,7 +543,7 @@ int RtpvAddPacket(PRTP_VIDEO_QUEUE queue, PRTP_PACKET packet, int length, PRTPV_
else {
Limelog("Unrecoverable frame %d: %d+%d=%d received < %d needed\n",
queue->currentFrameNumber, queue->receivedBufferDataPackets,
queue->pendingFecBlockList.count - queue->receivedBufferDataPackets,
queue->receivedParityPackets,
queue->pendingFecBlockList.count,
queue->bufferDataPackets);
}
@ -526,6 +562,9 @@ int RtpvAddPacket(PRTP_VIDEO_QUEUE queue, PRTP_PACKET packet, int length, PRTPV_
purgeListEntries(&queue->pendingFecBlockList);
purgeListEntries(&queue->completedFecBlockList);
// Notify the host of the loss of this frame
notifyFrameLost(queue->currentFrameNumber);
// We dropped a block of this frame, so we must skip to the next one.
queue->currentFrameNumber = nvPacket->frameIndex + 1;
queue->multiFecCurrentBlockNumber = 0;
@ -550,6 +589,9 @@ int RtpvAddPacket(PRTP_VIDEO_QUEUE queue, PRTP_PACKET packet, int length, PRTPV_
queue->bufferLowestSequenceNumber = U16(packet->sequenceNumber - fecIndex);
queue->nextContiguousSequenceNumber = queue->bufferLowestSequenceNumber;
queue->receivedBufferDataPackets = 0;
queue->receivedParityPackets = 0;
queue->receivedParityHighestSequenceNumber = 0;
queue->reportedLostFrame = false;
queue->bufferDataPackets = (nvPacket->fecInfo & 0xFFC00000) >> 22;
queue->fecPercentage = (nvPacket->fecInfo & 0xFF0) >> 4;
queue->bufferParityPackets = (queue->bufferDataPackets * queue->fecPercentage + 99) / 100;
@ -576,13 +618,19 @@ int RtpvAddPacket(PRTP_VIDEO_QUEUE queue, PRTP_PACKET packet, int length, PRTPV_
LC_ASSERT(((nvPacket->multiFecBlocks >> 6) & 0x3) == queue->multiFecLastBlockNumber);
LC_ASSERT((nvPacket->flags & FLAG_EOF) || length - dataOffset == StreamConfig.packetSize);
if (!queuePacket(queue, packetEntry, packet, length, !isBefore16(packet->sequenceNumber, queue->bufferFirstParitySequenceNumber))) {
if (!queuePacket(queue, packetEntry, packet, length, !isBefore16(packet->sequenceNumber, queue->bufferFirstParitySequenceNumber), false)) {
return RTPF_RET_REJECTED;
}
else {
if (isBefore16(packet->sequenceNumber, queue->bufferFirstParitySequenceNumber)) {
queue->receivedBufferDataPackets++;
}
else {
queue->receivedParityPackets++;
if (queue->receivedParityPackets == 1 || isBefore16(queue->receivedParityHighestSequenceNumber, packet->sequenceNumber)) {
queue->receivedParityHighestSequenceNumber = packet->sequenceNumber;
}
}
// Try to submit this frame. If we haven't received enough packets,
// this will fail and we'll keep waiting.

6
src/RtpVideoQueue.h
View file

@ -29,14 +29,20 @@ typedef struct _RTP_VIDEO_QUEUE {
uint32_t bufferDataPackets;
uint32_t bufferParityPackets;
uint32_t receivedBufferDataPackets;
uint32_t receivedParityPackets;
uint32_t receivedParityHighestSequenceNumber;
uint32_t fecPercentage;
uint32_t nextContiguousSequenceNumber;
bool reportedLostFrame;
uint32_t currentFrameNumber;
bool multiFecCapable;
uint8_t multiFecCurrentBlockNumber;
uint8_t multiFecLastBlockNumber;
uint16_t lastOosSequenceNumber;
bool receivedOosData;
} RTP_VIDEO_QUEUE, *PRTP_VIDEO_QUEUE;
#define RTPF_RET_QUEUED 0

23
src/VideoDepacketizer.c
View file

@ -884,6 +884,29 @@ static void processRtpPayload(PNV_VIDEO_PACKET videoPacket, int length,
}
}
// Called by the video RTP FEC queue to notify us of a lost frame
// if it determines the frame to be unrecoverable. This lets us
// avoid having to wait until the next received frame to determine
// that we lost a frame and submit an RFI request.
void notifyFrameLost(int frameNumber) {
// This may only be called at frame boundaries
LC_ASSERT(!decodingFrame);
// If RFI is enabled, we will notify the host PC now
if (isReferenceFrameInvalidationEnabled()) {
Limelog("Sending predictive RFI request for probable loss of frame %d\n", frameNumber);
// Advance the frame number since we won't be expecting this one anymore
nextFrameNumber = frameNumber + 1;
// Drop any existing frame state (shouldn't have any) and set the RFI wait flag
dropFrameState();
// Notify the host that we lost this one
connectionDetectedFrameLoss(startFrameNumber, frameNumber);
}
}
// Add an RTP Packet to the queue
void queueRtpPacket(PRTPV_QUEUE_ENTRY queueEntryPtr) {
int dataOffset;