Add presentation timestamp to video frame metadata

src/AudioStream.c

@@ -218,8 +218,10 @@ static void ReceiveThreadProc(void* context) {
             continue;
         }
 
-        // RTP sequence number must be in host order for the RTP queue
+        // Convert fields to host byte-order
         rtp->sequenceNumber = htons(rtp->sequenceNumber);
+        rtp->timestamp = htonl(rtp->timestamp);
+        rtp->ssrc = htonl(rtp->ssrc);
 
         queueStatus = RtpqAddPacket(&rtpReorderQueue, (PRTP_PACKET)packet, &packet->q.rentry);
         if (RTPQ_HANDLE_NOW(queueStatus)) {

src/Limelight.h

@@ -122,6 +122,11 @@ typedef struct _DECODE_UNIT {
     // shares the same epoch as this value.
     unsigned long long receiveTimeMs;
 
+    // Presentation time in milliseconds with the epoch at the first captured frame.
+    // This can be used to aid frame pacing or to drop old frames that were queued too
+    // long prior to display.
+    unsigned int presentationTimeMs;
+
     // Length of the entire buffer chain in bytes
     int fullLength;
 

src/RtpFecQueue.c

@@ -47,6 +47,9 @@ static int queuePacket(PRTP_FEC_QUEUE queue, PRTPFEC_QUEUE_ENTRY newEntry, int h
     newEntry->prev = NULL;
     newEntry->next = NULL;
 
+    // 90 KHz video clock
+    newEntry->presentationTimeMs = packet->timestamp / 90;
+
     if (queue->bufferHead == NULL) {
         LC_ASSERT(queue->bufferSize == 0);
         queue->bufferHead = queue->bufferTail = newEntry;
@@ -158,6 +161,8 @@ cleanup_packets:
                 PRTP_PACKET rtpPacket = (PRTP_PACKET) packets[i];
                 rtpPacket->sequenceNumber = U16(i + queue->bufferLowestSequenceNumber);
                 rtpPacket->header = queue->bufferHead->packet->header;
+                rtpPacket->timestamp = queue->bufferHead->packet->timestamp;
+                rtpPacket->ssrc = queue->bufferHead->packet->ssrc;
                 
                 int dataOffset = sizeof(*rtpPacket);
                 if (rtpPacket->header & FLAG_EXTENSION) {

src/RtpFecQueue.h

@@ -7,6 +7,7 @@ typedef struct _RTPFEC_QUEUE_ENTRY {
     int length;
     int isParity;
     unsigned long long receiveTimeMs;
+    unsigned int presentationTimeMs;
 
     struct _RTPFEC_QUEUE_ENTRY* next;
     struct _RTPFEC_QUEUE_ENTRY* prev;

src/Video.h

@@ -33,7 +33,8 @@ typedef struct _RTP_PACKET {
     char header;
     char packetType;
     unsigned short sequenceNumber;
-    char reserved[8];
+    unsigned int timestamp;
+    unsigned int ssrc;
 } RTP_PACKET, *PRTP_PACKET;
 
 #pragma pack(pop)

src/VideoDepacketizer.c

@@ -15,6 +15,7 @@ static unsigned int lastPacketInStream;
 static int decodingFrame;
 static int strictIdrFrameWait;
 static unsigned long long firstPacketReceiveTime;
+static unsigned int firstPacketPresentationTime;
 static int dropStatePending;
 static int idrFrameProcessed;
 
@@ -47,6 +48,7 @@ void initializeVideoDepacketizer(int pktSize) {
     lastPacketInStream = UINT32_MAX;
     decodingFrame = 0;
     firstPacketReceiveTime = 0;
+    firstPacketPresentationTime = 0;
     dropStatePending = 0;
     idrFrameProcessed = 0;
     strictIdrFrameWait = !isReferenceFrameInvalidationEnabled();
@@ -261,6 +263,7 @@ static void reassembleFrame(int frameNumber) {
             qdu->decodeUnit.fullLength = nalChainDataLength;
             qdu->decodeUnit.frameNumber = frameNumber;
             qdu->decodeUnit.receiveTimeMs = firstPacketReceiveTime;
+            qdu->decodeUnit.presentationTimeMs = firstPacketPresentationTime;
 
             // IDR frames will have leading CSD buffers
             if (nalChainHead->bufferType != BUFFER_TYPE_PICDATA) {
@@ -497,7 +500,9 @@ static int isFirstPacket(char flags) {
 
 // Process an RTP Payload
 // The caller will free *existingEntry unless we NULL it
-void processRtpPayload(PNV_VIDEO_PACKET videoPacket, int length, unsigned long long receiveTimeMs, PLENTRY_INTERNAL* existingEntry) {
+void processRtpPayload(PNV_VIDEO_PACKET videoPacket, int length,
+                       unsigned long long receiveTimeMs, unsigned int presentationTimeMs,
+                       PLENTRY_INTERNAL* existingEntry) {
     BUFFER_DESC currentPos;
     int frameIndex;
     char flags;
@@ -563,6 +568,7 @@ void processRtpPayload(PNV_VIDEO_PACKET videoPacket, int length, unsigned long l
         // We're now decoding a frame
         decodingFrame = 1;
         firstPacketReceiveTime = receiveTimeMs;
+        firstPacketPresentationTime = presentationTimeMs;
     }
 
     lastPacketInStream = streamPacketIndex;
@@ -694,6 +700,7 @@ void queueRtpPacket(PRTPFEC_QUEUE_ENTRY queueEntryPtr) {
     processRtpPayload((PNV_VIDEO_PACKET)(((char*)queueEntry.packet) + dataOffset),
                       queueEntry.length - dataOffset,
                       queueEntry.receiveTimeMs,
+                      queueEntry.presentationTimeMs,
                       &existingEntry);
 
     if (existingEntry != NULL) {

src/VideoStream.c

@@ -116,9 +116,11 @@ static void ReceiveThreadProc(void* context) {
         // as quickly, since we're now just keeping the NAT session open.
         receivedDataFromPeer = 1;
 
-        // RTP sequence number must be in host order for the RTP queue
+        // Convert fields to host byte-order
         packet = (PRTP_PACKET)&buffer[0];
         packet->sequenceNumber = htons(packet->sequenceNumber);
+        packet->timestamp = htonl(packet->timestamp);
+        packet->ssrc = htonl(packet->ssrc);
 
         queueStatus = RtpfAddPacket(&rtpQueue, packet, err, (PRTPFEC_QUEUE_ENTRY)&buffer[receiveSize]);