diff --git a/app/streaming/video/ffmpeg-renderers/d3d11va.cpp b/app/streaming/video/ffmpeg-renderers/d3d11va.cpp
index af50304a..3a6b24e7 100644
--- a/app/streaming/video/ffmpeg-renderers/d3d11va.cpp
+++ b/app/streaming/video/ffmpeg-renderers/d3d11va.cpp
@@ -568,6 +568,11 @@ bool D3D11VARenderer::prepareDecoderContextInGetFormat(AVCodecContext *context,
         d3d11vaFramesContext->BindFlags |= D3D11_BIND_SHADER_RESOURCE;
     }
 
+    // Mimic the logic in ff_decode_get_hw_frames_ctx() which adds an extra 3 frames
+    if (framesContext->initial_pool_size) {
+        framesContext->initial_pool_size += 3;
+    }
+
     err = av_hwframe_ctx_init(context->hw_frames_ctx);
     if (err < 0) {
         av_buffer_unref(&context->hw_frames_ctx);
diff --git a/app/streaming/video/ffmpeg-renderers/dxva2.h b/app/streaming/video/ffmpeg-renderers/dxva2.h
index a66ca7b8..af275748 100644
--- a/app/streaming/video/ffmpeg-renderers/dxva2.h
+++ b/app/streaming/video/ffmpeg-renderers/dxva2.h
@@ -57,8 +57,11 @@ private:
     int m_DisplayHeight;
 
     struct dxva_context m_DXVAContext;
-    std::array<Microsoft::WRL::ComPtr<IDirect3DSurface9>, 19> m_DecSurfaces;
-    std::array<IDirect3DSurface9*, 19> m_DecSurfacesRaw; // Referenced by m_DecSurfaces
+
+    // H.264 uses a maximum of 16 reference frames
+    std::array<Microsoft::WRL::ComPtr<IDirect3DSurface9>, 16 + PACER_MAX_OUTSTANDING_FRAMES> m_DecSurfaces;
+    std::array<IDirect3DSurface9*, 16 + PACER_MAX_OUTSTANDING_FRAMES> m_DecSurfacesRaw; // Referenced by m_DecSurfaces
+
     DXVA2_ConfigPictureDecode m_Config;
     Microsoft::WRL::ComPtr<IDirectXVideoDecoderService> m_DecService;
     Microsoft::WRL::ComPtr<IDirectXVideoDecoder> m_Decoder;
diff --git a/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp b/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp
index 688e45e0..e2e70f94 100644
--- a/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp
+++ b/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp
@@ -19,7 +19,9 @@
 // that the sum of all queued frames between both pacing and rendering queues
 // must not exceed the number buffer pool size to avoid running the decoder
 // out of available decoding surfaces.
-#define MAX_QUEUED_FRAMES 4
+#define MAX_QUEUED_FRAMES 3
+static_assert(PACER_MAX_OUTSTANDING_FRAMES == MAX_QUEUED_FRAMES + 2,
+              "PACER_MAX_OUTSTANDING_FRAMES and MAX_QUEUED_FRAMES must agree");
 
 // We may be woken up slightly late so don't go all the way
 // up to the next V-sync since we may accidentally step into
diff --git a/app/streaming/video/ffmpeg-renderers/pacer/pacer.h b/app/streaming/video/ffmpeg-renderers/pacer/pacer.h
index 0bdf8074..a1bb9eab 100644
--- a/app/streaming/video/ffmpeg-renderers/pacer/pacer.h
+++ b/app/streaming/video/ffmpeg-renderers/pacer/pacer.h
@@ -7,6 +7,12 @@
 #include <QMutex>
 #include <QWaitCondition>
 
+// The maximum number of frames pacer will ever hold is:
+// - 3 frames in the pacing queue
+// - 1 frame removed from the render queue in the process of rendering
+// - 1 frame for deferred free
+#define PACER_MAX_OUTSTANDING_FRAMES (3 + 1 + 1)
+
 class IVsyncSource {
 public:
     virtual ~IVsyncSource() {}
diff --git a/app/streaming/video/ffmpeg.cpp b/app/streaming/video/ffmpeg.cpp
index 464ae8ed..306c4fd1 100644
--- a/app/streaming/video/ffmpeg.cpp
+++ b/app/streaming/video/ffmpeg.cpp
@@ -542,6 +542,12 @@ bool FFmpegVideoDecoder::completeInitialization(const AVCodec* decoder, enum AVP
     m_VideoDecoderCtx->pkt_timebase.num = 1;
     m_VideoDecoderCtx->pkt_timebase.den = 90000;
 
+    // Allocate enough extra frames for Pacer to avoid stalling the decoder
+    //
+    // NB: Subtract 4 because FFmpeg always allocates 4 working surfaces when
+    // constructing a hwframes context (see ff_decode_get_hw_frames_ctx()).
+    m_VideoDecoderCtx->extra_hw_frames = std::max<int>(0, PACER_MAX_OUTSTANDING_FRAMES - 4);
+
     // For non-hwaccel decoders, set the pix_fmt to hint to the decoder which
     // format should be used. This is necessary for certain decoders like the
     // out-of-tree nvv4l2dec decoders for L4T platforms. We do not do this