From f25640778929216ea4cc21af2ce1286ff6793b3c Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Fri, 4 Feb 2022 21:51:34 -0600 Subject: [PATCH] Implement D3D11VA video rendering --- app/resources.qrc | 1 + app/shaders/build_hlsl.bat | 3 +- app/shaders/d3d11_video_pixel.fxc | Bin 0 -> 1408 bytes app/shaders/d3d11_video_pixel.hlsl | 30 ++ .../video/ffmpeg-renderers/d3d11va.cpp | 339 ++++++++++++++++-- .../video/ffmpeg-renderers/d3d11va.h | 11 +- .../video/ffmpeg-renderers/renderer.h | 5 + app/streaming/video/ffmpeg.cpp | 8 +- 8 files changed, 362 insertions(+), 35 deletions(-) create mode 100644 app/shaders/d3d11_video_pixel.fxc create mode 100644 app/shaders/d3d11_video_pixel.hlsl diff --git a/app/resources.qrc b/app/resources.qrc index 453902f7..9401fcd5 100644 --- a/app/resources.qrc +++ b/app/resources.qrc @@ -61,5 +61,6 @@ shaders/egl_overlay.vert shaders/d3d11_vertex.fxc shaders/d3d11_overlay_pixel.fxc + shaders/d3d11_video_pixel.fxc diff --git a/app/shaders/build_hlsl.bat b/app/shaders/build_hlsl.bat index 4c6ebd0d..2398134e 100644 --- a/app/shaders/build_hlsl.bat +++ b/app/shaders/build_hlsl.bat @@ -1,3 +1,4 @@ fxc /T vs_4_0_level_9_3 /Fo d3d11_vertex.fxc d3d11_vertex.hlsl -fxc /T ps_4_0_level_9_3 /Fo d3d11_overlay_pixel.fxc d3d11_overlay_pixel.hlsl \ No newline at end of file +fxc /T ps_4_0_level_9_3 /Fo d3d11_overlay_pixel.fxc d3d11_overlay_pixel.hlsl +fxc /T ps_4_0_level_9_3 /Fo d3d11_video_pixel.fxc d3d11_video_pixel.hlsl \ No newline at end of file diff --git a/app/shaders/d3d11_video_pixel.fxc b/app/shaders/d3d11_video_pixel.fxc new file mode 100644 index 0000000000000000000000000000000000000000..fcede8bf230ee860d338e849438f94f9744bb928 GIT binary patch literal 1408 zcma)6O=}ZT6g`t^(^Qni;=+aCATBBtNkycEVw(-PgZX`GNqn+cN$E<}d@ z0MWgRx^mG?H}x;*!i7s0?#vI+{(*Szn>QGX1#h@{_uP+j-@Tbkv7RqXF8-QsW#`a4 z{qfdMZl4+e!bxC?=MuF@$Cm_9Hi0_z`viTA=lt#>Bv~{u8hz%mpx~Ug7LdSgZA+RY z0xVZzu;vjkIf3*MJ1%RtI&PAYMhKWE+J1vn`Y5&y+Ia?nG;@Y0Mrx?Ap)rWrEny(Z z_}7qR%$*7i3_}yc_bFlnSEW|;D8e%eGe_fV3MYsou2jgoD;wps;u?(rr(CKNU*cF7+MZzxbs_^aEAy z#J`%VA5mzk*cF~1spsQyt;9c3|0QwGpzS%0gWkRuVE_3+r`zZ@z0Lhb*F&=%_UWfBo^Vz%es5q$ZsQ3jt;fMxKmpDJy literal 0 HcmV?d00001 diff --git a/app/shaders/d3d11_video_pixel.hlsl b/app/shaders/d3d11_video_pixel.hlsl new file mode 100644 index 00000000..b74f0f6c --- /dev/null +++ b/app/shaders/d3d11_video_pixel.hlsl @@ -0,0 +1,30 @@ +Texture2D luminancePlane : register(t0); +Texture2D chrominancePlane : register(t1); +SamplerState theSampler : register(s0); + +cbuffer CSC_CONST_BUF : register(b0) +{ + float3x3 cscMatrix; + float3 offsets; +}; + +struct ShaderInput +{ + float4 pos : SV_POSITION; + float2 tex : TEXCOORD0; +}; + +min16float4 main(ShaderInput input) : SV_TARGET +{ + float y = luminancePlane.Sample(theSampler, input.tex); + float2 uv = chrominancePlane.Sample(theSampler, input.tex); + float3 yuv = float3(y, uv); + + // Subtract the YUV offset for limited vs full range + yuv -= offsets; + + // Multiply by the conversion matrix for this colorspace + yuv = mul(yuv, cscMatrix); + + return min16float4(saturate(yuv), 1.0f); +} \ No newline at end of file diff --git a/app/streaming/video/ffmpeg-renderers/d3d11va.cpp b/app/streaming/video/ffmpeg-renderers/d3d11va.cpp index 4bf90bcc..8df7c3e2 100644 --- a/app/streaming/video/ffmpeg-renderers/d3d11va.cpp +++ b/app/streaming/video/ffmpeg-renderers/d3d11va.cpp @@ -19,18 +19,75 @@ typedef struct _VERTEX float tu, tv; } VERTEX, *PVERTEX; +#define CSC_MATRIX_RAW_ELEMENT_COUNT 9 +#define CSC_MATRIX_PACKED_ELEMENT_COUNT 12 + +static const float k_CscMatrix_Bt601Lim[CSC_MATRIX_RAW_ELEMENT_COUNT] = { + 1.1644f, 1.1644f, 1.1644f, + 0.0f, -0.3917f, 2.0172f, + 1.5960f, -0.8129f, 0.0f, +}; +static const float k_CscMatrix_Bt601Full[CSC_MATRIX_RAW_ELEMENT_COUNT] = { + 1.0f, 1.0f, 1.0f, + 0.0f, -0.3441f, 1.7720f, + 1.4020f, -0.7141f, 0.0f, +}; +static const float k_CscMatrix_Bt709Lim[CSC_MATRIX_RAW_ELEMENT_COUNT] = { + 1.1644f, 1.1644f, 1.1644f, + 0.0f, -0.2132f, 2.1124f, + 1.7927f, -0.5329f, 0.0f, +}; +static const float k_CscMatrix_Bt709Full[CSC_MATRIX_RAW_ELEMENT_COUNT] = { + 1.0f, 1.0f, 1.0f, + 0.0f, -0.1873f, 1.8556f, + 1.5748f, -0.4681f, 0.0f, +}; +static const float k_CscMatrix_Bt2020Lim[CSC_MATRIX_RAW_ELEMENT_COUNT] = { + 1.1644f, 1.1644f, 1.1644f, + 0.0f, -0.1874f, 2.1418f, + 1.6781f, -0.6505f, 0.0f, +}; +static const float k_CscMatrix_Bt2020Full[CSC_MATRIX_RAW_ELEMENT_COUNT] = { + 1.0f, 1.0f, 1.0f, + 0.0f, -0.1646f, 1.8814f, + 1.4746f, -0.5714f, 0.0f, +}; + +#define OFFSETS_ELEMENT_COUNT 3 + +static const float k_Offsets_Lim[OFFSETS_ELEMENT_COUNT] = { 16.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f }; +static const float k_Offsets_Full[OFFSETS_ELEMENT_COUNT] = { 0.0f, 128.0f / 255.0f, 128.0f / 255.0f }; + +typedef struct _CSC_CONST_BUF +{ + // CscMatrix value from above but packed appropriately + float cscMatrix[CSC_MATRIX_PACKED_ELEMENT_COUNT]; + + // YUV offset values from above + float offsets[OFFSETS_ELEMENT_COUNT]; + + // Padding float to be a multiple of 16 bytes + float padding; +} CSC_CONST_BUF, *PCSC_CONST_BUF; +static_assert(sizeof(CSC_CONST_BUF) % 16 == 0, "Constant buffer sizes must be a multiple of 16"); + D3D11VARenderer::D3D11VARenderer() : m_Factory(nullptr), m_Device(nullptr), m_SwapChain(nullptr), m_DeviceContext(nullptr), m_RenderTargetView(nullptr), + m_LastColorSpace(AVCOL_SPC_UNSPECIFIED), + m_LastColorRange(AVCOL_RANGE_UNSPECIFIED), m_AllowTearing(false), m_FrameWaitableObject(nullptr), m_VideoPixelShader(nullptr), + m_VideoVertexBuffer(nullptr), + m_VideoConstantBuffer(nullptr), m_OverlayLock(0), m_OverlayPixelShader(nullptr), - m_HwContext(nullptr) + m_HwDeviceContext(nullptr), + m_HwFramesContext(nullptr) { RtlZeroMemory(m_OverlayVertexBuffers, sizeof(m_OverlayVertexBuffers)); RtlZeroMemory(m_OverlayTextures, sizeof(m_OverlayTextures)); @@ -43,6 +100,8 @@ D3D11VARenderer::~D3D11VARenderer() { SDL_DestroyMutex(m_ContextLock); + SAFE_COM_RELEASE(m_VideoConstantBuffer); + SAFE_COM_RELEASE(m_VideoVertexBuffer); SAFE_COM_RELEASE(m_VideoPixelShader); for (int i = 0; i < ARRAYSIZE(m_OverlayVertexBuffers); i++) { @@ -65,16 +124,19 @@ D3D11VARenderer::~D3D11VARenderer() CloseHandle(m_FrameWaitableObject); } - if (m_SwapChain != nullptr) { + if (m_SwapChain != nullptr && !m_Windowed) { // It's illegal to destroy a full-screen swapchain. Make sure we're in windowed mode. m_SwapChain->SetFullscreenState(FALSE, nullptr); + } + SAFE_COM_RELEASE(m_SwapChain); - SAFE_COM_RELEASE(m_SwapChain); + if (m_HwFramesContext != nullptr) { + av_buffer_unref(&m_HwFramesContext); } - if (m_HwContext != nullptr) { + if (m_HwDeviceContext != nullptr) { // This will release m_Device and m_DeviceContext too - av_buffer_unref(&m_HwContext); + av_buffer_unref(&m_HwDeviceContext); } else { SAFE_COM_RELEASE(m_Device); @@ -293,31 +355,69 @@ bool D3D11VARenderer::initialize(PDECODER_PARAMETERS params) return false; } - m_HwContext = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_D3D11VA); - if (!m_HwContext) { - SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, - "Failed to allocate D3D11VA context"); - return false; + { + m_HwDeviceContext = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_D3D11VA); + if (!m_HwDeviceContext) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "Failed to allocate D3D11VA device context"); + return false; + } + + AVHWDeviceContext* deviceContext = (AVHWDeviceContext*)m_HwDeviceContext->data; + AVD3D11VADeviceContext* d3d11vaDeviceContext = (AVD3D11VADeviceContext*)deviceContext->hwctx; + + // AVHWDeviceContext takes ownership of these objects + d3d11vaDeviceContext->device = m_Device; + d3d11vaDeviceContext->device_context = m_DeviceContext; + + // Set lock functions that we will use to synchronize with FFmpeg's usage of our device context + d3d11vaDeviceContext->lock = lockContext; + d3d11vaDeviceContext->unlock = unlockContext; + d3d11vaDeviceContext->lock_ctx = this; + + int err = av_hwdevice_ctx_init(m_HwDeviceContext); + if (err < 0) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "Failed to initialize D3D11VA device context: %d", + err); + return false; + } } - AVHWDeviceContext* deviceContext = (AVHWDeviceContext*)m_HwContext->data; - AVD3D11VADeviceContext* d3d11vaDeviceContext = (AVD3D11VADeviceContext*)deviceContext->hwctx; + { + m_HwFramesContext = av_hwframe_ctx_alloc(m_HwDeviceContext); + if (!m_HwFramesContext) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "Failed to allocate D3D11VA frame context"); + return false; + } - // AVHWDeviceContext takes ownership of these objects - d3d11vaDeviceContext->device = m_Device; - d3d11vaDeviceContext->device_context = m_DeviceContext; + AVHWFramesContext* framesContext = (AVHWFramesContext*)m_HwFramesContext->data; - // Set lock functions that we will use to synchronize with FFmpeg's usage of our device context - d3d11vaDeviceContext->lock = lockContext; - d3d11vaDeviceContext->unlock = unlockContext; - d3d11vaDeviceContext->lock_ctx = this; + // We require NV12 or P010 textures for our shader + framesContext->format = AV_PIX_FMT_D3D11; + framesContext->sw_format = params->videoFormat == VIDEO_FORMAT_H265_MAIN10 ? + AV_PIX_FMT_P010 : AV_PIX_FMT_NV12; - int err = av_hwdevice_ctx_init(m_HwContext); - if (err < 0) { - SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, - "Failed to initialize D3D11VA context: %d", - err); - return false; + // Surfaces must be 128 pixel aligned for HEVC and 16 pixel aligned for H.264 + framesContext->width = FFALIGN(params->width, (params->videoFormat & VIDEO_FORMAT_MASK_H265) ? 128 : 16); + framesContext->height = FFALIGN(params->height, (params->videoFormat & VIDEO_FORMAT_MASK_H265) ? 128 : 16); + + // We can have up to 16 reference frames plus a working surface + framesContext->initial_pool_size = 17; + + AVD3D11VAFramesContext* d3d11vaFramesContext = (AVD3D11VAFramesContext*)framesContext->hwctx; + + // We need to override the default D3D11VA bind flags to bind the textures as a shader resources + d3d11vaFramesContext->BindFlags = D3D11_BIND_DECODER | D3D11_BIND_SHADER_RESOURCE; + + int err = av_hwframe_ctx_init(m_HwFramesContext); + if (err < 0) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "Failed to initialize D3D11VA frame context: %d", + err); + return false; + } } if (params->enableVsync && m_Windowed) { @@ -367,7 +467,7 @@ bool D3D11VARenderer::initialize(PDECODER_PARAMETERS params) bool D3D11VARenderer::prepareDecoderContext(AVCodecContext* context, AVDictionary**) { - context->hw_device_ctx = av_buffer_ref(m_HwContext); + context->hw_device_ctx = av_buffer_ref(m_HwDeviceContext); SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Using D3D11VA accelerated renderer"); @@ -375,6 +475,14 @@ bool D3D11VARenderer::prepareDecoderContext(AVCodecContext* context, AVDictionar return true; } +bool D3D11VARenderer::prepareDecoderContextInGetFormat(AVCodecContext *context, AVPixelFormat) +{ + // hw_frames_ctx must be initialized in ffGetFormat(). + context->hw_frames_ctx = av_buffer_ref(m_HwFramesContext); + + return true; +} + void D3D11VARenderer::setHdrMode(bool enabled) { HRESULT hr; @@ -446,6 +554,11 @@ void D3D11VARenderer::renderFrame(AVFrame* frame) { D3D11_VIEWPORT viewPort; + if (frame == nullptr) { + // End of stream - nothing to do for us + return; + } + // Acquire the context lock for rendering to prevent concurrent // access from inside FFmpeg's decoding code lockContext(this); @@ -476,7 +589,8 @@ void D3D11VARenderer::renderFrame(AVFrame* frame) viewPort.Height = dst.h; m_DeviceContext->RSSetViewports(1, &viewPort); - // TODO: Render video + // Render our video frame with the aspect-ratio adjusted viewport + renderVideo(frame); // Set the viewport to render overlays at the full window size viewPort.TopLeftX = viewPort.TopLeftY = 0; @@ -601,7 +715,7 @@ void D3D11VARenderer::renderOverlay(Overlay::OverlayType type) SDL_AtomicUnlock(&m_OverlayLock); - // Bind vertex buffer and shader + // Bind vertex buffer UINT stride = sizeof(VERTEX); UINT offset = 0; m_DeviceContext->IASetVertexBuffers(0, 1, &overlayVertexBuffer, &stride, &offset); @@ -618,6 +732,131 @@ void D3D11VARenderer::renderOverlay(Overlay::OverlayType type) overlayVertexBuffer->Release(); } +void D3D11VARenderer::updateColorConversionConstants(AVFrame* frame) +{ + // If nothing has changed since last frame, we're done + if (frame->colorspace == m_LastColorSpace && frame->color_range == m_LastColorRange) { + return; + } + + // Free any existing buffer + SAFE_COM_RELEASE(m_VideoConstantBuffer); + + D3D11_BUFFER_DESC constDesc = {}; + constDesc.ByteWidth = sizeof(CSC_CONST_BUF); + constDesc.Usage = D3D11_USAGE_IMMUTABLE; + constDesc.BindFlags = D3D11_BIND_CONSTANT_BUFFER; + constDesc.CPUAccessFlags = 0; + constDesc.MiscFlags = 0; + + // This handles the case where the color range is unknown, + // so that we use Limited color range which is the default + // behavior for Moonlight. + CSC_CONST_BUF constBuf = {}; + bool fullRange = (frame->color_range == AVCOL_RANGE_JPEG); + const float* rawCscMatrix; + switch (frame->colorspace) { + case AVCOL_SPC_SMPTE170M: + case AVCOL_SPC_BT470BG: + rawCscMatrix = fullRange ? k_CscMatrix_Bt601Full : k_CscMatrix_Bt601Lim; + break; + case AVCOL_SPC_BT709: + rawCscMatrix = fullRange ? k_CscMatrix_Bt709Full : k_CscMatrix_Bt709Lim; + break; + case AVCOL_SPC_BT2020_NCL: + case AVCOL_SPC_BT2020_CL: + rawCscMatrix = fullRange ? k_CscMatrix_Bt2020Full : k_CscMatrix_Bt2020Lim; + break; + default: + SDL_assert(false); + return; + } + + // We need to adjust our raw CSC matrix to be column-major and with float3 vectors + // padded with a float in between each of them to adhere to HLSL requirements. + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) { + constBuf.cscMatrix[i * 4 + j] = rawCscMatrix[j * 3 + i]; + } + } + + // No adjustments are needed to the float[3] array of offsets, so it can just + // be copied with memcpy(). + memcpy(constBuf.offsets, + fullRange ? k_Offsets_Full : k_Offsets_Lim, + sizeof(constBuf.offsets)); + + D3D11_SUBRESOURCE_DATA constData = {}; + constData.pSysMem = &constBuf; + + HRESULT hr = m_Device->CreateBuffer(&constDesc, &constData, &m_VideoConstantBuffer); + if (FAILED(hr)) { + m_VideoConstantBuffer = nullptr; + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "ID3D11Device::CreateBuffer() failed: %x", + hr); + return; + } + + m_LastColorSpace = frame->colorspace; + m_LastColorRange = frame->color_range; +} + +void D3D11VARenderer::renderVideo(AVFrame* frame) +{ + HRESULT hr; + + // Update our CSC constants if the colorspace has changed + updateColorConversionConstants(frame); + + // Bind video rendering vertex buffer + UINT stride = sizeof(VERTEX); + UINT offset = 0; + m_DeviceContext->IASetVertexBuffers(0, 1, &m_VideoVertexBuffer, &stride, &offset); + + // Create shader resource views for the video texture + D3D11_SHADER_RESOURCE_VIEW_DESC srvDesc = {}; + srvDesc.ViewDimension = D3D11_SRV_DIMENSION_TEXTURE2DARRAY; + srvDesc.Texture2DArray.MostDetailedMip = 0; + srvDesc.Texture2DArray.MipLevels = 1; + srvDesc.Texture2DArray.FirstArraySlice = (uintptr_t)frame->data[1]; + srvDesc.Texture2DArray.ArraySize = 1; + + // Bind the luminance plane + ID3D11ShaderResourceView* luminanceTextureView; + srvDesc.Format = m_DecoderParams.videoFormat == VIDEO_FORMAT_H265_MAIN10 ? DXGI_FORMAT_R16_UNORM : DXGI_FORMAT_R8_UNORM; + hr = m_Device->CreateShaderResourceView((ID3D11Resource*)frame->data[0], &srvDesc, &luminanceTextureView); + if (FAILED(hr)) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "ID3D11Device::CreateShaderResourceView() failed: %x", + hr); + return; + } + m_DeviceContext->PSSetShaderResources(0, 1, &luminanceTextureView); + luminanceTextureView->Release(); + + // Bind the chrominance plane + ID3D11ShaderResourceView* chrominanceTextureView; + srvDesc.Format = m_DecoderParams.videoFormat == VIDEO_FORMAT_H265_MAIN10 ? DXGI_FORMAT_R16G16_UNORM : DXGI_FORMAT_R8G8_UNORM; + hr = m_Device->CreateShaderResourceView((ID3D11Resource*)frame->data[0], &srvDesc, &chrominanceTextureView); + if (FAILED(hr)) { + luminanceTextureView->Release(); + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "ID3D11Device::CreateShaderResourceView() failed: %x", + hr); + return; + } + m_DeviceContext->PSSetShaderResources(1, 1, &chrominanceTextureView); + chrominanceTextureView->Release(); + + // Bind video pixel shader and CSC constants + m_DeviceContext->PSSetShader(m_VideoPixelShader, nullptr, 0); + m_DeviceContext->PSSetConstantBuffers(0, 1, &m_VideoConstantBuffer); + + // Draw the video + m_DeviceContext->DrawIndexed(6, 0, 0); +} + // This function must NOT use any DXGI or ID3D11DeviceContext methods // since it can be called on an arbitrary thread! void D3D11VARenderer::notifyOverlayUpdated(Overlay::OverlayType type) @@ -915,7 +1154,19 @@ bool D3D11VARenderer::setupRenderingResources() hr = m_Device->CreatePixelShader(overlayPixelShaderBytecode.constData(), overlayPixelShaderBytecode.length(), nullptr, &m_OverlayPixelShader); if (FAILED(hr)) { SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, - "ID3D11Device::CreateVertexShader() failed: %x", + "ID3D11Device::CreatePixelShader() failed: %x", + hr); + return false; + } + } + + { + QByteArray videoPixelShaderBytecode = Path::readDataFile("d3d11_video_pixel.fxc"); + + hr = m_Device->CreatePixelShader(videoPixelShaderBytecode.constData(), videoPixelShaderBytecode.length(), nullptr, &m_VideoPixelShader); + if (FAILED(hr)) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "ID3D11Device::CreatePixelShader() failed: %x", hr); return false; } @@ -998,6 +1249,36 @@ bool D3D11VARenderer::setupRenderingResources() } } + // Create our fixed vertex buffer for video rendering + { + VERTEX verts[] = + { + {-1, -1, 0, 1}, + {-1, 1, 0, 0}, + { 1, -1, 1, 1}, + { 1, 1, 1, 0}, + }; + + D3D11_BUFFER_DESC vbDesc = {}; + vbDesc.ByteWidth = sizeof(verts); + vbDesc.Usage = D3D11_USAGE_IMMUTABLE; + vbDesc.BindFlags = D3D11_BIND_VERTEX_BUFFER; + vbDesc.CPUAccessFlags = 0; + vbDesc.MiscFlags = 0; + vbDesc.StructureByteStride = sizeof(VERTEX); + + D3D11_SUBRESOURCE_DATA vbData = {}; + vbData.pSysMem = verts; + + hr = m_Device->CreateBuffer(&vbDesc, &vbData, &m_VideoVertexBuffer); + if (FAILED(hr)) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "ID3D11Device::CreateBuffer() failed: %x", + hr); + return false; + } + } + // Create our blend state { D3D11_BLEND_DESC blendDesc = {}; diff --git a/app/streaming/video/ffmpeg-renderers/d3d11va.h b/app/streaming/video/ffmpeg-renderers/d3d11va.h index e3528ae2..d5db7f58 100644 --- a/app/streaming/video/ffmpeg-renderers/d3d11va.h +++ b/app/streaming/video/ffmpeg-renderers/d3d11va.h @@ -17,6 +17,7 @@ public: virtual ~D3D11VARenderer() override; virtual bool initialize(PDECODER_PARAMETERS params) override; virtual bool prepareDecoderContext(AVCodecContext* context, AVDictionary**) override; + virtual bool prepareDecoderContextInGetFormat(AVCodecContext* context, AVPixelFormat pixelFormat) override; virtual void renderFrame(AVFrame* frame) override; virtual void notifyOverlayUpdated(Overlay::OverlayType) override; virtual void setHdrMode(bool enabled) override; @@ -28,6 +29,8 @@ private: bool setupRenderingResources(); void renderOverlay(Overlay::OverlayType type); + void updateColorConversionConstants(AVFrame* frame); + void renderVideo(AVFrame* frame); bool checkDecoderSupport(IDXGIAdapter* adapter); IDXGIFactory5* m_Factory; @@ -41,10 +44,15 @@ private: int m_DisplayWidth; int m_DisplayHeight; bool m_Windowed; + AVColorSpace m_LastColorSpace; + AVColorRange m_LastColorRange; bool m_AllowTearing; HANDLE m_FrameWaitableObject; + ID3D11PixelShader* m_VideoPixelShader; + ID3D11Buffer* m_VideoVertexBuffer; + ID3D11Buffer* m_VideoConstantBuffer; SDL_SpinLock m_OverlayLock; ID3D11Buffer* m_OverlayVertexBuffers[Overlay::OverlayMax]; @@ -52,6 +60,7 @@ private: ID3D11ShaderResourceView* m_OverlayTextureResourceViews[Overlay::OverlayMax]; ID3D11PixelShader* m_OverlayPixelShader; - AVBufferRef* m_HwContext; + AVBufferRef* m_HwDeviceContext; + AVBufferRef* m_HwFramesContext; }; diff --git a/app/streaming/video/ffmpeg-renderers/renderer.h b/app/streaming/video/ffmpeg-renderers/renderer.h index 262b9772..d8b19f4b 100644 --- a/app/streaming/video/ffmpeg-renderers/renderer.h +++ b/app/streaming/video/ffmpeg-renderers/renderer.h @@ -159,6 +159,11 @@ public: // Nothing } + virtual bool prepareDecoderContextInGetFormat(AVCodecContext*, AVPixelFormat) { + // Assume no further initialization is required + return true; + } + // IOverlayRenderer virtual void notifyOverlayUpdated(Overlay::OverlayType) override { // Nothing diff --git a/app/streaming/video/ffmpeg.cpp b/app/streaming/video/ffmpeg.cpp index 88a28bf4..04d00e81 100644 --- a/app/streaming/video/ffmpeg.cpp +++ b/app/streaming/video/ffmpeg.cpp @@ -114,9 +114,8 @@ enum AVPixelFormat FFmpegVideoDecoder::ffGetFormat(AVCodecContext* context, // format (if not using hardware decoding). It's crucial // to override the default get_format() which will try // to gracefully fall back to software decode and break us. - if (*p == (decoder->m_HwDecodeCfg ? - decoder->m_HwDecodeCfg->pix_fmt : - context->pix_fmt)) { + if (*p == (decoder->m_HwDecodeCfg ? decoder->m_HwDecodeCfg->pix_fmt : context->pix_fmt) && + decoder->m_BackendRenderer->prepareDecoderContextInGetFormat(context, *p)) { return *p; } } @@ -124,7 +123,8 @@ enum AVPixelFormat FFmpegVideoDecoder::ffGetFormat(AVCodecContext* context, // Failed to match the preferred pixel formats. Try non-preferred options for non-hwaccel decoders. if (decoder->m_HwDecodeCfg == nullptr) { for (p = pixFmts; *p != -1; p++) { - if (decoder->m_FrontendRenderer->isPixelFormatSupported(decoder->m_VideoFormat, *p)) { + if (decoder->m_FrontendRenderer->isPixelFormatSupported(decoder->m_VideoFormat, *p) && + decoder->m_BackendRenderer->prepareDecoderContextInGetFormat(context, *p)) { return *p; } }