From 5765c254cd428781580ba7f9f5857cb3361440bc Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Mon, 19 Aug 2024 19:20:17 -0500 Subject: [PATCH] Add software fallback for YUV444 using libswscale --- README.md | 4 +- app/app.pro | 6 +- .../video/ffmpeg-renderers/sdlvid.cpp | 162 +++++++++++++++--- app/streaming/video/ffmpeg-renderers/sdlvid.h | 11 ++ appveyor.yml | 2 +- libs | 2 +- scripts/clean-libs.sh | 8 +- 7 files changed, 157 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 2cea1e1f..86dbe223 100644 --- a/README.md +++ b/README.md @@ -52,10 +52,10 @@ Hosting for Moonlight's Debian and L4T package repositories is graciously provid ### Linux/Unix Build Requirements * Qt 6 is recommended, but Qt 5.9 or later is also supported (replace `qmake6` with `qmake` when using Qt 5). * GCC or Clang -* FFmpeg 4.0 or later +* FFmpeg 5.0 or later * Install the required packages: * Debian/Ubuntu: - * Base Requirements: `libegl1-mesa-dev libgl1-mesa-dev libopus-dev libsdl2-dev libsdl2-ttf-dev libssl-dev libavcodec-dev libavformat-dev libva-dev libvdpau-dev libxkbcommon-dev wayland-protocols libdrm-dev` + * Base Requirements: `libegl1-mesa-dev libgl1-mesa-dev libopus-dev libsdl2-dev libsdl2-ttf-dev libssl-dev libavcodec-dev libavformat-dev libswscale-dev libva-dev libvdpau-dev libxkbcommon-dev wayland-protocols libdrm-dev` * Qt 6 (Recommended): `qt6-base-dev qt6-declarative-dev libqt6svg6-dev qml6-module-qtquick-controls qml6-module-qtquick-templates qml6-module-qtquick-layouts qml6-module-qtqml-workerscript qml6-module-qtquick-window qml6-module-qtquick` * Qt 5: `qtbase5-dev qt5-qmake qtdeclarative5-dev qtquickcontrols2-5-dev qml-module-qtquick-controls2 qml-module-qtquick-layouts qml-module-qtquick-window2 qml-module-qtquick2 qtwayland5` * RedHat/Fedora (RPM Fusion repo required): diff --git a/app/app.pro b/app/app.pro index 073d08d2..08ed45fe 100644 --- a/app/app.pro +++ b/app/app.pro @@ -73,7 +73,7 @@ unix:if(!macx|disable-prebuilts) { !disable-ffmpeg { packagesExist(libavcodec) { - PKGCONFIG += libavcodec libavutil + PKGCONFIG += libavcodec libavutil libswscale CONFIG += ffmpeg !disable-libva { @@ -148,7 +148,7 @@ unix:if(!macx|disable-prebuilts) { } } win32 { - LIBS += -llibssl -llibcrypto -lSDL2 -lSDL2_ttf -lavcodec -lavutil -lopus -ldxgi -ld3d11 + LIBS += -llibssl -llibcrypto -lSDL2 -lSDL2_ttf -lavcodec -lavutil -lswscale -lopus -ldxgi -ld3d11 CONFIG += ffmpeg contains(QT_ARCH, x86_64) { LIBS += -llibplacebo @@ -160,7 +160,7 @@ win32:!winrt { } macx { !disable-prebuilts { - LIBS += -lssl -lcrypto -lavcodec.61 -lavutil.59 -lopus -framework SDL2 -framework SDL2_ttf + LIBS += -lssl -lcrypto -lavcodec.61 -lavutil.59 -lswscale.8 -lopus -framework SDL2 -framework SDL2_ttf CONFIG += discord-rpc } diff --git a/app/streaming/video/ffmpeg-renderers/sdlvid.cpp b/app/streaming/video/ffmpeg-renderers/sdlvid.cpp index be73e366..00418516 100644 --- a/app/streaming/video/ffmpeg-renderers/sdlvid.cpp +++ b/app/streaming/video/ffmpeg-renderers/sdlvid.cpp @@ -7,11 +7,19 @@ #include +extern "C" { +#include +#include +} + SdlRenderer::SdlRenderer() : m_VideoFormat(0), m_Renderer(nullptr), m_Texture(nullptr), m_ColorSpace(-1), + m_NeedsYuvToRgbConversion(false), + m_SwsContext(nullptr), + m_RgbFrame(av_frame_alloc()), m_SwFrameMapper(this) { SDL_zero(m_OverlayTextures); @@ -35,6 +43,9 @@ SdlRenderer::~SdlRenderer() } } + av_frame_free(&m_RgbFrame); + sws_freeContext(m_SwsContext); + if (m_Texture != nullptr) { SDL_DestroyTexture(m_Texture); } @@ -82,15 +93,27 @@ bool SdlRenderer::isRenderThreadSupported() bool SdlRenderer::isPixelFormatSupported(int videoFormat, AVPixelFormat pixelFormat) { - if (videoFormat & VIDEO_FORMAT_MASK_10BIT) { - // SDL2 doesn't support 10-bit pixel formats - return false; - } - else if (videoFormat & VIDEO_FORMAT_MASK_YUV444) { - // SDL2 doesn't support YUV444 pixel formats - return false; + if (videoFormat & (VIDEO_FORMAT_MASK_10BIT | VIDEO_FORMAT_MASK_YUV444)) { + // SDL2 can't natively handle textures with these formats, but we can perform + // conversion on the CPU using swscale then upload them as an RGB texture. + const AVPixFmtDescriptor* formatDesc = av_pix_fmt_desc_get(pixelFormat); + if (!formatDesc) { + SDL_assert(formatDesc); + return false; + } + + const int expectedPixelDepth = (videoFormat & VIDEO_FORMAT_MASK_10BIT) ? 10 : 8; + const int expectedLog2ChromaW = (videoFormat & VIDEO_FORMAT_MASK_YUV444) ? 0 : 1; + const int expectedLog2ChromaH = (videoFormat & VIDEO_FORMAT_MASK_YUV444) ? 0 : 1; + + return formatDesc->comp[0].depth == expectedPixelDepth && + formatDesc->log2_chroma_w == expectedLog2ChromaW && + formatDesc->log2_chroma_h == expectedLog2ChromaH; } else { + // The formats listed below are natively supported by SDL, so it can handle + // YUV to RGB conversion on the GPU using pixel shaders. + // // Remember to keep this in sync with SdlRenderer::renderFrame()! switch (pixelFormat) { case AV_PIX_FMT_YUV420P: @@ -112,8 +135,8 @@ bool SdlRenderer::initialize(PDECODER_PARAMETERS params) m_VideoFormat = params->videoFormat; m_SwFrameMapper.setVideoFormat(m_VideoFormat); - if (params->videoFormat & (VIDEO_FORMAT_MASK_10BIT | VIDEO_FORMAT_MASK_YUV444)) { - // SDL doesn't support rendering YUV444 or 10-bit textures yet + if (params->videoFormat & VIDEO_FORMAT_MASK_10BIT) { + // SDL doesn't support rendering HDR yet return false; } @@ -233,6 +256,11 @@ void SdlRenderer::renderOverlay(Overlay::OverlayType type) } } +void SdlRenderer::ffNoopFree(void*, uint8_t*) +{ + // Nothing +} + void SdlRenderer::renderFrame(AVFrame* frame) { int err; @@ -286,6 +314,7 @@ ReadbackRetry: Uint32 sdlFormat; // Remember to keep this in sync with SdlRenderer::isPixelFormatSupported()! + m_NeedsYuvToRgbConversion = false; switch (frame->format) { case AV_PIX_FMT_YUV420P: @@ -300,27 +329,75 @@ ReadbackRetry: sdlFormat = SDL_PIXELFORMAT_NV21; break; default: - SDL_assert(false); - goto Exit; + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "Performing color conversion on CPU due to lack of SDL support for format: %u", + frame->format); + sdlFormat = SDL_PIXELFORMAT_XRGB8888; + m_NeedsYuvToRgbConversion = true; + break; } - switch (colorspace) - { - case COLORSPACE_REC_709: - SDL_assert(!isFrameFullRange(frame)); - SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_BT709); - break; - case COLORSPACE_REC_601: - if (isFrameFullRange(frame)) { - // SDL's JPEG mode is Rec 601 Full Range - SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_JPEG); + if (m_NeedsYuvToRgbConversion) { + m_RgbFrame->width = frame->width; + m_RgbFrame->height = frame->height; + m_RgbFrame->format = AV_PIX_FMT_BGR0; + + sws_freeContext(m_SwsContext); + m_SwsContext = sws_alloc_context(); + if (!m_SwsContext) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "sws_alloc_context() failed"); + goto Exit; } - else { - SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_BT601); + + AVDictionary *options { nullptr }; + av_dict_set_int(&options, "srcw", frame->width, 0); + av_dict_set_int(&options, "srch", frame->height, 0); + av_dict_set_int(&options, "src_format", frame->format, 0); + av_dict_set_int(&options, "dstw", m_RgbFrame->width, 0); + av_dict_set_int(&options, "dsth", m_RgbFrame->height, 0); + av_dict_set_int(&options, "dst_format", m_RgbFrame->format, 0); + av_dict_set_int(&options, "threads", std::min(SDL_GetCPUCount(), 4), 0); // Up to 4 threads + + err = av_opt_set_dict(m_SwsContext, &options); + av_dict_free(&options); + if (err < 0) { + char string[AV_ERROR_MAX_STRING_SIZE]; + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "av_opt_set_dict() failed: %s", + av_make_error_string(string, sizeof(string), err)); + goto Exit; + } + + err = sws_init_context(m_SwsContext, nullptr, nullptr); + if (err < 0) { + char string[AV_ERROR_MAX_STRING_SIZE]; + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "sws_init_context() failed: %s", + av_make_error_string(string, sizeof(string), err)); + goto Exit; + } + } + else { + // SDL will perform YUV conversion on the GPU + switch (colorspace) + { + case COLORSPACE_REC_709: + SDL_assert(!isFrameFullRange(frame)); + SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_BT709); + break; + case COLORSPACE_REC_601: + if (isFrameFullRange(frame)) { + // SDL's JPEG mode is Rec 601 Full Range + SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_JPEG); + } + else { + SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_BT601); + } + break; + default: + break; } - break; - default: - break; } m_Texture = SDL_CreateTexture(m_Renderer, @@ -371,7 +448,7 @@ ReadbackRetry: frame->data[2], frame->linesize[2]); } - else { + else if (!m_NeedsYuvToRgbConversion) { #if SDL_VERSION_ATLEAST(2, 0, 15) // SDL_UpdateNVTexture is not supported on all renderer backends, // (notably not DX9), so we must have a fallback in case it's not @@ -430,6 +507,37 @@ ReadbackRetry: SDL_UnlockTexture(m_Texture); } } + else { + // We have a pixel format that SDL doesn't natively support, so we must use + // swscale to convert the YUV frame into an RGB frame to upload to the GPU. + uint8_t* pixels; + int texturePitch; + + err = SDL_LockTexture(m_Texture, nullptr, (void**)&pixels, &texturePitch); + if (err < 0) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "SDL_LockTexture() failed: %s", + SDL_GetError()); + goto Exit; + } + + // Create a buffer to wrap our locked texture buffer + m_RgbFrame->buf[0] = av_buffer_create(pixels, m_RgbFrame->height * texturePitch, ffNoopFree, nullptr, 0); + m_RgbFrame->data[0] = pixels; + m_RgbFrame->linesize[0] = texturePitch; + + // Perform multi-threaded color conversion into the locked texture buffer + err = sws_scale_frame(m_SwsContext, m_RgbFrame, frame); + av_buffer_unref(&m_RgbFrame->buf[0]); + SDL_UnlockTexture(m_Texture); + if (err < 0) { + char string[AV_ERROR_MAX_STRING_SIZE]; + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "sws_scale_frame() failed: %s", + av_make_error_string(string, AV_ERROR_MAX_STRING_SIZE, err)); + goto Exit; + } + } SDL_RenderClear(m_Renderer); diff --git a/app/streaming/video/ffmpeg-renderers/sdlvid.h b/app/streaming/video/ffmpeg-renderers/sdlvid.h index a47b5ce0..03e40d12 100644 --- a/app/streaming/video/ffmpeg-renderers/sdlvid.h +++ b/app/streaming/video/ffmpeg-renderers/sdlvid.h @@ -7,6 +7,10 @@ #include "cuda.h" #endif +extern "C" { +#include +} + class SdlRenderer : public IFFmpegRenderer { public: SdlRenderer(); @@ -23,6 +27,8 @@ public: private: void renderOverlay(Overlay::OverlayType type); + static void ffNoopFree(void *opaque, uint8_t *data); + int m_VideoFormat; SDL_Renderer* m_Renderer; SDL_Texture* m_Texture; @@ -30,6 +36,11 @@ private: SDL_Texture* m_OverlayTextures[Overlay::OverlayMax]; SDL_Rect m_OverlayRects[Overlay::OverlayMax]; + // Used for CPU conversion of YUV to RGB if needed + bool m_NeedsYuvToRgbConversion; + SwsContext* m_SwsContext; + AVFrame* m_RgbFrame; + SwFrameMapper m_SwFrameMapper; #ifdef HAVE_CUDA diff --git a/appveyor.yml b/appveyor.yml index a4784603..ce740f29 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -13,7 +13,7 @@ environment: BUILD_TARGET: steamlink - APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004 BUILD_TARGET: linux - FFMPEG_CONFIGURE_ARGS: --enable-pic --disable-static --enable-shared --disable-all --enable-avcodec --enable-avformat --enable-decoder=h264 --enable-decoder=hevc --enable-decoder=av1 --enable-hwaccel=h264_vaapi --enable-hwaccel=hevc_vaapi --enable-hwaccel=av1_vaapi --enable-hwaccel=h264_vdpau --enable-hwaccel=hevc_vdpau --enable-hwaccel=av1_vdpau --enable-libdrm --enable-hwaccel=h264_vulkan --enable-hwaccel=hevc_vulkan --enable-hwaccel=av1_vulkan --enable-libdav1d --enable-decoder=libdav1d + FFMPEG_CONFIGURE_ARGS: --enable-pic --disable-static --enable-shared --disable-all --enable-avcodec --enable-avformat --enable-swscale --enable-decoder=h264 --enable-decoder=hevc --enable-decoder=av1 --enable-hwaccel=h264_vaapi --enable-hwaccel=hevc_vaapi --enable-hwaccel=av1_vaapi --enable-hwaccel=h264_vdpau --enable-hwaccel=hevc_vdpau --enable-hwaccel=av1_vdpau --enable-libdrm --enable-hwaccel=h264_vulkan --enable-hwaccel=hevc_vulkan --enable-hwaccel=av1_vulkan --enable-libdav1d --enable-decoder=libdav1d install: - cmd: 'copy /y scripts\appveyor\qmake.bat %QTDIR%\msvc2019_arm64\bin\' diff --git a/libs b/libs index f04ef02a..f9b4803f 160000 --- a/libs +++ b/libs @@ -1 +1 @@ -Subproject commit f04ef02a95815a6a679f0d82ca88115edb413f26 +Subproject commit f9b4803fdfcfe12cb94076702bdf1ae70a7a4545 diff --git a/scripts/clean-libs.sh b/scripts/clean-libs.sh index 0fec4d89..c96cde25 100755 --- a/scripts/clean-libs.sh +++ b/scripts/clean-libs.sh @@ -46,8 +46,8 @@ while [[ "$#" -gt 0 ]]; do shift ;; --ffmpeg_win) - rm -r $LIB_PATH/windows/include/*/libavcodec $LIB_PATH/windows/include/*/libavutil $LIB_PATH/windows/include/*/libavformat - rm $LIB_PATH/windows/lib/*/avcodec* $LIB_PATH/windows/lib/*/avutil* $LIB_PATH/windows/lib/*/avformat* + rm -r $LIB_PATH/windows/include/*/libavcodec $LIB_PATH/windows/include/*/libavutil $LIB_PATH/windows/include/*/libavformat $LIB_PATH/windows/include/*/libswscale + rm $LIB_PATH/windows/lib/*/avcodec* $LIB_PATH/windows/lib/*/avutil* $LIB_PATH/windows/lib/*/avformat* $LIB_PATH/windows/lib/*/swscale* shift ;; --dav1d_win) @@ -55,8 +55,8 @@ while [[ "$#" -gt 0 ]]; do shift ;; --ffmpeg_mac) - rm -r $LIB_PATH/mac/include/libavcodec $LIB_PATH/mac/include/libavutil $LIB_PATH/mac/include/libavformat - rm $LIB_PATH/mac/lib/libavcodec* $LIB_PATH/mac/lib/libavutil* $LIB_PATH/mac/lib/libavformat* + rm -r $LIB_PATH/mac/include/libavcodec $LIB_PATH/mac/include/libavutil $LIB_PATH/mac/include/libavformat $LIB_PATH/mac/include/libswscale + rm $LIB_PATH/mac/lib/libavcodec* $LIB_PATH/mac/lib/libavutil* $LIB_PATH/mac/lib/libavformat* $LIB_PATH/mac/lib/libswscale* shift ;; --libplacebo_win)