From c989133d27bf040f59cf878dc34fa53dfad56157 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Sun, 24 Apr 2022 15:42:08 -0500 Subject: [PATCH] Allow renderers to apply size and display changes seamlessly --- app/streaming/session.cpp | 52 +++++++++++++++++-- app/streaming/video/decoder.h | 6 +++ .../video/ffmpeg-renderers/renderer.h | 6 +++ .../video/ffmpeg-renderers/sdlvid.cpp | 42 ++++++++++----- app/streaming/video/ffmpeg-renderers/sdlvid.h | 4 ++ app/streaming/video/ffmpeg.cpp | 5 ++ app/streaming/video/ffmpeg.h | 1 + app/streaming/video/slvid.h | 5 ++ 8 files changed, 106 insertions(+), 15 deletions(-) diff --git a/app/streaming/session.cpp b/app/streaming/session.cpp index 2704ccc1..ce77f5e8 100644 --- a/app/streaming/session.cpp +++ b/app/streaming/session.cpp @@ -1640,6 +1640,50 @@ void Session::execInternal() break; } + // If we have a window size or display change, ask the decoder if it can apply the change + if (m_VideoDecoder != nullptr && + (event.window.event == SDL_WINDOWEVENT_SIZE_CHANGED || SDL_GetWindowDisplayIndex(m_Window) != currentDisplayIndex)) { + int width, height; + bool handled; + int flags = 0; + + // If this is a resize, use the new width and height from the event itself + if (event.window.event == SDL_WINDOWEVENT_SIZE_CHANGED) { + flags |= WINDOW_SIZE_CHANGED; + width = event.window.data1; + height = event.window.data2; + } + else { + SDL_GetWindowSize(m_Window, &width, &height); + } + + if (SDL_GetWindowDisplayIndex(m_Window) != currentDisplayIndex) { + flags |= WINDOW_DISPLAY_CHANGED; + } + + // See if the decoder can apply this change + SDL_AtomicLock(&m_DecoderLock); + handled = m_VideoDecoder->applyWindowChange(width, height, flags); + SDL_AtomicUnlock(&m_DecoderLock); + + // If it did, we don't need to recreate the decoder + if (handled) { + SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, + "Applied seamless window change: %x", flags); + + if (currentDisplayIndex != SDL_GetWindowDisplayIndex(m_Window)) { + // Update the window display mode based on our current monitor + currentDisplayIndex = SDL_GetWindowDisplayIndex(m_Window); + updateOptimalWindowDisplayMode(); + } + + // After a window resize, we need to reset the pointer lock region + m_InputHandler->updatePointerRegionLock(); + + break; + } + } + SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Recreating renderer for window event: %d (%d %d)", event.window.event, @@ -1661,9 +1705,11 @@ void Session::execInternal() // important events to be lost. flushWindowEvents(); - // Update the window display mode based on our current monitor - currentDisplayIndex = SDL_GetWindowDisplayIndex(m_Window); - updateOptimalWindowDisplayMode(); + if (currentDisplayIndex != SDL_GetWindowDisplayIndex(m_Window)) { + // Update the window display mode based on our current monitor + currentDisplayIndex = SDL_GetWindowDisplayIndex(m_Window); + updateOptimalWindowDisplayMode(); + } // Now that the old decoder is dead, flush any events it may // have queued to reset itself (if this reset was the result diff --git a/app/streaming/video/decoder.h b/app/streaming/video/decoder.h index 650a021f..a53b41df 100644 --- a/app/streaming/video/decoder.h +++ b/app/streaming/video/decoder.h @@ -58,6 +58,11 @@ typedef struct _DECODER_PARAMETERS { HDR_MASTERING_METADATA hdrMetadata; } DECODER_PARAMETERS, *PDECODER_PARAMETERS; + +// Flags for applyWindowChange() +#define WINDOW_SIZE_CHANGED 0x01 +#define WINDOW_DISPLAY_CHANGED 0x02 + class IVideoDecoder { public: virtual ~IVideoDecoder() {} @@ -71,4 +76,5 @@ public: virtual int submitDecodeUnit(PDECODE_UNIT du) = 0; virtual void renderFrameOnMainThread() = 0; virtual void setHdrMode(bool enabled) = 0; + virtual bool applyWindowChange(int width, int height, int flags) = 0; }; diff --git a/app/streaming/video/ffmpeg-renderers/renderer.h b/app/streaming/video/ffmpeg-renderers/renderer.h index 244496b4..90f92bd4 100644 --- a/app/streaming/video/ffmpeg-renderers/renderer.h +++ b/app/streaming/video/ffmpeg-renderers/renderer.h @@ -167,6 +167,12 @@ public: return true; } + virtual bool applyWindowChange(int, int, int) { + // By default, we cannot apply any changes. The renderer + // will be recreated after any window change. + return false; + } + virtual AVPixelFormat getPreferredPixelFormat(int videoFormat) { if (videoFormat == VIDEO_FORMAT_H265_MAIN10) { // 10-bit YUV 4:2:0 diff --git a/app/streaming/video/ffmpeg-renderers/sdlvid.cpp b/app/streaming/video/ffmpeg-renderers/sdlvid.cpp index 132d78e1..50784bf8 100644 --- a/app/streaming/video/ffmpeg-renderers/sdlvid.cpp +++ b/app/streaming/video/ffmpeg-renderers/sdlvid.cpp @@ -91,6 +91,8 @@ bool SdlRenderer::initialize(PDECODER_PARAMETERS params) Uint32 rendererFlags = SDL_RENDERER_ACCELERATED; m_VideoFormat = params->videoFormat; + m_VideoWidth = params->width; + m_VideoHeight = params->height; if (params->videoFormat == VIDEO_FORMAT_H265_MAIN10) { // SDL doesn't support rendering YUV 10-bit textures yet @@ -139,18 +141,8 @@ bool SdlRenderer::initialize(PDECODER_PARAMETERS params) SDL_FlushEvent(SDL_WINDOWEVENT); } - // Calculate the video region size, scaling to fill the output size while - // preserving the aspect ratio of the video stream. - SDL_Rect src, dst; - src.x = src.y = 0; - src.w = params->width; - src.h = params->height; - dst.x = dst.y = 0; - SDL_GetRendererOutputSize(m_Renderer, &dst.w, &dst.h); - StreamUtils::scaleSourceToDestinationSurface(&src, &dst); - - // Ensure the viewport is set to the desired video region - SDL_RenderSetViewport(m_Renderer, &dst); + // Set the renderer viewport to draw the video while preserving aspect ratio + updateViewport(); // Draw a black frame until the video stream starts rendering SDL_SetRenderDrawColor(m_Renderer, 0, 0, 0, SDL_ALPHA_OPAQUE); @@ -339,6 +331,22 @@ AVFrame* SdlRenderer::getSwFrameFromHwFrame(AVFrame* hwFrame) return swFrame; } +void SdlRenderer::updateViewport() +{ + // Calculate the video region size, scaling to fill the output size while + // preserving the aspect ratio of the video stream. + SDL_Rect src, dst; + src.x = src.y = 0; + src.w = m_VideoWidth; + src.h = m_VideoHeight; + dst.x = dst.y = 0; + SDL_GetRendererOutputSize(m_Renderer, &dst.w, &dst.h); + StreamUtils::scaleSourceToDestinationSurface(&src, &dst); + + // Ensure the viewport is set to the desired video region + SDL_RenderSetViewport(m_Renderer, &dst); +} + void SdlRenderer::renderFrame(AVFrame* frame) { int err; @@ -580,3 +588,13 @@ bool SdlRenderer::testRenderFrame(AVFrame* frame) return true; } + +bool SdlRenderer::applyWindowChange(int, int, int flags) +{ + if (flags == WINDOW_SIZE_CHANGED) { + updateViewport(); + return true; + } + + return false; +} diff --git a/app/streaming/video/ffmpeg-renderers/sdlvid.h b/app/streaming/video/ffmpeg-renderers/sdlvid.h index 3223793a..05958c0a 100644 --- a/app/streaming/video/ffmpeg-renderers/sdlvid.h +++ b/app/streaming/video/ffmpeg-renderers/sdlvid.h @@ -16,13 +16,17 @@ public: virtual bool isRenderThreadSupported() override; virtual bool isPixelFormatSupported(int videoFormat, enum AVPixelFormat pixelFormat) override; virtual bool testRenderFrame(AVFrame* frame) override; + virtual bool applyWindowChange(int width, int height, int flags) override; private: void renderOverlay(Overlay::OverlayType type); bool initializeReadBackFormat(AVBufferRef* hwFrameCtxRef, AVFrame* testFrame); AVFrame* getSwFrameFromHwFrame(AVFrame* hwFrame); + void updateViewport(); int m_VideoFormat; + int m_VideoWidth; + int m_VideoHeight; SDL_Renderer* m_Renderer; SDL_Texture* m_Texture; enum AVPixelFormat m_SwPixelFormat; diff --git a/app/streaming/video/ffmpeg.cpp b/app/streaming/video/ffmpeg.cpp index 82d1ef55..88b58333 100644 --- a/app/streaming/video/ffmpeg.cpp +++ b/app/streaming/video/ffmpeg.cpp @@ -68,6 +68,11 @@ void FFmpegVideoDecoder::setHdrMode(bool enabled) m_FrontendRenderer->setHdrMode(enabled); } +bool FFmpegVideoDecoder::applyWindowChange(int width, int height, int flags) +{ + return m_FrontendRenderer->applyWindowChange(width, height, flags); +} + int FFmpegVideoDecoder::getDecoderCapabilities() { int capabilities = m_BackendRenderer->getDecoderCapabilities(); diff --git a/app/streaming/video/ffmpeg.h b/app/streaming/video/ffmpeg.h index a983569c..d7d35880 100644 --- a/app/streaming/video/ffmpeg.h +++ b/app/streaming/video/ffmpeg.h @@ -25,6 +25,7 @@ public: virtual int submitDecodeUnit(PDECODE_UNIT du) override; virtual void renderFrameOnMainThread() override; virtual void setHdrMode(bool enabled) override; + virtual bool applyWindowChange(int width, int height, int flags) override; virtual IFFmpegRenderer* getBackendRenderer(); diff --git a/app/streaming/video/slvid.h b/app/streaming/video/slvid.h index 9651e941..b3ffed74 100644 --- a/app/streaming/video/slvid.h +++ b/app/streaming/video/slvid.h @@ -26,6 +26,11 @@ public: return false; } + // SLVideo cannot apply any window changes (nor do we expect any) + virtual bool applyWindowChange(int, int, int) override { + return false; + } + private: static void slLogCallback(void* context, ESLVideoLog logLevel, const char* message);