fix(win/qsv): skip unsupported 4:4:4 codecs (#3029)

src/platform/windows/display_vram.cpp

@@ -1871,6 +1871,13 @@ namespace platf::dxgi {
       if (!boost::algorithm::ends_with(name, "_qsv")) {
         return false;
       }
+      if (config.chromaSamplingType == 1) {
+        if (config.videoFormat == 0 || config.videoFormat == 2) {
+          // QSV doesn't support 4:4:4 in H.264 or AV1
+          return false;
+        }
+        // TODO: Blacklist HEVC 4:4:4 based on adapter model
+      }
     }
     else if (adapter_desc.VendorId == 0x10de) {  // Nvidia
       // If it's not an NVENC encoder, it's not compatible with an Nvidia GPU

src/video.cpp

@@ -1434,9 +1434,7 @@ namespace video {
 
     bool hardware = platform_formats->avcodec_base_dev_type != AV_HWDEVICE_TYPE_NONE;
 
-    auto &video_format = config.videoFormat == 0 ? encoder.h264 :
+    auto &video_format = encoder.codec_from_config(config);
-                         config.videoFormat == 1 ? encoder.hevc :
-                                                   encoder.av1;
     if (!video_format[encoder_t::PASSED] || !disp->is_codec_supported(video_format.name, config)) {
       BOOST_LOG(error) << encoder.name << ": "sv << video_format.name << " mode not supported"sv;
       return nullptr;
@@ -1950,10 +1948,7 @@ namespace video {
     }
 
     {
-      auto encoder_name = config.videoFormat == 0 ? encoder.h264.name :
+      auto encoder_name = encoder.codec_from_config(config).name;
-                          config.videoFormat == 1 ? encoder.hevc.name :
-                          config.videoFormat == 2 ? encoder.av1.name :
-                                                    "unknown";
 
       BOOST_LOG(info) << "Creating encoder " << logging::bracket(encoder_name);
 
@@ -2516,7 +2511,8 @@ namespace video {
       // H.264 is special because encoders may support YUV 4:4:4 without supporting 10-bit color depth
       if (encoder.flags & YUV444_SUPPORT) {
         config_t config_h264_yuv444 { 1920, 1080, 60, 1000, 1, 0, 1, 0, 0, 1 };
-        encoder.h264[encoder_t::YUV444] = validate_config(disp, encoder, config_h264_yuv444);
+        encoder.h264[encoder_t::YUV444] = disp->is_codec_supported(encoder.h264.name, config_h264_yuv444) &&
+                                          validate_config(disp, encoder, config_h264_yuv444) >= 0;
       }
       else {
         encoder.h264[encoder_t::YUV444] = false;
@@ -2536,17 +2532,30 @@ namespace video {
 
         if (!flag_map[encoder_t::PASSED]) return;
 
+        auto encoder_codec_name = encoder.codec_from_config(config).name;
+
         // Test 4:4:4 HDR first. If 4:4:4 is supported, 4:2:0 should also be supported.
         config.chromaSamplingType = 1;
-        if ((encoder.flags & YUV444_SUPPORT) && validate_config(disp, encoder, config) >= 0) {
+        if ((encoder.flags & YUV444_SUPPORT) &&
+            disp->is_codec_supported(encoder_codec_name, config) &&
+            validate_config(disp, encoder, config) >= 0) {
           flag_map[encoder_t::DYNAMIC_RANGE] = true;
           flag_map[encoder_t::YUV444] = true;
           return;
         }
+        else {
+          flag_map[encoder_t::YUV444] = false;
+        }
 
         // Test 4:2:0 HDR
         config.chromaSamplingType = 0;
-        flag_map[encoder_t::DYNAMIC_RANGE] = validate_config(disp, encoder, config) >= 0;
+        if (disp->is_codec_supported(encoder_codec_name, config) &&
+            validate_config(disp, encoder, config) >= 0) {
+          flag_map[encoder_t::DYNAMIC_RANGE] = true;
+        }
+        else {
+          flag_map[encoder_t::DYNAMIC_RANGE] = false;
+        }
       };
 
       // HDR is not supported with H.264. Don't bother even trying it.

src/video.h

@@ -17,6 +17,29 @@ extern "C" {
 struct AVPacket;
 namespace video {
 
+  /* Encoding configuration requested by remote client */
+  struct config_t {
+    int width;  // Video width in pixels
+    int height;  // Video height in pixels
+    int framerate;  // Requested framerate, used in individual frame bitrate budget calculation
+    int bitrate;  // Video bitrate in kilobits (1000 bits) for requested framerate
+    int slicesPerFrame;  // Number of slices per frame
+    int numRefFrames;  // Max number of reference frames
+
+    /* Requested color range and SDR encoding colorspace, HDR encoding colorspace is always BT.2020+ST2084
+       Color range (encoderCscMode & 0x1) : 0 - limited, 1 - full
+       SDR encoding colorspace (encoderCscMode >> 1) : 0 - BT.601, 1 - BT.709, 2 - BT.2020 */
+    int encoderCscMode;
+
+    int videoFormat;  // 0 - H.264, 1 - HEVC, 2 - AV1
+
+    /* Encoding color depth (bit depth): 0 - 8-bit, 1 - 10-bit
+       HDR encoding activates when color depth is higher than 8-bit and the display which is being captured is operating in HDR mode */
+    int dynamicRange;
+
+    int chromaSamplingType;  // 0 - 4:2:0, 1 - 4:4:4
+  };
+
   platf::mem_type_e
   map_base_dev_type(AVHWDeviceType type);
   platf::pix_fmt_e
@@ -163,6 +186,21 @@ namespace video {
       }
     } av1, hevc, h264;
 
+    const codec_t &
+    codec_from_config(const config_t &config) const {
+      switch (config.videoFormat) {
+        default:
+          BOOST_LOG(error) << "Unknown video format " << config.videoFormat << ", falling back to H.264";
+          // fallthrough
+        case 0:
+          return h264;
+        case 1:
+          return hevc;
+        case 2:
+          return av1;
+      }
+    }
+
     uint32_t flags;
   };
 
@@ -309,29 +347,6 @@ namespace video {
 
   using hdr_info_t = std::unique_ptr<hdr_info_raw_t>;
 
-  /* Encoding configuration requested by remote client */
-  struct config_t {
-    int width;  // Video width in pixels
-    int height;  // Video height in pixels
-    int framerate;  // Requested framerate, used in individual frame bitrate budget calculation
-    int bitrate;  // Video bitrate in kilobits (1000 bits) for requested framerate
-    int slicesPerFrame;  // Number of slices per frame
-    int numRefFrames;  // Max number of reference frames
-
-    /* Requested color range and SDR encoding colorspace, HDR encoding colorspace is always BT.2020+ST2084
-       Color range (encoderCscMode & 0x1) : 0 - limited, 1 - full
-       SDR encoding colorspace (encoderCscMode >> 1) : 0 - BT.601, 1 - BT.709, 2 - BT.2020 */
-    int encoderCscMode;
-
-    int videoFormat;  // 0 - H.264, 1 - HEVC, 2 - AV1
-
-    /* Encoding color depth (bit depth): 0 - 8-bit, 1 - 10-bit
-       HDR encoding activates when color depth is higher than 8-bit and the display which is being captured is operating in HDR mode */
-    int dynamicRange;
-
-    int chromaSamplingType;  // 0 - 4:2:0, 1 - 4:4:4
-  };
-
   extern int active_hevc_mode;
   extern int active_av1_mode;
   extern bool last_encoder_probe_supported_ref_frames_invalidation;