fix(linux): fix issues with rendering and touchscreens when displays are scaled (#4607)

src/input.cpp

@@ -170,7 +170,7 @@ namespace input {
         touch_port_event {std::move(touch_port_event)},
         feedback_queue {std::move(feedback_queue)},
         mouse_left_button_timeout {},
-        touch_port {{0, 0, 0, 0}, 0, 0, 1.0f},
+        touch_port {{0, 0, 0, 0}, 0, 0, 1.0f, 1.0f, 0, 0},
         accumulated_vscroll_delta {},
         accumulated_hscroll_delta {} {
     }
@@ -480,7 +480,29 @@ namespace input {
     x = std::clamp(x, offsetX, (size.first * scalarX) - offsetX);
     y = std::clamp(y, offsetY, (size.second * scalarY) - offsetY);
 
-    return std::pair {(x - offsetX) * touch_port.scalar_inv, (y - offsetY) * touch_port.scalar_inv};
+    /*
+    x and y here below have the coordinates of the surface of the streaming resolution,
+    and are dependent on how that comes configured from the client (scalar_inv is calculated
+    from the proportion of that and the device's **physical** size).
+    */
+    x = (x - offsetX) * touch_port.scalar_inv;
+    y = (y - offsetY) * touch_port.scalar_inv;
+
+    /*
+    This final operation is a bit weird and has been brought about with lots of trial and error. A better
+    way to do this may exist.
+
+    Basically, this is what makes the touchscreen map to the coordinates inputtino expects properly.
+    Since inputtino's dimensions are now logical (because scaling breaks everything otherwise), using the previous
+    x and y coordinates would be incorrect when screens are scaled, because the touch port is smaller (or larger)
+    by a factor (that factor is touch_port.scalar_tpcoords), and that factor must be used to account for that difference
+    when moving the cursor. Otherwise, it will move either slower or faster than your finger proportionally to
+    scalar_tpcoords, and be offset *inversely* proportionally to scalar_tpcoords. So you must account for both differences
+    by multiplying and dividing.
+    */
+    float final_x = (x + touch_port.offset_x * touch_port.scalar_tpcoords) / touch_port.scalar_tpcoords;
+    float final_y = (y + touch_port.offset_y * touch_port.scalar_tpcoords) / touch_port.scalar_tpcoords;
+    return std::pair {final_x, final_y};
   }
 
   /**
@@ -545,11 +567,22 @@ namespace input {
     }
 
     auto &touch_port = input->touch_port;
+
+    int touch_port_dim_x;
+    int touch_port_dim_y;
+    if (touch_port.env_logical_width != 0 && touch_port.env_logical_height != 0) {
+      touch_port_dim_x = touch_port.env_logical_width;
+      touch_port_dim_y = touch_port.env_logical_height;
+    } else {
+      touch_port_dim_x = touch_port.env_width;
+      touch_port_dim_y = touch_port.env_height;
+    }
+
     platf::touch_port_t abs_port {
       touch_port.offset_x,
       touch_port.offset_y,
-      touch_port.env_width,
-      touch_port.env_height
+      touch_port_dim_x,
+      touch_port_dim_y
     };
 
     platf::abs_mouse(platf_input, abs_port, tpcoords->first, tpcoords->second);

src/input.h

@@ -30,7 +30,9 @@ namespace input {
     // Offset x and y coordinates of the client
     float client_offsetX, client_offsetY;
 
-    float scalar_inv;
+    float scalar_inv, scalar_tpcoords;
+
+    int env_logical_width, env_logical_height;
 
     explicit operator bool() const {
       return width != 0 && height != 0 && env_width != 0 && env_height != 0;

src/platform/common.h

@@ -270,6 +270,7 @@ namespace platf {
   struct touch_port_t {
     int offset_x, offset_y;
     int width, height;
+    int logical_width, logical_height;
   };
 
   // These values must match Limelight-internal.h's SS_FF_* constants!
@@ -535,8 +536,10 @@ namespace platf {
     // Offsets for when streaming a specific monitor. By default, they are 0.
     int offset_x, offset_y;
     int env_width, env_height;
+    int env_logical_width, env_logical_height;
 
     int width, height;
+    int logical_width, logical_height;
 
   protected:
     // collect capture timing data (at loglevel debug)

src/platform/linux/kmsgrab.cpp

@@ -123,6 +123,9 @@ namespace platf {
     static int env_width;
     static int env_height;
 
+    static int env_logical_width;
+    static int env_logical_height;
+
     std::string_view plane_type(std::uint64_t val) {
       switch (val) {
         case DRM_PLANE_TYPE_OVERLAY:
@@ -686,6 +689,9 @@ namespace platf {
             this->env_width = ::platf::kms::env_width;
             this->env_height = ::platf::kms::env_height;
 
+            this->env_logical_width = ::platf::kms::env_logical_width;
+            this->env_logical_height = ::platf::kms::env_logical_height;
+
             auto monitor = pos->crtc_to_monitor.find(plane->crtc_id);
             if (monitor != std::end(pos->crtc_to_monitor)) {
               auto &viewport = monitor->second.viewport;
@@ -693,6 +699,9 @@ namespace platf {
               width = viewport.width;
               height = viewport.height;
 
+              logical_width = viewport.logical_width;
+              logical_height = viewport.logical_height;
+
               switch (card.get_panel_orientation(plane->plane_id)) {
                 case DRM_MODE_ROTATE_270:
                   BOOST_LOG(debug) << "Detected panel orientation at 90, swapping width and height.";
@@ -1542,8 +1551,8 @@ namespace platf {
           if (monitor_descriptor.index == index && monitor_descriptor.type == type) {
             monitor_descriptor.viewport.offset_x = monitor->viewport.offset_x;
             monitor_descriptor.viewport.offset_y = monitor->viewport.offset_y;
-            monitor_descriptor.viewport.width = monitor->viewport.width;
-            monitor_descriptor.viewport.height = monitor->viewport.height;
+            monitor_descriptor.viewport.logical_width = monitor->viewport.logical_width;
+            monitor_descriptor.viewport.logical_height = monitor->viewport.logical_height;
 
             // A sanity check, it's guesswork after all.
             if (
@@ -1682,6 +1691,9 @@ namespace platf {
     kms::env_width = 0;
     kms::env_height = 0;
 
+    kms::env_logical_width = 0;
+    kms::env_logical_height = 0;
+
     for (auto &card_descriptor : cds) {
       for (auto &[_, monitor_descriptor] : card_descriptor.crtc_to_monitor) {
         BOOST_LOG(debug) << "Monitor description"sv;
@@ -1690,6 +1702,9 @@ namespace platf {
 
         kms::env_width = std::max(kms::env_width, (int) (monitor_descriptor.viewport.offset_x + monitor_descriptor.viewport.width));
         kms::env_height = std::max(kms::env_height, (int) (monitor_descriptor.viewport.offset_y + monitor_descriptor.viewport.height));
+
+        kms::env_logical_height = std::max(kms::env_logical_height, (int) (monitor_descriptor.viewport.offset_y + monitor_descriptor.viewport.logical_height));
+        kms::env_logical_width = std::max(kms::env_logical_width, (int) (monitor_descriptor.viewport.offset_x + monitor_descriptor.viewport.logical_width));
       }
     }
 

src/platform/linux/wayland.cpp

@@ -144,8 +144,8 @@ namespace wl {
   }
 
   void monitor_t::xdg_size(zxdg_output_v1 *, std::int32_t width, std::int32_t height) {
-    viewport.width = width;
-    viewport.height = height;
+    viewport.logical_width = width;
+    viewport.logical_height = height;
     BOOST_LOG(info) << "Logical size: "sv << width << 'x' << height;
   }
 

src/video.cpp

@@ -1993,6 +1993,18 @@ namespace video {
 
     auto scalar = std::fminf(wt / wd, ht / hd);
 
+    // we initialize scalar_tpcoords and logical dimensions to default values in case they are not set (non-KMS)
+    float scalar_tpcoords = 1.0f;
+    int display_env_logical_width = 0;
+    int display_env_logical_height = 0;
+    if (display->logical_width && display->logical_height && display->env_logical_width && display->env_logical_height) {
+      float lwd = display->logical_width;
+      float lhd = display->logical_height;
+      scalar_tpcoords = std::fminf(wd / lwd, hd / lhd);
+      display_env_logical_width = display->env_logical_width;
+      display_env_logical_height = display->env_logical_height;
+    }
+
     auto w2 = scalar * wd;
     auto h2 = scalar * hd;
 
@@ -2011,6 +2023,9 @@ namespace video {
       offsetX,
       offsetY,
       1.0f / scalar,
+      scalar_tpcoords,
+      display_env_logical_width,
+      display_env_logical_height
     };
   }