ci(tests): add test framework (#1603)

.github/workflows/CI.yml

@@ -169,12 +169,8 @@ jobs:
 
       - name: Checkout
         uses: actions/checkout@v4
-
-      - name: Checkout Flathub Shared Modules
-        uses: actions/checkout@v4
         with:
-          repository: flathub/shared-modules
-          path: build/shared-modules
+          submodules: recursive
 
       - name: Setup Dependencies Linux Flatpak
         run: |
@@ -185,8 +181,10 @@ jobs:
             cmake \
             flatpak \
             qemu-user-static
+
           sudo su $(whoami) -c "flatpak --user remote-add --if-not-exists flathub \
             https://flathub.org/repo/flathub.flatpakrepo"
+
           sudo su $(whoami) -c "flatpak --user install -y flathub \
             org.flatpak.Builder \
             org.freedesktop.Platform/${{ matrix.arch }}/${PLATFORM_VERSION} \
@@ -318,6 +316,7 @@ jobs:
           sudo rm /root/cuda.run
 
       - name: Setup Dependencies Linux
+        timeout-minutes: 5
         run: |
           # allow newer gcc
           sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y
@@ -353,7 +352,8 @@ jobs:
             libxcb1-dev \
             libxfixes-dev \
             libxrandr-dev \
-            libxtst-dev
+            libxtst-dev \
+            python3
 
           # clean apt cache
           sudo apt-get clean
@@ -368,6 +368,12 @@ jobs:
             --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-10 \
             --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-10
 
+      - name: Setup python
+        id: python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
       - name: Build Linux
         env:
           BRANCH: ${{ github.head_ref || github.ref_name }}
@@ -463,6 +469,50 @@ jobs:
           name: sunshine-linux-${{ matrix.type }}-${{ matrix.dist }}
           path: artifacts/
 
+      - name: Install test deps
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install -y \
+            doxygen \
+            graphviz \
+            python3-venv \
+            x11-xserver-utils \
+            xvfb
+
+          # clean apt cache
+          sudo apt-get clean
+          sudo rm -rf /var/lib/apt/lists/*
+
+      - name: Run tests
+        id: test
+        working-directory: build/tests
+        run: |
+          export DISPLAY=:1
+          Xvfb ${DISPLAY} -screen 0 1024x768x24 &
+
+          ./test_sunshine --gtest_color=yes
+
+      - name: Generate gcov report
+        # any except canceled or skipped
+        if: always() && (steps.test.outcome == 'success' || steps.test.outcome == 'failure')
+        id: test_report
+        working-directory: build
+        run: |
+          ${{ steps.python.outputs.python-path }} -m pip install gcovr
+          ${{ steps.python.outputs.python-path }} -m gcovr -r .. \
+            --exclude ../tests/ \
+            --exclude ../third-party/ \
+            --xml-pretty \
+            -o coverage.xml
+
+      - name: Upload coverage
+        # any except canceled or skipped
+        if: always() && (steps.test_report.outcome == 'success')
+        uses: codecov/codecov-action@v3
+        with:
+          files: ./build/coverage.xml
+          flags: ${{ runner.os }}
+
       - name: Create/Update GitHub Release
         if: ${{ needs.setup_release.outputs.create_release == 'true' }}
         uses: ncipollo/release-action@v1
@@ -605,6 +655,12 @@ jobs:
           # install dependencies using homebrew
           brew install cmake
 
+      - name: Setup python
+        id: python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
       - name: Configure Portfile
         run: |
           # variables for Portfile
@@ -662,6 +718,7 @@ jobs:
       - name: Build port
         env:
           subportlist: ${{ steps.subportlist.outputs.subportlist }}
+        id: build
         run: |
           subport="Sunshine"
 
@@ -683,6 +740,13 @@ jobs:
             "$subport"
           echo "::endgroup::"
 
+      - name: Build Logs
+        if: always()
+        run: |
+          logfile="/opt/local/var/macports/logs/_Users_runner_work_Sunshine_Sunshine_ports_multimedia_Sunshine/Sunshine/main.log"
+          cat "$logfile"
+          sudo mv "${logfile}" "${logfile}.bak"
+
       - name: Upload Artifacts
         if: ${{ matrix.release }}
         uses: actions/upload-artifact@v4
@@ -690,6 +754,86 @@ jobs:
           name: sunshine-macports
           path: artifacts/
 
+      - name: Fix screen capture permissions
+        if: ${{ matrix.os_version != 12 }}  # macOS-12 is okay
+        # can be removed if the following is fixed in the runner image
+        # https://github.com/actions/runner-images/issues/9529
+        # https://github.com/actions/runner-images/pull/9530
+        run: |
+          # https://apple.stackexchange.com/questions/362865/macos-list-apps-authorized-for-full-disk-access
+
+          # permissions for screen capture
+          values="'kTCCServiceScreenCapture','/opt/off/opt/runner/provisioner/provisioner',1,2,4,1,NULL,NULL,0,'UNUSED',NULL,0,1687786159"
+          if [[ "${{ matrix.os_version }}" == "14" ]]; then
+            # TCC access table in Sonoma has extra 4 columns: pid, pid_version, boot_uuid, last_reminded
+            values="${values},NULL,NULL,'UNUSED',${values##*,}"
+          fi
+
+          # system and user databases
+          dbPaths=(
+            "/Library/Application Support/com.apple.TCC/TCC.db"
+            "$HOME/Library/Application Support/com.apple.TCC/TCC.db"
+          )
+
+          sqlQuery="INSERT OR IGNORE INTO access VALUES($values);"
+
+          for dbPath in "${dbPaths[@]}"; do
+            echo "Column names for $dbPath"
+            echo "-------------------"
+            sudo sqlite3 "$dbPath" "PRAGMA table_info(access);"
+            echo "Current permissions for $dbPath"
+            echo "-------------------"
+            sudo sqlite3 "$dbPath" "SELECT * FROM access WHERE service='kTCCServiceScreenCapture';"
+            sudo sqlite3 "$dbPath" "$sqlQuery"
+            echo "Updated permissions for $dbPath"
+            echo "-------------------"
+            sudo sqlite3 "$dbPath" "SELECT * FROM access WHERE service='kTCCServiceScreenCapture';"
+          done
+
+      - name: Run tests
+        if: always()
+        id: test
+        timeout-minutes: 10
+        run: |
+          sudo port test "Sunshine"
+
+      - name: Test Logs
+        if: always()
+        run: |
+          logfile="/opt/local/var/macports/logs/_Users_runner_work_Sunshine_Sunshine_ports_multimedia_Sunshine/Sunshine/main.log"
+          cat "$logfile"
+
+      - name: Generate gcov report
+        # any except canceled or skipped
+        if: always() && (steps.test.outcome == 'success' || steps.test.outcome == 'failure')
+        id: test_report
+        working-directory:
+          /opt/local/var/macports/build/_Users_runner_work_Sunshine_Sunshine_ports_multimedia_Sunshine/Sunshine/work
+        run: |
+          base_dir=$(pwd)
+          build_dir=${base_dir}/build
+
+          # get the directory name that starts with Sunshine-*
+          dir=$(ls -d Sunshine-*)
+
+          cd ${build_dir}
+          ${{ steps.python.outputs.python-path }} -m pip install gcovr
+          sudo ${{ steps.python.outputs.python-path }} -m gcovr -r ../${dir} \
+            --exclude ../${dir}/tests/ \
+            --exclude ../${dir}/third-party/ \
+            --gcov-object-directory $(pwd) \
+            --verbose \
+            --xml-pretty \
+            -o ${{ github.workspace }}/build/coverage.xml
+
+      - name: Upload coverage
+        # any except canceled or skipped
+        if: always() && (steps.test_report.outcome == 'success')
+        uses: codecov/codecov-action@v3
+        with:
+          files: ./build/coverage.xml
+          flags: ${{ runner.os }}-${{ matrix.os_version }}
+
       - name: Create/Update GitHub Release
         if: ${{ needs.setup_release.outputs.create_release == 'true' && matrix.release }}
         uses: ncipollo/release-action@v1
@@ -715,6 +859,97 @@ jobs:
         with:
           submodules: recursive
 
+      - name: Prepare tests
+        id: prepare-tests
+        if: false  # todo: DirectX11 is not available, so even software encoder fails
+        run: |
+          # function to download and extract a zip file
+          function DownloadAndExtract {
+            param (
+              [string]$Uri,
+              [string]$OutFile
+            )
+
+            $maxRetries = 5
+            $retryCount = 0
+            $success = $false
+
+            while (-not $success -and $retryCount -lt $maxRetries) {
+              $retryCount++
+              Write-Host "Downloading $Uri to $OutFile, attempt $retryCount of $maxRetries"
+              try {
+                Invoke-WebRequest -Uri $Uri -OutFile $OutFile
+                $success = $true
+              } catch {
+                Write-Host "Attempt $retryCount of $maxRetries failed with error: $($_.Exception.Message). Retrying..."
+                Start-Sleep -Seconds 5
+              }
+            }
+
+            if (-not $success) {
+              Write-Host "Failed to download the file after $maxRetries attempts."
+              exit 1
+            }
+
+            # use .NET to get the base name of the file
+            $baseName = (Get-Item $OutFile).BaseName
+
+            # Extract the zip file
+            Expand-Archive -Path $OutFile -DestinationPath $baseName
+          }
+
+          # virtual display driver
+          DownloadAndExtract `
+            -Uri "https://www.amyuni.com/downloads/usbmmidd_v2.zip" `
+            -OutFile "usbmmidd_v2.zip"
+
+          # install
+          Set-Location -Path usbmmidd_v2/usbmmidd_v2
+          ./deviceinstaller64 install usbmmidd.inf usbmmidd
+
+          # create the virtual display
+          ./deviceinstaller64 enableidd 1
+
+          # install devcon
+          choco install devcon.portable
+
+          # disable Hyper-V Video
+          # https://stackoverflow.com/a/59490940
+          C:\ProgramData\chocolatey\lib\devcon.portable\devcon64.exe `
+            disable "VMBUS\{da0a7802-e377-4aac-8e77-0558eb1073f8}"
+
+          # move up a directory
+          Set-Location -Path ../..
+
+          # multi monitor tool
+          DownloadAndExtract `
+            -Uri "http://www.nirsoft.net/utils/multimonitortool-x64.zip" `
+            -OutFile "multimonitortool.zip"
+
+          # enable the virtual display
+          # http://www.nirsoft.net/utils/multi_monitor_tool.html
+          Set-Location -Path multimonitortool
+
+          # Original Hyper-V is \\.\DISPLAY1, it will recreate itself as \\.\DISPLAY6 (or something higher than 2)
+          # USB Mobile Monitor Virtual Display is \\.\DISPLAY2
+
+          # these don't seem to work if not using runAs
+          # todo: do they work if not using runAs?
+          Start-Process powershell -Verb runAs -ArgumentList '-Command ./MultiMonitorTool.exe /enable \\.\DISPLAY2'
+          Start-Process powershell -Verb runAs -ArgumentList '-Command ./MultiMonitorTool.exe /SetPrimary \\.\DISPLAY2'
+
+          # wait a few seconds
+          Start-Sleep -s 5
+
+          # list monitors
+          ./MultiMonitorTool.exe /stext monitor_list.txt
+
+          # wait a few seconds
+          Start-Sleep -s 5
+
+          # print the monitor list
+          Get-Content -Path monitor_list.txt
+
       - name: Setup Dependencies Windows
         uses: msys2/setup-msys2@v2
         with:
@@ -722,12 +957,14 @@ jobs:
           install: >-
             base-devel
             diffutils
+            doxygen
             git
             make
             mingw-w64-x86_64-binutils
             mingw-w64-x86_64-boost
             mingw-w64-x86_64-cmake
             mingw-w64-x86_64-curl
+            mingw-w64-x86_64-graphviz
             mingw-w64-x86_64-miniupnpc
             mingw-w64-x86_64-nlohmann-json
             mingw-w64-x86_64-nodejs
@@ -740,6 +977,24 @@ jobs:
             wget
             yasm
 
+      - name: Setup python
+        # use this instead of msys2 python due to known issues using wheels, https://www.msys2.org/docs/python/
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Python Path
+        id: python-path
+        shell: msys2 {0}
+        run: |
+          # replace backslashes with double backslashes
+          python_path=$(echo "${{ steps.setup-python.outputs.python-path }}" | sed 's/\\/\\\\/g')
+
+          # step output
+          echo "python-path=${python_path}"
+          echo "python-path=${python_path}" >> $GITHUB_OUTPUT
+
       - name: Build Windows
         shell: msys2 {0}
         env:
@@ -753,6 +1008,8 @@ jobs:
             -DBUILD_WERROR=ON \
             -DCMAKE_BUILD_TYPE=RelWithDebInfo \
             -DSUNSHINE_ASSETS_DIR=assets \
+            -DTESTS_PYTHON_EXECUTABLE='${{ steps.python-path.outputs.python-path }}' \
+            -DTESTS_SOFTWARE_ENCODER_UNAVAILABLE='skip' \
             -G "MinGW Makefiles" \
             ..
           mingw32-make -j$(nproc)
@@ -771,6 +1028,35 @@ jobs:
           mv ./cpack_artifacts/Sunshine.exe ../artifacts/sunshine-windows-installer.exe
           mv ./cpack_artifacts/Sunshine.zip ../artifacts/sunshine-windows-portable.zip
 
+      - name: Run tests
+        id: test
+        shell: msys2 {0}
+        working-directory: build/tests
+        run: |
+          ./test_sunshine.exe --gtest_color=yes
+
+      - name: Generate gcov report
+        # any except canceled or skipped
+        if: always() && (steps.test.outcome == 'success' || steps.test.outcome == 'failure')
+        id: test_report
+        shell: msys2 {0}
+        working-directory: build
+        run: |
+          ${{ steps.python-path.outputs.python-path }} -m pip install gcovr
+          ${{ steps.python-path.outputs.python-path }} -m gcovr -r .. \
+            --exclude ../tests/ \
+            --exclude ../third-party/ \
+            --xml-pretty \
+            -o coverage.xml
+
+      - name: Upload coverage
+        # any except canceled or skipped
+        if: always() && (steps.test_report.outcome == 'success')
+        uses: codecov/codecov-action@v3
+        with:
+          files: ./build/coverage.xml
+          flags: ${{ runner.os }}
+
       - name: Package Windows Debug Info
         working-directory: build
         run: |