From b1fe12182bc28a3c5c63597f5fdd9ded819c461c Mon Sep 17 00:00:00 2001 From: William S Fulton Date: Wed, 13 Oct 2021 11:54:25 +0100 Subject: [PATCH 01/12] GHA workflow break up It's useful to see the configure step by itself, so split it out from the installation of prerequisites. The make install is verbose, put it in it's own step Move checking of the errors test-suite and ccache checks from the Build to Test step. --- .github/workflows/ci.yml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b46500560..f0bf68377 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -192,7 +192,7 @@ jobs: with: key: ${{ matrix.os || 'ubuntu-20.04' }}-${{ matrix.compiler || 'gcc' }}${{ matrix.GCC }} - - name: Configure + - name: Install Dependencies run: | set -x export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" @@ -224,6 +224,9 @@ jobs: $CC --version $CXX --version + - name: Configure + run: | + set -x if test -n '${{ matrix.CONFIGOPTS }}'; then CONFIGOPTS=${{ matrix.CONFIGOPTS }} fi @@ -257,10 +260,6 @@ jobs: set -x make $SWIGJOBS ./swig -version && ./swig -pcreversion - if test -z "$SWIGLANG"; then make $SWIGJOBS check-ccache; fi - if test -z "$SWIGLANG"; then make $SWIGJOBS check-errors-test-suite; fi - echo 'Installing...' - if test -z "$SWIGLANG"; then sudo make install && swig -version && ccache-swig -V; fi - name: Test working-directory: build/build @@ -291,6 +290,10 @@ jobs: if test -n "$CPP11"; then export CPPSTD=c++11; fi if test -n "$CPP14"; then export CPPSTD=c++14; fi if test -n "$CPP17"; then export CPPSTD=c++17; fi + + if test -z "$SWIGLANG"; then make $SWIGJOBS check-ccache; fi + if test -z "$SWIGLANG"; then make $SWIGJOBS check-errors-test-suite; fi + # Stricter compile flags for examples. Various headers and SWIG generated code prevents full use of -pedantic. if test -n "$SWIGLANG"; then cflags=$($GITHUB_WORKSPACE/Tools/testflags.py --language $SWIGLANG --cflags --std=$CSTD --compiler=$CC) && echo $cflags; fi if test -n "$SWIGLANG"; then cxxflags=$($GITHUB_WORKSPACE/Tools/testflags.py --language $SWIGLANG --cxxflags --std=$CPPSTD --compiler=$CC) && echo $cxxflags; fi @@ -299,6 +302,12 @@ jobs: if test -n "$SWIGLANG"; then make $SWIGJOBS check-$SWIGLANG-examples CFLAGS="$cflags" CXXFLAGS="$cxxflags"; fi if test -n "$SWIGLANG"; then make $SWIGJOBS check-$SWIGLANG-test-suite CFLAGS="$cflags" CXXFLAGS="$cxxflags"; fi + - name: Test Installation + run: | + working-directory: build/build + set -x + if test -z "$SWIGLANG"; then sudo make install && swig -version && ccache-swig -V; fi + - name: Clean working-directory: build/build run: | From af68e7f61808cddf5ca55c4fe4138f6a8cf0fe37 Mon Sep 17 00:00:00 2001 From: William S Fulton Date: Wed, 13 Oct 2021 12:03:20 +0100 Subject: [PATCH 02/12] GHA: Restore silent flag to building SWIG There is too much cruft to wade through to find compiler warnings. I check these once in a while for all compilers. We could alternatively use -Werr, but at the risk of more broken builds. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0bf68377..4b4a03673 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -258,7 +258,7 @@ jobs: working-directory: build/build run: | set -x - make $SWIGJOBS + make -s $SWIGJOBS ./swig -version && ./swig -pcreversion - name: Test From b17416c1873b8a1f20a47b6cfd908a7989c8d7c7 Mon Sep 17 00:00:00 2001 From: William S Fulton Date: Wed, 13 Oct 2021 12:10:53 +0100 Subject: [PATCH 03/12] GHA: remove useless diagnostics This echo is out of sync with the script for some reason and is rather pointless subsequent to the breaking up of the steps --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b4a03673..a35133d8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -219,7 +219,6 @@ jobs: echo CC="$CC" >> $GITHUB_ENV echo CXX="$CXX" >> $GITHUB_ENV - echo "Compiler used:" ls -la $(which $CC) $(which $CXX) $CC --version $CXX --version From 7e9726945244c25c445fa76506d8345def2b887f Mon Sep 17 00:00:00 2001 From: William S Fulton Date: Wed, 13 Oct 2021 12:19:34 +0100 Subject: [PATCH 04/12] Fix Install step --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a35133d8f..5f0c70d46 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -301,9 +301,9 @@ jobs: if test -n "$SWIGLANG"; then make $SWIGJOBS check-$SWIGLANG-examples CFLAGS="$cflags" CXXFLAGS="$cxxflags"; fi if test -n "$SWIGLANG"; then make $SWIGJOBS check-$SWIGLANG-test-suite CFLAGS="$cflags" CXXFLAGS="$cxxflags"; fi - - name: Test Installation - run: | + - name: Install working-directory: build/build + run: | set -x if test -z "$SWIGLANG"; then sudo make install && swig -version && ccache-swig -V; fi From d7c2cbec38d44c1d76dd63e9433bc4db82c304a9 Mon Sep 17 00:00:00 2001 From: William S Fulton Date: Wed, 13 Oct 2021 12:27:31 +0100 Subject: [PATCH 05/12] GHA: clean up Configure step Setup Javascript before running configure - may fix broken Javascript testing. Configure step just configures now. --- .github/workflows/ci.yml | 78 ++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f0c70d46..c74ee45cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -200,6 +200,45 @@ jobs: source Tools/GHA-linux-install.sh + case $(uname) in + Linux) + cpu_count=$(nproc) + ;; + + Darwin) + cpu_count=$(sysctl -n hw.ncpu) + ;; + + *) + cpu_count=1 + ;; + esac + if [[ $cpu_count != 1 ]]; then + echo SWIGJOBS=-j$cpu_count >> $GITHUB_ENV + fi + + case "$SWIGLANG" in + javascript) + case "$ENGINE" in + node) + export NVM_DIR="$HOME/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + nvm use ${VER} + ;; + *) + # Running tests using v8 or jsc involves creating a custom + # interpreter in Tools/javascript, which is currently broken + # for parallel builds (we attempt to update this interpreter + # while running, resulting in "Text file busy" error). + unset SWIGJOBS + esac + ;; + + ruby) + source $HOME/.rvm/scripts/rvm + ;; + esac + if test '${{ matrix.compiler }}' = 'clang'; then CC="clang" CXX="clang++" @@ -236,23 +275,6 @@ jobs: echo "${CONFIGOPTS[@]}" ./autogen.sh && mkdir -p build/build && cd build/build && ../../configure "${CONFIGOPTS[@]}" - case $(uname) in - Linux) - cpu_count=$(nproc) - ;; - - Darwin) - cpu_count=$(sysctl -n hw.ncpu) - ;; - - *) - cpu_count=1 - ;; - esac - if [[ $cpu_count != 1 ]]; then - echo SWIGJOBS=-j$cpu_count >> $GITHUB_ENV - fi - - name: Build working-directory: build/build run: | @@ -263,28 +285,6 @@ jobs: - name: Test working-directory: build/build run: | - case "$SWIGLANG" in - javascript) - case "$ENGINE" in - node) - export NVM_DIR="$HOME/.nvm" - [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" - nvm use ${VER} - ;; - *) - # Running tests using v8 or jsc involves creating a custom - # interpreter in Tools/javascript, which is currently broken - # for parallel builds (we attempt to update this interpreter - # while running, resulting in "Text file busy" error). - unset SWIGJOBS - esac - ;; - - ruby) - source $HOME/.rvm/scripts/rvm - ;; - esac - set -x if test -n "$CPP11"; then export CPPSTD=c++11; fi if test -n "$CPP14"; then export CPPSTD=c++14; fi From 61983e04b078efde6a4c218b009e2653039a0feb Mon Sep 17 00:00:00 2001 From: William S Fulton Date: Wed, 13 Oct 2021 12:43:24 +0100 Subject: [PATCH 06/12] Try save WITHLANG --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c74ee45cc..d0ca72692 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -199,6 +199,7 @@ jobs: echo PATH="$PATH" >> $GITHUB_ENV source Tools/GHA-linux-install.sh + echo WITHLANG="$WITHLANG" >> $GITHUB_ENV case $(uname) in Linux) From aaf19fbf93e08d3ae0584096a66b0bad3a08aa59 Mon Sep 17 00:00:00 2001 From: William S Fulton Date: Wed, 13 Oct 2021 18:26:08 +0100 Subject: [PATCH 07/12] GHA: Fix regressions The environment variables are not kept between runs, arg. Temp hack to try and get export env working for Configure and Test steps. --- .github/workflows/ci.yml | 56 ++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0ca72692..c31ea2568 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -214,19 +214,11 @@ jobs: cpu_count=1 ;; esac - if [[ $cpu_count != 1 ]]; then - echo SWIGJOBS=-j$cpu_count >> $GITHUB_ENV - fi case "$SWIGLANG" in javascript) case "$ENGINE" in - node) - export NVM_DIR="$HOME/.nvm" - [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" - nvm use ${VER} - ;; - *) + v8 | jsc) # Running tests using v8 or jsc involves creating a custom # interpreter in Tools/javascript, which is currently broken # for parallel builds (we attempt to update this interpreter @@ -234,12 +226,12 @@ jobs: unset SWIGJOBS esac ;; - - ruby) - source $HOME/.rvm/scripts/rvm - ;; esac + if [[ $cpu_count != 1 ]]; then + echo SWIGJOBS=-j$cpu_count >> $GITHUB_ENV + fi + if test '${{ matrix.compiler }}' = 'clang'; then CC="clang" CXX="clang++" @@ -266,6 +258,25 @@ jobs: - name: Configure run: | set -x + + # Repeat language specific environment setup + case "$SWIGLANG" in + javascript) + case "$ENGINE" in + node) + export NVM_DIR="$HOME/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + nvm use ${VER} + ;; + *) + esac + ;; + + ruby) + source $HOME/.rvm/scripts/rvm + ;; + esac + if test -n '${{ matrix.CONFIGOPTS }}'; then CONFIGOPTS=${{ matrix.CONFIGOPTS }} fi @@ -287,6 +298,25 @@ jobs: working-directory: build/build run: | set -x + + # Repeat language specific environment setup + case "$SWIGLANG" in + javascript) + case "$ENGINE" in + node) + export NVM_DIR="$HOME/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + nvm use ${VER} + ;; + *) + esac + ;; + + ruby) + source $HOME/.rvm/scripts/rvm + ;; + esac + if test -n "$CPP11"; then export CPPSTD=c++11; fi if test -n "$CPP14"; then export CPPSTD=c++14; fi if test -n "$CPP17"; then export CPPSTD=c++17; fi From d0b6cfeca44d8a09e0cbfd3e754b4796fa429dd0 Mon Sep 17 00:00:00 2001 From: William S Fulton Date: Thu, 14 Oct 2021 11:40:55 +0100 Subject: [PATCH 08/12] Factor out common environment setup GHA steps --- .github/workflows/ci.yml | 38 ++--------------------------------- Tools/CI-linux-environment.sh | 27 +++++++++++++++++++++++++ Tools/CI-linux-install.sh | 2 +- 3 files changed, 30 insertions(+), 37 deletions(-) create mode 100644 Tools/CI-linux-environment.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c31ea2568..d1abbc404 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -258,24 +258,7 @@ jobs: - name: Configure run: | set -x - - # Repeat language specific environment setup - case "$SWIGLANG" in - javascript) - case "$ENGINE" in - node) - export NVM_DIR="$HOME/.nvm" - [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" - nvm use ${VER} - ;; - *) - esac - ;; - - ruby) - source $HOME/.rvm/scripts/rvm - ;; - esac + source Tools/CI-linux-environment.sh if test -n '${{ matrix.CONFIGOPTS }}'; then CONFIGOPTS=${{ matrix.CONFIGOPTS }} @@ -298,24 +281,7 @@ jobs: working-directory: build/build run: | set -x - - # Repeat language specific environment setup - case "$SWIGLANG" in - javascript) - case "$ENGINE" in - node) - export NVM_DIR="$HOME/.nvm" - [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" - nvm use ${VER} - ;; - *) - esac - ;; - - ruby) - source $HOME/.rvm/scripts/rvm - ;; - esac + source Tools/CI-linux-environment.sh if test -n "$CPP11"; then export CPPSTD=c++11; fi if test -n "$CPP14"; then export CPPSTD=c++14; fi diff --git a/Tools/CI-linux-environment.sh b/Tools/CI-linux-environment.sh new file mode 100644 index 000000000..d530f37a8 --- /dev/null +++ b/Tools/CI-linux-environment.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Expected to be called from elsewhere with same variables set as CI-linux-install.sh +# e.g. RETRY=travis-retry SWIGLANG=python +# Sets up environment for using various target languages +# For Github Actions where the environment is not preserved between steps +set -e # exit on failure (same as -o errexit) + +case "$SWIGLANG" in + "javascript") + case "$ENGINE" in + "node") + export NVM_DIR="$HOME/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && source "$NVM_DIR/nvm.sh" + nvm use ${VER} + ;; + *) ;; + esac + ;; + "ruby") + if ! command -v rvm; then + set +x + source $HOME/.rvm/scripts/rvm + set -x + fi + ;; + *) ;; +esac diff --git a/Tools/CI-linux-install.sh b/Tools/CI-linux-install.sh index ab5b10ddb..7daa55e3e 100644 --- a/Tools/CI-linux-install.sh +++ b/Tools/CI-linux-install.sh @@ -1,5 +1,5 @@ #!/bin/bash -# expected to be called from elsewhere with certain variables set +# Expected to be called from elsewhere with certain variables set # e.g. RETRY=travis-retry SWIGLANG=python GCC=7 set -e # exit on failure (same as -o errexit) From 4df45013d5c56fe940f9bdeb138bc6dff1a1e60c Mon Sep 17 00:00:00 2001 From: William S Fulton Date: Thu, 14 Oct 2021 12:26:04 +0100 Subject: [PATCH 09/12] Fix to find scripts in Tools --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1abbc404..32d5b53dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -198,7 +198,7 @@ jobs: export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" echo PATH="$PATH" >> $GITHUB_ENV - source Tools/GHA-linux-install.sh + source $GITHUB_WORKSPACE/Tools/GHA-linux-install.sh echo WITHLANG="$WITHLANG" >> $GITHUB_ENV case $(uname) in @@ -258,7 +258,7 @@ jobs: - name: Configure run: | set -x - source Tools/CI-linux-environment.sh + source $GITHUB_WORKSPACE/Tools/CI-linux-environment.sh if test -n '${{ matrix.CONFIGOPTS }}'; then CONFIGOPTS=${{ matrix.CONFIGOPTS }} @@ -281,7 +281,7 @@ jobs: working-directory: build/build run: | set -x - source Tools/CI-linux-environment.sh + source $GITHUB_WORKSPACE/Tools/CI-linux-environment.sh if test -n "$CPP11"; then export CPPSTD=c++11; fi if test -n "$CPP14"; then export CPPSTD=c++14; fi From 206105bc3f9ebd96c82cd51a8ee40d0518a125b6 Mon Sep 17 00:00:00 2001 From: William S Fulton Date: Thu, 14 Oct 2021 19:56:30 +0100 Subject: [PATCH 10/12] GHA: Correctly unset SWIGJOBS for broken parallel language checks --- .github/workflows/ci.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32d5b53dc..4e3eb7f45 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -215,19 +215,6 @@ jobs: ;; esac - case "$SWIGLANG" in - javascript) - case "$ENGINE" in - v8 | jsc) - # Running tests using v8 or jsc involves creating a custom - # interpreter in Tools/javascript, which is currently broken - # for parallel builds (we attempt to update this interpreter - # while running, resulting in "Text file busy" error). - unset SWIGJOBS - esac - ;; - esac - if [[ $cpu_count != 1 ]]; then echo SWIGJOBS=-j$cpu_count >> $GITHUB_ENV fi @@ -290,6 +277,19 @@ jobs: if test -z "$SWIGLANG"; then make $SWIGJOBS check-ccache; fi if test -z "$SWIGLANG"; then make $SWIGJOBS check-errors-test-suite; fi + case "$SWIGLANG" in + javascript) + case "$ENGINE" in + v8 | jsc) + # Running tests using v8 or jsc involves creating a custom + # interpreter in Tools/javascript, which is currently broken + # for parallel builds (we attempt to update this interpreter + # while running, resulting in "Text file busy" error). + unset SWIGJOBS + esac + ;; + esac + # Stricter compile flags for examples. Various headers and SWIG generated code prevents full use of -pedantic. if test -n "$SWIGLANG"; then cflags=$($GITHUB_WORKSPACE/Tools/testflags.py --language $SWIGLANG --cflags --std=$CSTD --compiler=$CC) && echo $cflags; fi if test -n "$SWIGLANG"; then cxxflags=$($GITHUB_WORKSPACE/Tools/testflags.py --language $SWIGLANG --cxxflags --std=$CPPSTD --compiler=$CC) && echo $cxxflags; fi From fc31b9ab0489cca0ead4f700233e7342f6449fb2 Mon Sep 17 00:00:00 2001 From: William S Fulton Date: Fri, 15 Oct 2021 08:15:14 +0100 Subject: [PATCH 11/12] Add gcc11 testing --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e3eb7f45..be949a136 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,9 @@ jobs: - SWIGLANG: "" GCC: 10 desc: gcc10 + - SWIGLANG: "" + GCC: 11 + desc: gcc11 - SWIGLANG: "" compiler: clang desc: clang From 6c4125db0c24c8ef79da58fe24d4a9362e758df4 Mon Sep 17 00:00:00 2001 From: William S Fulton Date: Mon, 18 Oct 2021 08:10:36 +0100 Subject: [PATCH 12/12] GHA: apt-get update for Octave To fix: Failed to fetch http://azure.archive.ubuntu.com/ubuntu/pool/main/m/mesa/libgl1-mesa-dev_21.0.3-0ubuntu0.3~20.04.2_amd64.deb 404 Not Found --- Tools/CI-linux-install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/CI-linux-install.sh b/Tools/CI-linux-install.sh index 7daa55e3e..80e7c1fb9 100644 --- a/Tools/CI-linux-install.sh +++ b/Tools/CI-linux-install.sh @@ -73,6 +73,7 @@ case "$SWIGLANG" in $RETRY sudo apt-get -qq install ocaml camlp4 ;; "octave") + $RETRY sudo apt-get -qq update $RETRY sudo apt-get -qq install liboctave-dev ;; "php")