refactor all code that builds csources (#17815)

* refactor all code that builds csources
* fixup
* nim_csourcesDir_v0 + nim_csourcesDir
* remove deprecated, unused scripts from ci/
* reuse nimCsourcesHash in ci
* simplify CI pipelines by reusing nimBuildCsourcesIfNeeded
* simplify ci_docs.yml by reusing nimBuildCsourcesIfNeeded
* cleanup
* use csources_v1 as destination dir
* fixup
* remove pushCsources
* address comment: remove build.sh support for now
* fixup

.builds/freebsd.yml

@@ -14,10 +14,10 @@ environment:
   CC: /usr/bin/clang
 tasks:
 - setup: |
+    set -e
     cd Nim
-    git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources
+    . ci/funs.sh && nimBuildCsourcesIfNeeded
-    gmake -C csources -j $(sysctl -n hw.ncpu)
+    $nim_csources c --skipUserCfg --skipParentCfg koch
-    bin/nim c --skipUserCfg --skipParentCfg koch
     echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
 - test: |
     cd Nim

.builds/openbsd_0.yml

@@ -17,15 +17,15 @@ environment:
   CC: /usr/bin/clang
 tasks:
 - setup: |
+    set -e
     cd Nim
-    git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources
+    . ci/funs.sh && nimBuildCsourcesIfNeeded
-    gmake -C csources -j $(sysctl -n hw.ncpuonline)
+    $nim_csources c koch
-    bin/nim c koch
     echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
 - test: |
     cd Nim
     if ! ./koch runCI; then
-      nim c -r tools/ci_testresults.nim
+      nim r tools/ci_testresults.nim
       exit 1
     fi
 triggers:

.builds/openbsd_1.yml

@@ -17,15 +17,15 @@ environment:
   CC: /usr/bin/clang
 tasks:
 - setup: |
+    set -e
     cd Nim
-    git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources
+    . ci/funs.sh && nimBuildCsourcesIfNeeded
-    gmake -C csources -j $(sysctl -n hw.ncpuonline)
+    $nim_csources c koch
-    bin/nim c koch
     echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
 - test: |
     cd Nim
     if ! ./koch runCI; then
-      nim c -r tools/ci_testresults.nim
+      nim r tools/ci_testresults.nim
       exit 1
     fi
 triggers:

.github/workflows/ci.yml.disabled

@@ -1,3 +1,5 @@
+# out of date and unused but revivable
+
 name: Continous Integration
 on: [push, pull_request]
 
@@ -30,11 +32,6 @@ jobs:
     steps:
       - name: 'Checkout'
         uses: actions/checkout@v2
-      - name: 'Checkout csources'
-        uses: actions/checkout@v2
-        with:
-          repository: nim-lang/csources
-          path: csources
 
       - name: 'Install node.js 8.x'
         uses: actions/setup-node@v1
@@ -91,24 +88,10 @@ jobs:
         shell: bash
         run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
-      - name: 'Build csources'
+      - name: 'Build csourcesAny'
         shell: bash
-        run: |
+        run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}'
-          ncpu=
-          case '${{ runner.os }}' in
-          'Linux')
-            ncpu=$(nproc)
-            ;;
-          'macOS')
-            ncpu=$(sysctl -n hw.ncpu)
-            ;;
-          'Windows')
-            ncpu=$NUMBER_OF_PROCESSORS
-            ;;
-          esac
-          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
 
-          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
       - name: 'Build koch'
         shell: bash
         run: nim c koch

.github/workflows/ci_docs.yml

@@ -70,53 +70,12 @@ jobs:
         shell: bash
         run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
-      - name: 'Get current csources version'
+      - name: 'Build csourcesAny'
-        id: csources-version
         shell: bash
-        run: |
+        run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc
-          sha=$(git ls-remote https://github.com/nim-lang/csources_v1 master | cut -f 1)
+          # was previously using caching via `actions/cache@v1` but this wasn't
-          echo "::set-output name=sha::$sha"
+          # used in other CI pipelines and it's unclear the added complexity
-
+          # was worth the saving; can be revisited if needed.
-      - name: 'Get prebuilt csources from cache'
-        id: csources-cache
-        uses: actions/cache@v1
-        with:
-          path: bin
-          key: '${{ matrix.os }}-${{ steps.csources-version.outputs.sha }}'
-
-      - name: 'Checkout csources'
-        if: steps.csources-cache.outputs.cache-hit != 'true'
-        uses: actions/checkout@v2
-        with:
-          repository: nim-lang/csources_v1
-          path: csources
-
-      - name: 'Build 1-stage compiler from csources'
-        shell: bash
-        run: |
-          ext=
-          [[ '${{ runner.os }}' == 'Windows' ]] && ext=.exe
-          if [[ ! -x bin/nim-csources$ext ]]; then
-            ncpu=
-            case '${{ runner.os }}' in
-            'Linux')
-              ncpu=$(nproc)
-              ;;
-            'macOS')
-              ncpu=$(sysctl -n hw.ncpu)
-              ;;
-            'Windows')
-              ncpu=$NUMBER_OF_PROCESSORS
-              ;;
-            esac
-            [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
-
-            make -C csources -j $ncpu CC=gcc
-            cp bin/nim{,-csources}$ext
-          else
-            echo 'Cache hit, using prebuilt csources'
-            cp bin/nim{-csources,}$ext
-          fi
 
       - name: 'Build koch'
         shell: bash

.github/workflows/ci_packages.yml

@@ -20,16 +20,11 @@ jobs:
         with:
           fetch-depth: 2
 
-      - name: 'Checkout csources'
-        uses: actions/checkout@v2
-        with:
-          repository: nim-lang/csources
-          path: csources
-
       - name: 'Install node.js 12.x'
         uses: actions/setup-node@v1
         with:
           node-version: '12.x'
+
       - name: 'Install dependencies (Linux amd64)'
         if: runner.os == 'Linux' && matrix.cpu == 'amd64'
         run: |
@@ -56,24 +51,10 @@ jobs:
         shell: bash
         run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
-      - name: 'Build csources'
+      - name: 'Build csourcesAny'
         shell: bash
-        run: |
+        run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}'
-          ncpu=
-          case '${{ runner.os }}' in
-          'Linux')
-            ncpu=$(nproc)
-            ;;
-          'macOS')
-            ncpu=$(sysctl -n hw.ncpu)
-            ;;
-          'Windows')
-            ncpu=$NUMBER_OF_PROCESSORS
-            ;;
-          esac
-          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
 
-          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
       - name: 'Build koch'
         shell: bash
         run: nim c koch

.gitignore

@@ -64,7 +64,10 @@ lib/**/*.html
 testament.db
 /tests/**/*.json
 /tests/**/*.js
+
 /csources
+/csources_v1
+
 /dist/
 # /lib/fusion # fusion is now unbundled; `git status` should reveal if it's there so users can act on it
 

.travis.yml

@@ -1,3 +1,4 @@
+# deprecated, out of date and unused
 sudo: false
 language: c
 
@@ -22,9 +23,8 @@ addons:
     - valgrind
 
 before_script:
-  - git clone --depth 1 https://github.com/nim-lang/csources.git
+  -  . ci/funs.sh && nimBuildCsourcesIfNeeded LD=$CC ucpu=$CPU
   - export PATH="$PWD/bin${PATH:+:$PATH}"
-  - make -C csources -j 2 LD=$CC ucpu=$CPU
 
 script:
   - echo "travis_fold:start:nim_c_koch"

appveyor.yml.disabled

@@ -1,3 +1,4 @@
+# deprecated, out of date and unused
 version: '{build}'
 
 environment:
@@ -23,10 +24,9 @@ install:
   - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%"
   - 7z x -y "%MINGW_ARCHIVE%" -o"%CD%\DIST"> nul
   - SET PATH=%CD%\DIST\%MINGW_DIR%\BIN;%CD%\BIN;%PATH%
-  - git clone --depth 1 https://github.com/nim-lang/csources
+  # may need adjustments, untested, see similar examples here:
-  - cd csources
+  # https://chromium.googlesource.com/external/github.com/swig/swig/+/1e36f51346d95f8b9848e682c2eb986e9cb9b4f4/appveyor.yml
-  - build64.bat
+  - bash -c ". ci/funs.sh && nimBuildCsourcesIfNeeded"
-  - cd ..
 
 build_script:
   - openssl version

azure-pipelines.yml

@@ -69,10 +69,6 @@ jobs:
         fi
       displayName: 'Check whether to skip CI'
 
-    - bash: git clone --depth 1 https://github.com/nim-lang/csources_v1 csources
-      displayName: 'Checkout Nim csources'
-      condition: and(succeeded(), eq(variables['skipci'], 'false'))
-
     - task: NodeTool@0
       inputs:
         versionSpec: '12.x'
@@ -163,45 +159,17 @@ jobs:
       condition: and(succeeded(), eq(variables['skipci'], 'false'))
       displayName: 'System information'
 
-    - bash: echo '##vso[task.setvariable variable=csources_version]'"$(git -C csources rev-parse HEAD)"
+    - bash: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu=$(CPU)
       condition: and(succeeded(), eq(variables['skipci'], 'false'))
-      displayName: 'Get csources version'
+      displayName: 'Build csourcesAny'
 
-    - task: Cache@2
+    # this could be revived if performance justifies it (needs a few updates)
-      inputs:
+    # - task: Cache@2
-        key: 'csources | "$(Agent.OS)" | $(CPU) | $(csources_version)'
+    #   inputs:
-        path: csources/bin
+    #     key: 'csourcesAny | "$(Agent.OS)" | $(CPU) | $(csources_version)'
-      condition: and(succeeded(), eq(variables['skipci'], 'false'))
+    #     path: $(nim_csources)
-      displayName: 'Restore built csources'
+    #   condition: and(succeeded(), eq(variables['skipci'], 'false'))
-
+    #   displayName: 'Restore built csourcesAny'
-    - bash: |
-        set -e
-        . ci/funs.sh
-        ncpu=
-        ext=
-        case '$(Agent.OS)' in
-        'Linux')
-          ncpu=$(nproc)
-          ;;
-        'Darwin')
-          ncpu=$(sysctl -n hw.ncpu)
-          ;;
-        'Windows_NT')
-          ncpu=$NUMBER_OF_PROCESSORS
-          ext=.exe
-          ;;
-        esac
-        [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
-
-        if [[ -x csources/bin/nim$ext ]]; then
-          echo_run echo "Found cached compiler, skipping build"
-        else
-          echo_run make -C csources -j $ncpu CC=gcc ucpu=$(CPU) koch=no
-        fi
-
-        echo_run cp csources/bin/nim$ext bin
-      condition: and(succeeded(), eq(variables['skipci'], 'false'))
-      displayName: 'Build 1-stage compiler from csources'
 
     - bash: nim c koch
       condition: and(succeeded(), eq(variables['skipci'], 'false'))

build_all.bat

@@ -1,10 +1,13 @@
 @echo off
 rem build development version of the compiler; can be rerun safely
-if not exist csources (
+rem TODO: call nimBuildCsourcesIfNeeded or auto-generate this file (from a nim script)
-  git clone --depth 1 https://github.com/nim-lang/csources.git
+rem to avoid duplication.
+if not exist csources_v1 (
+  git clone --depth 1 https://github.com/nim-lang/csources_v1.git
 )
 if not exist bin\nim.exe (
-  cd csources
+  cd csources_v1
+  git checkout a8a5241f9475099c823cfe1a5e0ca4022ac201ff
   if PROCESSOR_ARCHITECTURE == AMD64 (
     SET ARCH=64
   )

build_all.sh

@@ -1,52 +1,16 @@
 #! /bin/sh
 
 # build development version of the compiler; can be rerun safely.
-# arguments can be passed, e.g. `--os freebsd`
+# arguments can be passed, e.g.:
+# CC=gcc ucpu=amd64 uos=darwin
 
 set -u # error on undefined variables
 set -e # exit on first error
 
-echo_run(){
+. ci/funs.sh
-  echo "$*"
+nimBuildCsourcesIfNeeded "$@"
-  "$@"
-}
 
-[ -d csources ] || echo_run git clone -q --depth 1 https://github.com/nim-lang/csources.git
+# Note: if fails, may need to update csourcesAny manually
-
-nim_csources=bin/nim_csources
-
-build_nim_csources_via_script(){
-  echo_run cd csources
-  echo_run sh build.sh "$@"
-}
-
-build_nim_csources(){
-  # avoid changing dir in case of failure
-  (
-    if [ $# -ne 0 ]; then
-      # some args were passed (e.g.: `--cpu i386`), need to call build.sh
-      build_nim_csources_via_script "$@"
-    else
-      # no args, use multiple Make jobs (5X faster on 16 cores: 10s instead of 50s)
-      makeX=make
-      unamestr=$(uname)
-      if [ "$unamestr" = 'FreeBSD' ]; then
-        makeX=gmake
-      fi
-      nCPU=$(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || 1)
-      which $makeX && echo_run $makeX -C csources -j $((nCPU + 2)) -l $nCPU || build_nim_csources_via_script
-    fi
-  )
-  # keep $nim_csources in case needed to investigate bootstrap issues
-  # without having to rebuild from csources
-  echo_run cp bin/nim $nim_csources
-}
-
-[ -f $nim_csources ] || echo_run build_nim_csources "$@"
-
-# Note: if fails, may need to `cd csources && git pull`
 echo_run bin/nim c --skipUserCfg --skipParentCfg koch
-
 echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg
 echo_run ./koch tools --skipUserCfg --skipParentCfg # Compile Nimble and other tools.
-

ci/build.bat

@@ -1,14 +0,0 @@
-REM Some debug info
-echo "Running on %CI_RUNNER_ID% (%CI_RUNNER_DESCRIPTION%) with tags %CI_RUNNER_TAGS%."
-gcc -v
-
-git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
-cd csources
-call build64.bat
-cd ..
-set PATH=%CD%\bin;%PATH%
-nim -v
-nim c koch
-koch.exe boot
-copy bin/nim bin/nimd
-koch.exe boot -d:release

ci/build.sh

@@ -1,15 +0,0 @@
-sh ci/deps.sh
-
-# Build from C sources.
-git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
-cd csources
-sh build.sh
-cd ..
-# Add Nim to the PATH
-export PATH=$(pwd)/bin${PATH:+:$PATH}
-# Bootstrap.
-nim -v
-nim c koch
-./koch boot
-cp bin/nim bin/nimd
-./koch boot -d:release

ci/deps.bat

@@ -1,4 +0,0 @@
-nim e install_nimble.nims
-nim e tests/test_nimscript.nims
-nimble update
-nimble install -y zip opengl sdl1 jester@#head niminst

ci/deps.sh

@@ -1,16 +0,0 @@
-# Some debug info
-echo "Running on $CI_RUNNER_ID ($CI_RUNNER_DESCRIPTION) with tags $CI_RUNNER_TAGS."
-
-# Packages
-apt-get update -qq
-apt-get install -y -qq build-essential git libcurl4-openssl-dev libsdl1.2-dev libgc-dev nodejs
-
-gcc -v
-
-export PATH=$(pwd)/bin${PATH:+:$PATH}
-
-# Nimble deps
-nim e install_nimble.nims
-nim e tests/test_nimscript.nims
-nimble update
-nimble install zip opengl sdl1 jester@#head niminst

ci/funs.sh

@@ -1,5 +1,7 @@
-# utilities used in CI pipelines to avoid duplication.
+# Utilities used in CI pipelines and tooling to avoid duplication.
 # Avoid top-level statements.
+# Prefer nim scripts whenever possible.
+# functions starting with `_` are considered internal, less stable.
 
 echo_run () {
   # echo's a command before running it, which helps understanding logs
@@ -28,3 +30,72 @@ nimIsCiSkip(){
     return 1
   fi
 }
+
+nimDefineVars(){
+  nim_csourcesDir=csources_v1 # where we clone
+  nim_csourcesUrl=https://github.com/nim-lang/csources_v1.git
+  nim_csourcesHash=a8a5241f9475099c823cfe1a5e0ca4022ac201ff
+  nim_csources=bin/nim_csources_$nim_csourcesHash
+}
+
+_nimNumCpu(){
+  # linux: $(nproc)
+  # FreeBSD | macOS: $(sysctl -n hw.ncpu)
+  # OpenBSD: $(sysctl -n hw.ncpuonline)
+  # windows: $NUMBER_OF_PROCESSORS ?
+  echo $(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || 1)
+}
+
+_nimBuildCsourcesIfNeeded(){
+  # if some systems cannot use make or gmake, we could add support for calling `build.sh`
+  # but this is slower (not parallel jobs) and would require making build.sh
+  # understand the arguments passed to the makefile (e.g. `CC=gcc ucpu=amd64 uos=darwin`),
+  # instead of `--cpu amd64 --os darwin`.
+  unamestr=$(uname)
+  # uname values: https://en.wikipedia.org/wiki/Uname
+  if [ "$unamestr" = 'FreeBSD' ]; then
+    makeX=gmake
+  elif [ "$unamestr" = 'OpenBSD' ]; then
+    makeX=gmake
+  else
+    makeX=make
+  fi
+  nCPU=$(_nimNumCpu)
+  echo_run which $makeX
+  # parallel jobs (5X faster on 16 cores: 10s instead of 50s)
+  echo_run $makeX -C $nim_csourcesDir -j $((nCPU + 2)) -l $nCPU "$@"
+  # keep $nim_csources in case needed to investigate bootstrap issues
+  # without having to rebuild
+  echo_run cp bin/nim $nim_csources
+}
+
+nimCsourcesHash(){
+  nimDefineVars
+  echo $nim_csourcesHash
+}
+
+nimBuildCsourcesIfNeeded(){
+  # goal: allow cachine each tagged version independently
+  # to avoid rebuilding, so that tools like `git bisect`
+  # can grab a cached past version without rebuilding.
+  nimDefineVars
+  (
+    set -e
+    # avoid polluting caller scope with internal variable definitions.
+    if test -f "$nim_csources"; then
+      echo "$nim_csources exists."
+    else
+      if test -d "$nim_csourcesDir"; then
+        echo "$nim_csourcesDir exists."
+      else
+        # depth 1: adjust as needed in case useful for `git bisect`
+        echo_run git clone -q --depth 1 $nim_csourcesUrl "$nim_csourcesDir"
+        echo_run git -C "$nim_csourcesDir" checkout $nim_csourcesHash
+      fi
+      _nimBuildCsourcesIfNeeded "$@"
+    fi
+
+    echo_run cp $nim_csources bin/nim
+    echo_run $nim_csources -v
+  )
+}

ci/nsis_build.bat

@@ -1,59 +0,0 @@
-REM - Run the full testsuite;  testament\tester all
-
-REM - Uncomment the list of changes in news.txt
-REM - write a news ticker entry
-REM - Update the version
-
-REM - Generate the full docs;  koch web0
-REM - Generate the installers;
-REM - Update the version in system.nim
-REM - Test the installers
-REM - Tag the release
-REM - Merge devel into master
-REM - Update csources
-
-set NIMVER=%1
-
-Rem Build -docs file:
-koch web0
-cd web\upload
-7z a -tzip docs-%NIMVER%.zip *.html
-move /y docs-%NIMVER%.zip download
-cd ..\..
-
-Rem Build csources
-koch csources -d:release || exit /b
-
-rem Grab C sources and nimsuggest
-git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
-
-set PATH=%CD%\bin;%PATH%
-
-ReM Build Win32 version:
-
-set PATH=C:\Users\araq\projects\mingw32\bin;%PATH%
-cd csources
-call build.bat
-cd ..
-ReM Rebuilding koch is necessary because it uses its pointer size to determine
-ReM which mingw link to put in the NSIS installer.
-nim c --out:koch_temp koch || exit /b
-koch_temp boot -d:release || exit /b
-koch_temp nsis -d:release || exit /b
-koch_temp zip -d:release || exit /b
-dir build
-move /y build\nim_%NIMVER%.exe build\nim-%NIMVER%_x32.exe || exit /b
-move /y build\nim-%NIMVER%.zip build\nim-%NIMVER%_x32.zip || exit /b
-
-
-ReM Build Win64 version:
-set PATH=C:\Users\araq\projects\mingw64\bin;%PATH%
-cd csources
-call build64.bat
-cd ..
-nim c --out:koch_temp koch || exit /b
-koch_temp boot -d:release || exit /b
-koch_temp nsis -d:release || exit /b
-koch_temp zip -d:release || exit /b
-move /y build\nim_%NIMVER%.exe build\nim-%NIMVER%_x64.exe || exit /b
-move /y build\nim-%NIMVER%.zip build\nim-%NIMVER%_x64.zip || exit /b

koch.nim

@@ -89,7 +89,6 @@ Commands for core developers:
   tests [options]          run the testsuite (run a subset of tests by
                            specifying a category, e.g. `tests cat async`)
   temp options             creates a temporary compiler for testing
-  pushcsource              push generated C sources to its repo
 Web options:
   --googleAnalytics:UA-... add the given google analytics code to the docs. To
                            build the official docs, use UA-48159761-1
@@ -283,7 +282,7 @@ template doUseCpp(): bool = getEnv("NIM_COMPILE_TO_CPP", "false") == "true"
 
 proc boot(args: string) =
   ## bootstrapping is a process that involves 3 steps:
-  ## 1. use csources to produce nim1.exe. This nim1.exe is buggy but
+  ## 1. use csourcesAny to produce nim1.exe. This nim1.exe is buggy but
   ## rock solid for building a Nim compiler. It shouldn't be used for anything else.
   ## 2. use nim1.exe to produce nim2.exe. nim2.exe is the one you really need.
   ## 3. We use nim2.exe to build nim3.exe. nim3.exe is equal to nim2.exe except for timestamps.
@@ -315,13 +314,8 @@ proc boot(args: string) =
       let ret = execCmdEx(nimStart & " --version")
       doAssert ret.exitCode == 0
       let version = ret.output.splitLines[0]
-      # remove these when csources get updated
+      if version.startsWith "Nim Compiler Version 0.20.0":
-      template addLib() =
         extraOption.add " --lib:lib" # see https://github.com/nim-lang/Nim/pull/14291
-      if version.startsWith "Nim Compiler Version 0.19.0":
-        extraOption.add " -d:nimBoostrapCsources0_19_0"
-        addLib()
-      elif version.startsWith "Nim Compiler Version 0.20.0": addLib()
 
     # in order to use less memory, we split the build into two steps:
     # --compileOnly produces a $project.json file and does not run GCC/Clang.
@@ -587,29 +581,6 @@ proc runCI(cmd: string) =
     when defined(posix):
       execFold("Run nimsuggest tests", "nim c -r nimsuggest/tester")
 
-proc pushCsources() =
-  if not dirExists("../csources/.git"):
-    quit "[Error] no csources git repository found"
-  csource("-d:danger")
-  let cwd = getCurrentDir()
-  try:
-    copyDir("build/c_code", "../csources/c_code")
-    copyFile("build/build.sh", "../csources/build.sh")
-    copyFile("build/build.bat", "../csources/build.bat")
-    copyFile("build/build64.bat", "../csources/build64.bat")
-    copyFile("build/makefile", "../csources/makefile")
-
-    setCurrentDir("../csources")
-    for kind, path in walkDir("c_code"):
-      if kind == pcDir:
-        exec("git add " & path / "*.c")
-    exec("git commit -am \"updated csources to version " & NimVersion & "\"")
-    exec("git push origin master")
-    exec("git tag -am \"Version $1\" v$1" % NimVersion)
-    exec("git push origin v$1" % NimVersion)
-  finally:
-    setCurrentDir(cwd)
-
 proc testUnixInstall(cmdLineRest: string) =
   csource("-d:danger" & cmdLineRest)
   xz(false, cmdLineRest)
@@ -723,7 +694,8 @@ when isMainModule:
       of "tools":
         buildTools(op.cmdLineRest)
         bundleNimbleExe(latest, op.cmdLineRest)
-      of "pushcsource", "pushcsources": pushCsources()
+      of "pushcsource":
+        quit "use this instead: https://github.com/nim-lang/csources_v1/blob/master/push_c_code.nim"
       of "valgrind": valgrind(op.cmdLineRest)
       of "c2nim": bundleC2nim(op.cmdLineRest)
       of "drnim": buildDrNim(op.cmdLineRest)

lib/std/private/globs.nim

@@ -52,5 +52,5 @@ proc nativeToUnixPath*(path: string): string =
 
 when isMainModule:
   import sugar
-  for a in walkDirRecFilter(".", follow = a=>a.path.lastPathPart notin ["nimcache", ".git", ".csources", "bin"]):
+  for a in walkDirRecFilter(".", follow = a=>a.path.lastPathPart notin ["nimcache", ".git", "csources_v1", "csources", "bin"]):
     echo a

readme.md

@@ -45,7 +45,7 @@ Compiling the Nim compiler is quite straightforward if you follow these steps:
 First, the C source of an older version of the Nim compiler is needed to
 bootstrap the latest version because the Nim compiler itself is written in the
 Nim programming language. Those C sources are available within the
-[``nim-lang/csources``][csources-repo] repository.
+[``nim-lang/csources_v1``][csources-v1-repo] repository.
 
 Next, to build from source you will need:
 
@@ -217,7 +217,8 @@ Copyright © 2006-2021 Andreas Rumpf, all rights reserved.
 [nim-bitcoin]: https://blockchain.info/address/1BXfuKM2uvoD6mbx4g5xM3eQhLzkCK77tJ
 [nimble-repo]: https://github.com/nim-lang/nimble
 [nimsuggest-repo]: https://github.com/nim-lang/nimsuggest
-[csources-repo]: https://github.com/nim-lang/csources
+[csources-repo-deprecated]: https://github.com/nim-lang/csources
+[csources-v1-repo]: https://github.com/nim-lang/csources_v1
 [badge-nim-travisci]: https://img.shields.io/travis/nim-lang/Nim/devel.svg?style=flat-square
 [badge-nim-irc]: https://img.shields.io/badge/chat-on_irc-blue.svg?style=flat-square
 [badge-nim-discord]: https://img.shields.io/discord/371759389889003530?color=blue&label=discord&logo=discord&logoColor=gold&style=flat-square

tools/ci_generate.nim

@@ -33,15 +33,15 @@ environment:
   CC: /usr/bin/clang
 tasks:
 - setup: |
+    set -e
     cd Nim
-    git clone --depth 1 -q https://github.com/nim-lang/csources.git
+    . ci/funs.sh && nimBuildCsourcesIfNeeded
-    gmake -C csources -j $(sysctl -n hw.ncpuonline)
+    $nim_csources c koch
-    bin/nim c koch
     echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
 - test: |
     cd Nim
     if ! ./koch runCI; then
-      nim c -r tools/ci_testresults.nim
+      nim r tools/ci_testresults.nim
       exit 1
     fi
 triggers: