fixes explicit globals in macros (#21502)

* fixes #21496; Ambiguous calls compiles when module name are equal

* add a test case

.builds/freebsd.yml

@@ -1,37 +0,0 @@
-## DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`
-
-# see https://man.sr.ht/builds.sr.ht/compatibility.md#freebsd
-image: freebsd/latest
-packages:
-- databases/sqlite3
-- devel/boehm-gc-threaded
-- devel/pcre
-- devel/sdl20
-- devel/sfml
-- www/node
-- devel/gmake
-
-
-sources:
-- https://github.com/nim-lang/Nim
-environment:
-  NIM_TESTAMENT_BATCH: "0_1"
-  CC: /usr/bin/clang
-tasks:
-- setup: |
-    set -e
-    cd Nim
-    . ci/funs.sh && nimBuildCsourcesIfNeeded
-    $nim_csources c --skipUserCfg --skipParentCfg koch
-    echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
-- test: |
-    set -e
-    cd Nim
-    if ! ./koch runCI; then
-      nim r tools/ci_testresults.nim
-      exit 1
-    fi
-triggers:
-- action: email
-  condition: failure
-  to: Andreas Rumpf <rumpf_a@web.de>

.builds/openbsd_0.yml

@@ -1,37 +0,0 @@
-## DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`
-
-image: openbsd/latest
-packages:
-- gmake
-- sqlite3
-- node
-- boehm-gc
-- pcre
-- sfml
-- sdl2
-- libffi
-
-
-sources:
-- https://github.com/nim-lang/Nim
-environment:
-  NIM_TESTAMENT_BATCH: "0_2"
-  CC: /usr/bin/clang
-tasks:
-- setup: |
-    set -e
-    cd Nim
-    . ci/funs.sh && nimBuildCsourcesIfNeeded
-    $nim_csources c --skipUserCfg --skipParentCfg koch
-    echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
-- test: |
-    set -e
-    cd Nim
-    if ! ./koch runCI; then
-      nim r tools/ci_testresults.nim
-      exit 1
-    fi
-triggers:
-- action: email
-  condition: failure
-  to: Andreas Rumpf <rumpf_a@web.de>

.builds/openbsd_1.yml

@@ -1,37 +0,0 @@
-## DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`
-
-image: openbsd/latest
-packages:
-- gmake
-- sqlite3
-- node
-- boehm-gc
-- pcre
-- sfml
-- sdl2
-- libffi
-
-
-sources:
-- https://github.com/nim-lang/Nim
-environment:
-  NIM_TESTAMENT_BATCH: "1_2"
-  CC: /usr/bin/clang
-tasks:
-- setup: |
-    set -e
-    cd Nim
-    . ci/funs.sh && nimBuildCsourcesIfNeeded
-    $nim_csources c --skipUserCfg --skipParentCfg koch
-    echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
-- test: |
-    set -e
-    cd Nim
-    if ! ./koch runCI; then
-      nim r tools/ci_testresults.nim
-      exit 1
-    fi
-triggers:
-- action: email
-  condition: failure
-  to: Andreas Rumpf <rumpf_a@web.de>

.gitattributes

@@ -4,3 +4,6 @@
 # duplicated, which is easily identifiable and fixable.
 /changelog.md merge=union
 
+# bug https://github.com/dom96/choosenim/issues/256 for WSL CRLF
+*.sh text eol=lf
+/config/build_config.txt text eol=lf

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,51 +0,0 @@
----
-name: Bug report
-about: Have you found an unexpected behavior? Use this template.
-title: Think about the title, twice
-labels: ''
-assignees: ''
-
----
-
-Function `echo` outputs the wrong string.
-
-### Example
-```nim
-echo "Hello World!"
-# This code should be a minimum reproducible example:
-# try to simplify and minimize as much as possible. If it's a compiler
-# issue, try to minimize further by removing any imports if possible.
-```
-
-### Current Output
-please check whether the problem still exists in git head before posting,
-see [rebuilding the compiler](https://nim-lang.github.io/Nim/intern.html#rebuilding-the-compiler).
-```
-Hola mundo!
-```
-
-### Expected Output
-```
-Hello World!
-```
-
-### Possible Solution
-
-* In file xyz there is a call that might be the cause of it.
-
-### Additional Information
-If it's a regression, you can help us by identifying which version introduced
-the bug, see [Bisecting for regressions](https://nim-lang.github.io/Nim/intern.html#bisecting-for-regressions),
-or at least try known past releases (eg `choosenim 1.2.0`).
-
-If it's a pre-existing compiler bug, see [Debugging the compiler](https://nim-lang.github.io/Nim/intern.html#debugging-the-compiler)
-which should give more context on a compiler crash.
-
-* Issue #abc is related, but different because of ...
-* This issue is blocking my project xyz
-
-```
-$ nim -v
-Nim Compiler Version 0.1.2
-# make sure to include the git hash if not using a tagged release
-```

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,75 @@
+name:        "Bug Report"
+description: "Create a new bug report. Have you found an unexpected behavior? Use this form."
+title:       "Think about the title, twice."
+labels:      ["unconfirmed"]
+body:
+
+- type: markdown
+  attributes:
+    value: |
+      - **Please provide a minimal code example that reproduces the Bug!** :bug:
+        Reports with a reproducible example and descriptive detailed information will likely receive fixes faster.
+
+- type: textarea
+  id: description
+  attributes:
+    label: Description
+    description: |
+      Use DETAILED DESCRIPTIVE information about the problem.
+      Here, you go into more details about your Bug report. This section can be a few paragraphs long.
+    placeholder: Bug reports with reproducible code and detailed information will be fixed faster.
+  validations:
+    required: true
+    
+- type: textarea
+  id: nim-version
+  attributes:
+    label: Nim Version
+    description: Copy and paste the output of `nim -v` on the command line. For development versions, make sure to include the commit hash.
+  validations:
+    required: true
+
+- type: textarea
+  id: current-logs
+  attributes:
+    label: Current Output
+    description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
+    placeholder: Bug reports with reproducible code and detailed information will be fixed faster.
+    render: text
+
+- type: textarea
+  id: expected-logs
+  attributes:
+    label: Expected Output
+    description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
+    placeholder: Bug reports with reproducible code and detailed information will be fixed faster.
+    render: text
+
+- type: textarea
+  id: possible-solution
+  attributes:
+    label: Possible Solution
+    description: Propose a possible solution.
+  validations:
+    required: false
+
+- type: textarea
+  id: extra-info
+  attributes:
+    label: Additional Information
+    description: Any additional relevant information.
+  validations:
+    required: false
+
+- type: markdown
+  attributes:
+    value: |
+      - Thanks for your contributions!, your Bug report will receive feedback from the community soon...
+      - Please check whether the problem still exists in the devel branch, see [rebuilding the compiler](https://nim-lang.github.io/Nim/intern.html#rebuilding-the-compiler).
+      - Consider writing a PR targetting devel branch after filing this, see [contributing](https://nim-lang.github.io/Nim/contributing.html).
+      - If it's a pre-existing compiler bug, see [Debugging the compiler](https://nim-lang.github.io/Nim/intern.html#debugging-the-compiler)
+        which should give more context on a compiler crash.
+      - If it's a regression, you can help us by identifying which version introduced the bug,
+        see [Bisecting for regressions](https://nim-lang.github.io/Nim/intern.html#bisecting-for-regressions),
+        or at least try known past releases (eg `choosenim 1.2.0`).
+      - [Please, consider a Donation for the Nim project.](https://nim-lang.org/donate.html)

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,35 +0,0 @@
----
-name: Feature request
-about: Do you want to suggest a new feature? Use this template.
-title: ''
-labels: ''
-assignees: ''
-
----
-
-<!---
-Notice there is now a separate repository for the detailed RFCs and proposals:
-https://github.com/nim-lang/RFCs
-
-If you have a simple feature request, you can post it here using this template,
-but bear in mind that adding new features to the language is currently a low priority.
--->
-
-
-### Summary
-<!--- Short summary of your proposed feature -->
-
-
-### Description
-<!--- Describe your solution, what problem does it fix? -->
-
-
-### Alternatives
-<!--- Are there any alternatives you've considered? -->
-
-
-### Additional Information
-<!--- For Example:
-* A link to a project where the issue is relevant.
-* A link to a related issue or discussion.
--->

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,72 @@
+name:        "Feature request"
+description: "Create a new Feature Request. Do you want to suggest a new feature? Use this form."
+title:       "Think about the title, twice."
+labels:      ["unconfirmed"]
+body:
+
+- type: markdown
+  attributes:
+    value: |
+      - Please provide a minimal code example that illustrates the basic idea behind your feature request.
+        Reports with full repro code and descriptive detailed information will likely receive feedback faster.
+      - There is a separate repository for the detailed RFCs and proposals: https://github.com/nim-lang/RFCs.
+        If you have a simple feature request, you can post it here using this form,
+        but bear in mind that adding new features to the language is currently a low priority.
+
+- type: textarea
+  id: summary
+  attributes:
+    label: Summary
+    description: |
+      Use DETAILED DESCRIPTIVE information about the feature request.
+      Here, you go into more details about your ideas. This section can be a few paragraphs long.
+    placeholder: Short summary of your proposed feature.
+  validations:
+    required: true
+
+- type: textarea
+  id: description
+  attributes:
+    label: Description
+    description: Describe your solution, what problem does it fix?
+  validations:
+    required: true
+
+- type: textarea
+  id: alternatives
+  attributes:
+    label: Alternatives
+    description: Are there any alternatives you've considered?
+  validations:
+    required: false
+
+- type: textarea
+  id: Examples
+  attributes:
+    label: Examples
+    description: Provide examples for your feature request here.
+    placeholder: Example code here.
+
+- type: textarea
+  id: incompatibility
+  attributes:
+    label: Backwards Compatibility
+    description: If your Feature Request introduces backward-incompatible changes, describe them and propose how to deal with them.
+  validations:
+    required: false
+
+- type: textarea
+  id: links
+  attributes:
+    label: Links
+    description: Please copy and paste any relevant links to projects, issues, discussions, technical documentations, code samples, etc.
+  validations:
+    required: false
+
+- type: markdown
+  attributes:
+    value: |
+      - Thanks for your contributions!, your ideas will receive feedback from the community soon...
+      - **Remember to :star: Star the Nim project on GitHub!.**
+      - Consider writing a PR targetting devel branch after filing this, see [contributing](https://nim-lang.github.io/Nim/contributing.html).
+      - [Please, consider a Donation for the Nim project.](https://nim-lang.org/donate.html)

.github/workflows/ci_docs.yml

@@ -6,6 +6,7 @@ on:
       - 'compiler/renderverbatim.nim'
       - 'config/nimdoc.cfg'
       - 'doc/**.rst'
+      - 'doc/**.md'
       - 'doc/nimdoc.css'
       - 'lib/**.nim'
       - 'nimdoc/testproject/expected/testproject.html'
@@ -13,6 +14,7 @@ on:
       - 'tools/kochdocs.nim'
       - '.github/workflows/ci_docs.yml'
       - 'koch.nim'
+
   pull_request:
     # Run only on changes on these files.
     paths:
@@ -20,6 +22,7 @@ on:
       - 'compiler/renderverbatim.nim'
       - 'config/nimdoc.cfg'
       - 'doc/**.rst'
+      - 'doc/**.md'
       - 'doc/nimdoc.css'
       - 'lib/**.nim'
       - 'nimdoc/testproject/expected/testproject.html'
@@ -28,6 +31,10 @@ on:
       - '.github/workflows/ci_docs.yml'
       - 'koch.nim'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build:
     strategy:
@@ -40,14 +47,15 @@ jobs:
           - target: windows
             os: windows-2019
           - target: osx
-            os: macos-10.15
+            os: macos-11
 
     name: ${{ matrix.target }}
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 60 # refs bug #18178
 
     steps:
       - name: 'Checkout'
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 2
 
@@ -59,11 +67,9 @@ jobs:
         if: runner.os == 'Windows'
         shell: bash
         run: |
-          mkdir dist
-          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
-          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
-          7z x dist/mingw64.7z -odist
-          7z x dist/dlls.zip -obin
+          set -e
+          . ci/funs.sh
+          nimInternalInstallDepsWindows
           echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
 
       - name: 'Add build binaries to PATH'
@@ -105,7 +111,7 @@ jobs:
         if: |
           github.event_name == 'push' && github.ref == 'refs/heads/devel' &&
           matrix.target == 'linux'
-        uses: crazy-max/ghaction-github-pages@v1
+        uses: crazy-max/ghaction-github-pages@v3
         with:
           build_dir: doc/html
         env:

.github/workflows/ci_packages.yml

@@ -1,27 +1,39 @@
 name: Packages CI
-on: [push, pull_request]
+on:
+  pull_request:
+  push:
+    branches:
+      - 'devel'
+      - 'version-2-0'
+      - 'version-1-6'
+      - 'version-1-2'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
   build:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, macos-10.15]
+        os: [ubuntu-20.04, macos-11]
         cpu: [amd64]
         batch: ["allowed_failures", "0_3", "1_3", "2_3"] # list of `index_num`
     name: '${{ matrix.os }} (batch: ${{ matrix.batch }})'
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 60 # refs bug #18178
     env:
       NIM_TEST_PACKAGES: "1"
       NIM_TESTAMENT_BATCH: ${{ matrix.batch }}
     steps:
       - name: 'Checkout'
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 2
 
       - name: 'Install node.js 16.x'
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v3
         with:
           node-version: '16.x'
 
@@ -40,12 +52,10 @@ jobs:
         if: runner.os == 'Windows'
         shell: bash
         run: |
-          mkdir dist
-          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
-          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
-          7z x dist/mingw64.7z -odist
-          7z x dist/dlls.zip -obin
-          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
+          set -e
+          . ci/funs.sh
+          nimInternalInstallDepsWindows
+          echo_run echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
 
       - name: 'Add build binaries to PATH'
         shell: bash
@@ -59,14 +69,6 @@ jobs:
         shell: bash
         run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}'
 
-      - name: 'Build koch'
+      - name: 'koch, Run CI'
         shell: bash
-        run: nim c koch
-      - name: 'Run CI'
-        shell: bash
-        run: ./koch runCI
-
-      - name: 'Show failed tests'
-        if: failure()
-        shell: bash
-        run: nim c -r tools/ci_testresults.nim
+        run: . ci/funs.sh && nimInternalBuildKochAndRunCI

.github/workflows/ci_publish.yml

@@ -0,0 +1,90 @@
+name: Tracking orc-booting compiler memory usage
+
+on:
+  pull_request_target:
+    types: [closed]
+
+
+jobs:
+  build:
+    if: github.event.pull_request.merged == true
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-20.04]
+        cpu: [amd64]
+    name: '${{ matrix.os }}'
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+
+      - name: 'Install node.js 16.x'
+        uses: actions/setup-node@v3
+        with:
+          node-version: '16.x'
+
+      - name: 'Install dependencies (Linux amd64)'
+        if: runner.os == 'Linux' && matrix.cpu == 'amd64'
+        run: |
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends -yq \
+              libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev \
+              valgrind libc6-dbg libblas-dev xorg-dev
+      - name: 'Install dependencies (macOS)'
+        if: runner.os == 'macOS'
+        run: brew install boehmgc make sfml gtk+3
+      - name: 'Install dependencies (Windows)'
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          set -e
+          . ci/funs.sh
+          nimInternalInstallDepsWindows
+          echo_run echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Add build binaries to PATH'
+        shell: bash
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
+
+      - name: 'System information'
+        shell: bash
+        run: . ci/funs.sh && nimCiSystemInfo
+
+      - name: 'Build csourcesAny'
+        shell: bash
+        run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}'
+
+      - name: 'Build koch'
+        shell: bash
+        run: nim c koch
+
+      - name: 'Build Nim'
+        shell: bash
+        run: ./koch boot -d:release -d:nimStrictMode --lib:lib
+
+      - name: 'Action'
+        shell: bash
+        run: nim c -r -d:release ci/action.nim
+
+      - name: 'Comment'
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const fs = require('fs');
+
+            try {
+              const data = fs.readFileSync('ci/nimcache/results.txt', 'utf8');
+              github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: data
+            })
+            } catch (err) {
+              console.error(err);
+            }
+

.gitignore

@@ -3,9 +3,9 @@
 !*.*
 
 # Cache
-nimcache/
-rnimcache/
-dnimcache/
+nimcache*/
+rnimcache*/
+dnimcache*/
 
 *.o
 !/icons/*.o
@@ -67,6 +67,7 @@ testament.db
 
 /csources
 /csources_v1
+/csources_v2
 
 /dist/
 # /lib/fusion # fusion is now unbundled; `git status` should reveal if it's there so users can act on it

.gitlab-ci.yml

@@ -1,4 +1,6 @@
-image: ubuntu:16.04
+# xxx unused, out of date
+
+image: ubuntu:18.04
 
 stages:
   - pre-build

azure-pipelines.yml

@@ -1,7 +1,8 @@
 trigger:
   branches:
     include:
-    - '*'
+    - 'devel'
+    - 'version-*'
 pr:
   branches:
     include:
@@ -19,19 +20,19 @@ jobs:
   strategy:
     matrix:
       Linux_amd64:
-        vmImage: 'ubuntu-16.04'
+        vmImage: 'ubuntu-20.04'
         CPU: amd64
-      # pending bug #17325 (broken again)
-      # Linux_i386:
-      #   # bug #17325: fails on 'ubuntu-16.04' because it now errors with:
-      #   # g++-multilib : Depends: gcc-multilib (>= 4:5.3.1-1ubuntu1) but it is not going to be installed
-      #   vmImage: 'ubuntu-18.04'
-      #   CPU: i386
+      # regularly breaks, refs bug #17325
+      Linux_i386:
+        # on 'ubuntu-16.04' (not supported anymore anyways) it errored with:
+        # g++-multilib : Depends: gcc-multilib (>= 4:5.3.1-1ubuntu1) but it is not going to be installed
+        vmImage: 'ubuntu-18.04'
+        CPU: i386
       OSX_amd64:
-        vmImage: 'macOS-10.15'
+        vmImage: 'macOS-11'
         CPU: amd64
       OSX_amd64_cpp:
-        vmImage: 'macOS-10.15'
+        vmImage: 'macOS-11'
         CPU: amd64
         NIM_COMPILE_TO_CPP: true
       Windows_amd64_batch0_3:
@@ -133,11 +134,7 @@ jobs:
     - bash: |
         set -e
         . ci/funs.sh
-        echo_run mkdir dist
-        echo_run curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
-        echo_run curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
-        echo_run 7z x dist/mingw64.7z -odist
-        echo_run 7z x dist/dlls.zip -obin
+        nimInternalInstallDepsWindows
         echo_run echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/dist/mingw64/bin'
 
       displayName: 'Install dependencies (Windows)'
@@ -163,13 +160,9 @@ jobs:
     #   condition: and(succeeded(), eq(variables['skipci'], 'false'))
     #   displayName: 'Restore built csourcesAny'
 
-    - bash: nim c koch
+    - bash: . ci/funs.sh && nimInternalBuildKochAndRunCI
+      # would could also show on failure: echo '##vso[task.complete result=Failed]'
       condition: and(succeeded(), eq(variables['skipci'], 'false'))
-      displayName: 'Build koch'
-
-      # set result to omit the "bash exited with error code '1'" message
-    - bash: ./koch runCI || echo '##vso[task.complete result=Failed]'
-      condition: and(succeeded(), eq(variables['skipci'], 'false'))
-      displayName: 'Run CI'
+      displayName: 'koch, Run CI'
       env:
         SYSTEM_ACCESSTOKEN: $(System.AccessToken)

bin/nim-gdb

@@ -7,14 +7,14 @@ which nim > /dev/null || (echo "nim not in PATH"; exit 1)
 which gdb > /dev/null || (echo "gdb not in PATH"; exit 1)
 which readlink > /dev/null || (echo "readlink not in PATH."; exit 1)
 
-if [[ "$(uname -s)" == "Darwin" || "(uname -s)" == *"BSD" ]]; then
+if [[ $(uname -s) == Darwin || $(uname -s) == *BSD ]]; then
   NIM_SYSROOT=$(dirname $(dirname $(readlink -f $(which nim))))
 else
   NIM_SYSROOT=$(dirname $(dirname $(readlink -e $(which nim))))
 fi
 
 # Find out where the pretty printer Python module is
-GDB_PYTHON_MODULE_PATH="$NIM_SYSROOT/tools/nim-gdb.py"
+GDB_PYTHON_MODULE_PATH="$NIM_SYSROOT/tools/debug/nim-gdb.py"
 
 # Run GDB with the additional arguments that load the pretty printers
 # Set the environment variable `NIM_GDB` to overwrite the call to a

bin/nim-gdb.bash

@@ -1 +0,0 @@
-nim-gdb

bin/nim-gdb.bat

@@ -3,7 +3,7 @@ for %%i in (nim.exe) do (set NIM_BIN=%%~dp$PATH:i)
 
 for %%i in ("%NIM_BIN%\..\") do (set NIM_ROOT=%%~fi)
 
-set @GDB_PYTHON_MODULE_PATH=%NIM_ROOT%\tools\nim-gdb.py
+set @GDB_PYTHON_MODULE_PATH=%NIM_ROOT%\tools\debug\nim-gdb.py
 set @NIM_GDB=gdb.exe
 
 @echo source %@GDB_PYTHON_MODULE_PATH%> wingdbcommand.txt

build_all.bat

@@ -10,7 +10,7 @@ SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe
 echo "building from csources: %nim_csources%"
 
 if not exist %nim_csourcesDir% (
-  git clone -q --depth 1 %nim_csourcesUrl% %nim_csourcesDir%
+  git clone -q --depth 1 -b %nim_csourcesBranch% %nim_csourcesUrl% %nim_csourcesDir%
 )
 
 if not exist %nim_csources% (
@@ -24,6 +24,6 @@ if not exist %nim_csources% (
   cd ..
   copy /y bin\nim.exe  %nim_csources%
 )
- bin\nim.exe c --skipUserCfg --skipParentCfg --hints:off koch
+ bin\nim.exe c --noNimblePath --skipUserCfg --skipParentCfg --hints:off koch
  koch boot -d:release --skipUserCfg --skipParentCfg --hints:off
  koch tools --skipUserCfg --skipParentCfg --hints:off

build_all.sh

@@ -1,4 +1,4 @@
-#! /bin/sh
+#!/bin/sh
 # DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`
 
 # build development version of the compiler; can be rerun safely.
@@ -11,7 +11,7 @@ set -e # exit on first error
 . ci/funs.sh
 nimBuildCsourcesIfNeeded "$@"
 
-echo_run bin/nim c --skipUserCfg --skipParentCfg --hints:off koch
+echo_run bin/nim c --noNimblePath --skipUserCfg --skipParentCfg --hints:off koch
 echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg --hints:off
 echo_run ./koch tools --skipUserCfg --skipParentCfg --hints:off
 

changelog.md

@@ -1,428 +1,31 @@
-# v1.6.x - yyyy-mm-dd
-
+# v2.2.0 - yyyy-mm-dd
 
 
 ## Changes affecting backward compatibility
 
--  `cuchar` now aliases `uint8` instead of `char`
-
-- `repr` now doesn't insert trailing newline; previous behavior was very inconsistent,
-  see #16034. Use `-d:nimLegacyReprWithNewline` for previous behavior.
-
-- An enum now can't be converted to another enum directly, you must use `ord` (or `cast`, but
-  compiler won't help if you misuse it).
-  ```
-  type A = enum a1, a2
-  type B = enum b1, b2
-  doAssert not compiles(a1.B)
-  doAssert compiles(a1.ord.B)
-  ```
-  for a transition period, use `-d:nimLegacyConvEnumEnum`.
-
-- Type mismatch errors now show more context, use `-d:nimLegacyTypeMismatch` for previous
-  behavior.
-
-- `echo` and `debugEcho` will now raise `IOError` if writing to stdout fails.  Previous behavior
-  silently ignored errors.  See #16366.  Use `-d:nimLegacyEchoNoRaise` for previous behavior.
-
-- `math.round` now is rounded "away from zero" in JS backend which is consistent
-  with other backends. See #9125. Use `-d:nimLegacyJsRound` for previous behavior.
-
-- Changed the behavior of `uri.decodeQuery` when there are unencoded `=`
-  characters in the decoded values. Prior versions would raise an error. This is
-  no longer the case to comply with the HTML spec and other languages
-  implementations. Old behavior can be obtained with
-  `-d:nimLegacyParseQueryStrict`. `cgi.decodeData` which uses the same
-  underlying code is also updated the same way.
-- Custom pragma values have now an API for use in macros.
-
-- In `std/os`, `getHomeDir`, `expandTilde`, `getTempDir`, `getConfigDir` now do not include trailing `DirSep`,
-  unless `-d:nimLegacyHomeDir` is specified (for a transition period).
-
-- On POSIX systems, the default signal handlers used for Nim programs (it's
-  used for printing the stacktrace on fatal signals) will now re-raise the
-  signal for the OS default handlers to handle.
-
-  This lets the OS perform its default actions, which might include core
-  dumping (on select signals) and notifying the parent process about the cause
-  of termination.
-
-- On POSIX systems, we now ignore `SIGPIPE` signals, use `-d:nimLegacySigpipeHandler`
-  for previous behavior.
-
-- `hashes.hash` now supports `object` and `ref` (can be overloaded in user code).
-  For a transition period, use `-d:nimLegacyNoHashRef` to avoid defining `hash(ref)`.
-- `hashes.hash(proc|ptr|ref|pointer)` now calls `hash(int)` and honors `-d:nimIntHash1`,
-  `hashes.hash(closure)` has also been improved.
-
-- The unary slice `..b` was removed, use `0..b` instead or use `-d:nimLegacyUnarySlice`
-  for a deprecation period.
-
-- Removed `.travis.yml`, `appveyor.yml.disabled`, `.github/workflows/ci.yml.disabled`.
-
-- Nim compiler now adds ASCII unit separator `\31` before a newline for every generated
-  message (potentially multiline), so tooling can tell when messages start and end.
-
-- `random.initRand(seed)` now produces non-skewed values for the 1st call to `rand()` after
-  initialization with a small (< 30000) seed. Use `-d:nimLegacyRandomInitRand` to restore
-  previous behavior for a transition time, see PR #17467.
-
-- `jsonutils` now serializes/deserializes holey enums as regular enums (via `ord`) instead of as strings.
-  Use `-d:nimLegacyJsonutilsHoleyEnum` for a transition period.
-
-- `json` and `jsonutils` now serialize NaN, Inf, -Inf as strings, so that
-  `%[NaN, -Inf]` is the string `["nan","-inf"]` instead of `[nan,-inf]` which was invalid json.
 
 ## Standard library additions and changes
-- Added support for parenthesized expressions in `strformat`
 
-- Fixed buffer overflow bugs in `net`
+[//]: # "Changes:"
 
-- Exported `sslHandle` from `net` and `asyncnet`.
 
-- Added `sections` iterator in `parsecfg`.
+[//]: # "Additions:"
+- Added `parseutils.parseSize` - inverse to `strutils.formatSize` - to parse human readable sizes.
 
-- Make custom op in macros.quote work for all statements.
+[//]: # "Deprecations:"
 
-- On Windows the SSL library now checks for valid certificates.
-  It uses the `cacert.pem` file for this purpose which was extracted
-  from `https://curl.se/ca/cacert.pem`. Besides
-  the OpenSSL DLLs (e.g. libssl-1_1-x64.dll, libcrypto-1_1-x64.dll) you
-  now also need to ship `cacert.pem` with your `.exe` file.
 
-
-- Make `{.requiresInit.}` pragma to work for `distinct` types.
-
-- Added a macros `enumLen` for returning the number of items in an enum to the
-  `typetraits.nim` module.
-
-- `prelude` now works with the JavaScript target.
-  Added `sequtils` import to `prelude`.
-  `prelude` can now be used via `include std/prelude`, but `include prelude` still works.
-
-- Added `almostEqual` in `math` for comparing two float values using a machine epsilon.
-
-- Added `clamp` in `math` which allows using a `Slice` to clamp to a value.
-
-- The JSON module can now handle integer literals and floating point literals of
-  arbitrary length and precision.
-  Numbers that do not fit the underlying `BiggestInt` or `BiggestFloat` fields are
-  kept as string literals and one can use external BigNum libraries to handle these.
-  The `parseFloat` family of functions also has now optional `rawIntegers` and
-  `rawFloats` parameters that can be used to enforce that all integer or float
-  literals remain in the "raw" string form so that client code can easily treat
-  small and large numbers uniformly.
-
-- Added `BackwardsIndex` overload for `JsonNode`.
-
-- `json.%`,`json.to`, `jsonutils.formJson`,`jsonutils.toJson` now work with `uint|uint64`
-  instead of raising (as in 1.4) or giving wrong results (as in 1.2).
-
-- added `jsonutils.jsonTo` overload with `opt = Joptions()` param.
-
-- `jsonutils` now handles `cstring` (including as Table key), and `set`.
-
-- Added an overload for the `collect` macro that inferes the container type based
-  on the syntax of the last expression. Works with std seqs, tables and sets.
-
-- Added `randState` template that exposes the default random number generator.
-  Useful for library authors.
-
-- Added `random.initRand()` overload with no argument which uses the current time as a seed.
-
-- `random.initRand(seed)` now allows `seed == 0`.
-
-- Added `std/sysrand` module to get random numbers from a secure source
-  provided by the operating system.
-
-- Added `std/enumutils` module. Added `genEnumCaseStmt` macro that generates case statement to parse string to enum.
-  Added `items` for enums with holes.
-  Added `symbolName` to return the enum symbol name ignoring the human readable name.
-
-- Added `typetraits.HoleyEnum` for enums with holes, `OrdinalEnum` for enums without holes.
-
-- Removed deprecated `iup` module from stdlib, it has already moved to
-  [nimble](https://github.com/nim-lang/iup).
-
-- various functions in `httpclient` now accept `url` of type `Uri`. Moreover `request` function's
-  `httpMethod` argument of type `string` was deprecated in favor of `HttpMethod` enum type.
-
-- `nodejs` backend now supports osenv: `getEnv`, `putEnv`, `envPairs`, `delEnv`, `existsEnv`.
-
-- Added `cmpMem` to `system`.
-
-- `doAssertRaises` now correctly handles foreign exceptions.
-
-- Added `asyncdispatch.activeDescriptors` that returns the number of currently
-  active async event handles/file descriptors.
-
-- Added `getPort` to `asynchttpserver`.
-
-- `--gc:orc` is now 10% faster than previously for common workloads. If
-  you have trouble with its changed behavior, compile with `-d:nimOldOrc`.
-
-
-- `os.FileInfo` (returned by `getFileInfo`) now contains `blockSize`,
-  determining preferred I/O block size for this file object.
-
-- Added a simpler to use `io.readChars` overload.
-
-- Added `**` to jsffi.
-
-- `writeStackTrace` is available in JS backend now.
-
-- Added `decodeQuery` to `std/uri`.
-
-- `strscans.scanf` now supports parsing single characters.
-
-- `strscans.scanTuple` added which uses `strscans.scanf` internally,
-  returning a tuple which can be unpacked for easier usage of `scanf`.
-
-- Added `setutils.toSet` that can take any iterable and convert it to a built-in `set`,
-  if the iterable yields a built-in settable type.
-
-- Added `setutils.fullSet` which returns a full built-in `set` for a valid type.
-
-- Added `setutils.complement` which returns the complement of a built-in `set`.
-
-- Added `setutils.[]=`.
-
-- Added `math.isNaN`.
-
-- Added `jsbigints` module, arbitrary precision integers for JavaScript target.
-
-- Added `math.copySign`.
-
-- Added new operations for singly- and doubly linked lists: `lists.toSinglyLinkedList`
-  and `lists.toDoublyLinkedList` convert from `openArray`s; `lists.copy` implements
-  shallow copying; `lists.add` concatenates two lists - an O(1) variation that consumes
-  its argument, `addMoved`, is also supplied.
-
-- Added `euclDiv` and `euclMod` to `math`.
-
-- Added `httpcore.is1xx` and missing HTTP codes.
-
-- Added `jsconsole.jsAssert` for JavaScript target.
-
-- Added `posix_utils.osReleaseFile` to get system identification from `os-release` file on Linux and the BSDs.
-  https://www.freedesktop.org/software/systemd/man/os-release.html
-
-- Added `socketstream` module that wraps sockets in the stream interface
-
-- Added `sugar.dumpToString` which improves on `sugar.dump`.
-
-- Added `math.signbit`.
-
-- Removed the optional `longestMatch` parameter of the `critbits._WithPrefix` iterators (it never worked reliably)
-
-- In `lists`: renamed `append` to `add` and retained `append` as an alias;
-  added `prepend` and `prependMoved` analogously to `add` and `addMoved`;
-  added `remove` for `SinglyLinkedList`s.
-
-- Deprecated `any`. See https://github.com/nim-lang/RFCs/issues/281
-
-- Added optional `options` argument to `copyFile`, `copyFileToDir`, and
-  `copyFileWithPermissions`. By default, on non-Windows OSes, symlinks are
-  followed (copy files symlinks point to); on Windows, `options` argument is
-  ignored and symlinks are skipped.
-
-- On non-Windows OSes, `copyDir` and `copyDirWithPermissions` copy symlinks as
-  symlinks (instead of skipping them as it was before); on Windows symlinks are
-  skipped.
-
-- On non-Windows OSes, `moveFile` and `moveDir` move symlinks as symlinks
-  (instead of skipping them sometimes as it was before).
-
-- Added optional `followSymlinks` argument to `setFilePermissions`.
-
-- Added `os.isAdmin` to tell whether the caller's process is a member of the
-  Administrators local group (on Windows) or a root (on POSIX).
-
-- Added experimental `linenoise.readLineStatus` to get line and status (e.g. ctrl-D or ctrl-C).
-
-- Added `compilesettings.SingleValueSetting.libPath`.
-
-- `std/wrapnils` doesn't use `experimental:dotOperators` anymore, avoiding
-  issues like https://github.com/nim-lang/Nim/issues/13063 (which affected error messages)
-  for modules importing `std/wrapnils`.
-  Added `??.` macro which returns an `Option`.
-
-- Added `math.frexp` overload procs. Deprecated `c_frexp`, use `frexp` instead.
-
-- `parseopt.initOptParser` has been made available and `parseopt` has been
-  added back to `prelude` for all backends. Previously `initOptParser` was
-  unavailable if the `os` module did not have `paramCount` or `paramStr`,
-  but the use of these in `initOptParser` were conditionally to the runtime
-  arguments passed to it, so `initOptParser` has been changed to raise
-  `ValueError` when the real command line is not available. `parseopt` was
-  previously excluded from `prelude` for JS, as it could not be imported.
-
-- Added `system.prepareStrMutation` for better support of low
-  level `moveMem`, `copyMem` operations for Orc's copy-on-write string
-  implementation.
-
-- Added `std/strbasics` for high performance string operations.
-  Added `strip`, `setSlice`, `add(a: var string, b: openArray[char])`.
-
-
-- Added to `wrapnils` an option-like API via `??.`, `isSome`, `get`.
-
-- `std/options` changed `$some(3)` to `"some(3)"` instead of `"Some(3)"`
-  and `$none(int)` to `"none(int)"` instead of `"None[int]"`.
-
-- Added `algorithm.merge`.
-
-
-- Added `std/jsfetch` module [Fetch](https://developer.mozilla.org/docs/Web/API/Fetch_API) wrapper for JavaScript target.
-
-- Added `std/jsheaders` module [Headers](https://developer.mozilla.org/en-US/docs/Web/API/Headers) wrapper for JavaScript target.
-
-- Added `std/jsformdata` module [FormData](https://developer.mozilla.org/en-US/docs/Web/API/FormData) wrapper for JavaScript target.
-
-- `system.addEscapedChar` now renders `\r` as `\r` instead of `\c`, to be compatible
-  with most other languages.
-
-- Removed support for named procs in `sugar.=>`.
-
-- Added `jscore.debugger` to [call any available debugging functionality, such as breakpoints.](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/debugger).
-
-- Added `std/channels`.
-
-- Added `htmlgen.portal` for [making "SPA style" pages using HTML only](https://web.dev/hands-on-portals).
-
-- Added `ZZZ` and `ZZZZ` patterns to `times.nim` `DateTime` parsing, to match time
-  zone offsets without colons, e.g. `UTC+7 -> +0700`.
-
-- Added `jsconsole.dir`, `jsconsole.dirxml`, `jsconsole.timeStamp`.
-
-- Added dollar `$` and `len` for `jsre.RegExp`.
-
-- Added `std/tasks`.
-
-- Added `hasDataBuffered` to `asyncnet`.
-
-- Added `hasClosure` to `std/typetraits`.
-
-- Added `std/tempfiles`.
-
-- Added `genasts.genAst` that avoids the problems inherent with `quote do` and can
-  be used as a replacement.
-
-- Added `copyWithin` [for `seq` and `array` for JavaScript targets](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/copyWithin).
+[//]: # "Removals:"
 
 
 ## Language changes
 
-- `nimscript` now handles `except Exception as e`.
 
-- The `cstring` doesn't support `[]=` operator in JS backend.
-
-- nil dereference is not allowed at compile time. `cast[ptr int](nil)[]` is rejected at compile time.
-
-- `typetraits.distinctBase` now is identity instead of error for non distinct types.
-
-- `os.copyFile` is now 2.5x faster on OSX, by using `copyfile` from `copyfile.h`;
-  use `-d:nimLegacyCopyFile` for OSX < 10.5.
-
-- The required name of case statement macros for the experimental
-  `caseStmtMacros` feature has changed from `match` to `` `case` ``.
-
-- `typedesc[Foo]` now renders as such instead of `type Foo` in compiler messages.
-
-- The unary minus in `-1` is now part of the integer literal, it is now parsed as a single token.
-  This implies that edge cases like `-128'i8` finally work correctly.
-
-- Custom numeric literals (e.g. `-128'bignum`) are now supported.
-
-- Tuple expressions are now parsed consistently as
-  `nnkTupleConstr` node. Will affect macros expecting nodes to be of `nnkPar`.
-
-- `nim e` now accepts arbitrary file extensions for the nimscript file,
-  although `.nims` is still the preferred extension in general.
-
-- Added `iterable[T]` type class to match called iterators, which enables writing:
-  `template fn(a: iterable)` instead of `template fn(a: untyped)`
-
-- A new import syntax `import foo {.all.}` now allows to import all symbols (public or private)
-  from `foo`. It works in combination with all pre-existing import features.
-  This reduces or eliminates the need for workarounds such as using `include` (which has known issues)
-  when you need a private symbol for testing or making some internal APIs public just because
-  another internal module needs those.
-  It also helps mitigate the lack of cyclic imports in some cases.
-
-- Added a new module `std/importutils`, and an API `privateAccess`, which allows access to private fields
-  for an object type in the current scope.
-
-- `typeof(voidStmt)` now works and returns `void`.
 
 ## Compiler changes
 
-- Added `--declaredlocs` to show symbol declaration location in messages.
 
-- Deprecated `TaintedString` and `--taintmode`.
 
-- Deprecated `--nilseqs` which is now a noop.
-
-- Added `--spellSuggest` to show spelling suggestions on typos.
-
-- Added `--filenames:abs|canonical|legacyRelProj` which replaces --listFullPaths:on|off
-
-- Added `--processing:dots|filenames|off` which customizes `hintProcessing`
-
-- Source+Edit links now appear on top of every docgen'd page when
-  `nim doc --git.url:url ...` is given.
-
-- Added `nim --eval:cmd` to evaluate a command directly, see `nim --help`.
-
-- VM now supports `addr(mystring[ind])` (index + index assignment)
-
-- Added `--hintAsError` with similar semantics as `--warningAsError`.
-
-- TLS: OSX now uses native TLS (`--tlsEmulation:off`), TLS now works with importcpp non-POD types,
-  such types must use `.cppNonPod` and `--tlsEmulation:off`should be used.
-
-- Now array literals(JS backend) uses JS typed arrays when the corresponding js typed array exists, for example `[byte(1), 2, 3]` generates `new Uint8Array([1, 2, 3])`.
-
-- docgen: rst files can now use single backticks instead of double backticks and correctly render
-  in both rst2html (as before) as well as common tools rendering rst directly (e.g. github), by
-  adding: `default-role:: code` directive inside the rst file, which is now handled by rst2html.
-
-- Added `-d:nimStrictMode` in CI in several places to ensure code doesn't have certain hints/warnings
-
-- Added `then`, `catch` to `asyncjs`, for now hidden behind `-d:nimExperimentalAsyncjsThen`.
-
-- `--newruntime` and `--refchecks` are deprecated.
-
-- Added `unsafeIsolate` and `extract` to `std/isolation`.
-
-- `--hint:CC` now goes to stderr (like all other hints) instead of stdout.
-
-- json build instructions are now generated in `$nimcache/outFileBasename.json`
-  instead of `$nimcache/projectName.json`. This allows avoiding recompiling a given project
-  compiled with different options if the output file differs.
-
-- `--usenimcache` (implied by `nim r main`) now generates an output file that includes a hash of
-  some of the compilation options, which allows caching generated binaries:
-  nim r main # recompiles
-  nim r -d:foo main # recompiles
-  nim r main # uses cached binary
-  nim r main arg1 arg2 # ditto (runtime arguments are irrelevant)
-
-- The style checking of the compiler now supports a `--styleCheck:usages` switch. This switch
-  enforces that every symbol is written as it was declared, not enforcing
-  the official Nim style guide. To be enabled, this has to be combined either
-  with `--styleCheck:error` or `--styleCheck:hint`.
 
 ## Tool changes
 
-- The rst parser now supports markdown table syntax.
-  Known limitations:
-  - cell alignment is not supported, i.e. alignment annotations in a delimiter
-    row (`:---`, `:--:`, `---:`) are ignored,
-  - every table row must start with `|`, e.g. `| cell 1 | cell 2 |`.
-
-- `fusion` is now un-bundled from nim, `./koch fusion` will
-  install it via nimble at a fixed hash.
-
-- testament: added `nimoutFull: bool` spec to compare full output of compiler
-  instead of a subset.

changelogs/changelog_0_19_0.md

@@ -15,7 +15,7 @@
   at runtime. Use ``ref object`` with inheritance rather than ``object`` with
   inheritance to prevent this issue.
 - The ``not nil`` type annotation now has to be enabled explicitly
-  via ``{.experimental: "notnil"}`` as we are still not pleased with how this
+  via ``{.experimental: "notnil".}`` as we are still not pleased with how this
   feature works with Nim's containers.
 - The parser now warns about inconsistent spacing around binary operators as
   these can easily be confused with unary operators. This warning will likely

changelogs/changelog_1_4_0.md

@@ -282,7 +282,7 @@
 - Removed the deprecated `asyncdispatch.newAsyncNativeSocket`.
 - Removed the deprecated `dom.releaseEvents` and `dom.captureEvents`.
 
-- Removed `sharedlists.initSharedList`, was deprecated and produces undefined behaviour.
+- Removed `sharedlist.initSharedList`, was deprecated and produces undefined behaviour.
 
 - There is a new experimental feature called "strictFuncs" which makes the definition of
   `.noSideEffect` stricter. [See here](manual_experimental.html#stricts-funcs)

changelogs/changelog_1_6_0.md

@@ -0,0 +1,957 @@
+# v1.6.0 - 2021-10-14
+
+
+
+# Major new features
+
+With so many new features, pinpointing the most salient ones is a subjective exercise,
+but here are a select few:
+
+
+## `iterable[T]`
+
+The `iterable[T]` type class was added to match called iterators,
+which solves a number of long-standing issues related to iterators.
+Example:
+
+```nim
+iterator iota(n: int): int =
+  for i in 0..<n: yield i
+
+# previously, you'd need `untyped`, which caused other problems such as lack
+# of type inference, overloading issues, and MCS.
+template sumOld(a: untyped): untyped = # no type inference possible
+  var result: typeof(block:(for ai in a: ai))
+  for ai in a: result += ai
+  result
+
+assert sumOld(iota(3)) == 0 + 1 + 2
+
+# now, you can write:
+template sum[T](a: iterable[T]): T =
+  # `template sum(a: iterable): auto =` would also be possible
+  var result: T
+  for ai in a: result += ai
+  result
+
+assert sum(iota(3)) == 0 + 1 + 2 # or `iota(3).sum`
+```
+
+In particular iterable arguments can now be used with the method call syntax. For example:
+```nim
+import std/[sequtils, os]
+echo walkFiles("*").toSeq # now works
+```
+See PR [#17196](https://github.com/nim-lang/Nim/pull/17196) for additional details.
+
+
+## Strict effects
+
+The effect system was refined and there is a new `.effectsOf` annotation that does
+explicitly what was previously done implicitly. See the
+[manual](https://nim-lang.github.io/Nim/manual.html#effect-system-effectsof-annotation)
+for more details.
+To write code that is portable with older Nim versions, use this idiom:
+
+```nim
+when defined(nimHasEffectsOf):
+  {.experimental: "strictEffects".}
+else:
+  {.pragma: effectsOf.}
+
+proc mysort(s: seq; cmp: proc(a, b: T): int) {.effectsOf: cmp.}
+```
+
+To enable the new effect system, compile with `--experimental:strictEffects`.
+See also [#18777](https://github.com/nim-lang/Nim/pull/18777) and RFC
+[#408](https://github.com/nim-lang/RFCs/issues/408).
+
+
+## Private imports and private field access
+
+A new import syntax `import foo {.all.}` now allows importing all symbols
+(public or private) from `foo`.
+This can be useful for testing purposes or for more flexibility in project organization.
+
+Example:
+```nim
+from system {.all.} as system2 import nil
+echo system2.ThisIsSystem # ThisIsSystem is private in `system`
+import os {.all.} # weirdTarget is private in `os`
+echo weirdTarget # or `os.weirdTarget`
+```
+
+Added a new module `std/importutils`, and an API `privateAccess`, which allows access
+to private fields for an object type in the current scope.
+
+Example:
+```nim
+import times
+from std/importutils import privateAccess
+block:
+  let t = now()
+  # echo t.monthdayZero # Error: undeclared field: 'monthdayZero' for type times.DateTime
+  privateAccess(typeof(t)) # enables private access in this scope
+  echo t.monthdayZero # ok
+```
+
+See PR [#17706](https://github.com/nim-lang/Nim/pull/17706) for additional details.
+
+
+## `nim --eval:cmd`
+
+Added `nim --eval:cmd` to evaluate a command directly, e.g.: `nim --eval:"echo 1"`.
+It defaults to `e` (nimscript) but can also work with other commands, e.g.:
+```bash
+find . | nim r --eval:'import strutils; for a in stdin.lines: echo a.toUpper'
+```
+
+```bash
+# use as a calculator:
+nim --eval:'echo 3.1 / (1.2+7)'
+# explore a module's APIs, including private symbols:
+nim --eval:'import os {.all.}; echo weirdTarget'
+# use a custom backend:
+nim r -b:js --eval:"import std/jsbigints; echo 2'big ** 64'big"
+```
+
+See PR [#15687](https://github.com/nim-lang/Nim/pull/15687) for more details.
+
+
+## Round-trip float to string
+
+`system.addFloat` and `system.$` now can produce string representations of
+floating point numbers that are minimal in size and possess round-trip and correct
+rounding guarantees (via the
+[Dragonbox](https://raw.githubusercontent.com/jk-jeon/dragonbox/master/other_files/Dragonbox.pdf) algorithm).
+This currently has to be enabled via `-d:nimPreviewFloatRoundtrip`.
+It is expected that this behavior becomes the new default in upcoming versions,
+as with other `nimPreviewX` define flags.
+
+Example:
+```nim
+from math import round
+let a = round(9.779999999999999, 2)
+assert a == 9.78
+echo a # with `-d:nimPreviewFloatRoundtrip`: 9.78, like in python3 (instead of  9.779999999999999)
+```
+
+
+## New `std/jsbigints` module
+
+Provides arbitrary precision integers for the JS target. See PR
+[#16409](https://github.com/nim-lang/Nim/pull/16409).
+Example:
+```nim
+import std/jsbigints
+assert 2'big ** 65'big == 36893488147419103232'big
+echo 0xdeadbeef'big shl 4'big # 59774856944n
+```
+
+
+## New `std/sysrand` module
+Cryptographically secure pseudorandom number generator,
+allows generating random numbers from a secure source provided by the operating system.
+Example:
+```nim
+import std/sysrand
+assert urandom(1234) != urandom(1234) # unlikely to fail in practice
+```
+See PR [#16459](https://github.com/nim-lang/Nim/pull/16459).
+
+
+## New module: `std/tempfiles`
+
+Allows creating temporary files and directories, see PR
+[#17361](https://github.com/nim-lang/Nim/pull/17361) and followups.
+```nim
+import std/tempfiles
+let tmpPath = genTempPath("prefix", "suffix.log", "/tmp/")
+# tmpPath looks like: /tmp/prefixpmW1P2KLsuffix.log
+
+let dir = createTempDir("tmpprefix_", "_end")
+# created dir looks like: getTempDir() / "tmpprefix_YEl9VuVj_end"
+
+let (cfile, path) = createTempFile("tmpprefix_", "_end.tmp")
+# path looks like: getTempDir() / "tmpprefix_FDCIRZA0_end.tmp"
+cfile.write "foo"
+cfile.setFilePos 0
+assert readAll(cfile) == "foo"
+close cfile
+assert readFile(path) == "foo"
+```
+
+
+## User-defined literals
+
+Custom numeric literals (e.g. `-128'bignum`) are now supported.
+Additionally, the unary minus in `-1` is now part of the integer literal, i.e.
+it is now parsed as a single token.
+This implies that edge cases like `-128'i8` finally work correctly.
+Example:
+```nim
+func `'big`*(num: cstring): JsBigInt {.importjs: "BigInt(#)".}
+assert 0xffffffffffffffff'big == (1'big shl 64'big) - 1'big
+```
+
+
+## Dot-like operators
+
+With `-d:nimPreviewDotLikeOps`, dot-like operators (operators starting with `.`,
+but not with `..`) now have the same precedence as `.`, so that `a.?b.c` is now
+parsed as `(a.?b).c` instead of `a.?(b.c)`.
+A warning is generated when a dot-like operator is used without `-d:nimPreviewDotLikeOps`.
+
+An important use case is to enable dynamic fields without affecting the
+built-in `.` operator, e.g. for `std/jsffi`, `std/json`, `pkg/nimpy`. Example:
+```nim
+import std/json
+template `.?`(a: JsonNode, b: untyped{ident}): JsonNode =
+  a[astToStr(b)]
+let j = %*{"a1": {"a2": 10}}
+assert j.?a1.?a2.getInt == 10
+```
+
+
+## Block arguments now support optional parameters
+
+This solves a major pain point for routines accepting block parameters,
+see PR [#18631](https://github.com/nim-lang/Nim/pull/18631) for details:
+
+```nim
+template fn(a = 1, b = 2, body) = discard
+fn(1, 2): # already works
+  bar
+fn(a = 1): # now works
+  bar
+```
+
+Likewise with multiple block arguments via `do`:
+```nim
+template fn(a = 1, b = 2, body1, body2) = discard
+fn(a = 1): # now works
+  bar1
+do:
+  bar2
+```
+
+
+# Other new features
+
+## New and deprecated modules
+
+The following modules were added (they are discussed in the rest of the text):
+- `std/enumutils`
+- `std/genasts`
+- `std/importutils`
+- `std/jsbigints`
+- `std/jsfetch`
+- `std/jsformdata`
+- `std/jsheaders`
+- `std/packedsets`
+- `std/setutils`
+- `std/socketstreams`
+- `std/strbasics`
+- `std/sysrand`
+- `std/tasks`
+- `std/tempfiles`
+- `std/vmutils`
+
+- Deprecated `std/mersenne`.
+- Removed deprecated `std/iup` module from stdlib; it has already moved to
+  [nimble](https://github.com/nim-lang/iup).
+
+
+## New `std/jsfetch` module
+
+Provides a wrapper for JS Fetch API.
+Example:
+```nim
+# requires -d:nimExperimentalAsyncjsThen
+import std/[jsfetch, asyncjs, jsconsole, jsffi, sugar]
+proc fn {.async.} =
+  await fetch("https://api.github.com/users/torvalds".cstring)
+    .then((response: Response) => response.json())
+    .then((json: JsObject) => console.log(json))
+    .catch((err: Error) => console.log("Request Failed", err))
+discard fn()
+```
+
+
+## New `std/tasks` module
+
+Provides basic primitives for creating parallel programs.
+Example:
+```nim
+import std/tasks
+var num = 0
+proc hello(a: int) = num+=a
+
+let b = toTask hello(13) # arguments must be isolated, see `std/isolation`
+b.invoke()
+assert num == 13
+b.invoke() # can be called again
+assert num == 26
+```
+
+
+## New module: `std/genasts`
+
+Provides an API `genAst` that avoids the problems inherent with `quote do` and can
+be used as a replacement.
+Example showing how this could be used for writing a simplified version of `unittest.check`:
+```nim
+import std/[genasts, macros, strutils]
+macro check2(cond: bool): untyped =
+  assert cond.kind == nnkInfix, "$# not implemented" % $cond.kind
+  result = genAst(cond, s = repr(cond), lhs = cond[1], rhs = cond[2]):
+    # each local symbol we access must be explicitly captured
+    if not cond:
+      doAssert false, "'$#'' failed: lhs: '$#', rhs: '$#'" % [s, $lhs, $rhs]
+let a = 3
+check2 a*2 == a+3
+if false: check2 a*2 < a+1 # would error with: 'a * 2 < a + 1'' failed: lhs: '6', rhs: '4'
+```
+
+See PR [#17426](https://github.com/nim-lang/Nim/pull/17426) for details.
+
+
+## New module: `std/setutils`
+
+- Added `setutils.toSet` that can take any iterable and convert it to a built-in `set`,
+  if the iterable yields a built-in settable type.
+- Added `setutils.fullSet` which returns a full built-in `set` for a valid type.
+- Added `setutils.complement` which returns the complement of a built-in `set`.
+- Added `setutils.[]=`.
+
+
+## New module: `std/enumutils`
+
+- Added `genEnumCaseStmt` macro that generates
+  case statement to parse string to enum.
+- Added `items` for enums with holes.
+- Added `symbolName` to return the `enum` symbol name ignoring the human-readable name.
+- Added `symbolRank` to return the index in which an `enum` member is listed in an enum.
+
+
+## `system`
+
+- Added `system.prepareMutation` for better support of low
+  level `moveMem`, `copyMem` operations for `gc:orc`'s copy-on-write string
+  implementation.
+- `system.addEscapedChar` now renders `\r` as `\r` instead of `\c`, to be compatible
+  with most other languages.
+- Added `cmpMem` to `system`.
+- `doAssertRaises` now correctly handles foreign exceptions.
+- `addInt` now supports unsigned integers.
+
+Compatibility notes:
+- `system.delete` had surprising behavior when the index passed to it was out of
+  bounds (it would delete the last entry then). Compile with `-d:nimStrictDelete` so
+  that an index error is produced instead. Be aware however that your code might depend on
+  this quirky behavior so a review process is required on your part before you can
+  use `-d:nimStrictDelete`. To make this review easier, use the `-d:nimAuditDelete`
+  switch, which pretends that `system.delete` is deprecated so that it is easier
+  to see where it was used in your code.
+  `-d:nimStrictDelete` will become the default in upcoming versions.
+- `cuchar` is now deprecated as it aliased `char` where arguably it should have aliased `uint8`.
+  Please use `char` or `uint8` instead.
+- `repr` now doesn't insert trailing newlines; the previous behavior was very inconsistent,
+  see [#16034](https://github.com/nim-lang/Nim/pull/16034).
+  Use `-d:nimLegacyReprWithNewline` for the previous behavior.
+  `repr` now also renders ASTs correctly for user defined literals, setters, `do`, etc.
+- Deprecated `any`. See RFC [#281](https://github.com/nim-lang/RFCs/issues/281).
+- The unary slice `..b` was deprecated, use `0..b` instead.
+
+
+## `std/math`
+
+- Added `almostEqual` for comparing two float values using a machine epsilon.
+- Added `clamp` which allows using a `Slice` to clamp to a value.
+- Added `ceilDiv` for integer division that rounds up.
+- Added `isNaN`.
+- Added `copySign`.
+- Added `euclDiv` and `euclMod`.
+- Added `signbit`.
+- Added `frexp` overload procs. Deprecated `c_frexp`, use `frexp` instead.
+
+Compatibility notes:
+- `math.round` now rounds "away from zero" in the JS backend, which is consistent
+  with other backends. See [#9125](https://github.com/nim-lang/Nim/pull/9125).
+  Use `-d:nimLegacyJsRound` for the previous behavior.
+
+
+## Random number generators: `std/random`, `std/sysrand`, `std/oids`
+
+- Added `std/sysrand` module (see details above).
+- Added `randState` template that exposes the default random number generator.
+  Useful for library authors.
+- Added `initRand()` overload with no argument which uses the current time as a seed.
+- `initRand(seed)` now allows `seed == 0`.
+- Fixed overflow bugs.
+- Fix `initRand` to avoid random number sequences overlapping, refs
+  [#18744](https://github.com/nim-lang/Nim/pull/18744).
+- `std/oids` now uses `std/random`.
+
+Compatibility notes:
+- Deprecated `std/mersenne`.
+- `random.initRand(seed)` now produces non-skewed values for the first call to
+  `rand()` after initialization with a small (< 30000) seed.
+  Use `-d:nimLegacyRandomInitRand` to restore previous behavior for a transition
+  time, see PR [#17467](https://github.com/nim-lang/Nim/pull/17467).
+
+
+## `std/json`, `std/jsonutils`
+
+- With `-d:nimPreviewJsonutilsHoleyEnum`, `jsonutils` now can serialize/deserialize
+  holey enums as regular enums (via `ord`) instead of as strings.
+  It is expected that this behavior becomes the new default in upcoming versions.
+ `toJson` now serializes `JsonNode` as is via reference (without a deep copy)
+   instead of treating `JsonNode` as a regular ref object,
+  this can be customized via `jsonNodeMode`.
+- `std/json` and `std/jsonutils` now serialize `NaN`, `Inf`, `-Inf` as strings,
+  so that `%[NaN, -Inf]` is the string `["nan","-inf"]` instead of `[nan,-inf]`
+  which was invalid JSON.
+- `std/json` can now handle integer literals and floating point literals of
+  arbitrary length and precision.
+  Numbers that do not fit the underlying `BiggestInt` or `BiggestFloat` fields are
+  kept as string literals and one can use external BigNum libraries to handle these.
+  The `parseFloat` family of functions also has now optional `rawIntegers` and
+  `rawFloats` parameters that can be used to enforce that all integer or float
+  literals remain in the "raw" string form so that client code can easily treat
+  small and large numbers uniformly.
+- Added `BackwardsIndex` overload for `JsonNode`.
+- `json.%`,`json.to`, `jsonutils.fromJson`,`jsonutils.toJson` now work with `uint|uint64`
+  instead of raising (as in 1.4) or giving wrong results (as in 1.2).
+- `std/jsonutils` now handles `cstring` (including as Table key), and `set`.
+- Added `jsonutils.jsonTo` overload with `opt = Joptions()` param.
+- `jsonutils.toJson` now supports customization via `ToJsonOptions`.
+- `std/json`, `std/jsonutils` now support round-trip serialization when
+  `-d:nimPreviewFloatRoundtrip` is used.
+
+
+## `std/typetraits`, `std/compilesettings`
+
+- `distinctBase` now is identity instead of error for non distinct types.
+- `distinctBase` now allows controlling whether to be recursive or not.
+- Added `enumLen` to return the number of elements in an enum.
+- Added `HoleyEnum` for enums with holes, `OrdinalEnum` for enums without holes.
+- Added `hasClosure`.
+- Added `pointerBase` to return `T` for `ref T | ptr T`.
+- Added `compilesettings.SingleValueSetting.libPath`.
+
+
+## networking: `std/net`, `std/asyncnet`, `std/htmlgen`, `std/httpclient`, `std/asyncdispatch`, `std/asynchttpserver`, `std/httpcore`
+
+- Fixed buffer overflow bugs in `std/net`.
+- Exported `sslHandle` from `std/net` and `std/asyncnet`.
+- Added `hasDataBuffered` to `std/asyncnet`.
+- Various functions in `std/httpclient` now accept `url` of type `Uri`.
+  Moreover, the `request` function's `httpMethod` argument of type `string` was
+  deprecated in favor of `HttpMethod` `enum` type; see
+  [#15919](https://github.com/nim-lang/Nim/pull/15919).
+- Added `asyncdispatch.activeDescriptors` that returns the number of currently
+  active async event handles/file descriptors.
+- Added `getPort` to `std/asynchttpserver` to resolve OS-assigned `Port(0)`;
+  this is usually recommended instead of hardcoding ports which can lead to
+  "Address already in use" errors.
+- Fixed premature garbage collection in `std/asyncdispatch`, when a stacktrace
+  override is in place.
+- Added `httpcore.is1xx` and missing HTTP codes.
+- Added `htmlgen.portal` for [making "SPA style" pages using HTML only](https://web.dev/hands-on-portals).
+
+Compatibility notes:
+- On Windows, the SSL library now checks for valid certificates.
+  For this purpose it uses the `cacert.pem` file, which was extracted
+  from `https://curl.se/ca/cacert.pem`. Besides
+  the OpenSSL DLLs (e.g. `libssl-1_1-x64.dll`, `libcrypto-1_1-x64.dll`) you
+  now also need to ship `cacert.pem` with your `.exe` file.
+
+
+## `std/hashes`
+
+- `hashes.hash` can now support `object` and `ref` (can be overloaded in user code),
+  if `-d:nimPreviewHashRef` is used. It is expected that this behavior
+  becomes the new default in upcoming versions.
+- `hashes.hash(proc|ptr|ref|pointer)` now calls `hash(int)` and honors `-d:nimIntHash1`.
+  `hashes.hash(closure)` has also been improved.
+
+
+## OS: `std/os`, `std/io`, `std/socketstream`, `std/linenoise`, `std/tempfiles`
+
+- `os.FileInfo` (returned by `getFileInfo`) now contains `blockSize`,
+  determining preferred I/O block size for this file object.
+- Added `os.getCacheDir()` to return platform specific cache directory.
+- Improved `os.getTempDir()`, see PR [#16914](https://github.com/nim-lang/Nim/pull/16914).
+- Added `os.isAdmin` to tell whether the caller's process is a member of the
+  Administrators local group (on Windows) or a root (on POSIX).
+- Added optional `options` argument to `copyFile`, `copyFileToDir`, and
+  `copyFileWithPermissions`. By default, on non-Windows OSes, symlinks are
+  followed (copy files symlinks point to); on Windows, `options` argument is
+  ignored and symlinks are skipped.
+- On non-Windows OSes, `copyDir` and `copyDirWithPermissions` copy symlinks as
+  symlinks (instead of skipping them as it was before); on Windows symlinks are
+  skipped.
+- On non-Windows OSes, `moveFile` and `moveDir` move symlinks as symlinks
+  (instead of skipping them sometimes as it was before).
+- Added optional `followSymlinks` argument to `setFilePermissions`.
+- Added a simpler to use `io.readChars` overload.
+- Added `socketstream` module that wraps sockets in the stream interface.
+- Added experimental `linenoise.readLineStatus` to get line and status (e.g. ctrl-D or ctrl-C).
+
+
+## Environment variable handling
+
+- Empty environment variable values are now supported across OS's and backends.
+- Environment variable APIs now work in multithreaded scenarios, by delegating to
+  direct OS calls instead of trying to keep track of the environment.
+- `putEnv`, `delEnv` now work at compile time.
+- NodeJS backend now supports osenv: `getEnv`, `putEnv`, `envPairs`, `delEnv`, `existsEnv`.
+
+Compatibility notes:
+- `std/os`: `putEnv` now raises if the first argument contains a `=`.
+
+
+## POSIX
+
+- On POSIX systems, the default signal handlers used for Nim programs (it's
+  used for printing the stacktrace on fatal signals) will now re-raise the
+  signal for the OS default handlers to handle.
+  This lets the OS perform its default actions, which might include core
+  dumping (on select signals) and notifying the parent process about the cause
+  of termination.
+- On POSIX systems, we now ignore `SIGPIPE` signals, use `-d:nimLegacySigpipeHandler`
+  for previous behavior.
+- Added `posix_utils.osReleaseFile` to get system identification from `os-release`
+  file on Linux and the BSDs.
+  ([link](https://www.freedesktop.org/software/systemd/man/os-release.html))
+- Removed undefined behavior for `posix.open`.
+
+
+## `std/prelude`
+
+- `std/strformat` is now part of `include std/prelude`.
+- Added `std/sequtils` import to `std/prelude`.
+- `std/prelude` now works with the JS target.
+- `std/prelude` can now be used via `include std/prelude`, but `include prelude` still works.
+
+
+## String manipulation: `std/strformat`, `std/strbasics`
+
+- Added support for parenthesized expressions.
+- Added support for const strings instead of just string literals.
+- Added `std/strbasics` for high-performance string operations.
+- Added `strip`, `setSlice`, `add(a: var string, b: openArray[char])`.
+
+
+## `std/wrapnils`
+
+- `std/wrapnils` doesn't use `experimental:dotOperators` anymore, avoiding
+  issues like bug [#13063](https://github.com/nim-lang/Nim/issues/13063)
+  (which affected error messages) for modules importing `std/wrapnils`.
+- Added `??.` macro which returns an `Option`.
+- `std/wrapnils` can now be used to protect against `FieldDefect` errors in
+  case objects, generates optimal code (no overhead compared to manual
+  if-else branches), and preserves lvalue semantics which allows modifying
+  an expression.
+
+
+## Containers: `std/algorithm`, `std/lists`, `std/sequtils`, `std/options`, `std/packedsets`
+
+- Removed the optional `longestMatch` parameter of the `critbits._WithPrefix`
+  iterators (it never worked reliably).
+- Added `algorithm.merge`.
+- In `std/lists`: renamed `append` to `add` and retained `append` as an alias;
+  added `prepend` and `prependMoved` analogously to `add` and `addMoved`;
+  added `remove` for `SinglyLinkedList`s.
+- Added new operations for singly- and doubly linked lists: `lists.toSinglyLinkedList`
+  and `lists.toDoublyLinkedList` convert from `openArray`s; `lists.copy` implements
+  shallow copying; `lists.add` concatenates two lists - an O(1) variation that consumes
+  its argument, `addMoved`, is also supplied.
+  See PRs [#16362](https://github.com/nim-lang/Nim/pull/16362),
+  [#16536](https://github.com/nim-lang/Nim/pull/16536).
+- New module: `std/packedsets`.
+  Generalizes `std/intsets`, see PR [#15564](https://github.com/nim-lang/Nim/pull/15564).
+
+Compatibility notes:
+- Deprecated `sequtils.delete` and added an overload taking a `Slice` that raises
+  a defect if the slice is out of bounds, likewise with `strutils.delete`.
+- Deprecated `proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T]`
+  in `std/algorithm`.
+- `std/options` changed `$some(3)` to `"some(3)"` instead of `"Some(3)"`
+  and `$none(int)` to `"none(int)"` instead of `"None[int]"`.
+
+
+## `std/times`
+
+- Added `ZZZ` and `ZZZZ` patterns to `times.nim` `DateTime` parsing, to match time
+  zone offsets without colons, e.g. `UTC+7 -> +0700`.
+- Added `dateTime` and deprecated `initDateTime`.
+
+
+## `std/macros` and AST
+
+- New module `std/genasts`, see description above.
+- The required name of case statement macros for the experimental
+  `caseStmtMacros` feature has changed from `match` to `` `case` ``.
+- Tuple expressions are now parsed consistently as
+  `nnkTupleConstr` node. Will affect macros expecting nodes to be of `nnkPar`.
+- In `std/macros`, `treeRepr,lispRepr,astGenRepr` now represent SymChoice nodes
+  in a collapsed way.
+  Use `-d:nimLegacyMacrosCollapseSymChoice` to get the previous behavior.
+- Made custom op in `macros.quote` work for all statements.
+
+
+## `std/sugar`
+
+- Added `sugar.dumpToString` which improves on `sugar.dump`.
+- Added an overload for the `collect` macro that infers the container type based
+  on the syntax of the last expression. Works with std seqs, tables and sets.
+
+Compatibility notes:
+- Removed support for named procs in `sugar.=>`.
+
+
+## Parsing: `std/parsecfg`, `std/strscans`, `std/uri`
+
+- Added `sections` iterator in `parsecfg`.
+- `strscans.scanf` now supports parsing single characters.
+- Added `strscans.scanTuple` which uses `strscans.scanf` internally,
+  returning a tuple which can be unpacked for easier usage of `scanf`.
+- Added `decodeQuery` to `std/uri`.
+- `parseopt.initOptParser` has been made available and `parseopt` has been
+  added back to `std/prelude` for all backends. Previously `initOptParser` was
+  unavailable if the `std/os` module did not have `paramCount` or `paramStr`,
+  but the use of these in `initOptParser` were conditionally to the runtime
+  arguments passed to it, so `initOptParser` has been changed to raise
+  `ValueError` when the real command line is not available. `parseopt` was
+  previously excluded from `std/prelude` for JS, as it could not be imported.
+
+Compatibility notes:
+- Changed the behavior of `uri.decodeQuery` when there are unencoded `=`
+  characters in the decoded values. Prior versions would raise an error. This is
+  no longer the case to comply with the HTML spec and other languages'
+  implementations. Old behavior can be obtained with
+  `-d:nimLegacyParseQueryStrict`. `cgi.decodeData` which uses the same
+  underlying code is also updated the same way.
+
+
+## JS stdlib changes
+
+- Added `std/jsbigints` module, which provides arbitrary precision integers for the JS target.
+- Added `setCurrentException` for the JS backend.
+- `writeStackTrace` is available in the JS backend now.
+- Added `then`, `catch` to `std/asyncjs` for promise pipelining, for now hidden
+  behind `-d:nimExperimentalAsyncjsThen`.
+- Added `std/jsfetch` module [Fetch](https://developer.mozilla.org/docs/Web/API/Fetch_API)
+  wrapper for the JS target.
+- Added `std/jsheaders` module [Headers](https://developer.mozilla.org/en-US/docs/Web/API/Headers)
+  wrapper for the JS target.
+- Added `std/jsformdata` module [FormData](https://developer.mozilla.org/en-US/docs/Web/API/FormData)
+  wrapper for the JS target.
+- Added `jscore.debugger` to [call any available debugging functionality, such as breakpoints](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/debugger).
+- Added `jsconsole.dir`, `jsconsole.dirxml`, `jsconsole.timeStamp`.
+- Added dollar `$` and `len` for `jsre.RegExp`.
+- Added `jsconsole.jsAssert` for the JS target.
+- Added `**` to `std/jsffi`.
+- Added `copyWithin` [for `seq` and `array` for JS targets](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/copyWithin).
+- In `std/dom`, `Interval` is now a `ref object`, same as `Timeout`.
+  Definitions of `setTimeout`, `clearTimeout`, `setInterval`, `clearInterval` were updated.
+- Added `dom.scrollIntoView` proc with options.
+- Added `dom.setInterval`, `dom.clearInterval` overloads.
+- Merged `std/dom_extensions` into the `std/dom` module,
+  as it was a module with a single line, see RFC [#413](https://github.com/nim-lang/RFCs/issues/413).
+- `$` now gives more correct results on the JS backend.
+
+
+## JS compiler changes
+
+- `cstring` doesn't support the `[]=` operator anymore in the JS backend.
+- Array literals now use JS typed arrays when the corresponding JS typed array exists,
+  for example `[byte(1), 2, 3]` generates `new Uint8Array([1, 2, 3])`.
+
+
+## VM and nimscript backend
+
+- VM now supports `addr(mystring[ind])` (index + index assignment).
+- `nimscript` now handles `except Exception as e`.
+- `nil` dereference is not allowed at compile time. `cast[ptr int](nil)[]` is rejected at compile time.
+- `static[T]` now works better, refs [#17590](https://github.com/nim-lang/Nim/pull/17590),
+  [#15853](https://github.com/nim-lang/Nim/pull/15853).
+- `distinct T` conversions now work in VM.
+- `items(cstring)` now works in VM.
+- Fix `addr`, `len`, `high` in VM ([#16002](https://github.com/nim-lang/Nim/pull/16002),
+  [#16610](https://github.com/nim-lang/Nim/pull/16610)).
+- `std/cstrutils` now works in VM.
+
+
+## OS/platform-specific notes
+
+- Support for Apple silicon/M1.
+- Support for 32-bit RISC-V, refs [#16231](https://github.com/nim-lang/Nim/pull/16231).
+- Support for armv8l, refs [#18901](https://github.com/nim-lang/Nim/pull/18901).
+- Support for CROSSOS, refs [#18889](https://github.com/nim-lang/Nim/pull/18889).
+- The allocator for Nintendo Switch, which was nonfunctional because
+  of breaking changes in libnx, was removed, in favor of the new `-d:nimAllocPagesViaMalloc` option.
+- Allow reading parameters when compiling for Nintendo Switch.
+- `--nimcache` now correctly works in a cross-compilation setting.
+- Cross compilation targeting Windows was improved.
+- This now works from macOS/Linux: `nim r -d:mingw main`
+
+
+
+## Performance / memory optimizations
+
+- The comment field in PNode AST was moved to a side channel, reducing overall
+  memory usage during compilation by a factor 1.25x
+- `std/jsonutils` deserialization is now up to 20x faster.
+- `os.copyFile` is now 2.5x faster on macOS, by using `copyfile` from `copyfile.h`;
+  use `-d:nimLegacyCopyFile` for macOS < 10.5.
+- Float to string conversion is now 10x faster thanks to the Dragonbox algorithm,
+  with `-d:nimPreviewFloatRoundtrip`.
+- `newSeqWith` is 3x faster.
+- CI now supports batching (making Windows CI 2.3X faster).
+- Sets now uses the optimized `countSetBits` proc, see PR
+  [#17334](https://github.com/nim-lang/Nim/pull/17334).
+
+
+## Debugging
+
+- You can now enable/disable VM tracing in user code via `vmutils.vmTrace`.
+- `koch tools` now builds `bin/nim_dbg` which allows easy access to a debug version
+  of Nim without recompiling.
+- Added new module `compiler/debugutils` to help with debugging Nim compiler.
+- Renamed `-d:nimCompilerStackraceHints` to `-d:nimCompilerStacktraceHints` and
+  used it in more contexts; this flag which works in tandem with `--stackTraceMsgs`
+  to show user code context in compiler stacktraces.
+
+
+## Type system
+
+- `typeof(voidStmt)` now works and returns `void`.
+- `enum` values can now be overloaded. This needs to be enabled
+  via `{.experimental: "overloadableEnums".}`. We hope that this feature allows
+  for the development of more fluent (less ugly) APIs.
+  See RFC [#373](https://github.com/nim-lang/RFCs/issues/373) for more details.
+- A type conversion from one `enum` type to another now produces an `[EnumConv]` warning.
+  You should use `ord` (or `cast`, but the compiler won't help, if you misuse it) instead.
+  ```nim
+  type A = enum a1, a2
+  type B = enum b1, b2
+  echo a1.B # produces a warning
+  echo a1.ord.B # produces no warning
+  ```
+- A dangerous implicit conversion to `cstring` now triggers a `[CStringConv]` warning.
+  This warning will become an error in future versions! Use an explicit conversion
+  like `cstring(x)` in order to silence the warning.
+- There is a new warning for *any* type conversion to `enum` that can be enabled via
+  `.warning[AnyEnumConv]:on` or `--warning:AnyEnumConv:on`.
+- Reusing a type name in a different scope now works, refs [#17710](https://github.com/nim-lang/Nim/pull/17710).
+- Fixed implicit and explicit generics in procedures, refs [#18808](https://github.com/nim-lang/Nim/pull/18808).
+
+
+## New-style concepts
+
+Example:
+```nim
+type
+  Comparable = concept # no T, an atom
+    proc cmp(a, b: Self): int
+```
+The new design does not rely on `system.compiles` and may compile faster.
+See PR [#15251](https://github.com/nim-lang/Nim/pull/15251)
+and RFC [#168](https://github.com/nim-lang/RFCs/issues/168) for details.
+
+
+## Lexical / syntactic
+
+- Nim now supports a small subset of Unicode operators as operator symbols.
+  The supported symbols are: "∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔".
+  To enable this feature, use `--experimental:unicodeOperators`. Note that due
+  to parser limitations you **cannot** enable this feature via a
+  pragma `{.experimental: "unicodeOperators".}` reliably, you need to enable
+  it via the command line or in a configuration file.
+- `var a {.foo.} = expr` now works inside templates (except when `foo` is overloaded).
+
+
+## Compiler messages, error messages, hints, warnings
+
+- Significant improvement to error messages involving effect mismatches,
+  see PRs [#18384](https://github.com/nim-lang/Nim/pull/18384),
+  [#18418](https://github.com/nim-lang/Nim/pull/18418).
+- Added `--declaredLocs` to show symbol declaration location in error messages.
+- Added `--spellSuggest` to show spelling suggestions on typos.
+- Added `--processing:dots|filenames|off` which customizes `hintProcessing`;
+  `--processing:filenames` shows which include/import modules are being compiled as an import stack.
+- `FieldDefect` messages now shows discriminant value + lineinfo, in all backends (C, JS, VM)
+- Added `--hintAsError` with similar semantics as `--warningAsError`.
+- Added `--unitsep:on|off` to control whether to add ASCII unit separator `\31`
+  before a newline for every generated message (potentially multiline),
+  so tooling can tell when messages start and end.
+- Added `--filenames:abs|canonical|legacyRelProj` which replaces `--listFullPaths:on|off`
+- `--hint:all:on|off` is now supported to select or deselect all hints; it
+  differs from `--hints:on|off` which acts as a (reversible) gate.
+  Likewise with `--warning:all:on|off`.
+- The style checking of the compiler now supports a `--styleCheck:usages` switch.
+  This switch enforces that every symbol is written as it was declared, not enforcing
+  the official Nim style guide. To be enabled, this has to be combined either
+  with `--styleCheck:error` or `--styleCheck:hint`.
+- Type mismatch errors now show more context, use `-d:nimLegacyTypeMismatch` for
+  previous behavior.
+- `typedesc[Foo]` now renders as such instead of `type Foo` in compiler messages.
+- `runnableExamples` now show originating location in stacktraces on failure.
+- `SuccessX` message now shows more useful information.
+- New `DuplicateModuleImport` warning; improved `UnusedImport` and
+  `XDeclaredButNotUsed` accuracy.
+
+Compatibility notes:
+- `--hint:CC` now prints to stderr (like all other hints) instead of stdout.
+
+
+## Building and running Nim programs, configuration system
+
+- JSON build instructions are now generated in `$nimcache/outFileBasename.json`
+  instead of `$nimcache/projectName.json`. This allows avoiding recompiling a
+  given project compiled with different options if the output file differs.
+- `--usenimcache` (implied by `nim r main`) now generates an output file that includes
+  a hash of some of the compilation options, which allows caching generated binaries:
+  ```bash
+    nim r main # recompiles
+    nim r -d:foo main # recompiles
+    nim r main # uses cached binary
+    nim r main arg1 arg2 # likewise (runtime arguments are irrelevant)
+  ```
+- `nim r` now supports cross compilation from unix to windows when specifying
+  `-d:mingw` by using Wine, e.g.:
+  `nim r --eval:'import os; echo "a" / "b"'` prints `a\b`.
+- `nim` can compile version 1.4.0 as follows:
+  `nim c --lib:lib --stylecheck:off -d:nimVersion140 compiler/nim`.
+  `-d:nimVersion140` is not needed for bootstrapping, only for building 1.4.0 from devel.
+- `nim e` now accepts arbitrary file extensions for the nimscript file,
+  although `.nims` is still the preferred extension in general.
+- The configuration subsystem now allows for `-d:release` and `-d:danger` to work as expected.
+  The downside is that these defines now have custom logic that doesn't apply for
+  other defines.
+
+
+## Multithreading
+
+- TLS: macOS now uses native TLS (`--tlsEmulation:off`). TLS now works with
+  `importcpp` non-POD types; such types must use `.cppNonPod` and
+  `--tlsEmulation:off`should be used.
+- Added `unsafeIsolate` and `extract` to `std/isolation`.
+- Added `std/tasks`, see description above.
+
+
+## Memory management
+
+- `--gc:arc` now bootstraps (PR [#17342](https://github.com/nim-lang/Nim/pull/17342)).
+- Lots of improvements to `gc:arc`, `gc:orc`, see PR
+  [#15697](https://github.com/nim-lang/Nim/pull/15697),
+  [#16849](https://github.com/nim-lang/Nim/pull/16849),
+  [#17993](https://github.com/nim-lang/Nim/pull/17993).
+- `--gc:orc` is now 10% faster than previously for common workloads.
+  If you have trouble with its changed behavior, compile with `-d:nimOldOrc`.
+- The `--gc:orc` algorithm was refined so that custom container types can participate in the
+  cycle collection process. See the documentation of `=trace` for more details.
+- On embedded devices `malloc` can now be used instead of `mmap` via `-d:nimAllocPagesViaMalloc`.
+  This is only supported for `--gc:orc` or `--gc:arc`.
+
+Compatibility notes:
+- `--newruntime` and `--refchecks` are deprecated,
+  use `--gc:arc`, `--gc:orc`, or `--gc:none` as appropriate instead.
+
+
+## Docgen
+
+- docgen: RST files can now use single backticks instead of double backticks and
+  correctly render in both `nim rst2html` (as before) as well as common tools rendering
+  RST directly (e.g. GitHub).
+  This is done by adding the `default-role:: code` directive inside the RST file
+  (which is now handled by `nim rst2html`).
+- Source+Edit links now appear on top of every docgen'd page when
+  `nim doc --git.url:url ...` is given.
+- Latex doc generation is revised: output `.tex` files should be compiled
+  by `xelatex` (not by `pdflatex` as before). Now default Latex settings
+  provide support for Unicode and better avoid margin overflows.
+  The minimum required version is TeXLive 2018 (or an equivalent MikTeX version).
+- The RST parser now supports footnotes, citations, admonitions, and short style
+  references with symbols.
+- The RST parser now supports Markdown table syntax.
+  Known limitations:
+  - cell alignment is not supported, i.e. alignment annotations in a delimiter
+    row (`:---`, `:--:`, `---:`) are ignored
+  - every table row must start with `|`, e.g. `| cell 1 | cell 2 |`.
+- Implemented `doc2tex` compiler command which converts documentation in
+  `.nim` files to Latex.
+- docgen now supports syntax highlighting for inline code.
+- docgen now supports same-line doc comments:
+  ```nim
+  func fn*(a: int): int = 42  ## Doc comment
+  ```
+- docgen now renders `deprecated` and other pragmas.
+- `runnableExamples` now works with templates and nested templates.
+- `runnableExamples: "-r:off"` now works for examples that should compile but not run.
+- `runnableExamples` now renders code verbatim, and produces correct code in all cases.
+- docgen now shows correct, canonical import paths in docs.
+- docgen now shows all routines in sidebar, and the proc signature is now shown in sidebar.
+
+
+## Effects and checks
+
+- Significant improvement to error messages involving effect mismatches
+- There is a new `cast` section `{.cast(uncheckedAssign).}: body` that disables some
+  compiler checks regarding `case objects`. This allows serialization libraries
+  to avoid ugly, non-portable solutions. See RFC
+  [#407](https://github.com/nim-lang/RFCs/issues/407) for more details.
+
+Compatibility notes:
+- Fixed effect tracking for borrowed procs (see [#18882](https://github.com/nim-lang/Nim/pull/18882)).
+  One consequence is that, under some circumstances, Nim could previously permit a procedure with side effects to be written with `func` - you may need to change some occurrences of `func` to `proc`.
+  To illustrate, Nim versions before 1.6.0 compile the below without error
+  ```nim
+  proc print(s: string) =
+    echo s
+
+  type
+    MyString = distinct string
+
+  proc print(s: MyString) {.borrow.}
+
+  func foo(s: MyString) =
+    print(s)
+  ```
+  but Nim 1.6.0 produces the error
+  ```
+  Error: 'foo' can have side effects
+  ```
+  similar to how we expect that
+  ```nim
+  func print(s: string) =
+    echo s
+  ```
+  produces
+  ```
+  Error: 'print' can have side effects
+  ```
+
+
+## Tools
+
+- Major improvements to `nimgrep`, see PR [#15612
+](https://github.com/nim-lang/Nim/pull/15612).
+- `fusion` is now un-bundled from Nim, `./koch fusion` will
+  install it via Nimble at a fixed hash.
+- `testament`: added `nimoutFull: bool` spec to compare full output of compiler
+  instead of a subset; many bugfixes to testament.
+
+
+## Misc/cleanups
+
+- Deprecated `TaintedString` and `--taintmode`.
+- Deprecated `--nilseqs` which is now a noop.
+- Added `-d:nimStrictMode` in CI in several places to ensure code doesn't have
+  certain hints/warnings.
+- Removed `.travis.yml`, `appveyor.yml.disabled`, `.github/workflows/ci.yml.disabled`.
+- `[skip ci]` now works in azure and CI pipelines, see detail in PR
+  [#17561](https://github.com/nim-lang/Nim/pull/17561).

changelogs/changelog_2_0_0.md

@@ -0,0 +1,367 @@
+# v2.0.0 - yyyy-mm-dd
+
+
+## Changes affecting backward compatibility
+- `httpclient.contentLength` default to `-1` if the Content-Length header is not set in the response. It follows Apache HttpClient(Java), http(go) and .Net HttpWebResponse(C#) behavior. Previously it raised `ValueError`.
+
+- `addr` is now available for all addressable locations,
+  `unsafeAddr` is now deprecated and an alias for `addr`.
+
+- Certain definitions from the default `system` module have been moved to
+  the following new modules:
+
+  - `std/syncio`
+  - `std/assertions`
+  - `std/formatfloat`
+  - `std/objectdollar`
+  - `std/widestrs`
+  - `std/typedthreads`
+  - `std/sysatomics`
+
+  In the future, these definitions will be removed from the `system` module,
+  and their respective modules will have to be imported to use them.
+  Currently, to make these imports required, the `-d:nimPreviewSlimSystem` option
+  may be used.
+
+- Enabling `-d:nimPreviewSlimSystem` also removes the following deprecated
+  symbols in the `system` module:
+  - Aliases with `Error` suffix to exception types that have a `Defect` suffix
+    (see [exceptions](https://nim-lang.github.io/Nim/exceptions.html)):
+    `ArithmeticError`, `DivByZeroError`, `OverflowError`,
+    `AccessViolationError`, `AssertionError`, `OutOfMemError`, `IndexError`,
+    `FieldError`, `RangeError`, `StackOverflowError`, `ReraiseError`,
+    `ObjectAssignmentError`, `ObjectConversionError`, `FloatingPointError`,
+    `FloatOverflowError`, `FloatUnderflowError`, `FloatInexactError`,
+    `DeadThreadError`, `NilAccessError`
+  - `addQuitProc`, replaced by `exitprocs.addExitProc`
+  - Legacy unsigned conversion operations: `ze`, `ze64`, `toU8`, `toU16`, `toU32`
+  - `TaintedString`, formerly a distinct alias to `string`
+  - `PInt32`, `PInt64`, `PFloat32`, `PFloat64`, aliases to
+    `ptr int32`, `ptr int64`, `ptr float32`, `ptr float64`
+
+- Enabling `-d:nimPreviewSlimSystem` removes the import of `channels_builtin` in
+  in the `system` module.
+
+- Enabling `-d:nimPreviewCstringConversion`, `ptr char`, `ptr array[N, char]` and `ptr UncheckedArray[N, char]` don't support conversion to cstring anymore.
+
+- The `gc:v2` option is removed.
+
+- The `mainmodule` and `m` options are removed.
+
+- The `threads:on` option is now the default.
+
+- Optional parameters in combination with `: body` syntax (RFC #405) are now opt-in via
+  `experimental:flexibleOptionalParams`.
+
+- Automatic dereferencing (experimental feature) is removed.
+
+- The `Math.trunc` polyfill for targeting Internet Explorer was
+  previously included in most JavaScript output files.
+  Now, it is only included with `-d:nimJsMathTruncPolyfill`.
+  If you are targeting Internet Explorer, you may choose to enable this option
+  or define your own `Math.trunc` polyfill using the [`emit` pragma](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-emit-pragma).
+  Nim uses `Math.trunc` for the division and modulo operators for integers.
+
+- `shallowCopy` and `shallow` are removed for ARC/ORC. Use `move` when possible or combine assignment and
+`sink` for optimization purposes.
+
+- The experimental `nimPreviewDotLikeOps` switch is going to be removed or deprecated because it didn't fullfill its promises.
+
+- The `{.this.}` pragma, deprecated since 0.19, has been removed.
+- `nil` literals can no longer be directly assigned to variables or fields of `distinct` pointer types. They must be converted instead.
+  ```nim
+  type Foo = distinct ptr int
+
+  # Before:
+  var x: Foo = nil
+  # After:
+  var x: Foo = Foo(nil)
+  ```
+- Removed two type pragma syntaxes deprecated since 0.20, namely
+  `type Foo = object {.final.}`, and `type Foo {.final.} [T] = object`.
+
+- `foo a = b` now means `foo(a = b)` rather than `foo(a) = b`. This is consistent
+  with the existing behavior of `foo a, b = c` meaning `foo(a, b = c)`.
+  This decision was made with the assumption that the old syntax was used rarely;
+  if your code used the old syntax, please be aware of this change.
+
+- [Overloadable enums](https://nim-lang.github.io/Nim/manual.html#overloadable-enum-value-names) and Unicode Operators
+  are no longer experimental.
+
+- Removed the `nimIncrSeqV3` define.
+
+- `macros.getImpl` for `const` symbols now returns the full definition node
+  (as `nnkConstDef`) rather than the AST of the constant value.
+
+- Lock levels are deprecated, now a noop.
+
+- ORC is now the default memory management strategy. Use
+  `--mm:refc` for a transition period.
+
+- `strictEffects` are no longer experimental.
+  Use `legacy:laxEffects` to keep backward compatibility.
+
+- The `gorge`/`staticExec` calls will now return a descriptive message in the output
+  if the execution fails for whatever reason. To get back legacy behaviour use `-d:nimLegacyGorgeErrors`.
+
+- Pointer to `cstring` conversion now triggers a `[PtrToCstringConv]` warning.
+  This warning will become an error in future versions! Use a `cast` operation
+  like `cast[cstring](x)` instead.
+
+- `logging` will default to flushing all log level messages. To get the legacy behaviour of only flushing Error and Fatal messages, use `-d:nimV1LogFlushBehavior`.
+
+- Redefining templates with the same signature was previously
+  allowed to support certain macro code. To do this explicitly, the
+  `{.redefine.}` pragma has been added. Note that this is only for templates.
+  Implicit redefinition of templates is now deprecated and will give an error in the future.
+
+- Using an unnamed break in a block is deprecated. This warning will become an error in future versions! Use a named block with a named break instead.
+
+- Several Standard libraries are moved to nimble packages, use `nimble` to install them:
+  - `std/punycode` => `punycode`
+  - `std/asyncftpclient` => `asyncftpclient`
+  - `std/smtp` => `smtp`
+  - `std/db_common` => `db_connector/db_common`
+  - `std/db_sqlite` => `db_connector/db_sqlite`
+  - `std/db_mysql` => `db_connector/db_mysql`
+  - `std/db_postgres` => `db_connector/db_postgres`
+  - `std/db_odbc` => `db_connector/db_odbc`
+
+- Previously, calls like `foo(a, b): ...` or `foo(a, b) do: ...` where the final argument of
+  `foo` had type `proc ()` were assumed by the compiler to mean `foo(a, b, proc () = ...)`.
+  This behavior is now deprecated. Use `foo(a, b) do (): ...` or `foo(a, b, proc () = ...)` instead.
+
+- If no exception or any exception deriving from Exception but not Defect or CatchableError given in except, a `warnBareExcept` warning will be triggered.
+
+- The experimental strictFuncs feature now disallows a store to the heap via a `ref` or `ptr` indirection.
+
+- Underscores (`_`) as routine parameters are now ignored and cannot be used in the routine body.
+  The following code now does not compile:
+
+  ```nim
+  proc foo(_: int): int = _ + 1
+  echo foo(1)
+  ```
+
+  Instead, the following code now compiles:
+
+  ```nim
+  proc foo(_, _: int): int = 123
+  echo foo(1, 2)
+  ```
+- Underscores (`_`) as generic parameters are not supported and cannot be used.
+  Generics that use `_` as parameters will no longer compile requires you to replace `_` with something else:
+  
+  ```nim
+  proc foo[_](t: typedesc[_]): string = "BAR" # Can not compile
+  proc foo[T](t: typedesc[T]): string = "BAR" # Can compile
+  ```
+
+- - Added the `--legacy:verboseTypeMismatch` switch to get legacy type mismatch error messages.
+
+## Standard library additions and changes
+
+[//]: # "Changes:"
+- OpenSSL 3 is now supported.
+- `macros.parseExpr` and `macros.parseStmt` now accept an optional
+  filename argument for more informative errors.
+- Module `colors` expanded with missing colors from the CSS color standard.
+  `colPaleVioletRed` and `colMediumPurple` have also been changed to match the CSS color standard.
+- Fixed `lists.SinglyLinkedList` being broken after removing the last node ([#19353](https://github.com/nim-lang/Nim/pull/19353)).
+- The `md5` module now works at compile time and in JavaScript.
+- Changed `mimedb` to use an `OrderedTable` instead of `OrderedTableRef` to support `const` tables.
+- `strutils.find` now uses and defaults to `last = -1` for whole string searches,
+  making limiting it to just the first char (`last = 0`) valid.
+- `random.rand` now works with `Ordinal`s.
+- Undeprecated `os.isvalidfilename`.
+- `std/oids` now uses `int64` to store time internally (before it was int32).
+- `std/uri.Uri` dollar `$` improved, precalculates the `string` result length from the `Uri`.
+- `std/uri.Uri.isIpv6` is now exported.
+- `std/logging.ConsoleLogger` and `FileLogger` now have a `flushThreshold` attribute to set what log message levels are automatically flushed. For Nim v1 use `-d:nimFlushAllLogs` to automatically flush all message levels. Flushing all logs is the default behavior for Nim v2.
+
+
+- `std/net.IpAddress` dollar `$` improved, uses a fixed capacity for the `string` result based from the `IpAddressFamily`.
+- `std/jsfetch.newFetchOptions` now has default values for all parameters
+- `std/jsformdata` now accepts `Blob` data type.
+
+- `std/sharedlist` and `std/sharedtables` are now deprecated, see RFC [#433](https://github.com/nim-lang/RFCs/issues/433).
+
+- New compile flag (`-d:nimNoGetRandom`) when building `std/sysrand` to remove dependency on linux `getrandom` syscall.
+
+  This compile flag only affects linux builds and is necessary if either compiling on a linux kernel version < 3.17, or if code built will be executing on kernel < 3.17.
+
+  On linux kernels < 3.17 (such as kernel 3.10 in RHEL7 and CentOS7), the `getrandom` syscall was not yet introduced. Without this, the `std/sysrand` module will not build properly, and if code is built on a kernel >= 3.17 without the flag, any usage of the `std/sysrand` module will fail to execute on a kernel < 3.17 (since it attempts to perform a syscall to `getrandom`, which isn't present in the current kernel). A compile flag has been added to force the `std/sysrand` module to use /dev/urandom (available since linux kernel 1.3.30), rather than the `getrandom` syscall. This allows for use of a cryptographically secure PRNG, regardless of kernel support for the `getrandom` syscall.
+
+  When building for RHEL7/CentOS7 for example, the entire build process for nim from a source package would then be:
+  ```sh
+  $ yum install devtoolset-8 # Install GCC version 8 vs the standard 4.8.5 on RHEL7/CentOS7. Alternatively use -d:nimEmulateOverflowChecks. See issue #13692 for details
+  $ scl enable devtoolset-8 bash # Run bash shell with default toolchain of gcc 8
+  $ sh build.sh  # per unix install instructions
+  $ bin/nim c koch  # per unix install instructions
+  $ ./koch boot -d:release  # per unix install instructions
+  $ ./koch tools -d:nimNoGetRandom  # pass the nimNoGetRandom flag to compile std/sysrand without support for getrandom syscall
+  ```
+
+  This is necessary to pass when building nim on kernel versions < 3.17 in particular to avoid an error of "SYS_getrandom undeclared" during the build process for stdlib (sysrand in particular).
+
+[//]: # "Additions:"
+- Added ISO 8601 week date utilities in `times`:
+  - Added `IsoWeekRange`, a range type for weeks in a week-based year.
+  - Added `IsoYear`, a distinct type for a week-based year in contrast to a regular year.
+  - Added a `initDateTime` overload to create a datetime from an ISO week date.
+  - Added `getIsoWeekAndYear` to get an ISO week number and week-based year from a datetime.
+  - Added `getIsoWeeksInYear` to return the number of weeks in a week-based year.
+- Added new modules which were part of `std/os`:
+  - Added `std/oserrors` for OS error reporting. Added `std/envvars` for environment variables handling.
+  - Added `std/paths`, `std/dirs`, `std/files`, `std/symlinks` and `std/appdirs`.
+  - Added `std/cmdline` for reading command line parameters.
+- Added `sep` parameter in `std/uri` to specify the query separator.
+- Added bindings to [`Array.shift`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift)
+  and [`queueMicrotask`](https://developer.mozilla.org/en-US/docs/Web/API/queueMicrotask)
+  in `jscore` for JavaScript targets.
+- Added `UppercaseLetters`, `LowercaseLetters`, `PunctuationChars`, `PrintableChars` sets to `std/strutils`.
+- Added `complex.sgn` for obtaining the phase of complex numbers.
+- Added `insertAdjacentText`, `insertAdjacentElement`, `insertAdjacentHTML`,
+  `after`, `before`, `closest`, `append`, `hasAttributeNS`, `removeAttributeNS`,
+  `hasPointerCapture`, `releasePointerCapture`, `requestPointerLock`,
+  `replaceChildren`, `replaceWith`, `scrollIntoViewIfNeeded`, `setHTML`,
+  `toggleAttribute`, and `matches` to `std/dom`.
+- Added [`jsre.hasIndices`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/hasIndices)
+- Added `capacity` for `string` and `seq` to return the current capacity, see https://github.com/nim-lang/RFCs/issues/460
+- Added `openArray[char]` overloads for `std/parseutils` allowing more code reuse.
+- Added `openArray[char]` overloads for `std/unicode` allowing more code reuse.
+- Added `safe` parameter to `base64.encodeMime`.
+
+[//]: # "Deprecations:"
+- Deprecated `selfExe` for Nimscript.
+- Deprecated `std/sums`.
+- Deprecated `std/base64.encode` for collections of arbitrary integer element type.
+  Now only `byte` and `char` are supported.
+
+[//]: # "Removals:"
+- Removed deprecated module `parseopt2`.
+- Removed deprecated module `sharedstrings`.
+- Removed deprecated module `dom_extensions`.
+- Removed deprecated module `LockFreeHash`.
+- Removed deprecated module `events`.
+- Removed deprecated `oids.oidToString`.
+- Removed define `nimExperimentalAsyncjsThen` for `std/asyncjs.then` and `std/jsfetch`.
+- Removed deprecated `jsre.test` and `jsre.toString`.
+- Removed deprecated `math.c_frexp`.
+- Removed deprecated `` httpcore.`==` ``.
+- Removed deprecated `std/posix.CMSG_SPACE` and `std/posix.CMSG_LEN` that takes wrong argument types.
+- Removed deprecated `osproc.poDemon`, symbol with typo.
+- Removed deprecated `tables.rightSize`.
+
+
+- Removed deprecated `posix.CLONE_STOPPED`.
+
+
+## Language changes
+
+- [Tag tracking](https://nim-lang.github.io/Nim/manual.html#effect-system-tag-tracking) supports the definition of forbidden tags by the `.forbids` pragma
+  which can be used to disable certain effects in proc types.
+- [Case statement macros](https://nim-lang.github.io/Nim/manual.html#macros-case-statement-macros) are no longer experimental,
+  meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them.
+- Full command syntax and block arguments i.e. `foo a, b: c` are now allowed
+  for the right-hand side of type definitions in type sections. Previously
+  they would error with "invalid indentation".
+
+- Compile-time define changes:
+  - `defined` now accepts identifiers separated by dots, i.e. `defined(a.b.c)`.
+    In the command line, this is defined as `-d:a.b.c`. Older versions can
+    use accents as in ``defined(`a.b.c`)`` to access such defines.
+  - [Define pragmas for constants](https://nim-lang.github.io/Nim/manual.html#implementation-specific-pragmas-compileminustime-define-pragmas)
+    now support a string argument for qualified define names.
+
+    ```nim
+    # -d:package.FooBar=42
+    const FooBar {.intdefine: "package.FooBar".}: int = 5
+    echo FooBar # 42
+    ```
+
+    This was added to help disambiguate similar define names for different packages.
+    In older versions, this could only be achieved with something like the following:
+
+    ```nim
+    const FooBar = block:
+      const `package.FooBar` {.intdefine.}: int = 5
+      `package.FooBar`
+    ```
+  - A generic `define` pragma for constants has been added that interprets
+    the value of the define based on the type of the constant value.
+    See the [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#generic-define-pragma)
+    for a list of supported types.
+
+- [Macro pragmas](https://nim-lang.github.io/Nim/manual.html#userminusdefined-pragmas-macro-pragmas) changes:
+  - Templates now accept macro pragmas.
+  - Macro pragmas for var/let/const sections have been redesigned in a way that works
+    similarly to routine macro pragmas. The new behavior is documented in the
+    [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#extended-macro-pragmas).
+  - Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`,
+    allowing multiple type definitions to be injected in place of the original type definition.
+
+    ```nim
+    import macros
+    macro multiply(amount: static int, s: untyped): untyped =
+      let name = $s[0].basename
+      result = newNimNode(nnkTypeSection)
+      for i in 1 .. amount:
+        result.add(newTree(nnkTypeDef, ident(name & $i), s[1], s[2]))
+    type
+      Foo = object
+      Bar {.multiply: 3.} = object
+        x, y, z: int
+      Baz = object
+    # becomes
+    type
+      Foo = object
+      Bar1 = object
+        x, y, z: int
+      Bar2 = object
+        x, y, z: int
+      Bar3 = object
+        x, y, z: int
+      Baz = object
+    ```
+
+- A new form of type inference called [top-down inference](https://nim-lang.github.io/Nim/manual_experimental.html#topminusdown-type-inference)
+  has been implemented for a variety of basic cases. For example, code like the following now compiles:
+
+  ```nim
+  let foo: seq[(float, byte, cstring)] = @[(1, 2, "abc")]
+  ```
+
+- `cstring` is now accepted as a selector in `case` statements, removing the
+  need to convert to `string`. On the JS backend, this is translated directly
+  to a `switch` statement.
+
+- Nim now supports `out` parameters and ["strict definitions"](https://nim-lang.github.io/Nim/manual_experimental.html#strict-definitions-and-nimout-parameters).
+- Nim now offers a [strict mode](https://nim-lang.github.io/Nim/manual_experimental.html#strict-case-objects) for `case objects`.
+
+- IBM Z architecture and macOS m1 arm64 architecture are supported.
+
+- `=wasMoved` can be overridden by users.
+
+## Compiler changes
+
+- The `gc` switch has been renamed to `mm` ("memory management") in order to reflect the
+  reality better. (Nim moved away from all techniques based on "tracing".)
+
+- Defines the `gcRefc` symbol which allows writing specific code for the refc GC.
+
+- `nim` can now compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off compiler/nim`,
+  without requiring `-d:nimVersion140` which is now a noop.
+
+- `--styleCheck`, `--hintAsError` and `--warningAsError` now only apply to the current package.
+
+- The switch `--nimMainPrefix:prefix` has been added to add a prefix to the names of `NimMain` and
+  related functions produced on the backend. This prevents conflicts with other Nim
+  static libraries.
+
+- When compiling for Release the flag `-fno-math-errno` is used for GCC.
+
+
+## Tool changes
+
+- Nim now ships Nimble version 0.14 which added support for lock-files. Libraries are stored in `$nimbleDir/pkgs2` (it was `$nimbleDir/pkgs`).

ci/action.nim

@@ -0,0 +1,26 @@
+import std/[strutils, os, osproc, parseutils, strformat]
+
+
+proc main() =
+  var msg = ""
+  const cmd = "./koch boot --gc:orc -d:release"
+
+  let (output, exitCode) = execCmdEx(cmd)
+
+  doAssert exitCode == 0, output
+
+  let start = rfind(output, "Hint: mm")
+  doAssert parseUntil(output, msg, "; proj", start) > 0, output
+
+  let (commitHash, _) = execCmdEx("""git log --format="%H" -n 1""")
+
+  let welcomeMessage = fmt"""Thanks for your hard work on this PR!
+The lines below are statistics of the Nim compiler built from {commitHash}
+
+{msg}
+"""
+  createDir "ci/nimcache"
+  writeFile "ci/nimcache/results.txt", welcomeMessage
+
+when isMainModule:
+  main()

ci/build_autogen.bat

@@ -10,7 +10,7 @@ SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe
 echo "building from csources: %nim_csources%"
 
 if not exist %nim_csourcesDir% (
-  git clone -q --depth 1 %nim_csourcesUrl% %nim_csourcesDir%
+  git clone -q --depth 1 -b %nim_csourcesBranch% %nim_csourcesUrl% %nim_csourcesDir%
 )
 
 if not exist %nim_csources% (

ci/funs.sh

@@ -31,6 +31,23 @@ nimIsCiSkip(){
   fi
 }
 
+nimInternalInstallDepsWindows(){
+  echo_run mkdir dist
+  echo_run curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+  echo_run curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+  echo_run 7z x dist/mingw64.7z -odist
+  echo_run 7z x dist/dlls.zip -obin
+}
+
+nimInternalBuildKochAndRunCI(){
+  echo_run nim c koch
+  if ! echo_run ./koch runCI; then
+    echo_run echo "runCI failed"
+    echo_run nim r tools/ci_testresults.nim
+    return 1
+  fi
+}
+
 nimDefineVars(){
   . config/build_config.txt
   nim_csources=bin/nim_csources_$nim_csourcesHash
@@ -55,6 +72,12 @@ _nimBuildCsourcesIfNeeded(){
     makeX=gmake
   elif [ "$unamestr" = 'OpenBSD' ]; then
     makeX=gmake
+  elif [ "$unamestr" = 'NetBSD' ]; then
+    makeX=gmake
+  elif [ "$unamestr" = 'CROSSOS' ]; then
+    makeX=gmake
+  elif [ "$unamestr" = 'SunOS' ]; then
+    makeX=gmake
   else
     makeX=make
   fi
@@ -101,8 +124,12 @@ nimBuildCsourcesIfNeeded(){
       else
         # Note: using git tags would allow fetching just what's needed, unlike git hashes, e.g.
         # via `git clone -q --depth 1 --branch $tag $nim_csourcesUrl`.
-        echo_run git clone -q --depth 1 $nim_csourcesUrl "$nim_csourcesDir"
-        echo_run git -C "$nim_csourcesDir" checkout $nim_csourcesHash
+        echo_run git clone -q --depth 1 -b $nim_csourcesBranch \
+            $nim_csourcesUrl "$nim_csourcesDir"
+        # old `git` versions don't support -C option, using `cd` explicitly:
+        echo_run cd "$nim_csourcesDir"
+        echo_run git checkout $nim_csourcesHash
+        echo_run cd "$OLDPWD"
         # if needed we could also add: `git reset --hard $nim_csourcesHash`
       fi
       _nimBuildCsourcesIfNeeded "$@"

compiler.nimble

@@ -1,9 +0,0 @@
-
-version = system.NimVersion
-author = "Andreas Rumpf"
-description = "Compiler package providing the compiler sources as a library."
-license = "MIT"
-
-installDirs = @["compiler", "nimsuggest"]
-
-requires "nim >= 0.14.0"

compiler/aliasanalysis.nim

@@ -0,0 +1,128 @@
+
+import ast
+
+import std / assertions
+
+const
+  PathKinds0* = {nkDotExpr, nkCheckedFieldExpr,
+                 nkBracketExpr, nkDerefExpr, nkHiddenDeref,
+                 nkAddr, nkHiddenAddr,
+                 nkObjDownConv, nkObjUpConv}
+  PathKinds1* = {nkHiddenStdConv, nkHiddenSubConv}
+
+proc skipConvDfa*(n: PNode): PNode =
+  result = n
+  while true:
+    case result.kind
+    of nkObjDownConv, nkObjUpConv:
+      result = result[0]
+    of PathKinds1:
+      result = result[1]
+    else: break
+
+proc isAnalysableFieldAccess*(orig: PNode; owner: PSym): bool =
+  var n = orig
+  while true:
+    case n.kind
+    of PathKinds0 - {nkHiddenDeref, nkDerefExpr}:
+      n = n[0]
+    of PathKinds1:
+      n = n[1]
+    of nkHiddenDeref, nkDerefExpr:
+      # We "own" sinkparam[].loc but not ourVar[].location as it is a nasty
+      # pointer indirection.
+      # bug #14159, we cannot reason about sinkParam[].location as it can
+      # still be shared for tyRef.
+      n = n[0]
+      return n.kind == nkSym and n.sym.owner == owner and
+         (n.sym.typ.skipTypes(abstractInst-{tyOwned}).kind in {tyOwned})
+    else: break
+  # XXX Allow closure deref operations here if we know
+  # the owner controlled the closure allocation?
+  result = n.kind == nkSym and n.sym.owner == owner and
+    {sfGlobal, sfThread, sfCursor} * n.sym.flags == {} and
+    (n.sym.kind != skParam or isSinkParam(n.sym)) # or n.sym.typ.kind == tyVar)
+  # Note: There is a different move analyzer possible that checks for
+  # consume(param.key); param.key = newValue  for all paths. Then code like
+  #
+  #   let splited = split(move self.root, x)
+  #   self.root = merge(splited.lower, splited.greater)
+  #
+  # could be written without the ``move self.root``. However, this would be
+  # wrong! Then the write barrier for the ``self.root`` assignment would
+  # free the old data and all is lost! Lesson: Don't be too smart, trust the
+  # lower level C++ optimizer to specialize this code.
+
+type AliasKind* = enum
+  yes, no, maybe
+
+proc aliases*(obj, field: PNode): AliasKind =
+  # obj -> field:
+  # x -> x: true
+  # x -> x.f: true
+  # x.f -> x: false
+  # x.f -> x.f: true
+  # x.f -> x.v: false
+  # x -> x[]: true
+  # x[] -> x: false
+  # x -> x[0]: true
+  # x[0] -> x: false
+  # x[0] -> x[0]: true
+  # x[0] -> x[1]: false
+  # x -> x[i]: true
+  # x[i] -> x: false
+  # x[i] -> x[i]: maybe; Further analysis could make this return true when i is a runtime-constant
+  # x[i] -> x[j]: maybe; also returns maybe if only one of i or j is a compiletime-constant
+  template collectImportantNodes(result, n) =
+    var result: seq[PNode]
+    var n = n
+    while true:
+      case n.kind
+      of PathKinds0 - {nkDotExpr, nkBracketExpr, nkDerefExpr, nkHiddenDeref}:
+        n = n[0]
+      of PathKinds1:
+        n = n[1]
+      of nkDotExpr, nkBracketExpr, nkDerefExpr, nkHiddenDeref:
+        result.add n
+        n = n[0]
+      of nkSym:
+        result.add n
+        break
+      else: return no
+
+  collectImportantNodes(objImportantNodes, obj)
+  collectImportantNodes(fieldImportantNodes, field)
+
+  # If field is less nested than obj, then it cannot be part of/aliased by obj
+  if fieldImportantNodes.len < objImportantNodes.len: return no
+
+  result = yes
+  for i in 1..objImportantNodes.len:
+    # We compare the nodes leading to the location of obj and field
+    # with each other.
+    # We continue until they diverge, in which case we return no, or
+    # until we reach the location of obj, in which case we do not need
+    # to look further, since field must be part of/aliased by obj now.
+    # If we encounter an element access using an index which is a runtime value,
+    # we simply return maybe instead of yes; should further nodes not diverge.
+    let currFieldPath = fieldImportantNodes[^i]
+    let currObjPath = objImportantNodes[^i]
+
+    if currFieldPath.kind != currObjPath.kind:
+      return no
+
+    case currFieldPath.kind
+    of nkSym:
+      if currFieldPath.sym != currObjPath.sym: return no
+    of nkDotExpr:
+      if currFieldPath[1].sym != currObjPath[1].sym: return no
+    of nkDerefExpr, nkHiddenDeref:
+      discard
+    of nkBracketExpr:
+      if currFieldPath[1].kind in nkLiterals and currObjPath[1].kind in nkLiterals:
+        if currFieldPath[1].intVal != currObjPath[1].intVal:
+          return no
+      else:
+        result = maybe
+    else: assert false # unreachable
+

compiler/aliases.nim

@@ -12,6 +12,9 @@
 import
   ast, astalgo, types, trees, intsets
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 type
   TAnalysisResult* = enum
     arNo, arMaybe, arYes
@@ -74,24 +77,29 @@ proc isPartOf*(a, b: PNode): TAnalysisResult =
   ## cases:
   ##
   ## YES-cases:
+  ##  ```
   ##  x    <| x   # for general trees
   ##  x[]  <| x
   ##  x[i] <| x
   ##  x.f  <| x
+  ##  ```
   ##
   ## NO-cases:
+  ## ```
   ## x           !<| y    # depending on type and symbol kind
   ## x[constA]   !<| x[constB]
   ## x.f         !<| x.g
   ## x.f         !<| y.f  iff x !<= y
+  ## ```
   ##
   ## MAYBE-cases:
   ##
+  ##  ```
   ##  x[] ?<| y[]   iff compatible type
   ##
   ##
   ##  x[]  ?<| y  depending on type
-  ##
+  ##  ```
   if a.kind == b.kind:
     case a.kind
     of nkSym:

compiler/ast.nim

@@ -10,9 +10,12 @@
 # abstract syntax tree + symbol table
 
 import
-  lineinfos, hashes, options, ropes, idents, int128
+  lineinfos, hashes, options, ropes, idents, int128, tables
 from strutils import toLowerAscii
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 export int128
 
 type
@@ -204,12 +207,11 @@ type
     nkPtrTy,              # ``ptr T``
     nkVarTy,              # ``var T``
     nkConstTy,            # ``const T``
-    nkMutableTy,          # ``mutable T``
+    nkOutTy,              # ``out T``
     nkDistinctTy,         # distinct type
     nkProcTy,             # proc type
     nkIteratorTy,         # iterator type
-    nkSharedTy,           # 'shared T'
-                          # we use 'nkPostFix' for the 'not nil' addition
+    nkSinkAsgn,           # '=sink(x, y)'
     nkEnumTy,             # enum body
     nkEnumFieldDef,       # `ident = expr` in an enumeration
     nkArgList,            # argument list
@@ -229,7 +231,7 @@ type
   TNodeKinds* = set[TNodeKind]
 
 type
-  TSymFlag* = enum    # 46 flags!
+  TSymFlag* = enum    # 49 flags!
     sfUsed,           # read access of sym (for warnings) or simply used
     sfExported,       # symbol is exported from module
     sfFromGeneric,    # symbol is instantiation of a generic; this is needed
@@ -257,7 +259,8 @@ type
                       # *OR*: a proc is indirectly called (used as first class)
     sfCompilerProc,   # proc is a compiler proc, that is a C proc that is
                       # needed for the code generator
-    sfProcvar,        # proc can be passed to a proc var
+    sfEscapes         # param escapes
+                      # currently unimplemented
     sfDiscriminant,   # field is a discriminant in a record/object
     sfRequiresInit,   # field must be initialized during construction
     sfDeprecated,     # symbol is deprecated
@@ -296,9 +299,17 @@ type
     sfInjectDestructors # whether the proc needs the 'injectdestructors' transformation
     sfNeverRaises     # proc can never raise an exception, not even OverflowDefect
                       # or out-of-memory
+    sfSystemRaisesDefect # proc in the system can raise defects
     sfUsedInFinallyOrExcept  # symbol is used inside an 'except' or 'finally'
     sfSingleUsedTemp  # For temporaries that we know will only be used once
     sfNoalias         # 'noalias' annotation, means C's 'restrict'
+    sfEffectsDelayed  # an 'effectsDelayed' parameter
+    sfGeneratedType   # A anonymous generic type that is generated by the compiler for
+                      # objects that do not have generic parameters in case one of the
+                      # object fields has one.
+                      #
+                      # This is disallowed but can cause the typechecking to go into
+                      # an infinite loop, this flag is used as a sentinel to stop it.
 
   TSymFlags* = set[TSymFlag]
 
@@ -327,10 +338,10 @@ const
   sfExperimental* = sfOverriden       # module uses the .experimental switch
   sfGoto* = sfOverriden               # var is used for 'goto' code generation
   sfWrittenTo* = sfBorrow             # param is assigned to
-  sfEscapes* = sfProcvar              # param escapes
+                                      # currently unimplemented
   sfBase* = sfDiscriminant
-  sfIsSelf* = sfOverriden             # param is 'self'
   sfCustomPragma* = sfRegister        # symbol is custom pragma template
+  sfTemplateRedefinition* = sfExportc # symbol is a redefinition of an earlier template
 
 const
   # getting ready for the future expr/stmt merge
@@ -343,7 +354,8 @@ const
   ensuresEffects* = 2     # 'ensures' annotation
   tagEffects* = 3       # user defined tag ('gc', 'time' etc.)
   pragmasEffects* = 4    # not an effect, but a slot for pragmas in proc type
-  effectListLen* = 5    # list of effects list
+  forbiddenEffects* = 5    # list of illegal effects
+  effectListLen* = 6    # list of effects list
   nkLastBlockStmts* = {nkRaiseStmt, nkReturnStmt, nkBreakStmt, nkContinueStmt}
                         # these must be last statements in a block
 
@@ -496,10 +508,12 @@ type
                        # the flag is applied to proc default values and to calls
     nfExecuteOnReload  # A top-level statement that will be executed during reloads
     nfLastRead  # this node is a last read
-    nfFirstWrite# this node is a first write
+    nfFirstWrite # this node is a first write
+    nfHasComment # node has a comment
+    nfUseDefaultField # node has a default value (object constructor)
 
   TNodeFlags* = set[TNodeFlag]
-  TTypeFlag* = enum   # keep below 32 for efficiency reasons (now: ~40)
+  TTypeFlag* = enum   # keep below 32 for efficiency reasons (now: 46)
     tfVarargs,        # procedure has C styled varargs
                       # tyArray type represeting a varargs list
     tfNoSideEffect,   # procedure type does not allow side effects
@@ -566,6 +580,9 @@ type
       # (for importc types); type is fully specified, allowing to compute
       # sizeof, alignof, offsetof at CT
     tfExplicitCallConv
+    tfIsConstructor
+    tfEffectSystemWorkaround
+    tfIsOutParam
 
   TTypeFlags* = set[TTypeFlag]
 
@@ -616,7 +633,7 @@ const
   skError* = skUnknown
 
 var
-  eqTypeFlags* = {tfIterator, tfNotNil, tfVarIsPtr, tfGcSafe, tfNoSideEffect}
+  eqTypeFlags* = {tfIterator, tfNotNil, tfVarIsPtr, tfGcSafe, tfNoSideEffect, tfIsOutParam}
     ## type flags that are essential for type equality.
     ## This is now a variable because for emulation of version:1.0 we
     ## might exclude {tfGcSafe, tfNoSideEffect}.
@@ -651,7 +668,9 @@ type
     mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot,
     mUnaryPlusI, mBitnotI,
     mUnaryPlusF64, mUnaryMinusF64,
-    mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr,
+    mCharToStr, mBoolToStr,
+    mIntToStr, mInt64ToStr, mFloatToStr, # for compiling nimStdlibVersion < 1.5.1 (not bootstrapping)
+    mCStrToStr,
     mStrToStr, mEnumToStr,
     mAnd, mOr,
     mImplies, mIff, mExists, mForall, mOld,
@@ -664,10 +683,10 @@ type
     mInSet, mRepr, mExit,
     mSetLengthStr, mSetLengthSeq,
     mIsPartOf, mAstToStr, mParallel,
-    mSwap, mIsNil, mArrToSeq,
+    mSwap, mIsNil, mArrToSeq, mOpenArrayToSeq,
     mNewString, mNewStringOfCap, mParseBiggestFloat,
-    mMove, mWasMoved, mDestroy,
-    mDefault, mUnown, mIsolate, mAccessEnv, mReset,
+    mMove, mWasMoved, mDestroy, mTrace,
+    mDefault, mUnown, mFinished, mIsolate, mAccessEnv, mAccessTypeField, mReset,
     mArray, mOpenArray, mRange, mSet, mSeq, mVarargs,
     mRef, mPtr, mVar, mDistinct, mVoid, mTuple,
     mOrdinal, mIterableType,
@@ -687,19 +706,19 @@ type
     mNctPut, mNctLen, mNctGet, mNctHasNext, mNctNext,
 
     mNIntVal, mNFloatVal, mNSymbol, mNIdent, mNGetType, mNStrVal, mNSetIntVal,
-    mNSetFloatVal, mNSetSymbol, mNSetIdent, mNSetType, mNSetStrVal, mNLineInfo,
+    mNSetFloatVal, mNSetSymbol, mNSetIdent, mNSetStrVal, mNLineInfo,
     mNNewNimNode, mNCopyNimNode, mNCopyNimTree, mStrToIdent, mNSigHash, mNSizeOf,
     mNBindSym, mNCallSite,
     mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl, mNGenSym,
     mNHint, mNWarning, mNError,
     mInstantiationInfo, mGetTypeInfo, mGetTypeInfoV2,
-    mNimvm, mIntDefine, mStrDefine, mBoolDefine, mRunnableExamples,
+    mNimvm, mIntDefine, mStrDefine, mBoolDefine, mGenericDefine, mRunnableExamples,
     mException, mBuiltinType, mSymOwner, mUncheckedArray, mGetImplTransf,
-    mSymIsInstantiationOf, mNodeId, mPrivateAccess
+    mSymIsInstantiationOf, mNodeId, mPrivateAccess, mZeroDefault
 
 
-# things that we can evaluate safely at compile time, even if not asked for it:
 const
+  # things that we can evaluate safely at compile time, even if not asked for it:
   ctfeWhitelist* = {mNone, mSucc,
     mPred, mInc, mDec, mOrd, mLengthOpenArray,
     mLengthStr, mLengthArray, mLengthSeq,
@@ -719,19 +738,27 @@ const
     mEqRef, mEqProc, mLePtr, mLtPtr, mEqCString, mXor,
     mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot, mUnaryPlusI, mBitnotI,
     mUnaryPlusF64, mUnaryMinusF64,
-    mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr,
+    mCharToStr, mBoolToStr,
+    mIntToStr, mInt64ToStr, mFloatToStr,
+    mCStrToStr,
     mStrToStr, mEnumToStr,
     mAnd, mOr,
     mEqStr, mLeStr, mLtStr,
     mEqSet, mLeSet, mLtSet, mMulSet, mPlusSet, mMinusSet,
     mConStrStr, mAppendStrCh, mAppendStrStr, mAppendSeqElem,
-    mInSet, mRepr}
+    mInSet, mRepr, mOpenArrayToSeq}
+
+  generatedMagics* = {mNone, mIsolate, mFinished, mOpenArrayToSeq}
+    ## magics that are generated as normal procs in the backend
 
 type
   ItemId* = object
     module*: int32
     item*: int32
 
+proc `$`*(x: ItemId): string =
+  "(module: " & $x.module & ", item: " & $x.item & ")"
+
 proc `==`*(a, b: ItemId): bool {.inline.} =
   a.item == b.item and a.module == b.module
 
@@ -769,7 +796,8 @@ type
       ident*: PIdent
     else:
       sons*: TNodeSeq
-    comment*: string
+    when defined(nimsuggest):
+      endInfo*: TLineInfo
 
   TStrTable* = object         # a table[PIdent] of PSym
     counter*: int
@@ -855,7 +883,7 @@ type
     case kind*: TSymKind
     of routineKinds:
       #procInstCache*: seq[PInstantiation]
-      gcUnsafetyReason*: PSym  # for better error messages wrt gcsafe
+      gcUnsafetyReason*: PSym  # for better error messages regarding gcsafe
       transformedBody*: PNode  # cached body after transf pass
     of skLet, skVar, skField, skForVar:
       guard*: PSym
@@ -866,6 +894,8 @@ type
     typ*: PType
     name*: PIdent
     info*: TLineInfo
+    when defined(nimsuggest):
+      endInfo*: TLineInfo
     owner*: PSym
     flags*: TSymFlags
     ast*: PNode               # syntax tree of proc, iterator, etc.:
@@ -903,14 +933,13 @@ type
       allUsages*: seq[TLineInfo]
 
   TTypeSeq* = seq[PType]
-  TLockLevel* = distinct int16
 
   TTypeAttachedOp* = enum ## as usual, order is important here
+    attachedWasMoved,
     attachedDestructor,
     attachedAsgn,
     attachedSink,
     attachedTrace,
-    attachedDispose,
     attachedDeepCopy
 
   TType* {.acyclic.} = object of TIdObj # \
@@ -938,7 +967,6 @@ type
                               # -1 means that the size is unkwown
     align*: int16             # the type's alignment requirements
     paddingAtEnd*: int16      #
-    lockLevel*: TLockLevel    # lock level as required for deadlock checking
     loc*: TLoc
     typeInst*: PType          # for generic instantiations the tyGenericInst that led to this
                               # type.
@@ -987,13 +1015,45 @@ type
   TImplication* = enum
     impUnknown, impNo, impYes
 
+template nodeId(n: PNode): int = cast[int](n)
+
+type Gconfig = object
+  # we put comments in a side channel to avoid increasing `sizeof(TNode)`, which
+  # reduces memory usage given that `PNode` is the most allocated type by far.
+  comments: Table[int, string] # nodeId => comment
+  useIc*: bool
+
+var gconfig {.threadvar.}: Gconfig
+
+proc setUseIc*(useIc: bool) = gconfig.useIc = useIc
+
+proc comment*(n: PNode): string =
+  if nfHasComment in n.flags and not gconfig.useIc:
+    # IC doesn't track comments, see `packed_ast`, so this could fail
+    result = gconfig.comments[n.nodeId]
+
+proc `comment=`*(n: PNode, a: string) =
+  let id = n.nodeId
+  if a.len > 0:
+    # if needed, we could periodically cleanup gconfig.comments when its size increases,
+    # to ensure only live nodes (and with nfHasComment) have an entry in gconfig.comments;
+    # for compiling compiler, the waste is very small:
+    # num calls to newNodeImpl: 14984160 (num of PNode allocations)
+    # size of gconfig.comments: 33585
+    # num of nodes with comments that were deleted and hence wasted: 3081
+    n.flags.incl nfHasComment
+    gconfig.comments[id] = a
+  elif nfHasComment in n.flags:
+    n.flags.excl nfHasComment
+    gconfig.comments.del(id)
+
 # BUGFIX: a module is overloadable so that a proc can have the
 # same name as an imported module. This is necessary because of
 # the poor naming choices in the standard library.
 
 const
   OverloadableSyms* = {skProc, skFunc, skMethod, skIterator,
-    skConverter, skModule, skTemplate, skMacro}
+    skConverter, skModule, skTemplate, skMacro, skEnumField}
 
   GenericTypes*: TTypeKinds = {tyGenericInvocation, tyGenericBody,
     tyGenericParam}
@@ -1020,7 +1080,8 @@ const
                                       nfDotSetter, nfDotField,
                                       nfIsRef, nfIsPtr, nfPreventCg, nfLL,
                                       nfFromTemplate, nfDefaultRefsParam,
-                                      nfExecuteOnReload, nfLastRead, nfFirstWrite}
+                                      nfExecuteOnReload, nfLastRead,
+                                      nfFirstWrite, nfUseDefaultField}
   namePos* = 0
   patternPos* = 1    # empty except for term rewriting macros
   genericParamsPos* = 2
@@ -1064,24 +1125,9 @@ proc getPIdent*(a: PNode): PIdent {.inline.} =
   # which may simplify code.
   case a.kind
   of nkSym: a.sym.name
-  of nkIdent:  a.ident
+  of nkIdent: a.ident
   else: nil
 
-proc getnimblePkg*(a: PSym): PSym =
-  result = a
-  while result != nil:
-    case result.kind
-    of skModule:
-      result = result.owner
-      assert result.kind == skPackage
-    of skPackage:
-      if result.owner == nil:
-        break
-      else:
-        result = result.owner
-    else:
-      assert false, $result.kind
-
 const
   moduleShift = when defined(cpu32): 20 else: 24
 
@@ -1126,13 +1172,7 @@ when false:
     assert dest.ItemId.item <= src.ItemId.item
     dest = src
 
-proc getnimblePkgId*(a: PSym): int =
-  let b = a.getnimblePkg
-  result = if b == nil: -1 else: b.id
-
 var ggDebug* {.deprecated.}: bool ## convenience switch for trying out things
-#var
-#  gMainPackageId*: int
 
 proc isCallExpr*(n: PNode): bool =
   result = n.kind in nkCallKinds
@@ -1168,13 +1208,77 @@ template `[]=`*(n: Indexable, i: int; x: Indexable) = n.sons[i] = x
 template `[]`*(n: Indexable, i: BackwardsIndex): Indexable = n[n.len - i.int]
 template `[]=`*(n: Indexable, i: BackwardsIndex; x: Indexable) = n[n.len - i.int] = x
 
+proc getDeclPragma*(n: PNode): PNode =
+  ## return the `nkPragma` node for declaration `n`, or `nil` if no pragma was found.
+  ## Currently only supports routineDefs + {nkTypeDef}.
+  case n.kind
+  of routineDefs:
+    if n[pragmasPos].kind != nkEmpty: result = n[pragmasPos]
+  of nkTypeDef:
+    #[
+    type F3*{.deprecated: "x3".} = int
+
+    TypeSection
+      TypeDef
+        PragmaExpr
+          Postfix
+            Ident "*"
+            Ident "F3"
+          Pragma
+            ExprColonExpr
+              Ident "deprecated"
+              StrLit "x3"
+        Empty
+        Ident "int"
+    ]#
+    if n[0].kind == nkPragmaExpr:
+      result = n[0][1]
+  else:
+    # support as needed for `nkIdentDefs` etc.
+    result = nil
+  if result != nil:
+    assert result.kind == nkPragma, $(result.kind, n.kind)
+
+proc extractPragma*(s: PSym): PNode =
+  ## gets the pragma node of routine/type/var/let/const symbol `s`
+  if s.kind in routineKinds:
+    result = s.ast[pragmasPos]
+  elif s.kind in {skType, skVar, skLet, skConst}:
+    if s.ast != nil and s.ast.len > 0:
+      if s.ast[0].kind == nkPragmaExpr and s.ast[0].len > 1:
+        # s.ast = nkTypedef / nkPragmaExpr / [nkSym, nkPragma]
+        result = s.ast[0][1]
+  assert result == nil or result.kind == nkPragma
+
+proc skipPragmaExpr*(n: PNode): PNode =
+  ## if pragma expr, give the node the pragmas are applied to,
+  ## otherwise give node itself
+  if n.kind == nkPragmaExpr:
+    result = n[0]
+  else:
+    result = n
+
+proc setInfoRecursive*(n: PNode, info: TLineInfo) =
+  ## set line info recursively
+  if n != nil:
+    for i in 0..<n.safeLen: setInfoRecursive(n[i], info)
+    n.info = info
+
 when defined(useNodeIds):
   const nodeIdToDebug* = -1 # 2322968
   var gNodeId: int
 
-proc newNode*(kind: TNodeKind): PNode =
-  ## new node with unknown line info, no type, and no children
-  result = PNode(kind: kind, info: unknownLineInfo)
+template newNodeImpl(info2) =
+  result = PNode(kind: kind, info: info2)
+  when false:
+    # this would add overhead, so we skip it; it results in a small amount of leaked entries
+    # for old PNode that gets re-allocated at the same address as a PNode that
+    # has `nfHasComment` set (and an entry in that table). Only `nfHasComment`
+    # should be used to test whether a PNode has a comment; gconfig.comments
+    # can contain extra entries for deleted PNode's with comments.
+    gconfig.comments.del(cast[int](result))
+
+template setIdMaybe() =
   when defined(useNodeIds):
     result.id = gNodeId
     if result.id == nodeIdToDebug:
@@ -1182,27 +1286,22 @@ proc newNode*(kind: TNodeKind): PNode =
       writeStackTrace()
     inc gNodeId
 
+proc newNode*(kind: TNodeKind): PNode =
+  ## new node with unknown line info, no type, and no children
+  newNodeImpl(unknownLineInfo)
+  setIdMaybe()
+
 proc newNodeI*(kind: TNodeKind, info: TLineInfo): PNode =
   ## new node with line info, no type, and no children
-  result = PNode(kind: kind, info: info)
-  when defined(useNodeIds):
-    result.id = gNodeId
-    if result.id == nodeIdToDebug:
-      echo "KIND ", result.kind
-      writeStackTrace()
-    inc gNodeId
+  newNodeImpl(info)
+  setIdMaybe()
 
 proc newNodeI*(kind: TNodeKind, info: TLineInfo, children: int): PNode =
   ## new node with line info, type, and children
-  result = PNode(kind: kind, info: info)
+  newNodeImpl(info)
   if children > 0:
     newSeq(result.sons, children)
-  when defined(useNodeIds):
-    result.id = gNodeId
-    if result.id == nodeIdToDebug:
-      echo "KIND ", result.kind
-      writeStackTrace()
-    inc gNodeId
+  setIdMaybe()
 
 proc newNodeIT*(kind: TNodeKind, info: TLineInfo, typ: PType): PNode =
   ## new node with line info, type, and no children
@@ -1253,7 +1352,7 @@ proc newSym*(symKind: TSymKind, name: PIdent, id: ItemId, owner: PSym,
 
 proc astdef*(s: PSym): PNode =
   # get only the definition (initializer) portion of the ast
-  if s.ast != nil and s.ast.kind == nkIdentDefs:
+  if s.ast != nil and s.ast.kind in {nkIdentDefs, nkConstDef}:
     s.ast[2]
   else:
     s.ast
@@ -1403,16 +1502,8 @@ proc newProcNode*(kind: TNodeKind, info: TLineInfo, body: PNode,
                   pragmas, exceptions, body]
 
 const
-  UnspecifiedLockLevel* = TLockLevel(-1'i16)
-  MaxLockLevel* = 1000'i16
-  UnknownLockLevel* = TLockLevel(1001'i16)
   AttachedOpToStr*: array[TTypeAttachedOp, string] = [
-    "=destroy", "=copy", "=sink", "=trace", "=dispose", "=deepcopy"]
-
-proc `$`*(x: TLockLevel): string =
-  if x.ord == UnspecifiedLockLevel.ord: result = "<unspecified>"
-  elif x.ord == UnknownLockLevel.ord: result = "<unknown>"
-  else: result = $int16(x)
+    "=wasMoved", "=destroy", "=copy", "=sink", "=trace", "=deepcopy"]
 
 proc `$`*(s: PSym): string =
   if s != nil:
@@ -1423,7 +1514,6 @@ proc `$`*(s: PSym): string =
 proc newType*(kind: TTypeKind, id: ItemId; owner: PSym): PType =
   result = PType(kind: kind, owner: owner, size: defaultSize,
                  align: defaultAlignment, itemId: id,
-                 lockLevel: UnspecifiedLockLevel,
                  uniqueId: id)
   when false:
     if result.itemId.module == 55 and result.itemId.item == 2:
@@ -1436,7 +1526,7 @@ proc mergeLoc(a: var TLoc, b: TLoc) =
   if a.storage == low(typeof(a.storage)): a.storage = b.storage
   a.flags.incl b.flags
   if a.lode == nil: a.lode = b.lode
-  if a.r == nil: a.r = b.r
+  if a.r == "": a.r = b.r
 
 proc newSons*(father: Indexable, length: int) =
   setLen(father.sons, length)
@@ -1448,11 +1538,10 @@ proc assignType*(dest, src: PType) =
   dest.n = src.n
   dest.size = src.size
   dest.align = src.align
-  dest.lockLevel = src.lockLevel
   # this fixes 'type TLock = TSysLock':
   if src.sym != nil:
     if dest.sym != nil:
-      dest.sym.flags.incl src.sym.flags-{sfExported}
+      dest.sym.flags.incl src.sym.flags-{sfUsed, sfExported}
       if dest.sym.annex == nil: dest.sym.annex = src.sym.annex
       mergeLoc(dest.sym.loc, src.sym.loc)
     else:
@@ -1560,7 +1649,7 @@ proc propagateToOwner*(owner, elem: PType; propagateHasAsgn = true) =
   if mask != {} and propagateHasAsgn:
     let o2 = owner.skipTypes({tyGenericInst, tyAlias, tySink})
     if o2.kind in {tyTuple, tyObject, tyArray,
-                   tySequence, tySet, tyDistinct, tyOpenArray, tyVarargs}:
+                   tySequence, tySet, tyDistinct}:
       o2.flags.incl mask
       owner.flags.incl mask
 
@@ -1606,11 +1695,13 @@ proc copyNode*(src: PNode): PNode =
   of nkIdent: result.ident = src.ident
   of nkStrLit..nkTripleStrLit: result.strVal = src.strVal
   else: discard
+  when defined(nimsuggest):
+    result.endInfo = src.endInfo
 
 template transitionNodeKindCommon(k: TNodeKind) =
   let obj {.inject.} = n[]
-  n[] = TNode(kind: k, typ: obj.typ, info: obj.info, flags: obj.flags,
-              comment: obj.comment)
+  n[] = TNode(kind: k, typ: obj.typ, info: obj.info, flags: obj.flags)
+  # n.comment = obj.comment # shouldn't be needed, the address doesnt' change
   when defined(useNodeIds):
     n.id = obj.id
 
@@ -1622,6 +1713,10 @@ proc transitionIntKind*(n: PNode, kind: range[nkCharLit..nkUInt64Lit]) =
   transitionNodeKindCommon(kind)
   n.intVal = obj.intVal
 
+proc transitionIntToFloatKind*(n: PNode, kind: range[nkFloatLit..nkFloat128Lit]) =
+  transitionNodeKindCommon(kind)
+  n.floatVal = BiggestFloat(obj.intVal)
+
 proc transitionNoneToSym*(n: PNode) =
   transitionNodeKindCommon(nkSym)
 
@@ -1654,6 +1749,8 @@ template copyNodeImpl(dst, src, processSonsStmt) =
   if src == nil: return
   dst = newNode(src.kind)
   dst.info = src.info
+  when defined(nimsuggest):
+    result.endInfo = src.endInfo
   dst.typ = src.typ
   dst.flags = src.flags * PersistentNodeFlags
   dst.comment = src.comment
@@ -1711,7 +1808,7 @@ proc containsNode*(n: PNode, kinds: TNodeKinds): bool =
 
 proc hasSubnodeWith*(n: PNode, kind: TNodeKind): bool =
   case n.kind
-  of nkEmpty..nkNilLit: result = n.kind == kind
+  of nkEmpty..nkNilLit, nkFormalParams: result = n.kind == kind
   else:
     for i in 0..<n.len:
       if (n[i].kind == kind) or hasSubnodeWith(n[i], kind):
@@ -1887,6 +1984,26 @@ proc toObject*(typ: PType): PType =
   if t.kind == tyRef: t.lastSon
   else: typ
 
+proc toObjectFromRefPtrGeneric*(typ: PType): PType =
+  #[
+  See also `toObject`.
+  Finds the underlying `object`, even in cases like these:
+  type
+    B[T] = object f0: int
+    A1[T] = ref B[T]
+    A2[T] = ref object f1: int
+    A3 = ref object f2: int
+    A4 = object f3: int
+  ]#
+  result = typ
+  while true:
+    case result.kind
+    of tyGenericBody: result = result.lastSon
+    of tyRef, tyPtr, tyGenericInst, tyGenericInvocation, tyAlias: result = result[0]
+      # automatic dereferencing is deep, refs #18298.
+    else: break
+  assert result.sym != nil
+
 proc isImportedException*(t: PType; conf: ConfigRef): bool =
   assert t != nil
 
@@ -1939,6 +2056,9 @@ template detailedInfo*(sym: PSym): string =
 proc isInlineIterator*(typ: PType): bool {.inline.} =
   typ.kind == tyProc and tfIterator in typ.flags and typ.callConv != ccClosure
 
+proc isIterator*(typ: PType): bool {.inline.} =
+  typ.kind == tyProc and tfIterator in typ.flags
+
 proc isClosureIterator*(typ: PType): bool {.inline.} =
   typ.kind == tyProc and tfIterator in typ.flags and typ.callConv == ccClosure
 
@@ -2009,3 +2129,13 @@ proc skipAddr*(n: PNode): PNode {.inline.} =
 proc isNewStyleConcept*(n: PNode): bool {.inline.} =
   assert n.kind == nkTypeClassTy
   result = n[0].kind == nkEmpty
+
+proc isOutParam*(t: PType): bool {.inline.} = tfIsOutParam in t.flags
+
+const
+  nodesToIgnoreSet* = {nkNone..pred(nkSym), succ(nkSym)..nkNilLit,
+    nkTypeSection, nkProcDef, nkConverterDef,
+    nkMethodDef, nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo,
+    nkFuncDef, nkConstSection, nkConstDef, nkIncludeStmt, nkImportStmt,
+    nkExportStmt, nkPragma, nkCommentStmt, nkBreakState,
+    nkTypeOfExpr, nkMixinStmt, nkBindStmt}

compiler/astalgo.nim

@@ -12,9 +12,14 @@
 # the data structures here are used in various places of the compiler.
 
 import
-  ast, hashes, intsets, strutils, options, lineinfos, ropes, idents, rodutils,
+  ast, hashes, intsets, options, lineinfos, ropes, idents, rodutils,
   msgs
 
+import strutils except addf
+
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 proc hashNode*(p: RootRef): Hash
 proc treeToYaml*(conf: ConfigRef; n: PNode, indent: int = 0, maxRecDepth: int = - 1): Rope
   # Convert a tree into its YAML representation; this is used by the
@@ -221,11 +226,10 @@ proc getNamedParamFromList*(list: PNode, ident: PIdent): PSym =
   ## Named parameters are special because a named parameter can be
   ## gensym'ed and then they have '\`<number>' suffix that we need to
   ## ignore, see compiler / evaltempl.nim, snippet:
-  ##
-  ## .. code-block:: nim
-  ##
+  ##   ```
   ##   result.add newIdentNode(getIdent(c.ic, x.name.s & "\`gensym" & $x.id),
   ##            if c.instLines: actual.info else: templ.info)
+  ##   ```
   for i in 1..<list.len:
     let it = list[i].sym
     if it.name.id == ident.id or
@@ -253,7 +257,7 @@ proc makeYamlString*(s: string): Rope =
   # this could trigger InternalError(111). See the ropes module for
   # further information.
   const MaxLineLength = 64
-  result = nil
+  result = ""
   var res = "\""
   for i in 0..<s.len:
     if (i + 1) mod MaxLineLength == 0:
@@ -269,9 +273,9 @@ proc flagsToStr[T](flags: set[T]): Rope =
   if flags == {}:
     result = rope("[]")
   else:
-    result = nil
+    result = ""
     for x in items(flags):
-      if result != nil: result.add(", ")
+      if result != "": result.add(", ")
       result.add(makeYamlString($x))
     result = "[" & result & "]"
 
@@ -326,7 +330,7 @@ proc typeToYamlAux(conf: ConfigRef; n: PType, marker: var IntSet, indent: int,
     sonsRope = rope("null")
   elif containsOrIncl(marker, n.id):
     sonsRope = "\"$1 @$2\"" % [rope($n.kind), rope(
-        strutils.toHex(cast[ByteAddress](n), sizeof(n) * 2))]
+        strutils.toHex(cast[int](n), sizeof(n) * 2))]
   else:
     if n.len > 0:
       sonsRope = rope("[")
@@ -585,6 +589,9 @@ proc value(this: var DebugPrinter; value: PNode) =
   this.openCurly
   this.key "kind"
   this.value  value.kind
+  if value.comment.len > 0:
+    this.key "comment"
+    this.value  value.comment
   when defined(useNodeIds):
     this.key "id"
     this.value value.id
@@ -759,9 +766,9 @@ proc strTableAdd*(t: var TStrTable, n: PSym) =
 
 proc strTableInclReportConflict*(t: var TStrTable, n: PSym;
                                  onConflictKeepOld = false): PSym =
-  # returns true if n is already in the string table:
-  # It is essential that `n` is written nevertheless!
-  # This way the newest redefinition is picked by the semantic analyses!
+  # if `t` has a conflicting symbol (same identifier as `n`), return it
+  # otherwise return `nil`. Incl `n` to `t` unless `onConflictKeepOld = true`
+  # and a conflict was found.
   assert n.name != nil
   var h: Hash = n.name.h and high(t.data)
   var replaceSlot = -1
@@ -777,9 +784,10 @@ proc strTableInclReportConflict*(t: var TStrTable, n: PSym;
       replaceSlot = h
     h = nextTry(h, high(t.data))
   if replaceSlot >= 0:
+    result = t.data[replaceSlot] # found it
     if not onConflictKeepOld:
       t.data[replaceSlot] = n # overwrite it with newer definition!
-    return t.data[replaceSlot] # found it
+    return result # but return the old one
   elif mustRehash(t.data.len, t.counter):
     strTableEnlarge(t)
     strTableRawInsert(t.data, n)
@@ -808,16 +816,21 @@ type
     name*: PIdent
 
 proc nextIdentIter*(ti: var TIdentIter, tab: TStrTable): PSym =
+  # hot spots
   var h = ti.h and high(tab.data)
   var start = h
-  result = tab.data[h]
-  while result != nil:
-    if result.name.id == ti.name.id: break
+  var p {.cursor.} = tab.data[h]
+  while p != nil:
+    if p.name.id == ti.name.id: break
     h = nextTry(h, high(tab.data))
     if h == start:
-      result = nil
+      p = nil
       break
-    result = tab.data[h]
+    p = tab.data[h]
+  if p != nil:
+    result = p # increase the count
+  else:
+    result = nil
   ti.h = nextTry(h, high(tab.data))
 
 proc initIdentIter*(ti: var TIdentIter, tab: TStrTable, s: PIdent): PSym =

compiler/astmsgs.nim

@@ -2,6 +2,11 @@
 import std/strutils
 import options, ast, msgs
 
+proc typSym*(t: PType): PSym =
+  result = t.sym
+  if result == nil and t.kind == tyGenericInst: # this might need to be refined
+    result = t[0].sym
+
 proc addDeclaredLoc*(result: var string, conf: ConfigRef; sym: PSym) =
   result.add " [$1 declared in $2]" % [sym.kind.toHumanStr, toFileLineCol(conf, sym.info)]
 
@@ -13,7 +18,7 @@ proc addDeclaredLoc*(result: var string, conf: ConfigRef; typ: PType) =
   # xxx figure out how to resolve `tyGenericParam`, e.g. for
   # proc fn[T](a: T, b: T) = discard
   # fn(1.1, "a")
-  let typ = typ.skipTypes(abstractInst + {tyStatic} - {tyRange})
+  let typ = typ.skipTypes(abstractInst + {tyStatic, tySequence, tyArray, tySet, tyUserTypeClassInst, tyVar, tyRef, tyPtr} - {tyRange})
   result.add " [$1" % typ.kind.toHumanStr
   if typ.sym != nil:
     result.add " declared in " & toFileLineCol(conf, typ.sym.info)
@@ -21,3 +26,14 @@ proc addDeclaredLoc*(result: var string, conf: ConfigRef; typ: PType) =
 
 proc addDeclaredLocMaybe*(result: var string, conf: ConfigRef; typ: PType) =
   if optDeclaredLocs in conf.globalOptions: addDeclaredLoc(result, conf, typ)
+
+template quoteExpr*(a: string): untyped =
+  ## can be used for quoting expressions in error msgs.
+  "'" & a & "'"
+
+proc genFieldDefect*(conf: ConfigRef, field: string, disc: PSym): string =
+  let obj = disc.owner.name.s # `types.typeToString` might be better, eg for generics
+  result = "field '$#' is not accessible for type '$#'" % [field, obj]
+  if optDeclaredLocs in conf.globalOptions:
+    result.add " [discriminant declared in $#]" % toFileLineCol(conf, disc.info)
+  result.add " using '$# = " % disc.name.s

compiler/backendpragmas.nim

@@ -0,0 +1,25 @@
+import pragmas, options, ast, trees
+
+proc pushBackendOption(optionsStack: var seq[TOptions], options: var TOptions) =
+  optionsStack.add options
+
+proc popBackendOption(optionsStack: var seq[TOptions], options: var TOptions) =
+  options = optionsStack[^1]
+  optionsStack.setLen(optionsStack.len-1)
+
+proc processPushBackendOption*(optionsStack: var seq[TOptions], options: var TOptions,
+                           n: PNode, start: int) =
+  pushBackendOption(optionsStack, options)
+  for i in start..<n.len:
+    let it = n[i]
+    if it.kind in nkPragmaCallKinds and it.len == 2 and it[1].kind == nkIntLit:
+      let sw = whichPragma(it[0])
+      let opts = pragmaToOptions(sw)
+      if opts != {}:
+        if it[1].intVal != 0:
+          options.incl opts
+        else:
+          options.excl opts
+
+template processPopBackendOption*(optionsStack: var seq[TOptions], options: var TOptions) =
+  popBackendOption(optionsStack, options)

compiler/bitsets.nim

@@ -10,6 +10,9 @@
 # this unit handles Nim sets; it implements bit sets
 # the code here should be reused in the Nim standard library
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 type
   ElemType = byte
   TBitSet* = seq[ElemType]    # we use byte here to avoid issues with
@@ -23,53 +26,40 @@ const
 template modElemSize(arg: untyped): untyped = arg and 7
 template divElemSize(arg: untyped): untyped = arg shr 3
 
-proc bitSetInit*(b: var TBitSet, length: int)
-proc bitSetUnion*(x: var TBitSet, y: TBitSet)
-proc bitSetDiff*(x: var TBitSet, y: TBitSet)
-proc bitSetSymDiff*(x: var TBitSet, y: TBitSet)
-proc bitSetIntersect*(x: var TBitSet, y: TBitSet)
-proc bitSetIncl*(x: var TBitSet, elem: BiggestInt)
-proc bitSetExcl*(x: var TBitSet, elem: BiggestInt)
-proc bitSetIn*(x: TBitSet, e: BiggestInt): bool
-proc bitSetEquals*(x, y: TBitSet): bool
-proc bitSetContains*(x, y: TBitSet): bool
-proc bitSetCard*(x: TBitSet): BiggestInt
-# implementation
-
-proc bitSetIn(x: TBitSet, e: BiggestInt): bool =
+proc bitSetIn*(x: TBitSet, e: BiggestInt): bool =
   result = (x[int(e.divElemSize)] and (One shl e.modElemSize)) != Zero
 
-proc bitSetIncl(x: var TBitSet, elem: BiggestInt) =
+proc bitSetIncl*(x: var TBitSet, elem: BiggestInt) =
   assert(elem >= 0)
   x[int(elem.divElemSize)] = x[int(elem.divElemSize)] or
       (One shl elem.modElemSize)
 
-proc bitSetExcl(x: var TBitSet, elem: BiggestInt) =
+proc bitSetExcl*(x: var TBitSet, elem: BiggestInt) =
   x[int(elem.divElemSize)] = x[int(elem.divElemSize)] and
       not(One shl elem.modElemSize)
 
-proc bitSetInit(b: var TBitSet, length: int) =
+proc bitSetInit*(b: var TBitSet, length: int) =
   newSeq(b, length)
 
-proc bitSetUnion(x: var TBitSet, y: TBitSet) =
+proc bitSetUnion*(x: var TBitSet, y: TBitSet) =
   for i in 0..high(x): x[i] = x[i] or y[i]
 
-proc bitSetDiff(x: var TBitSet, y: TBitSet) =
+proc bitSetDiff*(x: var TBitSet, y: TBitSet) =
   for i in 0..high(x): x[i] = x[i] and not y[i]
 
-proc bitSetSymDiff(x: var TBitSet, y: TBitSet) =
+proc bitSetSymDiff*(x: var TBitSet, y: TBitSet) =
   for i in 0..high(x): x[i] = x[i] xor y[i]
 
-proc bitSetIntersect(x: var TBitSet, y: TBitSet) =
+proc bitSetIntersect*(x: var TBitSet, y: TBitSet) =
   for i in 0..high(x): x[i] = x[i] and y[i]
 
-proc bitSetEquals(x, y: TBitSet): bool =
+proc bitSetEquals*(x, y: TBitSet): bool =
   for i in 0..high(x):
     if x[i] != y[i]:
       return false
   result = true
 
-proc bitSetContains(x, y: TBitSet): bool =
+proc bitSetContains*(x, y: TBitSet): bool =
   for i in 0..high(x):
     if (x[i] and not y[i]) != Zero:
       return false
@@ -96,6 +86,6 @@ const populationCount: array[uint8, uint8] = block:
 
     arr
 
-proc bitSetCard(x: TBitSet): BiggestInt =
+proc bitSetCard*(x: TBitSet): BiggestInt =
   for it in x:
     result.inc int(populationCount[it])

compiler/btrees.nim

@@ -10,6 +10,9 @@
 ## BTree implementation with few features, but good enough for the
 ## Nim compiler's needs.
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 const
   M = 512    # max children per B-tree node = M-1
              # (must be even and greater than 2)
@@ -65,7 +68,10 @@ proc copyHalf[Key, Val](h, result: Node[Key, Val]) =
       result.links[j] = h.links[Mhalf + j]
   else:
     for j in 0..<Mhalf:
-      shallowCopy(result.vals[j], h.vals[Mhalf + j])
+      when defined(gcArc) or defined(gcOrc):
+        result.vals[j] = move h.vals[Mhalf + j]
+      else:
+        shallowCopy(result.vals[j], h.vals[Mhalf + j])
 
 proc split[Key, Val](h: Node[Key, Val]): Node[Key, Val] =
   ## split node in half
@@ -85,7 +91,10 @@ proc insert[Key, Val](h: Node[Key, Val], key: Key, val: Val): Node[Key, Val] =
       if less(key, h.keys[j]): break
       inc j
     for i in countdown(h.entries, j+1):
-      shallowCopy(h.vals[i], h.vals[i-1])
+      when defined(gcArc) or defined(gcOrc):
+        h.vals[i] = move h.vals[i-1]
+      else:
+        shallowCopy(h.vals[i], h.vals[i-1])
     h.vals[j] = val
   else:
     var newLink: Node[Key, Val] = nil

compiler/ccgcalls.nim

@@ -14,14 +14,15 @@ proc canRaiseDisp(p: BProc; n: PNode): bool =
   if n.kind == nkSym and {sfNeverRaises, sfImportc, sfCompilerProc} * n.sym.flags != {}:
     result = false
   elif optPanics in p.config.globalOptions or
-      (n.kind == nkSym and sfSystemModule in getModule(n.sym).flags):
+      (n.kind == nkSym and sfSystemModule in getModule(n.sym).flags and
+       sfSystemRaisesDefect notin n.sym.flags):
     # we know we can be strict:
     result = canRaise(n)
   else:
     # we have to be *very* conservative:
     result = canRaiseConservative(n)
 
-proc preventNrvo(p: BProc; le, ri: PNode): bool =
+proc preventNrvo(p: BProc; dest, le, ri: PNode): bool =
   proc locationEscapes(p: BProc; le: PNode; inTryStmt: bool): bool =
     var n = le
     while true:
@@ -54,6 +55,11 @@ proc preventNrvo(p: BProc; le, ri: PNode): bool =
     if canRaise(ri[0]) and
         locationEscapes(p, le, p.nestedTryStmts.len > 0):
       message(p.config, le.info, warnObservableStores, $le)
+  # bug #19613 prevent dangerous aliasing too:
+  if dest != nil and dest != le:
+    for i in 1..<ri.len:
+      let r = ri[i]
+      if isPartOf(dest, r) != arNo: return true
 
 proc hasNoInit(call: PNode): bool {.inline.} =
   result = call[0].kind == nkSym and sfNoInit in call[0].sym.flags
@@ -72,32 +78,32 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
                callee, params: Rope) =
   let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri[0])
   genLineDir(p, ri)
-  var pl = callee & ~"(" & params
+  var pl = callee & "(" & params
   # getUniqueType() is too expensive here:
   var typ = skipTypes(ri[0].typ, abstractInst)
   if typ[0] != nil:
-    if isInvalidReturnType(p.config, typ[0]):
-      if params != nil: pl.add(~", ")
+    if isInvalidReturnType(p.config, typ):
+      if params.len != 0: pl.add(", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, d.lode, le, ri):
         # Great, we can use 'd':
         if d.k == locNone: getTemp(p, typ[0], d, needsInit=true)
         elif d.k notin {locTemp} and not hasNoInit(ri):
           # reset before pass as 'result' var:
           discard "resetLoc(p, d)"
         pl.add(addrLoc(p.config, d))
-        pl.add(~");$n")
+        pl.add(");\n")
         line(p, cpsStmts, pl)
       else:
         var tmp: TLoc
         getTemp(p, typ[0], tmp, needsInit=true)
         pl.add(addrLoc(p.config, tmp))
-        pl.add(~");$n")
+        pl.add(");\n")
         line(p, cpsStmts, pl)
         genAssignment(p, d, tmp, {}) # no need for deep copying
       if canRaise: raiseExit(p)
     else:
-      pl.add(~")")
+      pl.add(")")
       if p.module.compileToCpp:
         if lfSingleUse in d.flags:
           # do not generate spurious temporaries for C++! For C we're better off
@@ -135,7 +141,7 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
         if canRaise: raiseExit(p)
         genAssignment(p, d, tmp, {})
   else:
-    pl.add(~");$n")
+    pl.add(");\n")
     line(p, cpsStmts, pl)
     if canRaise: raiseExit(p)
 
@@ -150,7 +156,7 @@ proc reifiedOpenArray(n: PNode): bool {.inline.} =
   else:
     result = true
 
-proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType): (Rope, Rope) =
+proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType; prepareForMutation = false): (Rope, Rope) =
   var a, b, c: TLoc
   initLocExpr(p, q[1], a)
   initLocExpr(p, q[2], b)
@@ -158,6 +164,8 @@ proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType): (Rope,
   # but first produce the required index checks:
   if optBoundsCheck in p.options:
     genBoundsCheck(p, a, b, c)
+  if prepareForMutation:
+    linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
   let ty = skipTypes(a.t, abstractVar+{tyPtr})
   let dest = getTypeDesc(p.module, destType)
   let lengthExpr = "($1)-($2)+1" % [rdLoc(c), rdLoc(b)]
@@ -168,8 +176,10 @@ proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType): (Rope,
       result = ("($3*)(($1)+($2))" % [rdLoc(a), rdLoc(b), dest],
                 lengthExpr)
     else:
+      var lit = newRopeAppender()
+      intLiteral(first, lit)
       result = ("($4*)($1)+(($2)-($3))" %
-        [rdLoc(a), rdLoc(b), intLiteral(first), dest],
+        [rdLoc(a), rdLoc(b), lit, dest],
         lengthExpr)
   of tyOpenArray, tyVarargs:
     if reifiedOpenArray(q[1]):
@@ -187,15 +197,17 @@ proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType): (Rope,
         optSeqDestructors in p.config.globalOptions:
       linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
     if atyp.kind in {tyVar} and not compileToCpp(p.module):
-      result = ("($4*)(*$1)$3+($2)" % [rdLoc(a), rdLoc(b), dataField(p), dest],
+      result = ("(($5) ? (($4*)(*$1)$3+($2)) : NIM_NIL)" %
+                  [rdLoc(a), rdLoc(b), dataField(p), dest, dataFieldAccessor(p, "*" & rdLoc(a))],
                 lengthExpr)
     else:
-      result = ("($4*)$1$3+($2)" % [rdLoc(a), rdLoc(b), dataField(p), dest],
+      result = ("(($5) ? (($4*)$1$3+($2)) : NIM_NIL)" %
+                  [rdLoc(a), rdLoc(b), dataField(p), dest, dataFieldAccessor(p, rdLoc(a))],
                 lengthExpr)
   else:
     internalError(p.config, "openArrayLoc: " & typeToString(a.t))
 
-proc openArrayLoc(p: BProc, formalType: PType, n: PNode): Rope =
+proc openArrayLoc(p: BProc, formalType: PType, n: PNode; result: var Rope) =
   var q = skipConv(n)
   var skipped = false
   while q.kind == nkStmtListExpr and q.len > 0:
@@ -210,19 +222,19 @@ proc openArrayLoc(p: BProc, formalType: PType, n: PNode): Rope =
           genStmts(p, q[i])
         q = q.lastSon
     let (x, y) = genOpenArraySlice(p, q, formalType, n.typ[0])
-    result = x & ", " & y
+    result.add x & ", " & y
   else:
     var a: TLoc
     initLocExpr(p, if n.kind == nkHiddenStdConv: n[1] else: n, a)
-    case skipTypes(a.t, abstractVar).kind
+    case skipTypes(a.t, abstractVar+{tyStatic}).kind
     of tyOpenArray, tyVarargs:
       if reifiedOpenArray(n):
         if a.t.kind in {tyVar, tyLent}:
-          result = "$1->Field0, $1->Field1" % [rdLoc(a)]
+          result.add "$1->Field0, $1->Field1" % [rdLoc(a)]
         else:
-          result = "$1.Field0, $1.Field1" % [rdLoc(a)]
+          result.add "$1.Field0, $1.Field1" % [rdLoc(a)]
       else:
-        result = "$1, $1Len_0" % [rdLoc(a)]
+        result.add "$1, $1Len_0" % [rdLoc(a)]
     of tyString, tySequence:
       let ntyp = skipTypes(n.typ, abstractInst)
       if formalType.skipTypes(abstractInst).kind in {tyVar} and ntyp.kind == tyString and
@@ -231,19 +243,24 @@ proc openArrayLoc(p: BProc, formalType: PType, n: PNode): Rope =
       if ntyp.kind in {tyVar} and not compileToCpp(p.module):
         var t: TLoc
         t.r = "(*$1)" % [a.rdLoc]
-        result = "(*$1)$3, $2" % [a.rdLoc, lenExpr(p, t), dataField(p)]
+        result.add "($4) ? ((*$1)$3) : NIM_NIL, $2" %
+                     [a.rdLoc, lenExpr(p, t), dataField(p),
+                      dataFieldAccessor(p, "*" & a.rdLoc)]
       else:
-        result = "$1$3, $2" % [a.rdLoc, lenExpr(p, a), dataField(p)]
+        result.add "($4) ? ($1$3) : NIM_NIL, $2" %
+                     [a.rdLoc, lenExpr(p, a), dataField(p), dataFieldAccessor(p, a.rdLoc)]
     of tyArray:
-      result = "$1, $2" % [rdLoc(a), rope(lengthOrd(p.config, a.t))]
+      result.add "$1, $2" % [rdLoc(a), rope(lengthOrd(p.config, a.t))]
     of tyPtr, tyRef:
       case lastSon(a.t).kind
       of tyString, tySequence:
         var t: TLoc
         t.r = "(*$1)" % [a.rdLoc]
-        result = "(*$1)$3, $2" % [a.rdLoc, lenExpr(p, t), dataField(p)]
+        result.add "($4) ? ((*$1)$3) : NIM_NIL, $2" %
+                     [a.rdLoc, lenExpr(p, t), dataField(p),
+                      dataFieldAccessor(p, "*" & a.rdLoc)]
       of tyArray:
-        result = "$1, $2" % [rdLoc(a), rope(lengthOrd(p.config, lastSon(a.t)))]
+        result.add "$1, $2" % [rdLoc(a), rope(lengthOrd(p.config, lastSon(a.t)))]
       else:
         internalError(p.config, "openArrayLoc: " & typeToString(a.t))
     else: internalError(p.config, "openArrayLoc: " & typeToString(a.t))
@@ -263,24 +280,24 @@ proc literalsNeedsTmp(p: BProc, a: TLoc): TLoc =
   getTemp(p, a.lode.typ, result, needsInit=false)
   genAssignment(p, result, a, {})
 
-proc genArgStringToCString(p: BProc, n: PNode, needsTmp: bool): Rope {.inline.} =
+proc genArgStringToCString(p: BProc, n: PNode; result: var Rope; needsTmp: bool) {.inline.} =
   var a: TLoc
   initLocExpr(p, n[0], a)
-  ropecg(p.module, "#nimToCStringConv($1)", [withTmpIfNeeded(p, a, needsTmp).rdLoc])
+  appcg(p.module, result, "#nimToCStringConv($1)", [withTmpIfNeeded(p, a, needsTmp).rdLoc])
 
-proc genArg(p: BProc, n: PNode, param: PSym; call: PNode, needsTmp = false): Rope =
+proc genArg(p: BProc, n: PNode, param: PSym; call: PNode; result: var Rope; needsTmp = false) =
   var a: TLoc
   if n.kind == nkStringToCString:
-    result = genArgStringToCString(p, n, needsTmp)
+    genArgStringToCString(p, n, result, needsTmp)
   elif skipTypes(param.typ, abstractVar).kind in {tyOpenArray, tyVarargs}:
     var n = if n.kind != nkHiddenAddr: n else: n[0]
-    result = openArrayLoc(p, param.typ, n)
+    openArrayLoc(p, param.typ, n, result)
   elif ccgIntroducedPtr(p.config, param, call[0].typ[0]):
     initLocExpr(p, n, a)
     if n.kind in {nkCharLit..nkNilLit}:
-      result = addrLoc(p.config, literalsNeedsTmp(p, a))
+      addAddrLoc(p.config, literalsNeedsTmp(p, a), result)
     else:
-      result = addrLoc(p.config, withTmpIfNeeded(p, a, needsTmp))
+      addAddrLoc(p.config, withTmpIfNeeded(p, a, needsTmp), result)
   elif p.module.compileToCpp and param.typ.kind in {tyVar} and
       n.kind == nkHiddenAddr:
     initLocExprSingleUse(p, n[0], a)
@@ -290,23 +307,23 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode, needsTmp = false): Rop
     if callee.kind == nkSym and
         {sfImportc, sfInfixCall, sfCompilerProc} * callee.sym.flags == {sfImportc} and
         {lfHeader, lfNoDecl} * callee.sym.loc.flags != {}:
-      result = addrLoc(p.config, a)
+      addAddrLoc(p.config, a, result)
     else:
-      result = rdLoc(a)
+      addRdLoc(a, result)
   else:
     initLocExprSingleUse(p, n, a)
-    result = rdLoc(withTmpIfNeeded(p, a, needsTmp))
+    addRdLoc(withTmpIfNeeded(p, a, needsTmp), result)
   #assert result != nil
 
-proc genArgNoParam(p: BProc, n: PNode, needsTmp = false): Rope =
+proc genArgNoParam(p: BProc, n: PNode; result: var Rope; needsTmp = false) =
   var a: TLoc
   if n.kind == nkStringToCString:
-    result = genArgStringToCString(p, n, needsTmp)
+    genArgStringToCString(p, n, result, needsTmp)
   else:
     initLocExprSingleUse(p, n, a)
-    result = rdLoc(withTmpIfNeeded(p, a, needsTmp))
+    addRdLoc(withTmpIfNeeded(p, a, needsTmp), result)
 
-from dfa import aliases, AliasKind
+import aliasanalysis
 
 proc potentialAlias(n: PNode, potentialWrites: seq[PNode]): bool =
   for p in potentialWrites:
@@ -317,38 +334,50 @@ proc skipTrivialIndirections(n: PNode): PNode =
   result = n
   while true:
     case result.kind
-    of {nkDerefExpr, nkHiddenDeref, nkAddr, nkHiddenAddr, nkObjDownConv, nkObjUpConv}:
+    of nkDerefExpr, nkHiddenDeref, nkAddr, nkHiddenAddr, nkObjDownConv, nkObjUpConv:
       result = result[0]
-    of {nkHiddenStdConv, nkHiddenSubConv}:
+    of nkHiddenStdConv, nkHiddenSubConv:
       result = result[1]
     else: break
 
-proc getPotentialWrites(n: PNode, mutate = false): seq[PNode] =
+proc getPotentialWrites(n: PNode; mutate: bool; result: var seq[PNode]) =
   case n.kind:
-  of nkLiterals, nkIdent: discard
+  of nkLiterals, nkIdent, nkFormalParams: discard
   of nkSym:
     if mutate: result.add n
-  of nkAsgn, nkFastAsgn:
-    result.add getPotentialWrites(n[0], true)
-    result.add getPotentialWrites(n[1], mutate)
+  of nkAsgn, nkFastAsgn, nkSinkAsgn:
+    getPotentialWrites(n[0], true, result)
+    getPotentialWrites(n[1], mutate, result)
   of nkAddr, nkHiddenAddr:
-    result.add getPotentialWrites(n[0], true)
-  of nkCallKinds: #TODO: Find out why in f += 1, f is a nkSym and not a nkHiddenAddr
-    for s in n.sons:
-      result.add getPotentialWrites(s, true)
+    getPotentialWrites(n[0], true, result)
+  of nkBracketExpr, nkDotExpr, nkCheckedFieldExpr:
+    getPotentialWrites(n[0], mutate, result)
+  of nkCallKinds:
+    case n.getMagic:
+    of mIncl, mExcl, mInc, mDec, mAppendStrCh, mAppendStrStr, mAppendSeqElem,
+        mAddr, mNew, mNewFinalize, mWasMoved, mDestroy, mReset:
+      getPotentialWrites(n[1], true, result)
+      for i in 2..<n.len:
+        getPotentialWrites(n[i], mutate, result)
+    of mSwap:
+      for i in 1..<n.len:
+        getPotentialWrites(n[i], true, result)
+    else:
+      for i in 1..<n.len:
+        getPotentialWrites(n[i], mutate, result)
   else:
-    for s in n.sons:
-      result.add getPotentialWrites(s, mutate)
+    for s in n:
+      getPotentialWrites(s, mutate, result)
 
-proc getPotentialReads(n: PNode): seq[PNode] =
+proc getPotentialReads(n: PNode; result: var seq[PNode]) =
   case n.kind:
-  of nkLiterals, nkIdent: discard
+  of nkLiterals, nkIdent, nkFormalParams: discard
   of nkSym: result.add n
   else:
-    for s in n.sons:
-      result.add getPotentialReads(s)
+    for s in n:
+      getPotentialReads(s, result)
 
-proc genParams(p: BProc, ri: PNode, typ: PType): Rope =
+proc genParams(p: BProc, ri: PNode, typ: PType; result: var Rope) =
   # We must generate temporaries in cases like #14396
   # to keep the strict Left-To-Right evaluation
   var needTmp = newSeq[bool](ri.len - 1)
@@ -357,24 +386,32 @@ proc genParams(p: BProc, ri: PNode, typ: PType): Rope =
     if ri[i].skipTrivialIndirections.kind == nkSym:
       needTmp[i - 1] = potentialAlias(ri[i], potentialWrites)
     else:
-      for n in getPotentialReads(ri[i]):
+      #if not ri[i].typ.isCompileTimeOnly:
+      var potentialReads: seq[PNode]
+      getPotentialReads(ri[i], potentialReads)
+      for n in potentialReads:
         if not needTmp[i - 1]:
           needTmp[i - 1] = potentialAlias(n, potentialWrites)
-      potentialWrites.add getPotentialWrites(ri[i])
-    if ri[i].kind == nkHiddenAddr:
-      # Optimization: don't use a temp, if we would only take the adress anyway
+      getPotentialWrites(ri[i], false, potentialWrites)
+    if ri[i].kind in {nkHiddenAddr, nkAddr}:
+      # Optimization: don't use a temp, if we would only take the address anyway
       needTmp[i - 1] = false
 
+  var oldLen = result.len
   for i in 1..<ri.len:
     if i < typ.len:
       assert(typ.n[i].kind == nkSym)
       let paramType = typ.n[i]
       if not paramType.typ.isCompileTimeOnly:
-        if result != nil: result.add(~", ")
-        result.add(genArg(p, ri[i], paramType.sym, ri, needTmp[i-1]))
+        if oldLen != result.len:
+          result.add(", ")
+          oldLen = result.len
+        genArg(p, ri[i], paramType.sym, ri, result, needTmp[i-1])
     else:
-      if result != nil: result.add(~", ")
-      result.add(genArgNoParam(p, ri[i], needTmp[i-1]))
+      if oldLen != result.len:
+        result.add(", ")
+        oldLen = result.len
+      genArgNoParam(p, ri[i], result, needTmp[i-1])
 
 proc addActualSuffixForHCR(res: var Rope, module: PSym, sym: PSym) =
   if sym.flags * {sfImportc, sfNonReloadable} == {} and sym.loc.k == locProc and
@@ -390,7 +427,8 @@ proc genPrefixCall(p: BProc, le, ri: PNode, d: var TLoc) =
   assert(typ.kind == tyProc)
   assert(typ.len == typ.n.len)
 
-  var params = genParams(p, ri, typ)
+  var params = newRopeAppender()
+  genParams(p, ri, typ, params)
 
   var callee = rdLoc(op)
   if p.hcrOn and ri[0].kind == nkSym:
@@ -400,7 +438,7 @@ proc genPrefixCall(p: BProc, le, ri: PNode, d: var TLoc) =
 proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
 
   proc addComma(r: Rope): Rope =
-    if r == nil: r else: r & ~", "
+    if r.len == 0: r else: r & ", "
 
   const PatProc = "$1.ClE_0? $1.ClP_0($3$1.ClE_0):(($4)($1.ClP_0))($2)"
   const PatIter = "$1.ClP_0($3$1.ClE_0)" # we know the env exists
@@ -413,7 +451,8 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
   assert(typ.kind == tyProc)
   assert(typ.len == typ.n.len)
 
-  var pl = genParams(p, ri, typ)
+  var pl = newRopeAppender()
+  genParams(p, ri, typ, pl)
 
   template genCallPattern {.dirty.} =
     if tfIterator in typ.flags:
@@ -424,10 +463,10 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
   let rawProc = getClosureType(p.module, typ, clHalf)
   let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri[0])
   if typ[0] != nil:
-    if isInvalidReturnType(p.config, typ[0]):
-      if ri.len > 1: pl.add(~", ")
+    if isInvalidReturnType(p.config, typ):
+      if ri.len > 1: pl.add(", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, d.lode, le, ri):
         # Great, we can use 'd':
         if d.k == locNone:
           getTemp(p, typ[0], d, needsInit=true)
@@ -471,24 +510,30 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
     genCallPattern()
     if canRaise: raiseExit(p)
 
-proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
+proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType; result: var Rope;
+                 argsCounter: var int) =
   if i < typ.len:
     # 'var T' is 'T&' in C++. This means we ignore the request of
     # any nkHiddenAddr when it's a 'var T'.
     let paramType = typ.n[i]
     assert(paramType.kind == nkSym)
     if paramType.typ.isCompileTimeOnly:
-      result = nil
+      discard
     elif typ[i].kind in {tyVar} and ri[i].kind == nkHiddenAddr:
-      result = genArgNoParam(p, ri[i][0])
+      if argsCounter > 0: result.add ", "
+      genArgNoParam(p, ri[i][0], result)
+      inc argsCounter
     else:
-      result = genArgNoParam(p, ri[i]) #, typ.n[i].sym)
+      if argsCounter > 0: result.add ", "
+      genArgNoParam(p, ri[i], result) #, typ.n[i].sym)
+      inc argsCounter
   else:
     if tfVarargs notin typ.flags:
       localError(p.config, ri.info, "wrong argument count")
-      result = nil
     else:
-      result = genArgNoParam(p, ri[i])
+      if argsCounter > 0: result.add ", "
+      genArgNoParam(p, ri[i], result)
+      inc argsCounter
 
 discard """
 Dot call syntax in C++
@@ -545,7 +590,7 @@ proc skipAddrDeref(node: PNode): PNode =
   else:
     result = node
 
-proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
+proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType; result: var Rope) =
   # for better or worse c2nim translates the 'this' argument to a 'var T'.
   # However manual wrappers may also use 'ptr T'. In any case we support both
   # for convenience.
@@ -559,75 +604,72 @@ proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
   if t.kind in {tyVar}:
     let x = if ri.kind == nkHiddenAddr: ri[0] else: ri
     if x.typ.kind == tyPtr:
-      result = genArgNoParam(p, x)
+      genArgNoParam(p, x, result)
       result.add("->")
     elif x.kind in {nkHiddenDeref, nkDerefExpr} and x[0].typ.kind == tyPtr:
-      result = genArgNoParam(p, x[0])
+      genArgNoParam(p, x[0], result)
       result.add("->")
     else:
-      result = genArgNoParam(p, x)
+      genArgNoParam(p, x, result)
       result.add(".")
   elif t.kind == tyPtr:
     if ri.kind in {nkAddr, nkHiddenAddr}:
-      result = genArgNoParam(p, ri[0])
+      genArgNoParam(p, ri[0], result)
       result.add(".")
     else:
-      result = genArgNoParam(p, ri)
+      genArgNoParam(p, ri, result)
       result.add("->")
   else:
     ri = skipAddrDeref(ri)
     if ri.kind in {nkAddr, nkHiddenAddr}: ri = ri[0]
-    result = genArgNoParam(p, ri) #, typ.n[i].sym)
+    genArgNoParam(p, ri, result) #, typ.n[i].sym)
     result.add(".")
 
-proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
+proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType; result: var Rope) =
   var i = 0
   var j = 1
   while i < pat.len:
     case pat[i]
     of '@':
-      var first = true
+      var argsCounter = 0
       for k in j..<ri.len:
-        let arg = genOtherArg(p, ri, k, typ)
-        if arg.len > 0:
-          if not first:
-            result.add(~", ")
-          first = false
-          result.add arg
+        genOtherArg(p, ri, k, typ, result, argsCounter)
       inc i
     of '#':
       if i+1 < pat.len and pat[i+1] in {'+', '@'}:
         let ri = ri[j]
         if ri.kind in nkCallKinds:
           let typ = skipTypes(ri[0].typ, abstractInst)
-          if pat[i+1] == '+': result.add genArgNoParam(p, ri[0])
-          result.add(~"(")
+          if pat[i+1] == '+': genArgNoParam(p, ri[0], result)
+          result.add("(")
           if 1 < ri.len:
-            result.add genOtherArg(p, ri, 1, typ)
+            var argsCounterB = 0
+            genOtherArg(p, ri, 1, typ, result, argsCounterB)
           for k in j+1..<ri.len:
-            result.add(~", ")
-            result.add genOtherArg(p, ri, k, typ)
-          result.add(~")")
+            var argsCounterB = 0
+            genOtherArg(p, ri, k, typ, result, argsCounterB)
+          result.add(")")
         else:
           localError(p.config, ri.info, "call expression expected for C++ pattern")
         inc i
       elif i+1 < pat.len and pat[i+1] == '.':
-        result.add genThisArg(p, ri, j, typ)
+        genThisArg(p, ri, j, typ, result)
         inc i
       elif i+1 < pat.len and pat[i+1] == '[':
         var arg = ri[j].skipAddrDeref
         while arg.kind in {nkAddr, nkHiddenAddr, nkObjDownConv}: arg = arg[0]
-        result.add genArgNoParam(p, arg)
+        genArgNoParam(p, arg, result)
         #result.add debugTree(arg, 0, 10)
       else:
-        result.add genOtherArg(p, ri, j, typ)
+        var argsCounter = 0
+        genOtherArg(p, ri, j, typ, result, argsCounter)
       inc j
       inc i
     of '\'':
       var idx, stars: int
       if scanCppGenericSlot(pat, i, idx, stars):
         var t = resolveStarsInCppType(typ, idx, stars)
-        if t == nil: result.add(~"void")
+        if t == nil: result.add("void")
         else: result.add(getTypeDesc(p.module, t))
     else:
       let start = i
@@ -645,10 +687,11 @@ proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) =
   assert(typ.kind == tyProc)
   assert(typ.len == typ.n.len)
   # don't call '$' here for efficiency:
-  let pat = ri[0].sym.loc.r.data
+  let pat = $ri[0].sym.loc.r
   internalAssert p.config, pat.len > 0
   if pat.contains({'#', '(', '@', '\''}):
-    var pl = genPatternCall(p, ri, pat, typ)
+    var pl = newRopeAppender()
+    genPatternCall(p, ri, pat, typ, pl)
     # simpler version of 'fixupCall' that works with the pl+params combination:
     var typ = skipTypes(ri[0].typ, abstractInst)
     if typ[0] != nil:
@@ -667,80 +710,79 @@ proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) =
         list.r = pl
         genAssignment(p, d, list, {}) # no need for deep copying
     else:
-      pl.add(~";$n")
+      pl.add(";\n")
       line(p, cpsStmts, pl)
   else:
-    var pl: Rope = nil
-    #var param = typ.n[1].sym
+    var pl = newRopeAppender()
+    var argsCounter = 0
     if 1 < ri.len:
-      pl.add(genThisArg(p, ri, 1, typ))
+      genThisArg(p, ri, 1, typ, pl)
     pl.add(op.r)
-    var params: Rope
+    var params = newRopeAppender()
     for i in 2..<ri.len:
-      if params != nil: params.add(~", ")
       assert(typ.len == typ.n.len)
-      params.add(genOtherArg(p, ri, i, typ))
+      genOtherArg(p, ri, i, typ, params, argsCounter)
     fixupCall(p, le, ri, d, pl, params)
 
 proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
   # generates a crappy ObjC call
   var op: TLoc
   initLocExpr(p, ri[0], op)
-  var pl = ~"["
+  var pl = "["
   # getUniqueType() is too expensive here:
   var typ = skipTypes(ri[0].typ, abstractInst)
   assert(typ.kind == tyProc)
   assert(typ.len == typ.n.len)
 
   # don't call '$' here for efficiency:
-  let pat = ri[0].sym.loc.r.data
+  let pat = $ri[0].sym.loc.r
   internalAssert p.config, pat.len > 0
   var start = 3
   if ' ' in pat:
     start = 1
     pl.add(op.r)
     if ri.len > 1:
-      pl.add(~": ")
-      pl.add(genArg(p, ri[1], typ.n[1].sym, ri))
+      pl.add(": ")
+      genArg(p, ri[1], typ.n[1].sym, ri, pl)
       start = 2
   else:
     if ri.len > 1:
-      pl.add(genArg(p, ri[1], typ.n[1].sym, ri))
-      pl.add(~" ")
+      genArg(p, ri[1], typ.n[1].sym, ri, pl)
+      pl.add(" ")
     pl.add(op.r)
     if ri.len > 2:
-      pl.add(~": ")
-      pl.add(genArg(p, ri[2], typ.n[2].sym, ri))
+      pl.add(": ")
+      genArg(p, ri[2], typ.n[2].sym, ri, pl)
   for i in start..<ri.len:
     assert(typ.len == typ.n.len)
     if i >= typ.len:
       internalError(p.config, ri.info, "varargs for objective C method?")
     assert(typ.n[i].kind == nkSym)
     var param = typ.n[i].sym
-    pl.add(~" ")
+    pl.add(" ")
     pl.add(param.name.s)
-    pl.add(~": ")
-    pl.add(genArg(p, ri[i], param, ri))
+    pl.add(": ")
+    genArg(p, ri[i], param, ri, pl)
   if typ[0] != nil:
-    if isInvalidReturnType(p.config, typ[0]):
-      if ri.len > 1: pl.add(~" ")
+    if isInvalidReturnType(p.config, typ):
+      if ri.len > 1: pl.add(" ")
       # beware of 'result = p(result)'. We always allocate a temporary:
       if d.k in {locTemp, locNone}:
         # We already got a temp. Great, special case it:
         if d.k == locNone: getTemp(p, typ[0], d, needsInit=true)
-        pl.add(~"Result: ")
+        pl.add("Result: ")
         pl.add(addrLoc(p.config, d))
-        pl.add(~"];$n")
+        pl.add("];\n")
         line(p, cpsStmts, pl)
       else:
         var tmp: TLoc
         getTemp(p, typ[0], tmp, needsInit=true)
         pl.add(addrLoc(p.config, tmp))
-        pl.add(~"];$n")
+        pl.add("];\n")
         line(p, cpsStmts, pl)
         genAssignment(p, d, tmp, {}) # no need for deep copying
     else:
-      pl.add(~"]")
+      pl.add("]")
       if d.k == locNone: getTemp(p, typ[0], d)
       assert(d.t != nil)        # generate an assignment to d:
       var list: TLoc
@@ -748,7 +790,7 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
       list.r = pl
       genAssignment(p, d, list, {}) # no need for deep copying
   else:
-    pl.add(~"];$n")
+    pl.add("];\n")
     line(p, cpsStmts, pl)
 
 proc notYetAlive(n: PNode): bool {.inline.} =
@@ -786,6 +828,5 @@ proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) =
     genNamedParamCall(p, ri, d)
   else:
     genPrefixCall(p, le, ri, d)
-  postStmtActions(p)
 
 proc genCall(p: BProc, e: PNode, d: var TLoc) = genAsgnCall(p, nil, e, d)

compiler/ccgliterals.nim

@@ -21,7 +21,7 @@ template detectVersion(field, corename) =
     if core == nil or core.kind != skConst:
       m.g.field = 1
     else:
-      m.g.field = toInt(ast.getInt(core.ast))
+      m.g.field = toInt(ast.getInt(core.astdef))
   result = m.g.field
 
 proc detectStrVersion(m: BModule): int =
@@ -32,82 +32,87 @@ proc detectSeqVersion(m: BModule): int =
 
 # ----- Version 1: GC'ed strings and seqs --------------------------------
 
-proc genStringLiteralDataOnlyV1(m: BModule, s: string): Rope =
-  discard cgsym(m, "TGenericSeq")
-  result = getTempName(m)
-  m.s[cfsData].addf("STRING_LITERAL($1, $2, $3);$n",
-       [result, makeCString(s), rope(s.len)])
+proc genStringLiteralDataOnlyV1(m: BModule, s: string; result: var Rope) =
+  cgsym(m, "TGenericSeq")
+  let tmp = getTempName(m)
+  result.add tmp
+  m.s[cfsStrData].addf("STRING_LITERAL($1, $2, $3);$n",
+       [tmp, makeCString(s), rope(s.len)])
 
-proc genStringLiteralV1(m: BModule; n: PNode): Rope =
+proc genStringLiteralV1(m: BModule; n: PNode; result: var Rope) =
   if s.isNil:
-    result = ropecg(m, "((#NimStringDesc*) NIM_NIL)", [])
+    appcg(m, result, "((#NimStringDesc*) NIM_NIL)", [])
   else:
     let id = nodeTableTestOrSet(m.dataCache, n, m.labels)
     if id == m.labels:
       # string literal not found in the cache:
-      result = ropecg(m, "((#NimStringDesc*) &$1)",
-                      [genStringLiteralDataOnlyV1(m, n.strVal)])
+      appcg(m, result, "((#NimStringDesc*) &", [])
+      genStringLiteralDataOnlyV1(m, n.strVal, result)
+      result.add ")"
     else:
-      result = ropecg(m, "((#NimStringDesc*) &$1$2)",
+      appcg(m, result, "((#NimStringDesc*) &$1$2)",
                       [m.tmpBase, id])
 
 # ------ Version 2: destructor based strings and seqs -----------------------
 
 proc genStringLiteralDataOnlyV2(m: BModule, s: string; result: Rope; isConst: bool) =
-  m.s[cfsData].addf("static $4 struct {$n" &
+  m.s[cfsStrData].addf("static $4 struct {$n" &
        "  NI cap; NIM_CHAR data[$2+1];$n" &
        "} $1 = { $2 | NIM_STRLIT_FLAG, $3 };$n",
        [result, rope(s.len), makeCString(s),
        rope(if isConst: "const" else: "")])
 
-proc genStringLiteralV2(m: BModule; n: PNode; isConst: bool): Rope =
+proc genStringLiteralV2(m: BModule; n: PNode; isConst: bool; result: var Rope) =
   let id = nodeTableTestOrSet(m.dataCache, n, m.labels)
   if id == m.labels:
     let pureLit = getTempName(m)
     genStringLiteralDataOnlyV2(m, n.strVal, pureLit, isConst)
-    result = getTempName(m)
-    discard cgsym(m, "NimStrPayload")
-    discard cgsym(m, "NimStringV2")
+    let tmp = getTempName(m)
+    result.add tmp
+    cgsym(m, "NimStrPayload")
+    cgsym(m, "NimStringV2")
     # string literal not found in the cache:
-    m.s[cfsData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
-          [result, rope(n.strVal.len), pureLit, rope(if isConst: "const" else: "")])
+    m.s[cfsStrData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
+          [tmp, rope(n.strVal.len), pureLit, rope(if isConst: "const" else: "")])
   else:
-    result = getTempName(m)
-    m.s[cfsData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
-          [result, rope(n.strVal.len), m.tmpBase & rope(id),
+    let tmp = getTempName(m)
+    result.add tmp
+    m.s[cfsStrData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
+          [tmp, rope(n.strVal.len), m.tmpBase & rope(id),
           rope(if isConst: "const" else: "")])
 
-proc genStringLiteralV2Const(m: BModule; n: PNode; isConst: bool): Rope =
+proc genStringLiteralV2Const(m: BModule; n: PNode; isConst: bool; result: var Rope) =
   let id = nodeTableTestOrSet(m.dataCache, n, m.labels)
   var pureLit: Rope
   if id == m.labels:
     pureLit = getTempName(m)
-    discard cgsym(m, "NimStrPayload")
-    discard cgsym(m, "NimStringV2")
+    cgsym(m, "NimStrPayload")
+    cgsym(m, "NimStringV2")
     # string literal not found in the cache:
     genStringLiteralDataOnlyV2(m, n.strVal, pureLit, isConst)
   else:
     pureLit = m.tmpBase & rope(id)
-  result = "{$1, (NimStrPayload*)&$2}" % [rope(n.strVal.len), pureLit]
+  result.addf "{$1, (NimStrPayload*)&$2}", [rope(n.strVal.len), pureLit]
 
 # ------ Version selector ---------------------------------------------------
 
 proc genStringLiteralDataOnly(m: BModule; s: string; info: TLineInfo;
-                              isConst: bool): Rope =
+                              isConst: bool; result: var Rope) =
   case detectStrVersion(m)
-  of 0, 1: result = genStringLiteralDataOnlyV1(m, s)
+  of 0, 1: genStringLiteralDataOnlyV1(m, s, result)
   of 2:
-    result = getTempName(m)
-    genStringLiteralDataOnlyV2(m, s, result, isConst)
+    let tmp = getTempName(m)
+    genStringLiteralDataOnlyV2(m, s, tmp, isConst)
+    result.add tmp
   else:
     localError(m.config, info, "cannot determine how to produce code for string literal")
 
-proc genNilStringLiteral(m: BModule; info: TLineInfo): Rope =
-  result = ropecg(m, "((#NimStringDesc*) NIM_NIL)", [])
+proc genNilStringLiteral(m: BModule; info: TLineInfo; result: var Rope) =
+  appcg(m, result, "((#NimStringDesc*) NIM_NIL)", [])
 
-proc genStringLiteral(m: BModule; n: PNode): Rope =
+proc genStringLiteral(m: BModule; n: PNode; result: var Rope) =
   case detectStrVersion(m)
-  of 0, 1: result = genStringLiteralV1(m, n)
-  of 2: result = genStringLiteralV2(m, n, isConst = true)
+  of 0, 1: genStringLiteralV1(m, n, result)
+  of 2: genStringLiteralV2(m, n, isConst = true, result)
   else:
     localError(m.config, n.info, "cannot determine how to produce code for string literal")

compiler/ccgmerge_unused.nim

@@ -19,13 +19,11 @@ import
 
 const
   CFileSectionNames: array[TCFileSection, string] = [
-    cfsMergeInfo: "",
     cfsHeaders: "NIM_merge_HEADERS",
     cfsFrameDefines: "NIM_merge_FRAME_DEFINES",
     cfsForwardTypes: "NIM_merge_FORWARD_TYPES",
     cfsTypes: "NIM_merge_TYPES",
     cfsSeqTypes: "NIM_merge_SEQ_TYPES",
-    cfsFieldInfo: "NIM_merge_FIELD_INFO",
     cfsTypeInfo: "NIM_merge_TYPE_INFO",
     cfsProcHeaders: "NIM_merge_PROC_HEADERS",
     cfsData: "NIM_merge_DATA",
@@ -34,11 +32,8 @@ const
     cfsInitProc: "NIM_merge_INIT_PROC",
     cfsDatInitProc: "NIM_merge_DATINIT_PROC",
     cfsTypeInit1: "NIM_merge_TYPE_INIT1",
-    cfsTypeInit2: "NIM_merge_TYPE_INIT2",
     cfsTypeInit3: "NIM_merge_TYPE_INIT3",
-    cfsDebugInit: "NIM_merge_DEBUG_INIT",
-    cfsDynLibInit: "NIM_merge_DYNLIB_INIT",
-    cfsDynLibDeinit: "NIM_merge_DYNLIB_DEINIT",
+    cfsDynLibInit: "NIM_merge_DYNLIB_INIT"
   ]
   CProcSectionNames: array[TCProcSection, string] = [
     cpsLocals: "NIM_merge_PROC_LOCALS",

compiler/ccgreset.nim

@@ -24,7 +24,7 @@ proc specializeResetN(p: BProc, accessor: Rope, n: PNode;
   of nkRecCase:
     if (n[0].kind != nkSym): internalError(p.config, n.info, "specializeResetN")
     let disc = n[0].sym
-    if disc.loc.r == nil: fillObjectFields(p.module, typ)
+    if disc.loc.r == "": fillObjectFields(p.module, typ)
     if disc.loc.t == nil:
       internalError(p.config, n.info, "specializeResetN()")
     lineF(p, cpsStmts, "switch ($1.$2) {$n", [accessor, disc.loc.r])
@@ -42,7 +42,7 @@ proc specializeResetN(p: BProc, accessor: Rope, n: PNode;
   of nkSym:
     let field = n.sym
     if field.typ.kind == tyVoid: return
-    if field.loc.r == nil: fillObjectFields(p.module, typ)
+    if field.loc.r == "": fillObjectFields(p.module, typ)
     if field.loc.t == nil:
       internalError(p.config, n.info, "specializeResetN()")
     specializeResetT(p, "$1.$2" % [accessor, field.loc.r], field.loc.t)
@@ -87,7 +87,20 @@ proc specializeResetT(p: BProc, accessor: Rope, typ: PType) =
     lineCg(p, cpsStmts, "$1 = 0;$n", [accessor])
   of tyCstring, tyPointer, tyPtr, tyVar, tyLent:
     lineCg(p, cpsStmts, "$1 = NIM_NIL;$n", [accessor])
-  else:
+  of tySet:
+    case mapSetType(p.config, typ)
+    of ctArray:
+      lineCg(p, cpsStmts, "#nimZeroMem($1, sizeof($2));$n",
+          [accessor, getTypeDesc(p.module, typ)])
+    of ctInt8, ctInt16, ctInt32, ctInt64:
+      lineCg(p, cpsStmts, "$1 = 0;$n", [accessor])
+    else:
+      doAssert false, "unexpected set type kind"
+  of {tyNone, tyEmpty, tyNil, tyUntyped, tyTyped, tyGenericInvocation,
+      tyGenericParam, tyOrdinal, tyRange, tyOpenArray, tyForward, tyVarargs,
+      tyUncheckedArray, tyProxy, tyBuiltInTypeClass, tyUserTypeClass,
+      tyUserTypeClassInst, tyCompositeTypeClass, tyAnd, tyOr, tyNot,
+      tyAnything, tyStatic, tyFromExpr, tyConcept, tyVoid, tyIterable}:
     discard
 
 proc specializeReset(p: BProc, a: TLoc) =

compiler/ccgstmts.nim

@@ -8,37 +8,44 @@
 #
 
 # included from cgen.nim
-
 const
   RangeExpandLimit = 256      # do not generate ranges
                               # over 'RangeExpandLimit' elements
   stringCaseThreshold = 8
     # above X strings a hash-switch for strings is generated
 
-proc getTraverseProc(p: BProc, v: PSym): Rope =
-  if p.config.selectedGC in {gcMarkAndSweep, gcHooks, gcV2, gcRefc} and
+proc registerTraverseProc(p: BProc, v: PSym) =
+  var traverseProc = ""
+  if p.config.selectedGC in {gcMarkAndSweep, gcHooks, gcRefc} and
       optOwnedRefs notin p.config.globalOptions and
       containsGarbageCollectedRef(v.loc.t):
     # we register a specialized marked proc here; this has the advantage
     # that it works out of the box for thread local storage then :-)
-    result = genTraverseProcForGlobal(p.module, v, v.info)
+    traverseProc = genTraverseProcForGlobal(p.module, v, v.info)
 
-proc registerTraverseProc(p: BProc, v: PSym, traverseProc: Rope) =
-  if sfThread in v.flags:
-    appcg(p.module, p.module.preInitProc.procSec(cpsInit),
-      "$n\t#nimRegisterThreadLocalMarker($1);$n$n", [traverseProc])
-  else:
-    appcg(p.module, p.module.preInitProc.procSec(cpsInit),
-      "$n\t#nimRegisterGlobalMarker($1);$n$n", [traverseProc])
+  if traverseProc.len != 0 and not p.hcrOn:
+    if sfThread in v.flags:
+      appcg(p.module, p.module.preInitProc.procSec(cpsInit),
+        "$n\t#nimRegisterThreadLocalMarker($1);$n$n", [traverseProc])
+    else:
+      appcg(p.module, p.module.preInitProc.procSec(cpsInit),
+        "$n\t#nimRegisterGlobalMarker($1);$n$n", [traverseProc])
 
 proc isAssignedImmediately(conf: ConfigRef; n: PNode): bool {.inline.} =
-  if n.kind == nkEmpty: return false
-  if isInvalidReturnType(conf, n.typ):
-    # var v = f()
-    # is transformed into: var v;  f(addr v)
-    # where 'f' **does not** initialize the result!
-    return false
-  result = true
+  if n.kind == nkEmpty:
+    result = false
+  elif n.kind in nkCallKinds and n[0] != nil and n[0].typ != nil and n[0].typ.skipTypes(abstractInst).kind == tyProc:
+    if isInvalidReturnType(conf, n[0].typ, true):
+      # var v = f()
+      # is transformed into: var v;  f(addr v)
+      # where 'f' **does not** initialize the result!
+      result = false
+    else:
+      result = true
+  elif isInvalidReturnType(conf, n.typ, false):
+    result = false
+  else:
+    result = true
 
 proc inExceptBlockLen(p: BProc): int =
   for x in p.nestedTryStmts:
@@ -47,7 +54,9 @@ proc inExceptBlockLen(p: BProc): int =
 proc startBlockInternal(p: BProc): int {.discardable.} =
   inc(p.labels)
   result = p.blocks.len
-  setLen(p.blocks, result + 1)
+
+  p.blocks.add initBlock()
+
   p.blocks[result].id = p.labels
   p.blocks[result].nestedTryStmts = p.nestedTryStmts.len.int16
   p.blocks[result].nestedExceptStmts = p.inExceptBlockLen.int16
@@ -71,7 +80,7 @@ proc genVarTuple(p: BProc, n: PNode) =
 
   # check only the first son
   var forHcr = treatGlobalDifferentlyForHCR(p.module, n[0].sym)
-  let hcrCond = if forHcr: getTempName(p.module) else: nil
+  let hcrCond = if forHcr: getTempName(p.module) else: ""
   var hcrGlobals: seq[tuple[loc: TLoc, tp: Rope]]
   # determine if the tuple is constructed at top-level scope or inside of a block (if/while/block)
   let isGlobalInBlock = forHcr and p.blocks.len > 2
@@ -90,13 +99,10 @@ proc genVarTuple(p: BProc, n: PNode) =
     let vn = n[i]
     let v = vn.sym
     if sfCompileTime in v.flags: continue
-    var traverseProc: Rope
     if sfGlobal in v.flags:
-      assignGlobalVar(p, vn, nil)
+      assignGlobalVar(p, vn, "")
       genObjectInit(p, cpsInit, v.typ, v.loc, constructObj)
-      traverseProc = getTraverseProc(p, v)
-      if traverseProc != nil and not p.hcrOn:
-        registerTraverseProc(p, v, traverseProc)
+      registerTraverseProc(p, v)
     else:
       assignLocalVar(p, vn)
       initLocalVar(p, v, immediateAsgn=isAssignedImmediately(p.config, n[^1]))
@@ -108,7 +114,7 @@ proc genVarTuple(p: BProc, n: PNode) =
       field.r = "$1.$2" % [rdLoc(tup), mangleRecFieldName(p.module, t.n[i].sym)]
     putLocIntoDest(p, v.loc, field)
     if forHcr or isGlobalInBlock:
-      hcrGlobals.add((loc: v.loc, tp: if traverseProc == nil: ~"NULL" else: traverseProc))
+      hcrGlobals.add((loc: v.loc, tp: "NULL"))
 
   if forHcr:
     # end the block where the tuple gets initialized
@@ -138,12 +144,12 @@ proc loadInto(p: BProc, le, ri: PNode, a: var TLoc) {.inline.} =
     a.flags.incl(lfEnforceDeref)
     expr(p, ri, a)
 
-proc assignLabel(b: var TBlock): Rope {.inline.} =
+proc assignLabel(b: var TBlock; result: var Rope) {.inline.} =
   b.label = "LA" & b.id.rope
-  result = b.label
+  result.add b.label
 
-proc blockBody(b: var TBlock): Rope =
-  result = b.sections[cpsLocals]
+proc blockBody(b: var TBlock; result: var Rope) =
+  result.add b.sections[cpsLocals]
   if b.frameLen > 0:
     result.addf("FR_.len+=$1;$n", [b.frameLen.rope])
   result.add(b.sections[cpsInit])
@@ -152,7 +158,7 @@ proc blockBody(b: var TBlock): Rope =
 proc endBlock(p: BProc, blockEnd: Rope) =
   let topBlock = p.blocks.len-1
   # the block is merged into the parent block
-  p.blocks[topBlock-1].sections[cpsStmts].add(p.blocks[topBlock].blockBody)
+  p.blocks[topBlock].blockBody(p.blocks[topBlock-1].sections[cpsStmts])
   setLen(p.blocks, topBlock)
   # this is done after the block is popped so $n is
   # properly indented when pretty printing is enabled
@@ -164,7 +170,7 @@ proc endBlock(p: BProc) =
   var blockEnd: Rope
   if frameLen > 0:
     blockEnd.addf("FR_.len-=$1;$n", [frameLen.rope])
-  if p.blocks[topBlock].label != nil:
+  if p.blocks[topBlock].label.len != 0:
     blockEnd.addf("} $1: ;$n", [p.blocks[topBlock].label])
   else:
     blockEnd.addf("}$n", [])
@@ -272,17 +278,15 @@ proc genGotoVar(p: BProc; value: PNode) =
   else:
     lineF(p, cpsStmts, "goto NIMSTATE_$#;$n", [value.intVal.rope])
 
-proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType): Rope
+proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType; result: var Rope)
 
-proc potentialValueInit(p: BProc; v: PSym; value: PNode): Rope =
+proc potentialValueInit(p: BProc; v: PSym; value: PNode; result: var Rope) =
   if lfDynamicLib in v.loc.flags or sfThread in v.flags or p.hcrOn:
-    result = nil
+    discard "nothing to do"
   elif sfGlobal in v.flags and value != nil and isDeepConstExpr(value, p.module.compileToCpp) and
       p.withinLoop == 0 and not containsGarbageCollectedRef(v.typ):
     #echo "New code produced for ", v.name.s, " ", p.config $ value.info
-    result = genBracedInit(p, value, isConst = false, v.typ)
-  else:
-    result = nil
+    genBracedInit(p, value, isConst = false, v.typ, result)
 
 proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
   if sfGoto in v.flags:
@@ -290,8 +294,8 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
     genGotoVar(p, value)
     return
   var targetProc = p
-  var traverseProc: Rope
-  let valueAsRope = potentialValueInit(p, v, value)
+  var valueAsRope = ""
+  potentialValueInit(p, v, value, valueAsRope)
   if sfGlobal in v.flags:
     if v.flags * {sfImportc, sfExportc} == {sfImportc} and
         value.kind == nkEmpty and
@@ -307,7 +311,7 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
     # That's why we are doing the construction inside the preInitProc.
     # genObjectInit relies on the C runtime's guarantees that
     # global variables will be initialized to zero.
-    if valueAsRope == nil:
+    if valueAsRope.len == 0:
       var loc = v.loc
 
       # When the native TLS is unavailable, a global thread-local variable needs
@@ -321,9 +325,7 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
     # if sfImportc notin v.flags: constructLoc(p.module.preInitProc, v.loc)
     if sfExportc in v.flags and p.module.g.generatedHeader != nil:
       genVarPrototype(p.module.g.generatedHeader, vn)
-    traverseProc = getTraverseProc(p, v)
-    if traverseProc != nil and not p.hcrOn:
-      registerTraverseProc(p, v, traverseProc)
+    registerTraverseProc(p, v)
   else:
     let imm = isAssignedImmediately(p.config, value)
     if imm and p.module.compileToCpp and p.splitDecls == 0 and
@@ -336,14 +338,14 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
       var tmp: TLoc
       if value.kind in nkCallKinds and value[0].kind == nkSym and
            sfConstructor in value[0].sym.flags:
-        var params: Rope
+        var params = newRopeAppender()
+        var argsCounter = 0
         let typ = skipTypes(value[0].typ, abstractInst)
         assert(typ.kind == tyProc)
         for i in 1..<value.len:
-          if params != nil: params.add(~", ")
           assert(typ.len == typ.n.len)
-          params.add(genOtherArg(p, value, i, typ))
-        if params == nil:
+          genOtherArg(p, value, i, typ, params, argsCounter)
+        if params.len == 0:
           lineF(p, cpsStmts, "$#;$n", [decl])
         else:
           lineF(p, cpsStmts, "$#($#);$n", [decl, params])
@@ -354,7 +356,7 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
     assignLocalVar(p, vn)
     initLocalVar(p, v, imm)
 
-  if traverseProc == nil: traverseProc = ~"NULL"
+  let traverseProc = "NULL"
   # If the var is in a block (control flow like if/while or a block) in global scope just
   # register the so called "global" so it can be used later on. There is no need to close
   # and reopen of if (nim_hcr_do_init_) blocks because we are in one already anyway.
@@ -374,7 +376,7 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
     lineCg(targetProc, cpsStmts, "if (hcrRegisterGlobal($3, \"$1\", sizeof($2), $4, (void**)&$1))$N",
            [v.loc.r, rdLoc(v.loc), getModuleDllPath(p.module, v), traverseProc])
     startBlock(targetProc)
-  if value.kind != nkEmpty and valueAsRope == nil:
+  if value.kind != nkEmpty and valueAsRope.len == 0:
     genLineDir(targetProc, vn)
     loadInto(targetProc, vn, value, v.loc)
   if forHcr:
@@ -398,7 +400,7 @@ proc genClosureVar(p: BProc, a: PNode) =
   genLineDir(p, a)
   if immediateAsgn:
     loadInto(p, a[0], a[2], v)
-  else:
+  elif sfNoInit notin a[0][1].sym.flags:
     constructLoc(p, v)
 
 proc genVarStmt(p: BProc, n: PNode) =
@@ -553,7 +555,9 @@ proc genComputedGoto(p: BProc; n: PNode) =
         return
 
       let val = getOrdValue(it[j])
-      lineF(p, cpsStmts, "TMP$#_:$n", [intLiteral(toInt64(val)+id+1)])
+      var lit = newRopeAppender()
+      intLiteral(toInt64(val)+id+1, lit)
+      lineF(p, cpsStmts, "TMP$#_:$n", [lit])
 
     genStmts(p, it.lastSon)
 
@@ -607,8 +611,9 @@ proc genWhileStmt(p: BProc, t: PNode) =
       p.blocks[p.breakIdx].isLoop = true
       initLocExpr(p, t[0], a)
       if (t[0].kind != nkIntLit) or (t[0].intVal == 0):
-        let label = assignLabel(p.blocks[p.breakIdx])
-        lineF(p, cpsStmts, "if (!$1) goto $2;$n", [rdLoc(a), label])
+        lineF(p, cpsStmts, "if (!$1) goto ", [rdLoc(a)])
+        assignLabel(p.blocks[p.breakIdx], p.s(cpsStmts))
+        lineF(p, cpsStmts, ";$n", [])
       genStmts(p, loopBody)
 
       if optProfiler in p.options:
@@ -695,12 +700,12 @@ proc genBreakStmt(p: BProc, t: PNode) =
     while idx >= 0 and not p.blocks[idx].isLoop: dec idx
     if idx < 0 or not p.blocks[idx].isLoop:
       internalError(p.config, t.info, "no loop to break")
-  let label = assignLabel(p.blocks[idx])
+  p.blocks[idx].label = "LA" & p.blocks[idx].id.rope
   blockLeaveActions(p,
     p.nestedTryStmts.len - p.blocks[idx].nestedTryStmts,
     p.inExceptBlockLen - p.blocks[idx].nestedExceptStmts)
   genLineDir(p, t)
-  lineF(p, cpsStmts, "goto $1;$n", [label])
+  lineF(p, cpsStmts, "goto $1;$n", [p.blocks[idx].label])
 
 proc raiseExit(p: BProc) =
   assert p.config.exc == excGoto
@@ -722,21 +727,19 @@ proc finallyActions(p: BProc) =
     if finallyBlock != nil:
       genSimpleBlock(p, finallyBlock[0])
 
-proc raiseInstr(p: BProc): Rope =
+proc raiseInstr(p: BProc; result: var Rope) =
   if p.config.exc == excGoto:
     let L = p.nestedTryStmts.len
     if L == 0:
       p.flags.incl beforeRetNeeded
       # easy case, simply goto 'ret':
-      result = ropecg(p.module, "goto BeforeRet_;$n", [])
+      result.add ropecg(p.module, "goto BeforeRet_;$n", [])
     else:
       # raise inside an 'except' must go to the finally block,
       # raise outside an 'except' block must go to the 'except' list.
-      result = ropecg(p.module, "goto LA$1_;$n",
+      result.add ropecg(p.module, "goto LA$1_;$n",
         [p.nestedTryStmts[L-1].label])
       # + ord(p.nestedTryStmts[L-1].inExcept)])
-  else:
-    result = nil
 
 proc genRaiseStmt(p: BProc, t: PNode) =
   if t[0].kind != nkEmpty:
@@ -765,12 +768,10 @@ proc genRaiseStmt(p: BProc, t: PNode) =
     genLineDir(p, t)
     # reraise the last exception:
     if p.config.exc == excCpp:
-      line(p, cpsStmts, ~"throw;$n")
+      line(p, cpsStmts, "throw;\n")
     else:
       linefmt(p, cpsStmts, "#reraiseException();$n", [])
-  let gotoInstr = raiseInstr(p)
-  if gotoInstr != nil:
-    line(p, cpsStmts, gotoInstr)
+  raiseInstr(p, p.s(cpsStmts))
 
 template genCaseGenericBranch(p: BProc, b: PNode, e: TLoc,
                           rangeFormat, eqFormat: FormatStr, labl: TLabel) =
@@ -830,17 +831,27 @@ template genCaseGeneric(p: BProc, t: PNode, d: var TLoc,
   fixLabel(p, lend)
 
 proc genCaseStringBranch(p: BProc, b: PNode, e: TLoc, labl: TLabel,
+                         stringKind: TTypeKind,
                          branches: var openArray[Rope]) =
   var x: TLoc
   for i in 0..<b.len - 1:
     assert(b[i].kind != nkRange)
     initLocExpr(p, b[i], x)
-    assert(b[i].kind in {nkStrLit..nkTripleStrLit})
-    var j = int(hashString(p.config, b[i].strVal) and high(branches))
-    appcg(p.module, branches[j], "if (#eqStrings($1, $2)) goto $3;$n",
+    var j: int
+    case b[i].kind
+    of nkStrLit..nkTripleStrLit:
+      j = int(hashString(p.config, b[i].strVal) and high(branches))
+    of nkNilLit: j = 0
+    else:
+      assert false, "invalid string case branch node kind"
+    if stringKind == tyCstring:
+      appcg(p.module, branches[j], "if (#eqCstrings($1, $2)) goto $3;$n",
+         [rdLoc(e), rdLoc(x), labl])
+    else:
+      appcg(p.module, branches[j], "if (#eqStrings($1, $2)) goto $3;$n",
          [rdLoc(e), rdLoc(x), labl])
 
-proc genStringCase(p: BProc, t: PNode, d: var TLoc) =
+proc genStringCase(p: BProc, t: PNode, stringKind: TTypeKind, d: var TLoc) =
   # count how many constant strings there are in the case:
   var strings = 0
   for i in 1..<t.len:
@@ -856,17 +867,23 @@ proc genStringCase(p: BProc, t: PNode, d: var TLoc) =
       inc(p.labels)
       if t[i].kind == nkOfBranch:
         genCaseStringBranch(p, t[i], a, "LA" & rope(p.labels) & "_",
-                            branches)
+                            stringKind, branches)
       else:
         # else statement: nothing to do yet
         # but we reserved a label, which we use later
         discard
-    linefmt(p, cpsStmts, "switch (#hashString($1) & $2) {$n",
-            [rdLoc(a), bitMask])
+    if stringKind == tyCstring:
+      linefmt(p, cpsStmts, "switch (#hashCstring($1) & $2) {$n",
+              [rdLoc(a), bitMask])
+    else:
+      linefmt(p, cpsStmts, "switch (#hashString($1) & $2) {$n",
+              [rdLoc(a), bitMask])
     for j in 0..high(branches):
-      if branches[j] != nil:
+      if branches[j] != "":
+        var lit = newRopeAppender()
+        intLiteral(j, lit)
         lineF(p, cpsStmts, "case $1: $n$2break;$n",
-             [intLiteral(j), branches[j]])
+             [lit, branches[j]])
     lineF(p, cpsStmts, "}$n", []) # else statement:
     if t[^1].kind != nkOfBranch:
       lineF(p, cpsStmts, "goto LA$1_;$n", [rope(p.labels)])
@@ -874,7 +891,10 @@ proc genStringCase(p: BProc, t: PNode, d: var TLoc) =
     var lend = genCaseSecondPass(p, t, d, labId, t.len-1)
     fixLabel(p, lend)
   else:
-    genCaseGeneric(p, t, d, "", "if (#eqStrings($1, $2)) goto $3;$n")
+    if stringKind == tyCstring:
+      genCaseGeneric(p, t, d, "", "if (#eqCstrings($1, $2)) goto $3;$n")
+    else:
+      genCaseGeneric(p, t, d, "", "if (#eqStrings($1, $2)) goto $3;$n")
 
 proc branchHasTooBigRange(b: PNode): bool =
   for it in b:
@@ -897,16 +917,22 @@ proc genCaseRange(p: BProc, branch: PNode) =
   for j in 0..<branch.len-1:
     if branch[j].kind == nkRange:
       if hasSwitchRange in CC[p.config.cCompiler].props:
-        lineF(p, cpsStmts, "case $1 ... $2:$n", [
-            genLiteral(p, branch[j][0]),
-            genLiteral(p, branch[j][1])])
+        var litA = newRopeAppender()
+        var litB = newRopeAppender()
+        genLiteral(p, branch[j][0], litA)
+        genLiteral(p, branch[j][1], litB)
+        lineF(p, cpsStmts, "case $1 ... $2:$n", [litA, litB])
       else:
         var v = copyNode(branch[j][0])
         while v.intVal <= branch[j][1].intVal:
-          lineF(p, cpsStmts, "case $1:$n", [genLiteral(p, v)])
+          var litA = newRopeAppender()
+          genLiteral(p, v, litA)
+          lineF(p, cpsStmts, "case $1:$n", [litA])
           inc(v.intVal)
     else:
-      lineF(p, cpsStmts, "case $1:$n", [genLiteral(p, branch[j])])
+      var litA = newRopeAppender()
+      genLiteral(p, branch[j], litA)
+      lineF(p, cpsStmts, "case $1:$n", [litA])
 
 proc genOrdinalCase(p: BProc, n: PNode, d: var TLoc) =
   # analyse 'case' statement:
@@ -918,7 +944,7 @@ proc genOrdinalCase(p: BProc, n: PNode, d: var TLoc) =
   var lend = if splitPoint > 0: genIfForCaseUntil(p, n, d,
                     rangeFormat = "if ($1 >= $2 && $1 <= $3) goto $4;$n",
                     eqFormat = "if ($1 == $2) goto $3;$n",
-                    splitPoint, a) else: nil
+                    splitPoint, a) else: ""
 
   # generate switch part (might be empty):
   if splitPoint+1 < n.len:
@@ -939,7 +965,7 @@ proc genOrdinalCase(p: BProc, n: PNode, d: var TLoc) =
     if (hasAssume in CC[p.config.cCompiler].props) and not hasDefault:
       lineF(p, cpsStmts, "default: __assume(0);$n", [])
     lineF(p, cpsStmts, "}$n", [])
-  if lend != nil: fixLabel(p, lend)
+  if lend != "": fixLabel(p, lend)
 
 proc genCase(p: BProc, t: PNode, d: var TLoc) =
   genLineDir(p, t)
@@ -947,7 +973,9 @@ proc genCase(p: BProc, t: PNode, d: var TLoc) =
     getTemp(p, t.typ, d)
   case skipTypes(t[0].typ, abstractVarRange).kind
   of tyString:
-    genStringCase(p, t, d)
+    genStringCase(p, t, tyString, d)
+  of tyCstring:
+    genStringCase(p, t, tyCstring, d)
   of tyFloat..tyFloat128:
     genCaseGeneric(p, t, d, "if ($1 >= $2 && $1 <= $3) goto $4;$n",
                             "if ($1 == $2) goto $3;$n")
@@ -1038,7 +1066,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
       linefmt(p, cpsStmts, "#popCurrentException();$n", [])
       endBlock(p)
     else:
-      var orExpr = Rope(nil)
+      var orExpr = newRopeAppender()
       var exvar = PNode(nil)
       for j in 0..<t[i].len - 1:
         var typeNode = t[i][j]
@@ -1049,22 +1077,24 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
         if isImportedException(typeNode.typ, p.config):
           hasImportedCppExceptions = true
         else:
-          if orExpr != nil: orExpr.add("||")
-          let checkFor = if optTinyRtti in p.config.globalOptions:
-            genTypeInfo2Name(p.module, typeNode.typ)
-          else:
-            genTypeInfoV1(p.module, typeNode.typ, typeNode.info)
+          if orExpr.len != 0: orExpr.add("||")
           let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type"
-          appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor])
+          if optTinyRtti in p.config.globalOptions:
+            let checkFor = $getObjDepth(typeNode.typ)
+            appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor, $genDisplayElem(MD5Digest(hashType(typeNode.typ, p.config)))])
+          else:
+            let checkFor = genTypeInfoV1(p.module, typeNode.typ, typeNode.info)
+            appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor])
 
-      if orExpr != nil:
+      if orExpr.len != 0:
         if hasIf:
           startBlock(p, "else if ($1) {$n", [orExpr])
         else:
           startBlock(p, "if ($1) {$n", [orExpr])
           hasIf = true
         if exvar != nil:
-          fillLoc(exvar.sym.loc, locTemp, exvar, mangleLocalName(p, exvar.sym), OnStack)
+          fillLocalName(p, exvar.sym)
+          fillLoc(exvar.sym.loc, locTemp, exvar, OnStack)
           linefmt(p, cpsStmts, "$1 $2 = T$3_;$n", [getTypeDesc(p.module, exvar.sym.typ),
             rdLoc(exvar.sym.loc), rope(etmp+1)])
         # we handled the error:
@@ -1105,7 +1135,8 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
             typeNode = t[i][j][1]
             if isImportedException(typeNode.typ, p.config):
               let exvar = t[i][j][2] # ex1 in `except ExceptType as ex1:`
-              fillLoc(exvar.sym.loc, locTemp, exvar, mangleLocalName(p, exvar.sym), OnStack)
+              fillLocalName(p, exvar.sym)
+              fillLoc(exvar.sym.loc, locTemp, exvar, OnStack)
               startBlock(p, "catch ($1& $2) {$n", getTypeDesc(p.module, typeNode.typ), rdLoc(exvar.sym.loc))
               genExceptBranchBody(t[i][^1])  # exception handler body will duplicated for every type
               endBlock(p)
@@ -1158,7 +1189,7 @@ proc genTryCppOld(p: BProc, t: PNode, d: var TLoc) =
   if not isEmptyType(t.typ) and d.k == locNone:
     getTemp(p, t.typ, d)
   genLineDir(p, t)
-  discard cgsym(p.module, "popCurrentExceptionEx")
+  cgsym(p.module, "popCurrentExceptionEx")
   let fin = if t[^1].kind == nkFinally: t[^1] else: nil
   p.nestedTryStmts.add((fin, false, 0.Natural))
   startBlock(p, "try {$n")
@@ -1184,7 +1215,8 @@ proc genTryCppOld(p: BProc, t: PNode, d: var TLoc) =
       for j in 0..<t[i].len-1:
         if t[i][j].isInfixAs():
           let exvar = t[i][j][2] # ex1 in `except ExceptType as ex1:`
-          fillLoc(exvar.sym.loc, locTemp, exvar, mangleLocalName(p, exvar.sym), OnUnknown)
+          fillLocalName(p, exvar.sym)
+          fillLoc(exvar.sym.loc, locTemp, exvar, OnUnknown)
           startBlock(p, "catch ($1& $2) {$n", getTypeDesc(p.module, t[i][j][1].typ), rdLoc(exvar.sym.loc))
         else:
           startBlock(p, "catch ($1&) {$n", getTypeDesc(p.module, t[i][j].typ))
@@ -1199,7 +1231,7 @@ proc genTryCppOld(p: BProc, t: PNode, d: var TLoc) =
       # finally requires catch all presence
       startBlock(p, "catch (...) {$n")
       genStmts(p, t[^1][0])
-      line(p, cpsStmts, ~"throw;$n")
+      line(p, cpsStmts, "throw;\n")
       endBlock(p)
 
     genSimpleBlock(p, t[^1][0])
@@ -1261,16 +1293,17 @@ proc genTryGoto(p: BProc; t: PNode; d: var TLoc) =
       linefmt(p, cpsStmts, "*nimErr_ = NIM_FALSE;$n", [])
       expr(p, t[i][0], d)
     else:
-      var orExpr: Rope = nil
+      var orExpr = newRopeAppender()
       for j in 0..<t[i].len - 1:
         assert(t[i][j].kind == nkType)
-        if orExpr != nil: orExpr.add("||")
-        let checkFor = if optTinyRtti in p.config.globalOptions:
-          genTypeInfo2Name(p.module, t[i][j].typ)
-        else:
-          genTypeInfoV1(p.module, t[i][j].typ, t[i][j].info)
+        if orExpr.len != 0: orExpr.add("||")
         let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type"
-        appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor])
+        if optTinyRtti in p.config.globalOptions:
+          let checkFor = $getObjDepth(t[i][j].typ)
+          appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor,  $genDisplayElem(MD5Digest(hashType(t[i][j].typ, p.config)))])
+        else:
+          let checkFor = genTypeInfoV1(p.module, t[i][j].typ, t[i][j].info)
+          appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor])
 
       if i > 1: line(p, cpsStmts, "else ")
       startBlock(p, "if ($1) {$n", [orExpr])
@@ -1346,7 +1379,7 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
   else:
     p.flags.incl noSafePoints
   genLineDir(p, t)
-  discard cgsym(p.module, "Exception")
+  cgsym(p.module, "Exception")
   var safePoint: Rope
   if not quirkyExceptions:
     safePoint = getTempName(p.module)
@@ -1356,8 +1389,24 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
       linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint])
     elif isDefined(p.config, "nimSigSetjmp"):
       linefmt(p, cpsStmts, "$1.status = sigsetjmp($1.context, 0);$n", [safePoint])
+    elif isDefined(p.config, "nimBuiltinSetjmp"):
+      linefmt(p, cpsStmts, "$1.status = __builtin_setjmp($1.context);$n", [safePoint])
     elif isDefined(p.config, "nimRawSetjmp"):
-      linefmt(p, cpsStmts, "$1.status = _setjmp($1.context);$n", [safePoint])
+      if isDefined(p.config, "mswindows"):
+        if isDefined(p.config, "vcc") or isDefined(p.config, "clangcl"):
+          # For the vcc compiler, use `setjmp()` with one argument.
+          # See https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setjmp?view=msvc-170
+          linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint])
+        else:
+          # The Windows `_setjmp()` takes two arguments, with the second being an
+          # undocumented buffer used by the SEH mechanism for stack unwinding.
+          # Mingw-w64 has been trying to get it right for years, but it's still
+          # prone to stack corruption during unwinding, so we disable that by setting
+          # it to NULL.
+          # More details: https://github.com/status-im/nimbus-eth2/issues/3121
+          linefmt(p, cpsStmts, "$1.status = _setjmp($1.context, 0);$n", [safePoint])
+      else:
+        linefmt(p, cpsStmts, "$1.status = _setjmp($1.context);$n", [safePoint])
     else:
       linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint])
     lineCg(p, cpsStmts, "if ($1.status == 0) {$n", [safePoint])
@@ -1389,16 +1438,17 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
       linefmt(p, cpsStmts, "#popCurrentException();$n", [])
       endBlock(p)
     else:
-      var orExpr: Rope = nil
+      var orExpr = newRopeAppender()
       for j in 0..<t[i].len - 1:
         assert(t[i][j].kind == nkType)
-        if orExpr != nil: orExpr.add("||")
-        let checkFor = if optTinyRtti in p.config.globalOptions:
-          genTypeInfo2Name(p.module, t[i][j].typ)
-        else:
-          genTypeInfoV1(p.module, t[i][j].typ, t[i][j].info)
+        if orExpr.len != 0: orExpr.add("||")
         let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type"
-        appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor])
+        if optTinyRtti in p.config.globalOptions:
+          let checkFor = $getObjDepth(t[i][j].typ)
+          appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor,  $genDisplayElem(MD5Digest(hashType(t[i][j].typ, p.config)))])
+        else:
+          let checkFor = genTypeInfoV1(p.module, t[i][j].typ, t[i][j].info)
+          appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor])
 
       if i > 1: line(p, cpsStmts, "else ")
       startBlock(p, "if ($1) {$n", [orExpr])
@@ -1424,7 +1474,7 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
   if not quirkyExceptions:
     linefmt(p, cpsStmts, "if ($1.status != 0) #reraiseException();$n", [safePoint])
 
-proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false): Rope =
+proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false; result: var Rope) =
   var res = ""
   for it in t.sons:
     case it.kind
@@ -1440,13 +1490,8 @@ proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false): Rope =
         res.add($getTypeDesc(p.module, sym.typ))
       else:
         discard getTypeDesc(p.module, skipTypes(sym.typ, abstractPtrs))
-        var r = sym.loc.r
-        if r == nil:
-          # if no name has already been given,
-          # it doesn't matter much:
-          r = mangleName(p.module, sym)
-          sym.loc.r = r       # but be consequent!
-        res.add($r)
+        fillBackendName(p.module, sym)
+        res.add($sym.loc.r)
     of nkTypeOfExpr:
       res.add($getTypeDesc(p.module, it.typ))
     else:
@@ -1471,12 +1516,13 @@ proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false): Rope =
           result.add("\\n\"\n")
   else:
     res.add("\L")
-    result = res.rope
+    result.add res.rope
 
 proc genAsmStmt(p: BProc, t: PNode) =
   assert(t.kind == nkAsmStmt)
   genLineDir(p, t)
-  var s = genAsmOrEmitStmt(p, t, isAsmStmt=true)
+  var s = newRopeAppender()
+  genAsmOrEmitStmt(p, t, isAsmStmt=true, s)
   # see bug #2362, "top level asm statements" seem to be a mis-feature
   # but even if we don't do this, the example in #2362 cannot possibly
   # work:
@@ -1484,18 +1530,20 @@ proc genAsmStmt(p: BProc, t: PNode) =
     # top level asm statement?
     p.module.s[cfsProcHeaders].add runtimeFormat(CC[p.config.cCompiler].asmStmtFrmt, [s])
   else:
-    p.s(cpsStmts).add indentLine(p, runtimeFormat(CC[p.config.cCompiler].asmStmtFrmt, [s]))
+    addIndent p, p.s(cpsStmts)
+    p.s(cpsStmts).add runtimeFormat(CC[p.config.cCompiler].asmStmtFrmt, [s])
 
 proc determineSection(n: PNode): TCFileSection =
   result = cfsProcHeaders
   if n.len >= 1 and n[0].kind in {nkStrLit..nkTripleStrLit}:
     let sec = n[0].strVal
-    if sec.startsWith("/*TYPESECTION*/"): result = cfsTypes
+    if sec.startsWith("/*TYPESECTION*/"): result = cfsForwardTypes # TODO WORKAROUND
     elif sec.startsWith("/*VARSECTION*/"): result = cfsVars
     elif sec.startsWith("/*INCLUDESECTION*/"): result = cfsHeaders
 
 proc genEmit(p: BProc, t: PNode) =
-  var s = genAsmOrEmitStmt(p, t[1])
+  var s = newRopeAppender()
+  genAsmOrEmitStmt(p, t[1], false, s)
   if p.prc == nil:
     # top level emit pragma?
     let section = determineSection(t[1])
@@ -1506,14 +1554,14 @@ proc genEmit(p: BProc, t: PNode) =
     line(p, cpsStmts, s)
 
 proc genPragma(p: BProc, n: PNode) =
-  for it in n.sons:
+  for i in 0..<n.len:
+    let it = n[i]
     case whichPragma(it)
     of wEmit: genEmit(p, it)
-    of wInjectStmt:
-      var p = newProc(nil, p.module)
-      p.options.excl {optLineTrace, optStackTrace}
-      genStmts(p, it[1])
-      p.module.injectStmt = p.s(cpsStmts)
+    of wPush:
+      processPushBackendOption(p.optionsStack, p.options, n, i+1)
+    of wPop:
+      processPopBackendOption(p.optionsStack, p.options)
     else: discard
 
 
@@ -1525,10 +1573,12 @@ proc genDiscriminantCheck(p: BProc, a, tmp: TLoc, objtype: PType,
   if not containsOrIncl(p.module.declaredThings, field.id):
     appcg(p.module, cfsVars, "extern $1",
           [discriminatorTableDecl(p.module, t, field)])
+  var lit = newRopeAppender()
+  intLiteral(toInt64(lengthOrd(p.config, field.typ))+1, lit)
   lineCg(p, cpsStmts,
         "#FieldDiscriminantCheck((NI)(NU)($1), (NI)(NU)($2), $3, $4);$n",
         [rdLoc(a), rdLoc(tmp), discriminatorTableName(p.module, t, field),
-         intLiteral(toInt64(lengthOrd(p.config, field.typ))+1)])
+         lit])
 
 when false:
   proc genCaseObjDiscMapping(p: BProc, e: PNode, t: PType, field: PSym; d: var TLoc) =
@@ -1553,7 +1603,7 @@ proc asgnFieldDiscriminant(p: BProc, e: PNode) =
   initLocExpr(p, e[0], a)
   getTemp(p, a.t, tmp)
   expr(p, e[1], tmp)
-  if optTinyRtti notin p.config.globalOptions:
+  if optTinyRtti notin p.config.globalOptions and p.inUncheckedAssignSection == 0:
     let field = dotExpr[1].sym
     genDiscriminantCheck(p, a, tmp, dotExpr[0].typ, field)
     message(p.config, e.info, warnCaseTransition)
@@ -1574,6 +1624,7 @@ proc genAsgn(p: BProc, e: PNode, fastAsgn: bool) =
     initLoc(a, locNone, le, OnUnknown)
     a.flags.incl(lfEnforceDeref)
     a.flags.incl(lfPrepareForMutation)
+    genLineDir(p, le) # it can be a nkBracketExpr, which may raise
     expr(p, le, a)
     a.flags.excl(lfPrepareForMutation)
     if fastAsgn: incl(a.flags, lfNoDeepCopy)

compiler/ccgthreadvars.nim

@@ -44,13 +44,13 @@ proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) =
     m.s[cfsVars].addf(" $1;$n", [s.loc.r])
 
 proc generateThreadLocalStorage(m: BModule) =
-  if m.g.nimtv != nil and (usesThreadVars in m.flags or sfMainModule in m.module.flags):
+  if m.g.nimtv != "" and (usesThreadVars in m.flags or sfMainModule in m.module.flags):
     for t in items(m.g.nimtvDeps): discard getTypeDesc(m, t)
     finishTypeDescriptions(m)
     m.s[cfsSeqTypes].addf("typedef struct {$1} NimThreadVars;$n", [m.g.nimtv])
 
 proc generateThreadVarsSize(m: BModule) =
-  if m.g.nimtv != nil:
+  if m.g.nimtv != "":
     let externc = if m.config.backend == backendCpp or
                        sfCompileToCpp in m.module.flags: "extern \"C\" "
                   else: ""

compiler/ccgtrav.nim

@@ -34,7 +34,7 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, n: PNode;
     if (n[0].kind != nkSym): internalError(c.p.config, n.info, "genTraverseProc")
     var p = c.p
     let disc = n[0].sym
-    if disc.loc.r == nil: fillObjectFields(c.p.module, typ)
+    if disc.loc.r == "": fillObjectFields(c.p.module, typ)
     if disc.loc.t == nil:
       internalError(c.p.config, n.info, "genTraverseProc()")
     lineF(p, cpsStmts, "switch ($1.$2) {$n", [accessor, disc.loc.r])
@@ -51,7 +51,7 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, n: PNode;
   of nkSym:
     let field = n.sym
     if field.typ.kind == tyVoid: return
-    if field.loc.r == nil: fillObjectFields(c.p.module, typ)
+    if field.loc.r == "": fillObjectFields(c.p.module, typ)
     if field.loc.t == nil:
       internalError(c.p.config, n.info, "genTraverseProc()")
     genTraverseProc(c, "$1.$2" % [accessor, field.loc.r], field.loc.t)
@@ -76,7 +76,8 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, typ: PType) =
     let arraySize = lengthOrd(c.p.config, typ[0])
     var i: TLoc
     getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo, tyInt), i)
-    let oldCode = p.s(cpsStmts)
+    var oldCode = p.s(cpsStmts)
+    freeze oldCode
     linefmt(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",
             [i.r, arraySize])
     let oldLen = p.s(cpsStmts).len
@@ -120,7 +121,8 @@ proc genTraverseProcSeq(c: TTraversalClosure, accessor: Rope, typ: PType) =
   assert typ.kind == tySequence
   var i: TLoc
   getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo, tyInt), i)
-  let oldCode = p.s(cpsStmts)
+  var oldCode = p.s(cpsStmts)
+  freeze oldCode
   var a: TLoc
   a.r = accessor
 

compiler/ccgtypes.nim

@@ -12,9 +12,7 @@
 # ------------------------- Name Mangling --------------------------------
 
 import sighashes, modulegraphs
-from lowerings import createObj
-
-proc genProcHeader(m: BModule, prc: PSym, asPtr: bool = false): Rope
+import std/md5
 
 proc isKeyword(w: PIdent): bool =
   # Nim and C++ share some keywords
@@ -36,27 +34,23 @@ proc mangleField(m: BModule; name: PIdent): string =
   if isKeyword(name):
     result.add "_0"
 
-proc mangleName(m: BModule; s: PSym): Rope =
-  result = s.loc.r
-  if result == nil:
-    result = s.name.s.mangle.rope
-    result.add "_"
+proc fillBackendName(m: BModule; s: PSym) =
+  if s.loc.r == "":
+    var result = s.name.s.mangle.rope
+    result.add "__"
     result.add m.g.graph.ifaces[s.itemId.module].uniqueName
     result.add "_"
     result.add rope s.itemId.item
     if m.hcrOn:
       result.add "_"
-      result.add(idOrSig(s, m.module.name.s.mangle, m.sigConflicts))
+      result.add(idOrSig(s, m.module.name.s.mangle, m.sigConflicts, m.config))
     s.loc.r = result
     writeMangledName(m.ndi, s, m.config)
 
-proc mangleParamName(m: BModule; s: PSym): Rope =
-  ## we cannot use 'sigConflicts' here since we have a BModule, not a BProc.
-  ## Fortunately C's scoping rules are sane enough so that that doesn't
-  ## cause any trouble.
-  result = s.loc.r
-  if result == nil:
+proc fillParamName(m: BModule; s: PSym) =
+  if s.loc.r == "":
     var res = s.name.s.mangle
+    res.add idOrSig(s, res, m.sigConflicts, m.config)
     # Take into account if HCR is on because of the following scenario:
     #   if a module gets imported and it has some more importc symbols in it,
     # some param names might receive the "_0" suffix to distinguish from what
@@ -73,21 +67,16 @@ proc mangleParamName(m: BModule; s: PSym): Rope =
     # and a function called in main or proxy uses `socket` as a parameter name.
     # That would lead to either needing to reload `proxy` or to overwrite the
     # executable file for the main module, which is running (or both!) -> error.
-    if m.hcrOn or isKeyword(s.name) or m.g.config.cppDefines.contains(res):
-      res.add "_0"
-    result = res.rope
-    s.loc.r = result
+    s.loc.r = res.rope
     writeMangledName(m.ndi, s, m.config)
 
-proc mangleLocalName(p: BProc; s: PSym): Rope =
+proc fillLocalName(p: BProc; s: PSym) =
   assert s.kind in skLocalVars+{skTemp}
   #assert sfGlobal notin s.flags
-  result = s.loc.r
-  if result == nil:
+  if s.loc.r == "":
     var key = s.name.s.mangle
-    shallow(key)
     let counter = p.sigConflicts.getOrDefault(key)
-    result = key.rope
+    var result = key.rope
     if s.kind == skTemp:
       # speed up conflict search for temps (these are quite common):
       if counter != 0: result.add "_" & rope(counter+1)
@@ -103,7 +92,6 @@ proc scopeMangledParam(p: BProc; param: PSym) =
   ## generate unique identifiers reliably (consider that ``var a = a`` is
   ## even an idiom in Nim).
   var key = param.name.s.mangle
-  shallow(key)
   p.sigConflicts.inc(key)
 
 const
@@ -111,13 +99,12 @@ const
                           tyDistinct, tyRange, tyStatic, tyAlias, tySink,
                           tyInferred, tyOwned}
 
-proc typeName(typ: PType): Rope =
+proc typeName(typ: PType; result: var Rope) =
   let typ = typ.skipTypes(irrelevantForBackend)
-  result =
-    if typ.sym != nil and typ.kind in {tyObject, tyEnum}:
-      rope($typ.kind & '_' & typ.sym.name.s.mangle)
-    else:
-      rope($typ.kind)
+  result.add $typ.kind
+  if typ.sym != nil and typ.kind in {tyObject, tyEnum}:
+    result.add "_"
+    result.add typ.sym.name.s.mangle
 
 proc getTypeName(m: BModule; typ: PType; sig: SigHash): Rope =
   var t = typ
@@ -130,14 +117,17 @@ proc getTypeName(m: BModule; typ: PType; sig: SigHash): Rope =
     else:
       break
   let typ = if typ.kind in {tyAlias, tySink, tyOwned}: typ.lastSon else: typ
-  if typ.loc.r == nil:
-    typ.loc.r = typ.typeName & $sig
+  if typ.loc.r == "":
+    typ.typeName(typ.loc.r)
+    typ.loc.r.add $sig
   else:
     when defined(debugSigHashes):
       # check consistency:
-      assert($typ.loc.r == $(typ.typeName & $sig))
+      var tn = newRopeAppender()
+      typ.typeName(tn)
+      assert($typ.loc.r == $(tn & $sig))
   result = typ.loc.r
-  if result == nil: internalError(m.config, "getTypeName: " & $typ.kind)
+  if result == "": internalError(m.config, "getTypeName: " & $typ.kind)
 
 proc mapSetType(conf: ConfigRef; typ: PType): TCTypeKind =
   case int(getSize(conf, typ))
@@ -216,12 +206,19 @@ proc isObjLackingTypeField(typ: PType): bool {.inline.} =
   result = (typ.kind == tyObject) and ((tfFinal in typ.flags) and
       (typ[0] == nil) or isPureObject(typ))
 
-proc isInvalidReturnType(conf: ConfigRef; rettype: PType): bool =
+proc isInvalidReturnType(conf: ConfigRef; typ: PType, isProc = true): bool =
   # Arrays and sets cannot be returned by a C procedure, because C is
   # such a poor programming language.
   # We exclude records with refs too. This enhances efficiency and
   # is necessary for proper code generation of assignments.
-  if rettype == nil: result = true
+  var rettype = typ
+  var isAllowedCall = true
+  if isProc:
+    rettype = rettype[0]
+    isAllowedCall = typ.callConv in {ccClosure, ccInline, ccNimCall}
+  if rettype == nil or (isAllowedCall and
+                    getSize(conf, rettype) > conf.target.floatSize*3):
+    result = true
   else:
     case mapType(conf, rettype, skResult)
     of ctArray:
@@ -256,14 +253,12 @@ proc addAbiCheck(m: BModule, t: PType, name: Rope) =
     m.s[cfsTypeInfo].addf("NIM_STATIC_ASSERT(sizeof($1) == $2, $3);$n", [name, rope(size), msg2.rope])
     # see `testCodegenABICheck` for example error message it generates
 
-from ccgutils import ccgIntroducedPtr
-export ccgIntroducedPtr
 
-proc fillResult(conf: ConfigRef; param: PNode) =
-  fillLoc(param.sym.loc, locParam, param, ~"Result",
+proc fillResult(conf: ConfigRef; param: PNode, proctype: PType) =
+  fillLoc(param.sym.loc, locParam, param, "Result",
           OnStack)
   let t = param.sym.typ
-  if mapReturnType(conf, t) != ctArray and isInvalidReturnType(conf, t):
+  if mapReturnType(conf, t) != ctArray and isInvalidReturnType(conf, proctype):
     incl(param.sym.loc.flags, lfIndirect)
     param.sym.loc.storage = OnUnknown
 
@@ -286,11 +281,11 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope =
   of tyString:
     case detectStrVersion(m)
     of 2:
-      discard cgsym(m, "NimStrPayload")
-      discard cgsym(m, "NimStringV2")
+      cgsym(m, "NimStrPayload")
+      cgsym(m, "NimStringV2")
       result = typeNameOrLiteral(m, typ, "NimStringV2")
     else:
-      discard cgsym(m, "NimStringDesc")
+      cgsym(m, "NimStringDesc")
       result = typeNameOrLiteral(m, typ, "NimStringDesc*")
   of tyCstring: result = typeNameOrLiteral(m, typ, "NCSTRING")
   of tyBool: result = typeNameOrLiteral(m, typ, "NIM_BOOL")
@@ -304,11 +299,11 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope =
     else: internalError(m.config, "tyStatic for getSimpleTypeDesc")
   of tyGenericInst, tyAlias, tySink, tyOwned:
     result = getSimpleTypeDesc(m, lastSon typ)
-  else: result = nil
+  else: result = ""
 
-  if result != nil and typ.isImportedType():
-    let sig = hashType typ
-    if cacheGetType(m.typeCache, sig) == nil:
+  if result != "" and typ.isImportedType():
+    let sig = hashType(typ, m.config)
+    if cacheGetType(m.typeCache, sig) == "":
       m.typeCache[sig] = result
 
 proc pushType(m: BModule, typ: PType) =
@@ -321,7 +316,7 @@ proc getTypePre(m: BModule, typ: PType; sig: SigHash): Rope =
   if typ == nil: result = rope("void")
   else:
     result = getSimpleTypeDesc(m, typ)
-    if result == nil: result = cacheGetType(m.typeCache, sig)
+    if result == "": result = cacheGetType(m.typeCache, sig)
 
 proc structOrUnion(t: PType): Rope =
   let cachedUnion = rope("union")
@@ -342,9 +337,9 @@ proc seqStar(m: BModule): string =
 
 proc getTypeForward(m: BModule, typ: PType; sig: SigHash): Rope =
   result = cacheGetType(m.forwTypeCache, sig)
-  if result != nil: return
+  if result != "": return
   result = getTypePre(m, typ, sig)
-  if result != nil: return
+  if result != "": return
   let concrete = typ.skipTypes(abstractInst)
   case concrete.kind
   of tySequence, tyTuple, tyObject:
@@ -367,16 +362,16 @@ proc getTypeDescWeak(m: BModule; t: PType; check: var IntSet; kind: TSymKind): R
     if isImportedCppType(etB) and t.kind == tyGenericInst:
       result = getTypeDescAux(m, t, check, kind)
     else:
-      result = getTypeForward(m, t, hashType(t))
+      result = getTypeForward(m, t, hashType(t, m.config))
       pushType(m, t)
   of tySequence:
-    let sig = hashType(t)
+    let sig = hashType(t, m.config)
     if optSeqDestructors in m.config.globalOptions:
       if skipTypes(etB[0], typedescInst).kind == tyEmpty:
         internalError(m.config, "cannot map the empty seq type to a C type")
 
       result = cacheGetType(m.forwTypeCache, sig)
-      if result == nil:
+      if result == "":
         result = getTypeName(m, t, sig)
         if not isImportedType(t):
           m.forwTypeCache[sig] = result
@@ -384,7 +379,7 @@ proc getTypeDescWeak(m: BModule; t: PType; check: var IntSet; kind: TSymKind): R
           let payload = result & "_Content"
           addForwardStructFormat(m, rope"struct", payload)
 
-      if cacheGetType(m.typeCache, sig) == nil:
+      if cacheGetType(m.typeCache, sig) == "":
         m.typeCache[sig] = result
         #echo "adding ", sig, " ", typeToString(t), " ", m.module.name.s
         appcg(m, m.s[cfsTypes],
@@ -403,9 +398,9 @@ proc getSeqPayloadType(m: BModule; t: PType): Rope =
   #result = getTypeForward(m, t, hashType(t)) & "_Content"
 
 proc seqV2ContentType(m: BModule; t: PType; check: var IntSet) =
-  let sig = hashType(t)
+  let sig = hashType(t, m.config)
   let result = cacheGetType(m.typeCache, sig)
-  if result == nil:
+  if result == "":
     discard getTypeDescAux(m, t, check, skVar)
   else:
     # little hack for now to prevent multiple definitions of the same
@@ -427,30 +422,31 @@ proc paramStorageLoc(param: PSym): TStorageLoc =
 proc genProcParams(m: BModule, t: PType, rettype, params: var Rope,
                    check: var IntSet, declareEnvironment=true;
                    weakDep=false) =
-  params = nil
-  if t[0] == nil or isInvalidReturnType(m.config, t[0]):
-    rettype = ~"void"
+  params = "("
+  if t[0] == nil or isInvalidReturnType(m.config, t):
+    rettype = "void"
   else:
     rettype = getTypeDescAux(m, t[0], check, skResult)
   for i in 1..<t.n.len:
     if t.n[i].kind != nkSym: internalError(m.config, t.n.info, "genProcParams")
     var param = t.n[i].sym
     if isCompileTimeOnly(param.typ): continue
-    if params != nil: params.add(~", ")
-    fillLoc(param.loc, locParam, t.n[i], mangleParamName(m, param),
+    if params != "(": params.add(", ")
+    fillParamName(m, param)
+    fillLoc(param.loc, locParam, t.n[i],
             param.paramStorageLoc)
     if ccgIntroducedPtr(m.config, param, t[0]):
       params.add(getTypeDescWeak(m, param.typ, check, skParam))
-      params.add(~"*")
+      params.add("*")
       incl(param.loc.flags, lfIndirect)
       param.loc.storage = OnUnknown
     elif weakDep:
       params.add(getTypeDescWeak(m, param.typ, check, skParam))
     else:
       params.add(getTypeDescAux(m, param.typ, check, skParam))
-    params.add(~" ")
+    params.add(" ")
     if sfNoalias in param.flags:
-      params.add(~"NIM_NOALIAS ")
+      params.add("NIM_NOALIAS ")
     params.add(param.loc.r)
     # declare the len field for open arrays:
     var arr = param.typ.skipTypes({tyGenericInst})
@@ -463,54 +459,58 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope,
       params.addf(", NI $1Len_$2", [param.loc.r, j.rope])
       inc(j)
       arr = arr[0].skipTypes({tySink})
-  if t[0] != nil and isInvalidReturnType(m.config, t[0]):
+  if t[0] != nil and isInvalidReturnType(m.config, t):
     var arr = t[0]
-    if params != nil: params.add(", ")
+    if params != "(": params.add(", ")
     if mapReturnType(m.config, t[0]) != ctArray:
-      params.add(getTypeDescWeak(m, arr, check, skResult))
-      params.add("*")
+      if isHeaderFile in m.flags:
+        # still generates types for `--header`
+        params.add(getTypeDescAux(m, arr, check, skResult))
+        params.add("*")
+      else:
+        params.add(getTypeDescWeak(m, arr, check, skResult))
+        params.add("*")
     else:
       params.add(getTypeDescAux(m, arr, check, skResult))
     params.addf(" Result", [])
   if t.callConv == ccClosure and declareEnvironment:
-    if params != nil: params.add(", ")
+    if params != "(": params.add(", ")
     params.add("void* ClE_0")
   if tfVarargs in t.flags:
-    if params != nil: params.add(", ")
+    if params != "(": params.add(", ")
     params.add("...")
-  if params == nil: params.add("void)")
+  if params == "(": params.add("void)")
   else: params.add(")")
-  params = "(" & params
 
 proc mangleRecFieldName(m: BModule; field: PSym): Rope =
   if {sfImportc, sfExportc} * field.flags != {}:
     result = field.loc.r
   else:
     result = rope(mangleField(m, field.name))
-  if result == nil: internalError(m.config, field.info, "mangleRecFieldName")
+  if result == "": internalError(m.config, field.info, "mangleRecFieldName")
 
 proc genRecordFieldsAux(m: BModule, n: PNode,
                         rectype: PType,
-                        check: var IntSet, unionPrefix = ""): Rope =
-  result = nil
+                        check: var IntSet; result: var Rope; unionPrefix = "") =
   case n.kind
   of nkRecList:
     for i in 0..<n.len:
-      result.add(genRecordFieldsAux(m, n[i], rectype, check, unionPrefix))
+      genRecordFieldsAux(m, n[i], rectype, check, result, unionPrefix)
   of nkRecCase:
     if n[0].kind != nkSym: internalError(m.config, n.info, "genRecordFieldsAux")
-    result.add(genRecordFieldsAux(m, n[0], rectype, check, unionPrefix))
+    genRecordFieldsAux(m, n[0], rectype, check, result, unionPrefix)
     # prefix mangled name with "_U" to avoid clashes with other field names,
     # since identifiers are not allowed to start with '_'
-    var unionBody: Rope = nil
+    var unionBody: Rope = ""
     for i in 1..<n.len:
       case n[i].kind
       of nkOfBranch, nkElse:
         let k = lastSon(n[i])
         if k.kind != nkSym:
           let structName = "_" & mangleRecFieldName(m, n[0].sym) & "_" & $i
-          let a = genRecordFieldsAux(m, k, rectype, check, unionPrefix & $structName & ".")
-          if a != nil:
+          var a = newRopeAppender()
+          genRecordFieldsAux(m, k, rectype, check, a, unionPrefix & $structName & ".")
+          if a != "":
             if tfPacked notin rectype.flags:
               unionBody.add("struct {")
             else:
@@ -523,9 +523,9 @@ proc genRecordFieldsAux(m: BModule, n: PNode,
             if tfPacked in rectype.flags and hasAttribute notin CC[m.config.cCompiler].props:
               unionBody.addf("#pragma pack(pop)$n", [])
         else:
-          unionBody.add(genRecordFieldsAux(m, k, rectype, check, unionPrefix))
+          genRecordFieldsAux(m, k, rectype, check, unionBody, unionPrefix)
       else: internalError(m.config, "genRecordFieldsAux(record case branch)")
-    if unionBody != nil:
+    if unionBody != "":
       result.addf("union{$n$1};$n", [unionBody])
   of nkSym:
     let field = n.sym
@@ -539,7 +539,7 @@ proc genRecordFieldsAux(m: BModule, n: PNode,
     # have to recurse via 'getTypeDescAux'. And not doing so prevents problems
     # with heavily templatized C++ code:
     if not isImportedCppType(rectype):
-      let noAlias = if sfNoalias in field.flags: ~" NIM_NOALIAS" else: nil
+      let noAlias = if sfNoalias in field.flags: " NIM_NOALIAS" else: ""
 
       let fieldType = field.loc.lode.typ.skipTypes(abstractInst)
       if fieldType.kind == tyUncheckedArray:
@@ -557,7 +557,8 @@ proc genRecordFieldsAux(m: BModule, n: PNode,
   else: internalError(m.config, n.info, "genRecordFieldsAux()")
 
 proc getRecordFields(m: BModule, typ: PType, check: var IntSet): Rope =
-  result = genRecordFieldsAux(m, typ.n, typ, check)
+  result = newRopeAppender()
+  genRecordFieldsAux(m, typ.n, typ, check, result)
 
 proc fillObjectFields*(m: BModule; typ: PType) =
   # sometimes generic objects are not consistently merged. We patch over
@@ -585,7 +586,7 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope,
 
   if typ.kind == tyObject:
     if typ[0] == nil:
-      if (typ.sym != nil and sfPure in typ.sym.flags) or tfFinal in typ.flags:
+      if lacksMTypeField(typ):
         appcg(m, result, " {$n", [])
       else:
         if optTinyRtti in m.config.globalOptions:
@@ -614,7 +615,7 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope,
     result.addf(" {$n", [name])
 
   let desc = getRecordFields(m, typ, check)
-  if desc == nil and not hasField:
+  if desc == "" and not hasField:
     result.addf("char dummy;$n", [])
   else:
     result.add(desc)
@@ -625,11 +626,11 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope,
 proc getTupleDesc(m: BModule, typ: PType, name: Rope,
                   check: var IntSet): Rope =
   result = "$1 $2 {$n" % [structOrUnion(typ), name]
-  var desc: Rope = nil
+  var desc: Rope = ""
   for i in 0..<typ.len:
     desc.addf("$1 Field$2;$n",
          [getTypeDescAux(m, typ[i], check, skField), rope(i)])
-  if desc == nil: result.add("char dummy;\L")
+  if desc == "": result.add("char dummy;\L")
   else: result.add(desc)
   result.add("};\L")
 
@@ -663,12 +664,12 @@ proc resolveStarsInCppType(typ: PType, idx, stars: int): PType =
                else: result.elemType
 
 proc getOpenArrayDesc(m: BModule, t: PType, check: var IntSet; kind: TSymKind): Rope =
-  let sig = hashType(t)
+  let sig = hashType(t, m.config)
   if kind == skParam:
     result = getTypeDescWeak(m, t[0], check, kind) & "*"
   else:
     result = cacheGetType(m.typeCache, sig)
-    if result == nil:
+    if result == "":
       result = getTypeName(m, t, sig)
       m.typeCache[sig] = result
       let elemType = getTypeDescWeak(m, t[0], check, kind)
@@ -687,14 +688,14 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin
     # C type generation into an analysis and a code generation phase somehow.
   if t.sym != nil: useHeader(m, t.sym)
   if t != origTyp and origTyp.sym != nil: useHeader(m, origTyp.sym)
-  let sig = hashType(origTyp)
+  let sig = hashType(origTyp, m.config)
 
   defer: # defer is the simplest in this case
     if isImportedType(t) and not m.typeABICache.containsOrIncl(sig):
       addAbiCheck(m, t, result)
 
   result = getTypePre(m, t, sig)
-  if result != nil and t.kind != tyOpenArray:
+  if result != "" and t.kind != tyOpenArray:
     excl(check, t.id)
     return
   case t.kind
@@ -716,7 +717,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin
         result = getTypeDescAux(m, et, check, kind) & star
       else:
         # no restriction! We have a forward declaration for structs
-        let name = getTypeForward(m, et, hashType et)
+        let name = getTypeForward(m, et, hashType(et, m.config))
         result = name & star
         m.typeCache[sig] = result
     of tySequence:
@@ -725,7 +726,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin
         m.typeCache[sig] = result
       else:
         # no restriction! We have a forward declaration for structs
-        let name = getTypeForward(m, et, hashType et)
+        let name = getTypeForward(m, et, hashType(et, m.config))
         result = name & seqStar(m) & star
         m.typeCache[sig] = result
         pushType(m, et)
@@ -737,7 +738,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin
     result = getOpenArrayDesc(m, t, check, kind)
   of tyEnum:
     result = cacheGetType(m.typeCache, sig)
-    if result == nil:
+    if result == "":
       result = getTypeName(m, origTyp, sig)
       if not (isImportedCppType(t) or
           (sfImportc in t.sym.flags and t.sym.magic == mNone)):
@@ -785,12 +786,12 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin
       # we cannot use getTypeForward here because then t would be associated
       # with the name of the struct, not with the pointer to the struct:
       result = cacheGetType(m.forwTypeCache, sig)
-      if result == nil:
+      if result == "":
         result = getTypeName(m, origTyp, sig)
         if not isImportedType(t):
           addForwardStructFormat(m, structOrUnion(t), result)
         m.forwTypeCache[sig] = result
-      assert(cacheGetType(m.typeCache, sig) == nil)
+      assert(cacheGetType(m.typeCache, sig) == "")
       m.typeCache[sig] = result & seqStar(m)
       if not isImportedType(t):
         if skipTypes(t[0], typedescInst).kind != tyEmpty:
@@ -825,40 +826,42 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin
       m.s[cfsTypes].addf("typedef $1 $2[$3];$n",
            [foo, result, rope(n)])
   of tyObject, tyTuple:
-    if isImportedCppType(t) and origTyp.kind == tyGenericInst:
-      let cppName = getTypeName(m, t, sig)
+    let tt = origTyp.skipTypes({tyDistinct})
+    if isImportedCppType(t) and tt.kind == tyGenericInst:
+      let cppNameAsRope = getTypeName(m, t, sig)
+      let cppName = $cppNameAsRope
       var i = 0
       var chunkStart = 0
 
       template addResultType(ty: untyped) =
         if ty == nil or ty.kind == tyVoid:
-          result.add(~"void")
+          result.add("void")
         elif ty.kind == tyStatic:
           internalAssert m.config, ty.n != nil
           result.add ty.n.renderTree
         else:
           result.add getTypeDescAux(m, ty, check, kind)
 
-      while i < cppName.data.len:
-        if cppName.data[i] == '\'':
+      while i < cppName.len:
+        if cppName[i] == '\'':
           var chunkEnd = i-1
           var idx, stars: int
-          if scanCppGenericSlot(cppName.data, i, idx, stars):
-            result.add cppName.data.substr(chunkStart, chunkEnd)
+          if scanCppGenericSlot(cppName, i, idx, stars):
+            result.add cppName.substr(chunkStart, chunkEnd)
             chunkStart = i
 
-            let typeInSlot = resolveStarsInCppType(origTyp, idx + 1, stars)
+            let typeInSlot = resolveStarsInCppType(tt, idx + 1, stars)
             addResultType(typeInSlot)
         else:
           inc i
 
       if chunkStart != 0:
-        result.add cppName.data.substr(chunkStart)
+        result.add cppName.substr(chunkStart)
       else:
-        result = cppName & "<"
-        for i in 1..<origTyp.len-1:
+        result = cppNameAsRope & "<"
+        for i in 1..<tt.len-1:
           if i > 1: result.add(" COMMA ")
-          addResultType(origTyp[i])
+          addResultType(tt[i])
         result.add("> ")
       # always call for sideeffects:
       assert t.kind != tyTuple
@@ -866,13 +869,13 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin
       # The resulting type will include commas and these won't play well
       # with the C macros for defining procs such as N_NIMCALL. We must
       # create a typedef for the type and use it in the proc signature:
-      let typedefName = ~"TY" & $sig
+      let typedefName = "TY" & $sig
       m.s[cfsTypes].addf("typedef $1 $2;$n", [result, typedefName])
       m.typeCache[sig] = typedefName
       result = typedefName
     else:
       result = cacheGetType(m.forwTypeCache, sig)
-      if result == nil:
+      if result == "":
         result = getTypeName(m, origTyp, sig)
         m.forwTypeCache[sig] = result
         if not isImportedType(t):
@@ -888,7 +891,9 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin
           discard # addAbiCheck(m, t, result) # already handled elsewhere
   of tySet:
     # Don't use the imported name as it may be scoped: 'Foo::SomeKind'
-    result = $t.kind & '_' & t.lastSon.typeName & $t.lastSon.hashType
+    result = rope("tySet_")
+    t.lastSon.typeName(result)
+    result.add $t.lastSon.hashType(m.config)
     m.typeCache[sig] = result
     if not isImportedType(t):
       let s = int(getSize(m.config, t))
@@ -901,7 +906,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin
     result = getTypeDescAux(m, lastSon(t), check, kind)
   else:
     internalError(m.config, "getTypeDescAux(" & $t.kind & ')')
-    result = nil
+    result = ""
   # fixes bug #145:
   excl(check, t.id)
 
@@ -952,21 +957,12 @@ proc isReloadable(m: BModule, prc: PSym): bool =
 proc isNonReloadable(m: BModule, prc: PSym): bool =
   return m.hcrOn and sfNonReloadable in prc.flags
 
-proc genProcHeader(m: BModule, prc: PSym, asPtr: bool = false): Rope =
-  var
-    rettype, params: Rope
+proc genProcHeader(m: BModule, prc: PSym; result: var Rope; asPtr: bool = false) =
   # using static is needed for inline procs
-  if lfExportLib in prc.loc.flags:
-    if isHeaderFile in m.flags:
-      result.add "N_LIB_IMPORT "
-    else:
-      result.add "N_LIB_EXPORT "
-  elif prc.typ.callConv == ccInline or asPtr or isNonReloadable(m, prc):
-    result.add "static "
-  elif sfImportc notin prc.flags:
-    result.add "N_LIB_PRIVATE "
   var check = initIntSet()
-  fillLoc(prc.loc, locProc, prc.ast[namePos], mangleName(m, prc), OnUnknown)
+  fillBackendName(m, prc)
+  fillLoc(prc.loc, locProc, prc.ast[namePos], OnUnknown)
+  var rettype, params: Rope
   genProcParams(m, prc.typ, rettype, params, check)
   # handle the 2 options for hotcodereloading codegen - function pointer
   # (instead of forward declaration) or header for function body with "_actual" postfix
@@ -977,12 +973,21 @@ proc genProcHeader(m: BModule, prc: PSym, asPtr: bool = false): Rope =
   # careful here! don't access ``prc.ast`` as that could reload large parts of
   # the object graph!
   if prc.constraint.isNil:
+    if lfExportLib in prc.loc.flags:
+      if isHeaderFile in m.flags:
+        result.add "N_LIB_IMPORT "
+      else:
+        result.add "N_LIB_EXPORT "
+    elif prc.typ.callConv == ccInline or asPtr or isNonReloadable(m, prc):
+      result.add "static "
+    elif sfImportc notin prc.flags:
+      result.add "N_LIB_PRIVATE "
     result.addf("$1$2($3, $4)$5",
          [rope(CallingConvToStr[prc.typ.callConv]), asPtrStr, rettype, name,
          params])
   else:
     let asPtrStr = if asPtr: (rope("(*") & name & ")") else: name
-    result = runtimeFormat(prc.cgDeclFrmt, [rettype, asPtrStr, params])
+    result.add runtimeFormat(prc.cgDeclFrmt, [rettype, asPtrStr, params])
 
 # ------------------ type info generation -------------------------------------
 
@@ -1021,7 +1026,7 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType;
   #else echo("can contain a cycle: " & typeToString(typ))
   if flags != 0:
     m.s[cfsTypeInit3].addf("$1.flags = $2;$n", [nameHcr, rope(flags)])
-  discard cgsym(m, "TNimType")
+  cgsym(m, "TNimType")
   if isDefined(m.config, "nimTypeNames"):
     var typename = typeToString(if origType.typeInst != nil: origType.typeInst
                                 else: origType, preferName)
@@ -1029,16 +1034,16 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType;
       typename = "anon ref object from " & m.config$origType.skipTypes(skipPtrs).sym.info
     m.s[cfsTypeInit3].addf("$1.name = $2;$n",
         [nameHcr, makeCString typename])
-    discard cgsym(m, "nimTypeRoot")
+    cgsym(m, "nimTypeRoot")
     m.s[cfsTypeInit3].addf("$1.nextType = nimTypeRoot; nimTypeRoot=&$1;$n",
          [nameHcr])
 
   if m.hcrOn:
-    m.s[cfsData].addf("static TNimType* $1;$n", [name])
+    m.s[cfsStrData].addf("static TNimType* $1;$n", [name])
     m.hcrCreateTypeInfosProc.addf("\thcrRegisterGlobal($2, \"$1\", sizeof(TNimType), NULL, (void**)&$1);$n",
          [name, getModuleDllPath(m, m.module)])
   else:
-    m.s[cfsData].addf("N_LIB_PRIVATE TNimType $1;$n", [name])
+    m.s[cfsStrData].addf("N_LIB_PRIVATE TNimType $1;$n", [name])
 
 proc genTypeInfoAux(m: BModule, typ, origType: PType, name: Rope;
                     info: TLineInfo) =
@@ -1061,12 +1066,12 @@ proc discriminatorTableName(m: BModule, objtype: PType, d: PSym): Rope =
     objtype = objtype[0].skipTypes(abstractPtrs)
   if objtype.sym == nil:
     internalError(m.config, d.info, "anonymous obj with discriminator")
-  result = "NimDT_$1_$2" % [rope($hashType(objtype)), rope(d.name.s.mangle)]
+  result = "NimDT_$1_$2" % [rope($hashType(objtype, m.config)), rope(d.name.s.mangle)]
 
 proc rope(arg: Int128): Rope = rope($arg)
 
 proc discriminatorTableDecl(m: BModule, objtype: PType, d: PSym): Rope =
-  discard cgsym(m, "TNimNode")
+  cgsym(m, "TNimNode")
   var tmp = discriminatorTableName(m, objtype, d)
   result = "TNimNode* $1[$2];$n" % [tmp, rope(lengthOrd(m.config, d.typ)+1)]
 
@@ -1101,7 +1106,7 @@ proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope;
     var tmp = discriminatorTableName(m, typ, field)
     var L = lengthOrd(m.config, field.typ)
     assert L > 0
-    if field.loc.r == nil: fillObjectFields(m, typ)
+    if field.loc.r == "": fillObjectFields(m, typ)
     if field.loc.t == nil:
       internalError(m.config, n.info, "genObjectFields")
     m.s[cfsTypeInit3].addf("$1.kind = 3;$n" &
@@ -1139,7 +1144,7 @@ proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope;
     # Do not produce code for void types
     if isEmptyType(field.typ): return
     if field.bitsize == 0:
-      if field.loc.r == nil: fillObjectFields(m, typ)
+      if field.loc.r == "": fillObjectFields(m, typ)
       if field.loc.t == nil:
         internalError(m.config, n.info, "genObjectFields")
       m.s[cfsTypeInit3].addf("$1.kind = 1;$n" &
@@ -1233,7 +1238,7 @@ proc genSetInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) =
   assert(typ[0] != nil)
   genTypeInfoAux(m, typ, typ, name, info)
   var tmp = getNimNode(m)
-  m.s[cfsTypeInit3].addf("$1.len = $2; $1.kind = 0;$n" & "$3.node = &$1;$n",
+  m.s[cfsTypeInit3].addf("$1.len = $2; $1.kind = 0;$n$3.node = &$1;$n",
        [tmp, rope(firstOrd(m.config, typ)), tiNameForHcr(m, name)])
 
 proc genArrayInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) =
@@ -1258,38 +1263,34 @@ proc genDeepCopyProc(m: BModule; s: PSym; result: Rope) =
 proc declareNimType(m: BModule, name: string; str: Rope, module: int) =
   let nr = rope(name)
   if m.hcrOn:
-    m.s[cfsData].addf("static $2* $1;$n", [str, nr])
+    m.s[cfsStrData].addf("static $2* $1;$n", [str, nr])
     m.s[cfsTypeInit1].addf("\t$1 = ($3*)hcrGetGlobal($2, \"$1\");$n",
           [str, getModuleDllPath(m, module), nr])
   else:
-    m.s[cfsData].addf("extern $2 $1;$n", [str, nr])
+    m.s[cfsStrData].addf("extern $2 $1;$n", [str, nr])
 
 proc genTypeInfo2Name(m: BModule; t: PType): Rope =
-  var res = "|"
   var it = t
-  while it != nil:
-    it = it.skipTypes(skipPtrs)
-    if it.sym != nil:
-      var m = it.sym.owner
-      while m != nil and m.kind != skModule: m = m.owner
-      if m == nil or sfSystemModule in m.flags:
-        # produce short names for system types:
-        res.add it.sym.name.s
-      else:
-        var p = m.owner
-        if p != nil and p.kind == skPackage:
-          res.add p.name.s & "."
-        res.add m.name.s & "."
-        res.add it.sym.name.s
+  it = it.skipTypes(skipPtrs)
+  if it.sym != nil and tfFromGeneric notin it.flags:
+    var m = it.sym.owner
+    while m != nil and m.kind != skModule: m = m.owner
+    if m == nil or sfSystemModule in m.flags:
+      # produce short names for system types:
+      result = it.sym.name.s
     else:
-      res.add $hashType(it)
-    res.add "|"
-    it = it[0]
-  result = makeCString(res)
+      var p = m.owner
+      if p != nil and p.kind == skPackage:
+        result.add p.name.s & "."
+      result.add m.name.s & "."
+      result.add it.sym.name.s
+  else:
+    result = $hashType(it, m.config)
+  result = makeCString(result)
 
 proc isTrivialProc(g: ModuleGraph; s: PSym): bool {.inline.} = getBody(g, s).len == 0
 
-proc genHook(m: BModule; t: PType; info: TLineInfo; op: TTypeAttachedOp): Rope =
+proc genHook(m: BModule; t: PType; info: TLineInfo; op: TTypeAttachedOp; result: var Rope) =
   let theProc = getAttachedOp(m.g.graph, t, op)
   if theProc != nil and not isTrivialProc(m.g.graph, theProc):
     # the prototype of a destructor is ``=destroy(x: var T)`` and that of a
@@ -1300,7 +1301,7 @@ proc genHook(m: BModule; t: PType; info: TLineInfo; op: TTypeAttachedOp): Rope =
         theProc.name.s & " needs to have the 'nimcall' calling convention")
 
     genProc(m, theProc)
-    result = theProc.loc.r
+    result.add theProc.loc.r
 
     when false:
       if not canFormAcycle(t) and op == attachedTrace:
@@ -1312,30 +1313,117 @@ proc genHook(m: BModule; t: PType; info: TLineInfo; op: TTypeAttachedOp): Rope =
         # unfortunately this check is wrong for an object type that only contains
         # .cursor fields like 'Node' inside 'cycleleak'.
         internalError(m.config, info, "no attached trace proc found")
-    result = rope("NIM_NIL")
+    result.add rope("NIM_NIL")
 
-proc genTypeInfoV2Impl(m: BModule, t, origType: PType, name: Rope; info: TLineInfo) =
-  var typeName: Rope
-  if t.kind in {tyObject, tyDistinct}:
-    if incompleteType(t):
-      localError(m.config, info, "request for RTTI generation for incomplete object: " &
-                 typeToString(t))
-    typeName = genTypeInfo2Name(m, t)
-  else:
-    typeName = rope("NIM_NIL")
+proc getObjDepth(t: PType): int16 =
+  var x = t
+  result = -1
+  while x != nil:
+    x = skipTypes(x, skipPtrs)
+    x = x[0]
+    inc(result)
 
-  discard cgsym(m, "TNimTypeV2")
-  m.s[cfsData].addf("N_LIB_PRIVATE TNimTypeV2 $1;$n", [name])
-  let destroyImpl = genHook(m, t, info, attachedDestructor)
-  let traceImpl = genHook(m, t, info, attachedTrace)
-  let disposeImpl = genHook(m, t, info, attachedDispose)
+proc genDisplayElem(d: MD5Digest): uint32 =
+  result = 0
+  for i in 0..3:
+    result += uint32(d[i])
+    result = result shl 8
+
+proc genDisplay(m: BModule, t: PType, depth: int): Rope =
+  result = Rope"{"
+  var x = t
+  var seqs = newSeq[string](depth+1)
+  var i = 0
+  while x != nil:
+    x = skipTypes(x, skipPtrs)
+    seqs[i] = $genDisplayElem(MD5Digest(hashType(x, m.config)))
+    x = x[0]
+    inc i
+
+  for i in countdown(depth, 1):
+    result.add seqs[i] & ", "
+  result.add seqs[0]
+  result.add "}"
+
+proc genTypeInfoV2OldImpl(m: BModule; t, origType: PType, name: Rope; info: TLineInfo) =
+  cgsym(m, "TNimTypeV2")
+  m.s[cfsStrData].addf("N_LIB_PRIVATE TNimTypeV2 $1;$n", [name])
 
   var flags = 0
   if not canFormAcycle(t): flags = flags or 1
 
-  addf(m.s[cfsTypeInit3], "$1.destructor = (void*)$2; $1.size = sizeof($3); $1.align = NIM_ALIGNOF($3); $1.name = $4;$n; $1.traceImpl = (void*)$5; $1.disposeImpl = (void*)$6; $1.flags = $7;", [
-    name, destroyImpl, getTypeDesc(m, t), typeName,
-    traceImpl, disposeImpl, rope(flags)])
+  var typeEntry = newRopeAppender()
+  addf(typeEntry, "$1.destructor = (void*)", [name])
+  genHook(m, t, info, attachedDestructor, typeEntry)
+
+  addf(typeEntry, "; $1.traceImpl = (void*)", [name])
+  genHook(m, t, info, attachedTrace, typeEntry)
+
+  let objDepth = if t.kind == tyObject: getObjDepth(t) else: -1
+
+  if t.kind in {tyObject, tyDistinct} and incompleteType(t):
+    localError(m.config, info, "request for RTTI generation for incomplete object: " &
+              typeToString(t))
+
+  if isDefined(m.config, "nimTypeNames"):
+    var typeName: Rope
+    if t.kind in {tyObject, tyDistinct}:
+      typeName = genTypeInfo2Name(m, t)
+    else:
+      typeName = rope("NIM_NIL")
+    addf(typeEntry, "; $1.name = $2", [name, typeName])
+  addf(typeEntry, "; $1.size = sizeof($2); $1.align = (NI16) NIM_ALIGNOF($2); $1.depth = $3; $1.flags = $4;",
+    [name, getTypeDesc(m, t), rope(objDepth), rope(flags)])
+
+  if objDepth >= 0:
+    let objDisplay = genDisplay(m, t, objDepth)
+    let objDisplayStore = getTempName(m)
+    m.s[cfsVars].addf("static $1 $2[$3] = $4;$n", [getTypeDesc(m, getSysType(m.g.graph, unknownLineInfo, tyUInt32), skVar), objDisplayStore, rope(objDepth+1), objDisplay])
+    addf(typeEntry, "$1.display = $2;$n", [name, rope(objDisplayStore)])
+
+  m.s[cfsTypeInit3].add typeEntry
+
+  if t.kind == tyObject and t.len > 0 and t[0] != nil and optEnableDeepCopy in m.config.globalOptions:
+    discard genTypeInfoV1(m, t, info)
+
+proc genTypeInfoV2Impl(m: BModule; t, origType: PType, name: Rope; info: TLineInfo) =
+  cgsym(m, "TNimTypeV2")
+  m.s[cfsStrData].addf("N_LIB_PRIVATE TNimTypeV2 $1;$n", [name])
+
+  var flags = 0
+  if not canFormAcycle(t): flags = flags or 1
+
+  var typeEntry = newRopeAppender()
+  addf(typeEntry, "N_LIB_PRIVATE TNimTypeV2 $1 = {", [name])
+  add(typeEntry, ".destructor = (void*)")
+  genHook(m, t, info, attachedDestructor, typeEntry)
+
+  let objDepth = if t.kind == tyObject: getObjDepth(t) else: -1
+
+  if t.kind in {tyObject, tyDistinct} and incompleteType(t):
+    localError(m.config, info, "request for RTTI generation for incomplete object: " &
+              typeToString(t))
+
+  addf(typeEntry, ", .size = sizeof($1), .align = (NI16) NIM_ALIGNOF($1), .depth = $2",
+    [getTypeDesc(m, t), rope(objDepth)])
+
+  if objDepth >= 0:
+    let objDisplay = genDisplay(m, t, objDepth)
+    let objDisplayStore = getTempName(m)
+    m.s[cfsVars].addf("static NIM_CONST $1 $2[$3] = $4;$n", [getTypeDesc(m, getSysType(m.g.graph, unknownLineInfo, tyUInt32), skVar), objDisplayStore, rope(objDepth+1), objDisplay])
+    addf(typeEntry, ", .display = $1", [rope(objDisplayStore)])
+  if isDefined(m.config, "nimTypeNames"):
+    var typeName: Rope
+    if t.kind in {tyObject, tyDistinct}:
+      typeName = genTypeInfo2Name(m, t)
+    else:
+      typeName = rope("NIM_NIL")
+    addf(typeEntry, ", .name = $1", [typeName])
+  add(typeEntry, ", .traceImpl = (void*)")
+  genHook(m, t, info, attachedTrace, typeEntry)
+
+  addf(typeEntry, ", .flags = $1};$n", [rope(flags)])
+  m.s[cfsVars].add typeEntry
 
   if t.kind == tyObject and t.len > 0 and t[0] != nil and optEnableDeepCopy in m.config.globalOptions:
     discard genTypeInfoV1(m, t, info)
@@ -1347,14 +1435,14 @@ proc genTypeInfoV2(m: BModule, t: PType; info: TLineInfo): Rope =
 
   let prefixTI = if m.hcrOn: "(" else: "(&"
 
-  let sig = hashType(origType)
+  let sig = hashType(origType, m.config)
   result = m.typeInfoMarkerV2.getOrDefault(sig)
-  if result != nil:
+  if result != "":
     return prefixTI.rope & result & ")".rope
 
   let marker = m.g.typeInfoMarkerV2.getOrDefault(sig)
-  if marker.str != nil:
-    discard cgsym(m, "TNimTypeV2")
+  if marker.str != "":
+    cgsym(m, "TNimTypeV2")
     declareNimType(m, "TNimTypeV2", marker.str, marker.owner)
     # also store in local type section:
     m.typeInfoMarkerV2[sig] = marker.str
@@ -1368,12 +1456,15 @@ proc genTypeInfoV2(m: BModule, t: PType; info: TLineInfo): Rope =
     # make sure the type info is created in the owner module
     discard genTypeInfoV2(m.g.modules[owner], origType, info)
     # reference the type info as extern here
-    discard cgsym(m, "TNimTypeV2")
+    cgsym(m, "TNimTypeV2")
     declareNimType(m, "TNimTypeV2", result, owner)
     return prefixTI.rope & result & ")".rope
 
   m.g.typeInfoMarkerV2[sig] = (str: result, owner: owner)
-  genTypeInfoV2Impl(m, t, origType, result, info)
+  if m.compileToCpp:
+    genTypeInfoV2OldImpl(m, t, origType, result, info)
+  else:
+    genTypeInfoV2Impl(m, t, origType, result, info)
   result = prefixTI.rope & result & ")".rope
 
 proc openArrayToTuple(m: BModule; t: PType): PType =
@@ -1418,15 +1509,15 @@ proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope =
 
   let prefixTI = if m.hcrOn: "(" else: "(&"
 
-  let sig = hashType(origType)
+  let sig = hashType(origType, m.config)
   result = m.typeInfoMarker.getOrDefault(sig)
-  if result != nil:
+  if result != "":
     return prefixTI.rope & result & ")".rope
 
   let marker = m.g.typeInfoMarker.getOrDefault(sig)
-  if marker.str != nil:
-    discard cgsym(m, "TNimType")
-    discard cgsym(m, "TNimNode")
+  if marker.str != "":
+    cgsym(m, "TNimType")
+    cgsym(m, "TNimNode")
     declareNimType(m, "TNimType", marker.str, marker.owner)
     # also store in local type section:
     m.typeInfoMarker[sig] = marker.str
@@ -1437,8 +1528,8 @@ proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope =
 
   let old = m.g.graph.emittedTypeInfo.getOrDefault($result)
   if old != FileIndex(0):
-    discard cgsym(m, "TNimType")
-    discard cgsym(m, "TNimNode")
+    cgsym(m, "TNimType")
+    cgsym(m, "TNimNode")
     declareNimType(m, "TNimType", result, old.int)
     return prefixTI.rope & result & ")".rope
 
@@ -1447,8 +1538,8 @@ proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope =
     # make sure the type info is created in the owner module
     discard genTypeInfoV1(m.g.modules[owner], origType, info)
     # reference the type info as extern here
-    discard cgsym(m, "TNimType")
-    discard cgsym(m, "TNimNode")
+    cgsym(m, "TNimType")
+    cgsym(m, "TNimNode")
     declareNimType(m, "TNimType", result, owner)
     return prefixTI.rope & result & ")".rope
   else:
@@ -1475,12 +1566,12 @@ proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope =
       genTupleInfo(m, x, x, result, info)
   of tySequence:
     genTypeInfoAux(m, t, t, result, info)
-    if m.config.selectedGC in {gcMarkAndSweep, gcRefc, gcV2, gcGo}:
+    if m.config.selectedGC in {gcMarkAndSweep, gcRefc, gcGo}:
       let markerProc = genTraverseProc(m, origType, sig)
       m.s[cfsTypeInit3].addf("$1.marker = $2;$n", [tiNameForHcr(m, result), markerProc])
   of tyRef:
     genTypeInfoAux(m, t, t, result, info)
-    if m.config.selectedGC in {gcMarkAndSweep, gcRefc, gcV2, gcGo}:
+    if m.config.selectedGC in {gcMarkAndSweep, gcRefc, gcGo}:
       let markerProc = genTraverseProc(m, origType, sig)
       m.s[cfsTypeInit3].addf("$1.marker = $2;$n", [tiNameForHcr(m, result), markerProc])
   of tyPtr, tyRange, tyUncheckedArray: genTypeInfoAux(m, t, t, result, info)
@@ -1515,3 +1606,9 @@ proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope =
 
 proc genTypeSection(m: BModule, n: PNode) =
   discard
+
+proc genTypeInfo*(config: ConfigRef, m: BModule, t: PType; info: TLineInfo): Rope =
+  if optTinyRtti in config.globalOptions:
+    result = genTypeInfoV2(m, t, info)
+  else:
+    result = genTypeInfoV1(m, t, info)

compiler/ccgutils.nim

@@ -13,6 +13,9 @@ import
   ast, types, hashes, strutils, msgs, wordrecg,
   platform, trees, options, cgendata
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 proc getPragmaStmt*(n: PNode, w: TSpecialWord): PNode =
   case n.kind
   of nkStmtList:
@@ -50,7 +53,7 @@ proc hashString*(conf: ConfigRef; s: string): BiggestInt =
     a = a + (a shl 3)
     a = a xor (a shr 11)
     a = a + (a shl 15)
-    result = cast[Hash](a)
+    result = cast[Hash](uint(a))
 
 template getUniqueType*(key: PType): PType = key
 

compiler/cgendata.nim

@@ -16,7 +16,6 @@ import
 type
   TLabel* = Rope              # for the C generator a label is just a rope
   TCFileSection* = enum       # the sections a generated C file consists of
-    cfsMergeInfo,             # section containing merge information
     cfsHeaders,               # section for C include file headers
     cfsFrameDefines           # section for nim frame macros
     cfsForwardTypes,          # section for C forward typedefs
@@ -24,21 +23,17 @@ type
     cfsSeqTypes,              # section for sequence types only
                               # this is needed for strange type generation
                               # reasons
-    cfsFieldInfo,             # section for field information
     cfsTypeInfo,              # section for type information (ag ABI checks)
     cfsProcHeaders,           # section for C procs prototypes
+    cfsStrData,               # section for constant string literals
     cfsData,                  # section for C constant data
     cfsVars,                  # section for C variable declarations
     cfsProcs,                 # section for C procs that are not inline
     cfsInitProc,              # section for the C init proc
     cfsDatInitProc,           # section for the C datInit proc
     cfsTypeInit1,             # section 1 for declarations of type information
-    cfsTypeInit2,             # section 2 for init of type information
     cfsTypeInit3,             # section 3 for init of type information
-    cfsDebugInit,             # section for init of debug information
     cfsDynLibInit,            # section for init of dynamic library binding
-    cfsDynLibDeinit           # section for deinitialization of dynamic
-                              # libraries
   TCTypeKind* = enum          # describes the type kind of a C type
     ctVoid, ctChar, ctBool,
     ctInt, ctInt8, ctInt16, ctInt32, ctInt64,
@@ -91,6 +86,7 @@ type
     options*: TOptions        # options that should be used for code
                               # generation; this is the same as prc.options
                               # unless prc == nil
+    optionsStack*: seq[TOptions]
     module*: BModule          # used to prevent excessive parameter passing
     withinLoop*: int          # > 0 if we are within a loop
     splitDecls*: int          # > 0 if we are in some context for C++ that
@@ -99,6 +95,7 @@ type
     withinTryWithExcept*: int # required for goto based exception handling
     withinBlockLeaveActions*: int # complex to explain
     sigConflicts*: CountTable[string]
+    inUncheckedAssignSection*: int
 
   TTypeSeq* = seq[PType]
   TypeCache* = Table[SigHash, Rope]
@@ -170,13 +167,13 @@ type
     labels*: Natural          # for generating unique module-scope names
     extensionLoaders*: array['0'..'9', Rope] # special procs for the
                                              # OpenGL wrapper
-    injectStmt*: Rope
     sigConflicts*: CountTable[SigHash]
     g*: BModuleList
     ndi*: NdiFile
 
 template config*(m: BModule): ConfigRef = m.g.config
 template config*(p: BProc): ConfigRef = p.module.g.config
+template vccAndC*(p: BProc): bool = p.module.config.cCompiler == ccVcc and p.module.config.backend == backendC
 
 proc includeHeader*(this: BModule; header: string) =
   if not this.headerFiles.contains header:
@@ -190,13 +187,18 @@ proc procSec*(p: BProc, s: TCProcSection): var Rope {.inline.} =
   # top level proc sections
   result = p.blocks[0].sections[s]
 
+proc initBlock*(): TBlock =
+  result = TBlock()
+  for i in low(result.sections)..high(result.sections):
+    result.sections[i] = newRopeAppender()
+
 proc newProc*(prc: PSym, module: BModule): BProc =
   new(result)
   result.prc = prc
   result.module = module
   result.options = if prc != nil: prc.options
                    else: module.config.options
-  newSeq(result.blocks, 1)
+  result.blocks = @[initBlock()]
   result.nestedTryStmts = @[]
   result.finallySafePoints = @[]
   result.sigConflicts = initCountTable[string]()

compiler/cgmeth.nim

@@ -11,7 +11,11 @@
 
 import
   intsets, options, ast, msgs, idents, renderer, types, magicsys,
-  sempass2, strutils, modulegraphs, lineinfos
+  sempass2, modulegraphs, lineinfos
+
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 
 proc genConv(n: PNode, d: PType, downcast: bool; conf: ConfigRef): PNode =
   var dest = skipTypes(d, abstractPtrs)
@@ -46,6 +50,7 @@ proc methodCall*(n: PNode; conf: ConfigRef): PNode =
   # replace ordinary method by dispatcher method:
   let disp = getDispatcher(result[0].sym)
   if disp != nil:
+    result[0].typ = disp.typ
     result[0].sym = disp
     # change the arguments to up/downcasts to fit the dispatcher's parameters:
     for i in 1..<result.len:
@@ -119,7 +124,7 @@ proc createDispatcher(s: PSym; g: ModuleGraph; idgen: IdGenerator): PSym =
   if disp.typ.callConv == ccInline: disp.typ.callConv = ccNimCall
   disp.ast = copyTree(s.ast)
   disp.ast[bodyPos] = newNodeI(nkEmpty, s.info)
-  disp.loc.r = nil
+  disp.loc.r = ""
   if s.typ[0] != nil:
     if disp.ast.len > resultPos:
       disp.ast[resultPos].sym = copySym(s.ast[resultPos].sym, nextSymId(idgen))
@@ -143,23 +148,6 @@ proc fixupDispatcher(meth, disp: PSym; conf: ConfigRef) =
      disp.ast[resultPos].kind == nkEmpty:
     disp.ast[resultPos] = copyTree(meth.ast[resultPos])
 
-  # The following code works only with lock levels, so we disable
-  # it when they're not available.
-  when declared(TLockLevel):
-    proc `<`(a, b: TLockLevel): bool {.borrow.}
-    proc `==`(a, b: TLockLevel): bool {.borrow.}
-    if disp.typ.lockLevel == UnspecifiedLockLevel:
-      disp.typ.lockLevel = meth.typ.lockLevel
-    elif meth.typ.lockLevel != UnspecifiedLockLevel and
-         meth.typ.lockLevel != disp.typ.lockLevel:
-      message(conf, meth.info, warnLockLevel,
-        "method has lock level $1, but another method has $2" %
-        [$meth.typ.lockLevel, $disp.typ.lockLevel])
-      # XXX The following code silences a duplicate warning in
-      # checkMethodeffects() in sempass2.nim for now.
-      if disp.typ.lockLevel < meth.typ.lockLevel:
-        disp.typ.lockLevel = meth.typ.lockLevel
-
 proc methodDef*(g: ModuleGraph; idgen: IdGenerator; s: PSym) =
   var witness: PSym
   for i in 0..<g.methods.len:

compiler/closureiters.nim

@@ -121,7 +121,10 @@
 #   yield 2
 #   if :unrollFinally: # This node is created by `newEndFinallyNode`
 #     if :curExc.isNil:
-#       return :tmpResult
+#       if nearestFinally == 0:
+#         return :tmpResult
+#       else:
+#         :state = nearestFinally # bubble up
 #     else:
 #       closureIterSetupExc(nil)
 #       raise
@@ -133,6 +136,9 @@ import
   renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos,
   tables, options
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 type
   Ctx = object
     g: ModuleGraph
@@ -414,8 +420,11 @@ proc exprToStmtList(n: PNode): tuple[s, res: PNode] =
 
 
 proc newEnvVarAsgn(ctx: Ctx, s: PSym, v: PNode): PNode =
-  result = newTree(nkFastAsgn, ctx.newEnvVarAccess(s), v)
-  result.info = v.info
+  if isEmptyType(v.typ):
+    result = v
+  else:
+    result = newTree(nkFastAsgn, ctx.newEnvVarAccess(s), v)
+    result.info = v.info
 
 proc addExprAssgn(ctx: Ctx, output, input: PNode, sym: PSym) =
   if input.kind == nkStmtListExpr:
@@ -427,8 +436,7 @@ proc addExprAssgn(ctx: Ctx, output, input: PNode, sym: PSym) =
 
 proc convertExprBodyToAsgn(ctx: Ctx, exprBody: PNode, res: PSym): PNode =
   result = newNodeI(nkStmtList, exprBody.info)
-  if exprBody.typ != nil:
-    ctx.addExprAssgn(result, exprBody, res)
+  ctx.addExprAssgn(result, exprBody, res)
 
 proc newNotCall(g: ModuleGraph; e: PNode): PNode =
   result = newTree(nkCall, newSymNode(g.getSysMagic(e.info, "not", mNot), e.info), e)
@@ -606,6 +614,12 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
             internalError(ctx.g.config, "lowerStmtListExpr(nkCaseStmt): " & $branch.kind)
         result.add(n)
         result.add(ctx.newEnvVarAccess(tmp))
+      elif n[0].kind == nkStmtListExpr:
+        result = newNodeI(nkStmtList, n.info)
+        let (st, ex) = exprToStmtList(n[0])
+        result.add(st)
+        n[0] = ex
+        result.add(n)
 
   of nkCallKinds, nkChckRange, nkChckRangeF, nkChckRange64:
     var ns = false
@@ -704,7 +718,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       n[^1] = ex
       result.add(n)
 
-  of nkAsgn, nkFastAsgn:
+  of nkAsgn, nkFastAsgn, nkSinkAsgn:
     var ns = false
     for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
@@ -796,7 +810,10 @@ proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode =
   # Generate the following code:
   #   if :unrollFinally:
   #       if :curExc.isNil:
-  #         return :tmpResult
+  #         if nearestFinally == 0:
+  #           return :tmpResult
+  #         else:
+  #           :state = nearestFinally # bubble up
   #       else:
   #         raise
   let curExc = ctx.newCurExcAccess()
@@ -805,11 +822,20 @@ proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode =
   let cmp = newTree(nkCall, newSymNode(ctx.g.getSysMagic(info, "==", mEqRef), info), curExc, nilnode)
   cmp.typ = ctx.g.getSysType(info, tyBool)
 
-  let asgn = newTree(nkFastAsgn,
-    newSymNode(getClosureIterResult(ctx.g, ctx.fn, ctx.idgen), info),
-    ctx.newTmpResultAccess())
+  let retStmt =
+    if ctx.nearestFinally == 0:
+      # last finally, we can return
+      let retValue = if ctx.fn.typ[0].isNil:
+                   ctx.g.emptyNode
+                 else:
+                   newTree(nkFastAsgn,
+                           newSymNode(getClosureIterResult(ctx.g, ctx.fn, ctx.idgen), info),
+                           ctx.newTmpResultAccess())
+      newTree(nkReturnStmt, retValue)
+    else:
+      # bubble up to next finally
+      newTree(nkGotoState, ctx.g.newIntLit(info, ctx.nearestFinally))
 
-  let retStmt = newTree(nkReturnStmt, asgn)
   let branch = newTree(nkElifBranch, cmp, retStmt)
 
   let nullifyExc = newTree(nkCall, newSymNode(ctx.g.getCompilerProc("closureIterSetupExc")), nilnode)
@@ -842,7 +868,7 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
     if n[0].kind != nkEmpty:
       let asgnTmpResult = newNodeI(nkAsgn, n.info)
       asgnTmpResult.add(ctx.newTmpResultAccess())
-      let x = if n[0].kind in {nkAsgn, nkFastAsgn}: n[0][1] else: n[0]
+      let x = if n[0].kind in {nkAsgn, nkFastAsgn, nkSinkAsgn}: n[0][1] else: n[0]
       asgnTmpResult.add(x)
       result.add(asgnTmpResult)
 
@@ -853,6 +879,13 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
 
   of nkSkip:
     discard
+  of nkTryStmt:
+    if n.hasYields:
+      # the inner try will handle these transformations
+      discard
+    else:
+      for i in 0..<n.len:
+        n[i] = ctx.transformReturnsInTry(n[i])
   else:
     for i in 0..<n.len:
       n[i] = ctx.transformReturnsInTry(n[i])
@@ -1122,7 +1155,7 @@ proc newArrayType(g: ModuleGraph; n: int, t: PType; idgen: IdGenerator; owner: P
   result = newType(tyArray, nextTypeId(idgen), owner)
 
   let rng = newType(tyRange, nextTypeId(idgen), owner)
-  rng.n = newTree(nkRange, g.newIntLit(owner.info, 0), g.newIntLit(owner.info, n))
+  rng.n = newTree(nkRange, g.newIntLit(owner.info, 0), g.newIntLit(owner.info, n - 1))
   rng.rawAddSon(t)
 
   result.rawAddSon(rng)
@@ -1339,20 +1372,24 @@ proc preprocess(c: var PreprocessContext; n: PNode): PNode =
   # detect: 'finally: raises X' which is currently not supported. We produce
   # an error for this case for now. All this will be done properly with Yuriy's
   # patch.
+
   result = n
   case n.kind
   of nkTryStmt:
     let f = n.lastSon
+    var didAddSomething = false
     if f.kind == nkFinally:
       c.finallys.add f.lastSon
+      didAddSomething = true
 
     for i in 0 ..< n.len:
       result[i] = preprocess(c, n[i])
 
-    if f.kind == nkFinally:
+    if didAddSomething:
       discard c.finallys.pop()
 
   of nkWhileStmt, nkBlockStmt:
+    if n.hasYields == false: return n
     c.blocks.add((n, c.finallys.len))
     for i in 0 ..< n.len:
       result[i] = preprocess(c, n[i])
@@ -1376,7 +1413,7 @@ proc preprocess(c: var PreprocessContext; n: PNode): PNode =
         result = newNodeI(nkStmtList, n.info)
         for i in countdown(c.finallys.high, fin):
           var vars = FreshVarsContext(tab: initTable[int, PSym](), config: c.config, info: n.info, idgen: c.idgen)
-          result.add freshVars(preprocess(c, c.finallys[i]), vars)
+          result.add freshVars(copyTree(c.finallys[i]), vars)
           c.idgen = vars.idgen
         result.add n
   of nkSkip: discard

compiler/cmdlinehelper.nim

@@ -44,14 +44,7 @@ proc processCmdLineAndProjectPath*(self: NimProg, conf: ConfigRef) =
   elif self.supportsStdinFile and conf.projectName == "-":
     handleStdinInput(conf)
   elif conf.projectName != "":
-    try:
-      conf.projectFull = canonicalizePath(conf, AbsoluteFile conf.projectName)
-    except OSError:
-      conf.projectFull = AbsoluteFile conf.projectName
-    let p = splitFile(conf.projectFull)
-    let dir = if p.dir.isEmpty: AbsoluteDir getCurrentDir() else: p.dir
-    conf.projectPath = AbsoluteDir canonicalizePath(conf, AbsoluteFile dir)
-    conf.projectName = p.name
+    setFromProjectName(conf, conf.projectName)
   else:
     conf.projectPath = AbsoluteDir canonicalizePath(conf, AbsoluteFile getCurrentDir())
 
@@ -71,7 +64,8 @@ proc loadConfigsAndProcessCmdLine*(self: NimProg, cache: IdentCache; conf: Confi
       if conf.cmd == cmdNimscript: return false
   # now process command line arguments again, because some options in the
   # command line can overwrite the config file's settings
-  extccomp.initVars(conf)
+  if conf.backend != backendJs: # bug #19059
+    extccomp.initVars(conf)
   self.processCmdLine(passCmd2, "", conf)
   if conf.cmd == cmdNone:
     rawMessage(conf, errGenerated, "command missing")

compiler/commands.nim

@@ -9,7 +9,6 @@
 
 # This module handles the parsing of command line arguments.
 
-
 # We do this here before the 'import' statement so 'defined' does not get
 # confused with 'TGCMode.gcMarkAndSweep' etc.
 template bootSwitch(name, expr, userString) =
@@ -25,12 +24,15 @@ bootSwitch(usedMarkAndSweep, defined(gcmarkandsweep), "--gc:markAndSweep")
 bootSwitch(usedGoGC, defined(gogc), "--gc:go")
 bootSwitch(usedNoGC, defined(nogc), "--gc:none")
 
+import std/[setutils, os, strutils, parseutils, parseopt, sequtils, strtabs]
 import
-  os, msgs, options, nversion, condsyms, strutils, extccomp, platform,
-  wordrecg, parseutils, nimblecmd, parseopt, sequtils, lineinfos,
-  pathutils, strtabs, pathnorm
+  msgs, options, nversion, condsyms, extccomp, platform,
+  wordrecg, nimblecmd, lineinfos, pathutils, pathnorm
 
-from ast import eqTypeFlags, tfGcSafe, tfNoSideEffect
+from ast import setUseIc, eqTypeFlags, tfGcSafe, tfNoSideEffect
+
+when defined(nimPreviewSlimSystem):
+  import std/assertions
 
 # but some have deps to imported modules. Yay.
 bootSwitch(usedTinyC, hasTinyCBackend, "-d:tinyc")
@@ -114,7 +116,7 @@ const
   errInvalidCmdLineOption = "invalid command line option: '$1'"
   errOnOrOffExpectedButXFound = "'on' or 'off' expected, but '$1' found"
   errOnOffOrListExpectedButXFound = "'on', 'off' or 'list' expected, but '$1' found"
-  errOffHintsError = "'off', 'hint' or 'error' expected, but '$1' found"
+  errOffHintsError = "'off', 'hint', 'error' or 'usages' expected, but '$1' found"
 
 proc invalidCmdLineOption(conf: ConfigRef; pass: TCmdLinePass, switch: string, info: TLineInfo) =
   if switch == " ": localError(conf, info, errInvalidCmdLineOption % "-")
@@ -182,7 +184,7 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass,
                          info: TLineInfo; orig: string; conf: ConfigRef) =
   var id = ""  # arg = key or [key] or key:val or [key]:val;  with val=on|off
   var i = 0
-  var n = hintMin
+  var notes: set[TMsgKind]
   var isBracket = false
   if i < arg.len and arg[i] == '[':
     isBracket = true
@@ -197,37 +199,38 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass,
   if i == arg.len: discard
   elif i < arg.len and (arg[i] in {':', '='}): inc(i)
   else: invalidCmdLineOption(conf, pass, orig, info)
-  # unfortunately, hintUser and warningUser clash
-  if state in {wHint, wHintAsError}:
-    let x = findStr(hintMin, hintMax, id, errUnknown)
-    if x != errUnknown: n = TNoteKind(x)
-    else: localError(conf, info, "unknown hint: " & id)
-  else:
-    let x = findStr(warnMin, warnMax, id, errUnknown)
-    if x != errUnknown: n = TNoteKind(x)
-    else: localError(conf, info, "unknown warning: " & id)
 
+  let isSomeHint = state in {wHint, wHintAsError}
+  template findNote(noteMin, noteMax, name) =
+    # unfortunately, hintUser and warningUser clash, otherwise implementation would simplify a bit
+    let x = findStr(noteMin, noteMax, id, errUnknown)
+    if x != errUnknown: notes = {TNoteKind(x)}
+    else: localError(conf, info, "unknown $#: $#" % [name, id])
+  case id.normalize
+  of "all": # other note groups would be easy to support via additional cases
+    notes = if isSomeHint: {hintMin..hintMax} else: {warnMin..warnMax}
+  elif isSomeHint: findNote(hintMin, hintMax, "hint")
+  else: findNote(warnMin, warnMax, "warning")
   var val = substr(arg, i).normalize
   if val == "": val = "on"
   if val notin ["on", "off"]:
+    # xxx in future work we should also allow users to have control over `foreignPackageNotes`
+    # so that they can enable `hints|warnings|warningAsErrors` for all the code they depend on.
     localError(conf, info, errOnOrOffExpectedButXFound % arg)
-  elif n notin conf.cmdlineNotes or pass == passCmd1:
-    if pass == passCmd1: incl(conf.cmdlineNotes, n)
-    incl(conf.modifiedyNotes, n)
-    case val
-    of "on":
-      if state in {wWarningAsError, wHintAsError}:
-        incl(conf.warningAsErrors, n) # xxx rename warningAsErrors to noteAsErrors
-      else:
-        incl(conf.notes, n)
-        incl(conf.mainPackageNotes, n)
-    of "off":
-      if state in {wWarningAsError, wHintAsError}:
-        excl(conf.warningAsErrors, n)
-      else:
-        excl(conf.notes, n)
-        excl(conf.mainPackageNotes, n)
-        excl(conf.foreignPackageNotes, n)
+  else:
+    let isOn = val == "on"
+    if isOn and id.normalize == "all":
+      localError(conf, info, "only 'all:off' is supported")
+    for n in notes:
+      if n notin conf.cmdlineNotes or pass == passCmd1:
+        if pass == passCmd1: incl(conf.cmdlineNotes, n)
+        incl(conf.modifiedyNotes, n)
+        if state in {wWarningAsError, wHintAsError}:
+          conf.warningAsErrors[n] = isOn # xxx rename warningAsErrors to noteAsErrors
+        else:
+          conf.notes[n] = isOn
+          conf.mainPackageNotes[n] = isOn
+        if not isOn: excl(conf.foreignPackageNotes, n)
 
 proc processCompile(conf: ConfigRef; filename: string) =
   var found = findFile(conf, filename)
@@ -238,7 +241,7 @@ const
   errNoneBoehmRefcExpectedButXFound = "'arc', 'orc', 'markAndSweep', 'boehm', 'go', 'none', 'regions', or 'refc' expected, but '$1' found"
   errNoneSpeedOrSizeExpectedButXFound = "'none', 'speed' or 'size' expected, but '$1' found"
   errGuiConsoleOrLibExpectedButXFound = "'gui', 'console' or 'lib' expected, but '$1' found"
-  errInvalidExceptionSystem = "'goto', 'setjump', 'cpp' or 'quirky' expected, but '$1' found"
+  errInvalidExceptionSystem = "'goto', 'setjmp', 'cpp' or 'quirky' expected, but '$1' found"
 
 template warningOptionNoop(switch: string) =
   warningDeprecated(conf, info, "'$#' is deprecated, now a noop" % switch)
@@ -248,7 +251,7 @@ template deprecatedAlias(oldName, newName: string) =
 
 proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo): bool =
   case switch.normalize
-  of "gc":
+  of "gc", "mm":
     case arg.normalize
     of "boehm": result = conf.selectedGC == gcBoehm
     of "refc": result = conf.selectedGC == gcRefc
@@ -259,7 +262,6 @@ proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo
     of "go": result = conf.selectedGC == gcGo
     of "none": result = conf.selectedGC == gcNone
     of "stack", "regions": result = conf.selectedGC == gcRegions
-    of "v2", "generational": warningOptionNoop(arg)
     else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg)
   of "opt":
     case arg.normalize
@@ -324,6 +326,7 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
   of "run", "r": result = contains(conf.globalOptions, optRun)
   of "symbolfiles": result = conf.symbolFiles != disabledSf
   of "genscript": result = contains(conf.globalOptions, optGenScript)
+  of "gencdeps": result = contains(conf.globalOptions, optGenCDeps)
   of "threads": result = contains(conf.globalOptions, optThreads)
   of "tlsemulation": result = contains(conf.globalOptions, optTlsEmulation)
   of "implicitstatic": result = contains(conf.options, optImplicitStatic)
@@ -332,6 +335,7 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
     result = contains(conf.options, optTrMacros)
   of "excessivestacktrace": result = contains(conf.globalOptions, optExcessiveStackTrace)
   of "nilseqs", "nilchecks", "taintmode": warningOptionNoop(switch)
+  of "panics": result = contains(conf.globalOptions, optPanics)
   else: invalidCmdLineOption(conf, passCmd1, switch, info)
 
 proc processPath(conf: ConfigRef; path: string, info: TLineInfo,
@@ -443,9 +447,11 @@ proc parseCommand*(command: string): Command =
   of "check": cmdCheck
   of "e": cmdNimscript
   of "doc0": cmdDoc0
-  of "doc2", "doc": cmdDoc2
+  of "doc2", "doc": cmdDoc
   of "doc2tex": cmdDoc2tex
   of "rst2html": cmdRst2html
+  of "md2tex": cmdMd2tex
+  of "md2html": cmdMd2html
   of "rst2tex": cmdRst2tex
   of "jsondoc0": cmdJsondoc0
   of "jsondoc2", "jsondoc": cmdJsondoc
@@ -477,11 +483,108 @@ proc setCommandEarly*(conf: ConfigRef, command: string) =
   # command early customizations
   # must be handled here to honor subsequent `--hint:x:on|off`
   case conf.cmd
-  of cmdRst2html, cmdRst2tex: # xxx see whether to add others: cmdGendepend, etc.
+  of cmdRst2html, cmdRst2tex, cmdMd2html, cmdMd2tex:
+      # xxx see whether to add others: cmdGendepend, etc.
     conf.foreignPackageNotes = {hintSuccessX}
   else:
     conf.foreignPackageNotes = foreignPackageNotesDefault
 
+proc specialDefine(conf: ConfigRef, key: string; pass: TCmdLinePass) =
+  # Keep this syncronized with the default config/nim.cfg!
+  if cmpIgnoreStyle(key, "nimQuirky") == 0:
+    conf.exc = excQuirky
+  elif cmpIgnoreStyle(key, "release") == 0 or cmpIgnoreStyle(key, "danger") == 0:
+    if pass in {passCmd1, passPP}:
+      conf.options.excl {optStackTrace, optLineTrace, optLineDir, optOptimizeSize}
+      conf.globalOptions.excl {optExcessiveStackTrace, optCDebug}
+      conf.options.incl optOptimizeSpeed
+  if cmpIgnoreStyle(key, "danger") == 0 or cmpIgnoreStyle(key, "quick") == 0:
+    if pass in {passCmd1, passPP}:
+      conf.options.excl {optObjCheck, optFieldCheck, optRangeCheck, optBoundsCheck,
+        optOverflowCheck, optAssert, optStackTrace, optLineTrace, optLineDir}
+      conf.globalOptions.excl {optCDebug}
+
+proc initOrcDefines*(conf: ConfigRef) =
+  conf.selectedGC = gcOrc
+  defineSymbol(conf.symbols, "gcorc")
+  defineSymbol(conf.symbols, "gcdestructors")
+  incl conf.globalOptions, optSeqDestructors
+  incl conf.globalOptions, optTinyRtti
+  defineSymbol(conf.symbols, "nimSeqsV2")
+  defineSymbol(conf.symbols, "nimV2")
+  if conf.exc == excNone and conf.backend != backendCpp:
+    conf.exc = excGoto
+
+proc registerArcOrc(pass: TCmdLinePass, conf: ConfigRef, isOrc: bool) =
+  if isOrc:
+    conf.selectedGC = gcOrc
+    defineSymbol(conf.symbols, "gcorc")
+  else:
+    conf.selectedGC = gcArc
+    defineSymbol(conf.symbols, "gcarc")
+
+  defineSymbol(conf.symbols, "gcdestructors")
+  incl conf.globalOptions, optSeqDestructors
+  incl conf.globalOptions, optTinyRtti
+  if pass in {passCmd2, passPP}:
+    defineSymbol(conf.symbols, "nimSeqsV2")
+    defineSymbol(conf.symbols, "nimV2")
+  if conf.exc == excNone and conf.backend != backendCpp:
+    conf.exc = excGoto
+
+proc unregisterArcOrc(conf: ConfigRef) =
+  undefSymbol(conf.symbols, "gcdestructors")
+  undefSymbol(conf.symbols, "gcarc")
+  undefSymbol(conf.symbols, "gcorc")
+  undefSymbol(conf.symbols, "nimSeqsV2")
+  undefSymbol(conf.symbols, "nimV2")
+  excl conf.globalOptions, optSeqDestructors
+  excl conf.globalOptions, optTinyRtti
+
+proc processMemoryManagementOption(switch, arg: string, pass: TCmdLinePass,
+                info: TLineInfo; conf: ConfigRef) =
+  if conf.backend == backendJs: return # for: bug #16033
+  expectArg(conf, switch, arg, pass, info)
+  if pass in {passCmd2, passPP}:
+    case arg.normalize
+    of "boehm":
+      unregisterArcOrc(conf)
+      conf.selectedGC = gcBoehm
+      defineSymbol(conf.symbols, "boehmgc")
+      incl conf.globalOptions, optTlsEmulation # Boehm GC doesn't scan the real TLS
+    of "refc":
+      unregisterArcOrc(conf)
+      defineSymbol(conf.symbols, "gcrefc")
+      conf.selectedGC = gcRefc
+    of "markandsweep":
+      unregisterArcOrc(conf)
+      conf.selectedGC = gcMarkAndSweep
+      defineSymbol(conf.symbols, "gcmarkandsweep")
+    of "destructors", "arc":
+      registerArcOrc(pass, conf, false)
+    of "orc":
+      registerArcOrc(pass, conf, true)
+    of "hooks":
+      conf.selectedGC = gcHooks
+      defineSymbol(conf.symbols, "gchooks")
+      incl conf.globalOptions, optSeqDestructors
+      processOnOffSwitchG(conf, {optSeqDestructors}, arg, pass, info)
+      if pass in {passCmd2, passPP}:
+        defineSymbol(conf.symbols, "nimSeqsV2")
+    of "go":
+      unregisterArcOrc(conf)
+      conf.selectedGC = gcGo
+      defineSymbol(conf.symbols, "gogc")
+    of "none":
+      unregisterArcOrc(conf)
+      conf.selectedGC = gcNone
+      defineSymbol(conf.symbols, "nogc")
+    of "stack", "regions":
+      unregisterArcOrc(conf)
+      conf.selectedGC = gcRegions
+      defineSymbol(conf.symbols, "gcregions")
+    else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg)
+
 proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
                     conf: ConfigRef) =
   var
@@ -523,6 +626,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     conf.lazyPaths.keepItIf(it != path)
   of "nimcache":
     expectArg(conf, switch, arg, pass, info)
+    var arg = arg
+    # refs bug #18674, otherwise `--os:windows` messes up with `--nimcache` set
+    # in config nims files, e.g. via: `import os; switch("nimcache", "/tmp/somedir")`
+    if conf.target.targetOS == osWindows and DirSep == '/': arg = arg.replace('\\', '/')
     conf.nimcacheDir = processPath(conf, arg, info, notRelativeToProj=true)
   of "out", "o":
     expectArg(conf, switch, arg, pass, info)
@@ -542,18 +649,18 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "backend", "b":
     let backend = parseEnum(arg.normalize, TBackend.default)
     if backend == TBackend.default: localError(conf, info, "invalid backend: '$1'" % arg)
+    if backend == backendJs: # bug #21209
+      conf.globalOptions.excl {optThreadAnalysis, optThreads}
     conf.backend = backend
   of "doccmd": conf.docCmd = arg
   of "define", "d":
     expectArg(conf, switch, arg, pass, info)
     if {':', '='} in arg:
       splitSwitch(conf, arg, key, val, pass, info)
-      if cmpIgnoreStyle(key, "nimQuirky") == 0:
-        conf.exc = excQuirky
+      specialDefine(conf, key, pass)
       defineSymbol(conf.symbols, key, val)
     else:
-      if cmpIgnoreStyle(arg, "nimQuirky") == 0:
-        conf.exc = excQuirky
+      specialDefine(conf, arg, pass)
       defineSymbol(conf.symbols, arg)
   of "undef", "u":
     expectArg(conf, switch, arg, pass, info)
@@ -580,59 +687,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "project":
     processOnOffSwitchG(conf, {optWholeProject, optGenIndex}, arg, pass, info)
   of "gc":
-    if conf.backend == backendJs: return # for: bug #16033
-    expectArg(conf, switch, arg, pass, info)
-    if pass in {passCmd2, passPP}:
-      case arg.normalize
-      of "boehm":
-        conf.selectedGC = gcBoehm
-        defineSymbol(conf.symbols, "boehmgc")
-        incl conf.globalOptions, optTlsEmulation # Boehm GC doesn't scan the real TLS
-      of "refc":
-        conf.selectedGC = gcRefc
-      of "markandsweep":
-        conf.selectedGC = gcMarkAndSweep
-        defineSymbol(conf.symbols, "gcmarkandsweep")
-      of "destructors", "arc":
-        conf.selectedGC = gcArc
-        defineSymbol(conf.symbols, "gcdestructors")
-        defineSymbol(conf.symbols, "gcarc")
-        incl conf.globalOptions, optSeqDestructors
-        incl conf.globalOptions, optTinyRtti
-        if pass in {passCmd2, passPP}:
-          defineSymbol(conf.symbols, "nimSeqsV2")
-          defineSymbol(conf.symbols, "nimV2")
-        if conf.exc == excNone and conf.backend != backendCpp:
-          conf.exc = excGoto
-      of "orc":
-        conf.selectedGC = gcOrc
-        defineSymbol(conf.symbols, "gcdestructors")
-        defineSymbol(conf.symbols, "gcorc")
-        incl conf.globalOptions, optSeqDestructors
-        incl conf.globalOptions, optTinyRtti
-        if pass in {passCmd2, passPP}:
-          defineSymbol(conf.symbols, "nimSeqsV2")
-          defineSymbol(conf.symbols, "nimV2")
-        if conf.exc == excNone and conf.backend != backendCpp:
-          conf.exc = excGoto
-      of "hooks":
-        conf.selectedGC = gcHooks
-        defineSymbol(conf.symbols, "gchooks")
-        incl conf.globalOptions, optSeqDestructors
-        processOnOffSwitchG(conf, {optSeqDestructors}, arg, pass, info)
-        if pass in {passCmd2, passPP}:
-          defineSymbol(conf.symbols, "nimSeqsV2")
-      of "go":
-        conf.selectedGC = gcGo
-        defineSymbol(conf.symbols, "gogc")
-      of "none":
-        conf.selectedGC = gcNone
-        defineSymbol(conf.symbols, "nogc")
-      of "stack", "regions":
-        conf.selectedGC = gcRegions
-        defineSymbol(conf.symbols, "gcregions")
-      of "v2": warningOptionNoop(arg)
-      else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg)
+    warningDeprecated(conf, info, "`gc:option` is deprecated; use `mm:option` instead")
+    processMemoryManagementOption(switch, arg, pass, info, conf)
+  of "mm":
+    processMemoryManagementOption(switch, arg, pass, info, conf)
   of "warnings", "w":
     if processOnOffSwitchOrList(conf, {optWarns}, arg, pass, info): listWarnings(conf)
   of "warning": processSpecificNote(arg, wWarning, pass, info, switch, conf)
@@ -700,7 +758,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "linedir": processOnOffSwitch(conf, {optLineDir}, arg, pass, info)
   of "assertions", "a": processOnOffSwitch(conf, {optAssert}, arg, pass, info)
   of "threads":
-    if conf.backend == backendJs: discard
+    if conf.backend == backendJs or conf.cmd == cmdNimscript: discard
     else: processOnOffSwitchG(conf, {optThreads}, arg, pass, info)
     #if optThreads in conf.globalOptions: conf.setNote(warnGcUnsafe)
   of "tlsemulation": processOnOffSwitchG(conf, {optTlsEmulation}, arg, pass, info)
@@ -759,12 +817,18 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "clib":
     expectArg(conf, switch, arg, pass, info)
     if pass in {passCmd2, passPP}:
-      conf.cLinkedLibs.add processPath(conf, arg, info).string
+      conf.cLinkedLibs.add arg
   of "header":
     if conf != nil: conf.headerFile = arg
     incl(conf.globalOptions, optGenIndex)
   of "index":
-    processOnOffSwitchG(conf, {optGenIndex}, arg, pass, info)
+    case arg.normalize
+    of "", "on": conf.globalOptions.incl {optGenIndex}
+    of "only": conf.globalOptions.incl {optGenIndexOnly, optGenIndex}
+    of "off": conf.globalOptions.excl {optGenIndex, optGenIndexOnly}
+    else: localError(conf, info, errOnOrOffExpectedButXFound % arg)
+  of "noimportdoc":
+    processOnOffSwitchG(conf, {optNoImportdoc}, arg, pass, info)
   of "import":
     expectArg(conf, switch, arg, pass, info)
     if pass in {passCmd2, passPP}:
@@ -846,6 +910,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       of "v2": conf.symbolFiles = v2Sf
       of "stress": conf.symbolFiles = stressTest
       else: localError(conf, info, "invalid option for --incremental: " & arg)
+    setUseIc(conf.symbolFiles != disabledSf)
   of "skipcfg":
     processOnOffSwitchG(conf, {optSkipSystemConfigFile}, arg, pass, info)
   of "skipprojcfg":
@@ -858,6 +923,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if switch.normalize == "gendeps": deprecatedAlias(switch, "genscript")
     processOnOffSwitchG(conf, {optGenScript}, arg, pass, info)
     processOnOffSwitchG(conf, {optCompileOnly}, arg, pass, info)
+  of "gencdeps":
+    processOnOffSwitchG(conf, {optGenCDeps}, arg, pass, info)
   of "colors": processOnOffSwitchG(conf, {optUseColors}, arg, pass, info)
   of "lib":
     expectArg(conf, switch, arg, pass, info)
@@ -867,8 +934,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     splitSwitch(conf, arg, key, val, pass, info)
     os.putEnv(key, val)
   of "cc":
-    expectArg(conf, switch, arg, pass, info)
-    setCC(conf, arg, info)
+    if conf.backend != backendJs: # bug #19330
+      expectArg(conf, switch, arg, pass, info)
+      setCC(conf, arg, info)
   of "track":
     expectArg(conf, switch, arg, pass, info)
     track(conf, arg, info)
@@ -908,6 +976,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       excl(conf.notes, hintProcessing)
       excl(conf.mainPackageNotes, hintProcessing)
     else: localError(conf, info, "expected: dots|filenames|off, got: $1" % arg)
+  of "unitsep":
+    conf.unitSep = if switchOn(arg): "\31" else: ""
   of "listfullpaths":
     # xxx in future work, use `warningDeprecated`
     conf.filenameOption = if switchOn(arg): foAbs else: foCanonical
@@ -938,6 +1008,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     expectNoArg(conf, switch, arg, pass, info)
     conf.exc = low(ExceptionSystem)
     defineSymbol(conf.symbols, "noCppExceptions")
+  of "shownonexports":
+    expectNoArg(conf, switch, arg, pass, info)
+    showNonExportedFields(conf)
   of "exceptions":
     case arg.normalize
     of "cpp": conf.exc = excCpp
@@ -1007,6 +1080,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       defineSymbol(conf.symbols, "NimMajor", "1")
       defineSymbol(conf.symbols, "NimMinor", "2")
       conf.globalOptions.incl optNimV12Emulation
+    of "1.6":
+      defineSymbol(conf.symbols, "NimMajor", "1")
+      defineSymbol(conf.symbols, "NimMinor", "6")
+      conf.globalOptions.incl optNimV16Emulation
     else:
       localError(conf, info, "unknown Nim version; currently supported values are: `1.0`, `1.2`")
     # always be compatible with 1.x.100:
@@ -1031,7 +1108,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     processOnOffSwitchG(conf, {optEnableDeepCopy}, arg, pass, info)
   of "": # comes from "-" in for example: `nim c -r -` (gets stripped from -)
     handleStdinInput(conf)
-  of "nilseqs", "nilchecks", "mainmodule", "m", "symbol", "taintmode", "cs", "deadcodeelim": warningOptionNoop(switch)
+  of "nilseqs", "nilchecks", "symbol", "taintmode", "cs", "deadcodeelim": warningOptionNoop(switch)
+  of "nimmainprefix": conf.nimMainPrefix = arg
   else:
     if strutils.find(switch, '.') >= 0: options.setConfigVar(conf, switch, arg)
     else: invalidCmdLineOption(conf, pass, switch, info)
@@ -1068,6 +1146,8 @@ proc processArgument*(pass: TCmdLinePass; p: OptParser;
   else:
     if pass == passCmd1: config.commandArgs.add p.key
     if argsCount == 1:
+      if p.key.endsWith(".nims"):
+        incl(config.globalOptions, optWasNimscript)
       # support UNIX style filenames everywhere for portable build scripts:
       if config.projectName.len == 0:
         config.projectName = unixToNativePath(p.key)

compiler/concepts.nim

@@ -11,11 +11,13 @@
 ## for details. Note this is a first implementation and only the "Concept matching"
 ## section has been implemented.
 
-import ast, astalgo, semdata, lookups, lineinfos, idents, msgs, renderer,
-  types, intsets
+import ast, astalgo, semdata, lookups, lineinfos, idents, msgs, renderer, types, intsets
 
 from magicsys import addSonSkipIntLit
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 const
   logBindings = false
 
@@ -23,7 +25,7 @@ const
 ## --------------------------------------
 
 proc declareSelf(c: PContext; info: TLineInfo) =
-  ## adds the magical 'Self' symbols to the current scope.
+  ## Adds the magical 'Self' symbols to the current scope.
   let ow = getCurrOwner(c)
   let s = newSym(skType, getIdent(c.cache, "Self"), nextSymId(c.idgen), ow, info)
   s.typ = newType(tyTypeDesc, nextTypeId(c.idgen), ow)
@@ -32,7 +34,7 @@ proc declareSelf(c: PContext; info: TLineInfo) =
   addDecl(c, s, info)
 
 proc isSelf*(t: PType): bool {.inline.} =
-  ## is this the magical 'Self' type?
+  ## Is this the magical 'Self' type?
   t.kind == tyTypeDesc and tfPacked in t.flags
 
 proc makeTypeDesc*(c: PContext, typ: PType): PType =
@@ -45,8 +47,8 @@ proc makeTypeDesc*(c: PContext, typ: PType): PType =
 
 proc semConceptDecl(c: PContext; n: PNode): PNode =
   ## Recursive helper for semantic checking for the concept declaration.
-  ## Currently we only support lists of statements containing 'proc'
-  ## declarations and the like.
+  ## Currently we only support (possibly empty) lists of statements
+  ## containing 'proc' declarations and the like.
   case n.kind
   of nkStmtList, nkStmtListExpr:
     result = shallowCopy(n)
@@ -59,8 +61,10 @@ proc semConceptDecl(c: PContext; n: PNode): PNode =
     for i in 0..<n.len-1:
       result[i] = n[i]
     result[^1] = semConceptDecl(c, n[^1])
+  of nkCommentStmt:
+    discard
   else:
-    localError(c.config, n.info, "unexpected construct in the new-styled concept " & renderTree(n))
+    localError(c.config, n.info, "unexpected construct in the new-styled concept: " & renderTree(n))
     result = n
 
 proc semConceptDeclaration*(c: PContext; n: PNode): PNode =
@@ -97,7 +101,7 @@ proc existingBinding(m: MatchCon; key: PType): PType =
 proc conceptMatchNode(c: PContext; n: PNode; m: var MatchCon): bool
 
 proc matchType(c: PContext; f, a: PType; m: var MatchCon): bool =
-  ## the heart of the concept matching process. 'f' is the formal parameter of some
+  ## The heart of the concept matching process. 'f' is the formal parameter of some
   ## routine inside the concept that we're looking for. 'a' is the formal parameter
   ## of a routine that might match.
   const
@@ -270,7 +274,7 @@ proc matchSym(c: PContext; candidate: PSym, n: PNode; m: var MatchCon): bool =
 proc matchSyms(c: PContext, n: PNode; kinds: set[TSymKind]; m: var MatchCon): bool =
   ## Walk the current scope, extract candidates which the same name as 'n[namePos]',
   ## 'n' is the nkProcDef or similar from the concept that we try to match.
-  let candidates = searchInScopesFilterBy(c, n[namePos].sym.name, kinds)
+  let candidates = searchInScopesAllCandidatesFilterBy(c, n[namePos].sym.name, kinds)
   for candidate in candidates:
     #echo "considering ", typeToString(candidate.typ), " ", candidate.magic
     m.magic = candidate.magic
@@ -312,7 +316,7 @@ proc conceptMatch*(c: PContext; concpt, arg: PType; bindings: var TIdTable; invo
   ## concept's requirements. If so, we return true and fill the 'bindings' with pairs of
   ## (typeVar, instance) pairs. ('typeVar' is usually simply written as a generic 'T'.)
   ## 'invocation' can be nil for atomic concepts. For non-atomic concepts, it contains the
-  ## 'C[S, T]' parent type that we look for. We need this because we need to store bindings
+  ## `C[S, T]` parent type that we look for. We need this because we need to store bindings
   ## for 'S' and 'T' inside 'bindings' on a successful match. It is very important that
   ## we do not add any bindings at all on an unsuccessful match!
   var m = MatchCon(inferred: @[], potentialImplementation: arg)

compiler/condsyms.nim

@@ -25,7 +25,7 @@ proc undefSymbol*(symbols: StringTableRef; symbol: string) =
 #  result = if isDefined(symbol): gSymbols[symbol] else: nil
 
 iterator definedSymbolNames*(symbols: StringTableRef): string =
-  for key, val in pairs(symbols):
+  for key in keys(symbols):
     yield key
 
 proc countDefinedSymbols*(symbols: StringTableRef): int =
@@ -70,7 +70,7 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimVmExportFixed") # deadcode
   defineSymbol("nimHasSymOwnerInMacro") # deadcode
   defineSymbol("nimNewRuntime") # deadcode
-  defineSymbol("nimIncrSeqV3") # xxx: turn this into deadcode
+  defineSymbol("nimIncrSeqV3") # deadcode
   defineSymbol("nimAshr") # deadcode
   defineSymbol("nimNoNilSeqs") # deadcode
   defineSymbol("nimNoNilSeqs2") # deadcode
@@ -84,10 +84,24 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimHasSignatureHashInMacro") # deadcode
   defineSymbol("nimHasDefault") # deadcode
   defineSymbol("nimMacrosSizealignof") # deadcode
+  defineSymbol("nimNoZeroExtendMagic") # deadcode
+  defineSymbol("nimMacrosGetNodeId") # deadcode
+  defineSymbol("nimFixedForwardGeneric") # deadcode
+  defineSymbol("nimToOpenArrayCString") # deadcode
+  defineSymbol("nimHasUsed") # deadcode
+  defineSymbol("nimnomagic64") # deadcode
+  defineSymbol("nimNewShiftOps") # deadcode
+  defineSymbol("nimHasCursor") # deadcode
+  defineSymbol("nimAlignPragma") # deadcode
+  defineSymbol("nimHasExceptionsQuery") # deadcode
+  defineSymbol("nimHasIsNamedTuple") # deadcode
+  defineSymbol("nimHashOrdinalFixed") # deadcode
+  defineSymbol("nimHasSinkInference") # deadcode
+  defineSymbol("nimNewIntegerOps") # deadcode
+  defineSymbol("nimHasInvariant") # deadcode
+
+
 
-  # > 0.20.0
-  defineSymbol("nimNoZeroExtendMagic")
-  defineSymbol("nimMacrosGetNodeId")
   for f in Feature:
     defineSymbol("nimHas" & $f)
 
@@ -98,31 +112,18 @@ proc initDefines*(symbols: StringTableRef) =
 
   defineSymbol("nimFixedOwned")
   defineSymbol("nimHasStyleChecks")
-  defineSymbol("nimToOpenArrayCString")
-  defineSymbol("nimHasUsed")
-  defineSymbol("nimFixedForwardGeneric")
-  defineSymbol("nimnomagic64")
-  defineSymbol("nimNewShiftOps")
-  defineSymbol("nimHasCursor")
-  defineSymbol("nimAlignPragma")
-  defineSymbol("nimHasExceptionsQuery")
-  defineSymbol("nimHasIsNamedTuple")
-  defineSymbol("nimHashOrdinalFixed")
 
   when defined(nimHasLibFFI):
     # Renaming as we can't conflate input vs output define flags; e.g. this
     # will report the right thing regardless of whether user adds
     # `-d:nimHasLibFFI` in his user config.
-    defineSymbol("nimHasLibFFIEnabled")
+    defineSymbol("nimHasLibFFIEnabled") # deadcode
 
-  defineSymbol("nimHasSinkInference")
-  defineSymbol("nimNewIntegerOps")
-  defineSymbol("nimHasInvariant")
-  defineSymbol("nimHasStacktraceMsgs")
+  defineSymbol("nimHasStacktraceMsgs") # deadcode
   defineSymbol("nimDoesntTrackDefects")
-  defineSymbol("nimHasLentIterators")
-  defineSymbol("nimHasDeclaredMagic")
-  defineSymbol("nimHasStacktracesModule")
+  defineSymbol("nimHasLentIterators") # deadcode
+  defineSymbol("nimHasDeclaredMagic") # deadcode
+  defineSymbol("nimHasStacktracesModule") # deadcode
   defineSymbol("nimHasEffectTraitsModule")
   defineSymbol("nimHasCastPragmaBlocks")
   defineSymbol("nimHasDeclaredLocs")
@@ -133,4 +134,23 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimHasCustomLiterals")
   defineSymbol("nimHasUnifiedTuple")
   defineSymbol("nimHasIterable")
-  defineSymbol("nimHasTypeofVoid")
+  defineSymbol("nimHasTypeofVoid") # deadcode
+  defineSymbol("nimHasDragonBox") # deadcode
+  defineSymbol("nimHasHintAll")
+  defineSymbol("nimHasTrace")
+  defineSymbol("nimHasEffectsOf")
+
+  defineSymbol("nimHasEnforceNoRaises")
+  defineSymbol("nimHasTopDownInference")
+  defineSymbol("nimHasTemplateRedefinitionPragma")
+  defineSymbol("nimHasCstringCase")
+  defineSymbol("nimHasCallsitePragma")
+  defineSymbol("nimHasAmbiguousEnumHint")
+
+  defineSymbol("nimHasWarnCastSizes")
+  defineSymbol("nimHasOutParams")
+  defineSymbol("nimHasSystemRaisesDefect")
+  defineSymbol("nimHasWarnUnnamedBreak")
+  defineSymbol("nimHasGenericDefine")
+  defineSymbol("nimHasDefineAliases")
+  defineSymbol("nimHasWarnBareExcept")

compiler/debugutils.nim

@@ -4,7 +4,26 @@ Utilities to help with debugging nim compiler.
 Experimental API, subject to change.
 ]##
 
+#[
+## example
+useful debugging flags:
+--stacktrace -d:debug -d:nimDebugUtils
+ nim c -o:bin/nim_temp --stacktrace -d:debug -d:nimDebugUtils compiler/nim
+
+## future work
+* expose and improve astalgo.debug, replacing it by std/prettyprints,
+  refs https://github.com/nim-lang/RFCs/issues/385
+]#
+
 import options
+import std/wrapnils
+export wrapnils
+  # allows using things like: `?.n.sym.typ.len`
+
+import std/stackframes
+export stackframes
+  # allows using things like: `setFrameMsg c.config$n.info & " " & $n.kind`
+  # which doesn't log, but augments stacktrace with side channel information
 
 var conf0: ConfigRef
 
@@ -17,3 +36,37 @@ proc onNewConfigRef*(conf: ConfigRef) {.inline.} =
 proc getConfigRef*(): ConfigRef =
   ## nil, if -d:nimDebugUtils wasn't specified
   result = conf0
+
+proc isCompilerDebug*(): bool =
+  ##[
+  Provides a simple way for user code to enable/disable logging in the compiler
+  in a granular way. This can then be used in the compiler as follows:
+  ```nim
+  if isCompilerDebug():
+    echo ?.n.sym.typ.len
+  ```
+  ]##
+  runnableExamples:
+    proc main =
+      echo 2
+      {.define(nimCompilerDebug).}
+      echo 3.5 # code section in which `isCompilerDebug` will be true
+      {.undef(nimCompilerDebug).}
+      echo 'x'
+  conf0.isDefined("nimCompilerDebug")
+
+proc enteringDebugSection*() {.exportc, dynlib.} =
+  ## Provides a way for native debuggers to enable breakpoints, watchpoints, etc
+  ## when code of interest is being compiled.
+  ## 
+  ## Set your debugger to break on entering `nimCompilerIsEnteringDebugSection`
+  ## and then execute a desired command.
+  discard
+
+proc exitingDebugSection*() {.exportc, dynlib.} =
+  ## Provides a way for native debuggers to disable breakpoints, watchpoints, etc
+  ## when code of interest is no longer being compiled.
+  ## 
+  ## Set your debugger to break on entering `exitingDebugSection`
+  ## and then execute a desired command.
+  discard

compiler/depends.nim

@@ -9,10 +9,17 @@
 
 # This module implements a dependency file generator.
 
-import
-  options, ast, ropes, idents, passes, modulepaths, pathutils
+import options, ast, ropes, pathutils, msgs, lineinfos
+
+import modulegraphs
+
+import std/[os, parseutils]
+import strutils except addf
+import std/private/globs
+
+when defined(nimPreviewSlimSystem):
+  import std/assertions
 
-from modulegraphs import ModuleGraph, PPassContext
 
 type
   TGen = object of PPassContext
@@ -25,21 +32,63 @@ type
     dotGraph: Rope
 
 proc addDependencyAux(b: Backend; importing, imported: string) =
-  b.dotGraph.addf("$1 -> \"$2\";$n", [rope(importing), rope(imported)])
+  b.dotGraph.addf("\"$1\" -> \"$2\";$n", [rope(importing), rope(imported)])
   # s1 -> s2_4[label="[0-9]"];
 
-proc addDotDependency(c: PPassContext, n: PNode): PNode =
+proc toNimblePath(s: string, isStdlib: bool): string =
+  const stdPrefix = "std/"
+  const pkgPrefix = "pkg/"
+  if isStdlib:
+    let sub = "lib/"
+    var start = s.find(sub)
+    if start < 0:
+      doAssert false
+    else:
+      start += sub.len
+      let base = s[start..^1]
+
+      if base.startsWith("system") or base.startsWith("std"):
+        result = base
+      else:
+        for dir in stdlibDirs:
+          if base.startsWith(dir):
+            return stdPrefix & base.splitFile.name
+
+        result = stdPrefix & base
+  else:
+    var sub = getEnv("NIMBLE_DIR")
+    if sub.len == 0:
+      sub = ".nimble/pkgs/"
+    else:
+      sub.add "/pkgs/"
+    var start = s.find(sub)
+    if start < 0:
+      result = s
+    else:
+      start += sub.len
+      start += skipUntil(s, '/', start)
+      start += 1
+      result = pkgPrefix & s[start..^1]
+
+proc addDependency(c: PPassContext, g: PGen, b: Backend, n: PNode) =
+  doAssert n.kind == nkSym, $n.kind
+
+  let path = splitFile(toProjPath(g.config, n.sym.position.FileIndex))
+  let modulePath = splitFile(toProjPath(g.config, g.module.position.FileIndex))
+  let parent = nativeToUnixPath(modulePath.dir / modulePath.name).toNimblePath(belongsToStdlib(g.graph, g.module))
+  let child = nativeToUnixPath(path.dir / path.name).toNimblePath(belongsToStdlib(g.graph, n.sym))
+  addDependencyAux(b, parent, child)
+
+proc addDotDependency*(c: PPassContext, n: PNode): PNode =
   result = n
   let g = PGen(c)
   let b = Backend(g.graph.backend)
   case n.kind
   of nkImportStmt:
     for i in 0..<n.len:
-      var imported = getModuleName(g.config, n[i])
-      addDependencyAux(b, g.module.name.s, imported)
+      addDependency(c, g, b, n[i])
   of nkFromStmt, nkImportExceptStmt:
-    var imported = getModuleName(g.config, n[0])
-    addDependencyAux(b, g.module.name.s, imported)
+    addDependency(c, g, b, n[0])
   of nkStmtList, nkBlockStmt, nkStmtListExpr, nkBlockExpr:
     for i in 0..<n.len: discard addDotDependency(c, n[i])
   else:
@@ -51,18 +100,12 @@ proc generateDot*(graph: ModuleGraph; project: AbsoluteFile) =
       rope(project.splitFile.name), b.dotGraph],
             changeFileExt(project, "dot"))
 
-when not defined(nimHasSinkInference):
-  {.pragma: nosinks.}
-
-proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} =
+proc setupDependPass*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
   var g: PGen
   new(g)
   g.module = module
   g.config = graph.config
   g.graph = graph
   if graph.backend == nil:
-    graph.backend = Backend(dotGraph: nil)
+    graph.backend = Backend(dotGraph: "")
   result = g
-
-const gendependPass* = makePass(open = myOpen, process = addDotDependency)
-

compiler/dfa.nim

@@ -9,37 +9,33 @@
 
 ## Data flow analysis for Nim.
 ## We transform the AST into a linear list of instructions first to
-## make this easier to handle: There are only 2 different branching
+## make this easier to handle: There are only 3 different branching
 ## instructions: 'goto X' is an unconditional goto, 'fork X'
 ## is a conditional goto (either the next instruction or 'X' can be
-## taken). Exhaustive case statements are translated
+## taken), 'loop X' is the only jump that jumps back.
+##
+## Exhaustive case statements are translated
 ## so that the last branch is transformed into an 'else' branch.
 ## ``return`` and ``break`` are all covered by 'goto'.
 ##
-## Control flow through exception handling:
-## Contrary to popular belief, exception handling doesn't cause
-## many problems for this DFA representation, ``raise`` is a statement
-## that ``goes to`` the outer ``finally`` or ``except`` if there is one,
-## otherwise it is the same as ``return``. Every call is treated as
-## a call that can potentially ``raise``. However, without a surrounding
-## ``try`` we don't emit these ``fork ReturnLabel`` instructions in order
-## to speed up the dataflow analysis passes.
-##
 ## The data structures and algorithms used here are inspired by
 ## "A Graph–Free Approach to Data–Flow Analysis" by Markus Mohnen.
 ## https://link.springer.com/content/pdf/10.1007/3-540-45937-5_6.pdf
 
-import ast, types, intsets, lineinfos, renderer
+import ast, intsets, lineinfos, renderer, aliasanalysis
 import std/private/asciitables
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 type
   InstrKind* = enum
-    goto, fork, def, use
+    goto, loop, fork, def, use
   Instr* = object
-    n*: PNode # contains the def/use location.
     case kind*: InstrKind
-    of goto, fork: dest*: int
-    else: discard
+    of goto, fork, loop: dest*: int
+    of def, use:
+      n*: PNode # contains the def/use location.
 
   ControlFlowGraph* = seq[Instr]
 
@@ -56,9 +52,10 @@ type
 
   Con = object
     code: ControlFlowGraph
-    inTryStmt: int
+    inTryStmt, interestingInstructions: int
     blocks: seq[TBlock]
     owner: PSym
+    root: PSym
 
 proc codeListing(c: ControlFlowGraph, start = 0; last = -1): string =
   # for debugging purposes
@@ -66,7 +63,7 @@ proc codeListing(c: ControlFlowGraph, start = 0; last = -1): string =
   var jumpTargets = initIntSet()
   let last = if last < 0: c.len-1 else: min(last, c.len-1)
   for i in start..last:
-    if c[i].kind in {goto, fork}:
+    if c[i].kind in {goto, fork, loop}:
       jumpTargets.incl(i+c[i].dest)
   var i = start
   while i <= last:
@@ -77,12 +74,12 @@ proc codeListing(c: ControlFlowGraph, start = 0; last = -1): string =
     case c[i].kind
     of def, use:
       result.add renderTree(c[i].n)
-    of goto, fork:
+      result.add("\t#")
+      result.add($c[i].n.info.line)
+      result.add("\n")
+    of goto, fork, loop:
       result.add "L"
       result.addInt c[i].dest+i
-    result.add("\t#")
-    result.add($c[i].n.info.line)
-    result.add("\n")
     inc i
   if i in jumpTargets: result.add("L" & $i & ": End\n")
 
@@ -90,181 +87,13 @@ proc echoCfg*(c: ControlFlowGraph; start = 0; last = -1) {.deprecated.} =
   ## echos the ControlFlowGraph for debugging purposes.
   echo codeListing(c, start, last).alignTable
 
-proc forkI(c: var Con; n: PNode): TPosition =
+proc forkI(c: var Con): TPosition =
   result = TPosition(c.code.len)
-  c.code.add Instr(n: n, kind: fork, dest: 0)
+  c.code.add Instr(kind: fork, dest: 0)
 
-proc gotoI(c: var Con; n: PNode): TPosition =
+proc gotoI(c: var Con): TPosition =
   result = TPosition(c.code.len)
-  c.code.add Instr(n: n, kind: goto, dest: 0)
-
-#[
-
-Join is no more
-===============
-Instead of generating join instructions we adapt our traversal of the CFG.
-
-When encountering a fork we split into two paths, we follow the path
-starting at "pc + 1" until it encounters the joinpoint: "pc + forkInstr.dest".
-If we encounter gotos that would jump further than the current joinpoint,
-as can happen with gotos generated by unstructured controlflow such as break, raise or return,
-we simply suspend following the current path, and follow the other path until the new joinpoint
-which is simply the instruction pointer returned to us by the now suspended path.
-If the path we are following now, also encounters a goto that exceeds the joinpoint
-we repeat the process; suspending the current path and evaluating the other one with a new joinpoint.
-If we eventually reach a common joinpoint we join the two paths.
-This new "ping-pong" approach has the obvious advantage of not requiring join instructions, as such
-cutting down on the CFG size but is also mandatory for correctly handling complicated cases
-of unstructured controlflow.
-
-
-Design of join
-==============
-
-block:
-  if cond: break
-  def(x)
-
-use(x)
-
-Generates:
-
-L0: fork lab1
-  join L0  # patched.
-  goto Louter
-lab1:
-  def x
-  join L0
-Louter:
-  use x
-
-
-block outer:
-  while a:
-    while b:
-      if foo:
-        if bar:
-          break outer # --> we need to 'join' every pushed 'fork' here
-
-
-This works and then our abstract interpretation needs to deal with 'fork'
-differently. It really causes a split in execution. Two threads are
-"spawned" and both need to reach the 'join L' instruction. Afterwards
-the abstract interpretations are joined and execution resumes single
-threaded.
-
-
-Abstract Interpretation
------------------------
-
-proc interpret(pc, state, comesFrom): state =
-  result = state
-  # we need an explicit 'create' instruction (an explicit heap), in order
-  # to deal with 'var x = create(); var y = x; var z = y; destroy(z)'
-  while true:
-    case pc
-    of fork:
-      let a = interpret(pc+1, result, pc)
-      let b = interpret(forkTarget, result, pc)
-      result = a ++ b # ++ is a union operation
-      inc pc
-    of join:
-      if joinTarget == comesFrom: return result
-      else: inc pc
-    of use X:
-      if not result.contains(x):
-        error "variable not initialized " & x
-      inc pc
-    of def X:
-      if not result.contains(x):
-        result.incl X
-      else:
-        error "overwrite of variable causes memory leak " & x
-      inc pc
-    of destroy X:
-      result.excl X
-
-This is correct but still can lead to false positives:
-
-proc p(cond: bool) =
-  if cond:
-    new(x)
-  otherThings()
-  if cond:
-    destroy x
-
-Is not a leak. We should find a way to model *data* flow, not just
-control flow. One solution is to rewrite the 'if' without a fork
-instruction. The unstructured aspect can now be easily dealt with
-the 'goto' and 'join' instructions.
-
-proc p(cond: bool) =
-  L0: fork Lend
-    new(x)
-    # do not 'join' here!
-
-  Lend:
-    otherThings()
-    join L0  # SKIP THIS FOR new(x) SOMEHOW
-  destroy x
-  join L0 # but here.
-
-
-
-But if we follow 'goto Louter' we will never come to the join point.
-We restore the bindings after popping pc from the stack then there
-"no" problem?!
-
-
-while cond:
-  prelude()
-  if not condB: break
-  postlude()
-
---->
-var setFlag = true
-while cond and not setFlag:
-  prelude()
-  if not condB:
-    setFlag = true   # BUT: Dependency
-  if not setFlag:    # HERE
-    postlude()
-
---->
-var setFlag = true
-while cond and not setFlag:
-  prelude()
-  if not condB:
-    postlude()
-    setFlag = true
-
-
--------------------------------------------------
-
-while cond:
-  prelude()
-  if more:
-    if not condB: break
-    stuffHere()
-  postlude()
-
--->
-var setFlag = true
-while cond and not setFlag:
-  prelude()
-  if more:
-    if not condB:
-      setFlag = false
-    else:
-      stuffHere()
-      postlude()
-  else:
-    postlude()
-
-This is getting complicated. Instead we keep the whole 'join' idea but
-duplicate the 'join' instructions on breaks and return exits!
-
-]#
+  c.code.add Instr(kind: goto, dest: 0)
 
 proc genLabel(c: Con): TPosition = TPosition(c.code.len)
 
@@ -272,8 +101,8 @@ template checkedDistance(dist): int =
   doAssert low(int) div 2 + 1 < dist and dist < high(int) div 2
   dist
 
-proc jmpBack(c: var Con, n: PNode, p = TPosition(0)) =
-  c.code.add Instr(n: n, kind: goto, dest: checkedDistance(p.int - c.code.len))
+proc jmpBack(c: var Con, p = TPosition(0)) =
+  c.code.add Instr(kind: loop, dest: checkedDistance(p.int - c.code.len))
 
 proc patch(c: var Con, p: TPosition) =
   # patch with current index
@@ -283,12 +112,12 @@ proc gen(c: var Con; n: PNode)
 
 proc popBlock(c: var Con; oldLen: int) =
   var exits: seq[TPosition]
-  exits.add c.gotoI(newNode(nkEmpty))
+  exits.add c.gotoI()
   for f in c.blocks[oldLen].breakFixups:
     c.patch(f[0])
     for finale in f[1]:
       c.gen(finale)
-    exits.add c.gotoI(newNode(nkEmpty))
+    exits.add c.gotoI()
   for e in exits:
     c.patch e
   c.blocks.setLen(oldLen)
@@ -303,86 +132,29 @@ proc isTrue(n: PNode): bool =
   n.kind == nkSym and n.sym.kind == skEnumField and n.sym.position != 0 or
     n.kind == nkIntLit and n.intVal != 0
 
-when true:
-  proc genWhile(c: var Con; n: PNode) =
-    # We unroll every loop 3 times. We emulate 0, 1, 2 iterations
-    # through the loop. We need to prove this is correct for our
-    # purposes. But Herb Sutter claims it is. (Proof by authority.)
-    #[
-    while cond:
-      body
-
-    Becomes:
-
-    block:
-      if cond:
-        body
-        if cond:
-          body
-          if cond:
-            body
-
-    We still need to ensure 'break' resolves properly, so an AST to AST
-    translation is impossible.
-
-    So the code to generate is:
-
-      cond
-      fork L4  # F1
-      body
-      cond
-      fork L5  # F2
-      body
-      cond
-      fork L6  # F3
-      body
-    L6:
-      join F3
-    L5:
-      join F2
-    L4:
-      join F1
-    ]#
-    if isTrue(n[0]):
-      # 'while true' is an idiom in Nim and so we produce
-      # better code for it:
-      withBlock(nil):
-        for i in 0..2:
-          c.gen(n[1])
-    else:
-      withBlock(nil):
-        var endings: array[3, TPosition]
-        for i in 0..2:
-          c.gen(n[0])
-          endings[i] = c.forkI(n)
-          c.gen(n[1])
-        for i in countdown(endings.high, 0):
-          c.patch(endings[i])
-
-else:
-  proc genWhile(c: var Con; n: PNode) =
-    # lab1:
-    #   cond, tmp
-    #   fork tmp, lab2
-    #   body
-    #   jmp lab1
-    # lab2:
-    let lab1 = c.genLabel
-    withBlock(nil):
-      if isTrue(n[0]):
-        c.gen(n[1])
-        c.jmpBack(n, lab1)
-      else:
-        c.gen(n[0])
-        forkT(n):
-          c.gen(n[1])
-          c.jmpBack(n, lab1)
-
-template forkT(n, body) =
-  let lab1 = c.forkI(n)
+template forkT(body) =
+  let lab1 = c.forkI()
   body
   c.patch(lab1)
 
+proc genWhile(c: var Con; n: PNode) =
+  # lab1:
+  #   cond, tmp
+  #   fork tmp, lab2
+  #   body
+  #   jmp lab1
+  # lab2:
+  let lab1 = c.genLabel
+  withBlock(nil):
+    if isTrue(n[0]):
+      c.gen(n[1])
+      c.jmpBack(lab1)
+    else:
+      c.gen(n[0])
+      forkT:
+        c.gen(n[1])
+        c.jmpBack(lab1)
+
 proc genIf(c: var Con, n: PNode) =
   #[
 
@@ -421,15 +193,22 @@ proc genIf(c: var Con, n: PNode) =
 
   ]#
   var endings: seq[TPosition] = @[]
+  let oldInteresting = c.interestingInstructions
+  let oldLen = c.code.len
+
   for i in 0..<n.len:
     let it = n[i]
     c.gen(it[0])
     if it.len == 2:
-      forkT(it[1]):
-        c.gen(it[1])
-        endings.add c.gotoI(it[1])
-  for i in countdown(endings.high, 0):
-    c.patch(endings[i])
+      forkT:
+        c.gen(it.lastSon)
+        endings.add c.gotoI()
+
+  if oldInteresting == c.interestingInstructions:
+    setLen c.code, oldLen
+  else:
+    for i in countdown(endings.high, 0):
+      c.patch(endings[i])
 
 proc genAndOr(c: var Con; n: PNode) =
   #   asgn dest, a
@@ -438,7 +217,7 @@ proc genAndOr(c: var Con; n: PNode) =
   # lab1:
   #   join F1
   c.gen(n[1])
-  forkT(n):
+  forkT:
     c.gen(n[2])
 
 proc genCase(c: var Con; n: PNode) =
@@ -453,34 +232,34 @@ proc genCase(c: var Con; n: PNode) =
   #    elsePart
   #  Lend:
   let isExhaustive = skipTypes(n[0].typ,
-    abstractVarRange-{tyTypeDesc}).kind notin {tyFloat..tyFloat128, tyString}
+    abstractVarRange-{tyTypeDesc}).kind notin {tyFloat..tyFloat128, tyString, tyCstring}
 
-  # we generate endings as a set of chained gotos, this is a bit awkward but it
-  # ensures when recursively traversing the CFG for various analysis, we don't
-  # artificially extended the life of each branch (for the purposes of DFA)
-  # beyond the minimum amount.
   var endings: seq[TPosition] = @[]
   c.gen(n[0])
+  let oldInteresting = c.interestingInstructions
+  let oldLen = c.code.len
   for i in 1..<n.len:
     let it = n[i]
     if it.len == 1 or (i == n.len-1 and isExhaustive):
       # treat the last branch as 'else' if this is an exhaustive case statement.
       c.gen(it.lastSon)
-      if endings.len != 0:
-        c.patch(endings[^1])
     else:
-      forkT(it.lastSon):
+      forkT:
         c.gen(it.lastSon)
-        if endings.len != 0:
-          c.patch(endings[^1])
-        endings.add c.gotoI(it.lastSon)
+        endings.add c.gotoI()
+
+  if oldInteresting == c.interestingInstructions:
+    setLen c.code, oldLen
+  else:
+    for i in countdown(endings.high, 0):
+      c.patch(endings[i])
 
 proc genBlock(c: var Con; n: PNode) =
   withBlock(n[0].sym):
     c.gen(n[1])
 
 proc genBreakOrRaiseAux(c: var Con, i: int, n: PNode) =
-  let lab1 = c.gotoI(n)
+  let lab1 = c.gotoI()
   if c.blocks[i].isTryBlock:
     c.blocks[i].raiseFixups.add lab1
   else:
@@ -493,6 +272,7 @@ proc genBreakOrRaiseAux(c: var Con, i: int, n: PNode) =
     c.blocks[i].breakFixups.add (lab1, trailingFinales)
 
 proc genBreak(c: var Con; n: PNode) =
+  inc c.interestingInstructions
   if n[0].kind == nkSym:
     for i in countdown(c.blocks.high, 0):
       if not c.blocks[i].isTryBlock and c.blocks[i].label == n[0].sym:
@@ -523,9 +303,9 @@ proc genTry(c: var Con; n: PNode) =
   for i in 1..<n.len:
     let it = n[i]
     if it.kind != nkFinally:
-      forkT(it):
+      forkT:
         c.gen(it.lastSon)
-        endings.add c.gotoI(it)
+        endings.add c.gotoI()
   for i in countdown(endings.high, 0):
     c.patch(endings[i])
 
@@ -533,11 +313,12 @@ proc genTry(c: var Con; n: PNode) =
   if fin.kind == nkFinally:
     c.gen(fin[0])
 
-template genNoReturn(c: var Con; n: PNode) =
+template genNoReturn(c: var Con) =
   # leave the graph
-  c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
+  c.code.add Instr(kind: goto, dest: high(int) - c.code.len)
 
 proc genRaise(c: var Con; n: PNode) =
+  inc c.interestingInstructions
   gen(c, n[0])
   if c.inTryStmt > 0:
     for i in countdown(c.blocks.high, 0):
@@ -546,13 +327,14 @@ proc genRaise(c: var Con; n: PNode) =
         return
     assert false #Unreachable
   else:
-    genNoReturn(c, n)
+    genNoReturn(c)
 
 proc genImplicitReturn(c: var Con) =
   if c.owner.kind in {skProc, skFunc, skMethod, skIterator, skConverter} and resultPos < c.owner.ast.len:
     gen(c, c.owner.ast[resultPos])
 
 proc genReturn(c: var Con; n: PNode) =
+  inc c.interestingInstructions
   if n[0].kind != nkEmpty:
     gen(c, n[0])
   else:
@@ -561,124 +343,6 @@ proc genReturn(c: var Con; n: PNode) =
 
 const
   InterestingSyms = {skVar, skResult, skLet, skParam, skForVar, skTemp}
-  PathKinds0 = {nkDotExpr, nkCheckedFieldExpr,
-                nkBracketExpr, nkDerefExpr, nkHiddenDeref,
-                nkAddr, nkHiddenAddr,
-                nkObjDownConv, nkObjUpConv}
-  PathKinds1 = {nkHiddenStdConv, nkHiddenSubConv}
-
-proc skipConvDfa*(n: PNode): PNode =
-  result = n
-  while true:
-    case result.kind
-    of nkObjDownConv, nkObjUpConv:
-      result = result[0]
-    of PathKinds1:
-      result = result[1]
-    else: break
-
-type AliasKind* = enum
-  yes, no, maybe
-
-proc aliases*(obj, field: PNode): AliasKind =
-  # obj -> field:
-  # x -> x: true
-  # x -> x.f: true
-  # x.f -> x: false
-  # x.f -> x.f: true
-  # x.f -> x.v: false
-  # x -> x[0]: true
-  # x[0] -> x: false
-  # x[0] -> x[0]: true
-  # x[0] -> x[1]: false
-  # x -> x[i]: true
-  # x[i] -> x: false
-  # x[i] -> x[i]: maybe; Further analysis could make this return true when i is a runtime-constant
-  # x[i] -> x[j]: maybe; also returns maybe if only one of i or j is a compiletime-constant
-  template collectImportantNodes(result, n) =
-    var result: seq[PNode]
-    var n = n
-    while true:
-      case n.kind
-      of PathKinds0 - {nkDotExpr, nkCheckedFieldExpr, nkBracketExpr}:
-        n = n[0]
-      of PathKinds1:
-        n = n[1]
-      of nkDotExpr, nkCheckedFieldExpr, nkBracketExpr:
-        result.add n
-        n = n[0]
-      of nkSym:
-        result.add n; break
-      else: return no
-
-  collectImportantNodes(objImportantNodes, obj)
-  collectImportantNodes(fieldImportantNodes, field)
-
-  # If field is less nested than obj, then it cannot be part of/aliased by obj
-  if fieldImportantNodes.len < objImportantNodes.len: return no
-
-  result = yes
-  for i in 1..objImportantNodes.len:
-    # We compare the nodes leading to the location of obj and field
-    # with each other.
-    # We continue until they diverge, in which case we return no, or
-    # until we reach the location of obj, in which case we do not need
-    # to look further, since field must be part of/aliased by obj now.
-    # If we encounter an element access using an index which is a runtime value,
-    # we simply return maybe instead of yes; should further nodes not diverge.
-    let currFieldPath = fieldImportantNodes[^i]
-    let currObjPath = objImportantNodes[^i]
-
-    if currFieldPath.kind != currObjPath.kind:
-      return no
-
-    case currFieldPath.kind
-    of nkSym:
-      if currFieldPath.sym != currObjPath.sym: return no
-    of nkDotExpr:
-      if currFieldPath[1].sym != currObjPath[1].sym: return no
-    of nkCheckedFieldExpr:
-      if currFieldPath[0][1].sym != currObjPath[0][1].sym: return no
-    of nkBracketExpr:
-      if currFieldPath[1].kind in nkLiterals and currObjPath[1].kind in nkLiterals:
-        if currFieldPath[1].intVal != currObjPath[1].intVal:
-          return no
-      else:
-        result = maybe
-    else: assert false # unreachable
-
-proc isAnalysableFieldAccess*(orig: PNode; owner: PSym): bool =
-  var n = orig
-  while true:
-    case n.kind
-    of PathKinds0 - {nkHiddenDeref, nkDerefExpr}:
-      n = n[0]
-    of PathKinds1:
-      n = n[1]
-    of nkHiddenDeref, nkDerefExpr:
-      # We "own" sinkparam[].loc but not ourVar[].location as it is a nasty
-      # pointer indirection.
-      # bug #14159, we cannot reason about sinkParam[].location as it can
-      # still be shared for tyRef.
-      n = n[0]
-      return n.kind == nkSym and n.sym.owner == owner and
-         (n.sym.typ.skipTypes(abstractInst-{tyOwned}).kind in {tyOwned})
-    else: break
-  # XXX Allow closure deref operations here if we know
-  # the owner controlled the closure allocation?
-  result = n.kind == nkSym and n.sym.owner == owner and
-    {sfGlobal, sfThread, sfCursor} * n.sym.flags == {} and
-    (n.sym.kind != skParam or isSinkParam(n.sym)) # or n.sym.typ.kind == tyVar)
-  # Note: There is a different move analyzer possible that checks for
-  # consume(param.key); param.key = newValue  for all paths. Then code like
-  #
-  #   let splited = split(move self.root, x)
-  #   self.root = merge(splited.lower, splited.greater)
-  #
-  # could be written without the ``move self.root``. However, this would be
-  # wrong! Then the write barrier for the ``self.root`` assignment would
-  # free the old data and all is lost! Lesson: Don't be too smart, trust the
-  # lower level C++ optimizer to specialize this code.
 
 proc skipTrivials(c: var Con, n: PNode): PNode =
   result = n
@@ -697,8 +361,9 @@ proc genUse(c: var Con; orig: PNode) =
   let n = c.skipTrivials(orig)
 
   if n.kind == nkSym:
-    if n.sym.kind in InterestingSyms:
-      c.code.add Instr(n: orig, kind: use)
+    if n.sym.kind in InterestingSyms and n.sym == c.root:
+      c.code.add Instr(kind: use, n: orig)
+      inc c.interestingInstructions
   else:
     gen(c, n)
 
@@ -706,7 +371,9 @@ proc genDef(c: var Con; orig: PNode) =
   let n = c.skipTrivials(orig)
 
   if n.kind == nkSym and n.sym.kind in InterestingSyms:
-    c.code.add Instr(n: orig, kind: def)
+    if n.sym == c.root:
+      c.code.add Instr(kind: def, n: orig)
+      inc c.interestingInstructions
 
 proc genCall(c: var Con; n: PNode) =
   gen(c, n[0])
@@ -714,18 +381,17 @@ proc genCall(c: var Con; n: PNode) =
   if t != nil: t = t.skipTypes(abstractInst)
   for i in 1..<n.len:
     gen(c, n[i])
-    when false:
-      if t != nil and i < t.len and t[i].kind == tyOut:
-        # Pass by 'out' is a 'must def'. Good enough for a move optimizer.
-        genDef(c, n[i])
+    if t != nil and i < t.len and isOutParam(t[i]):
+      # Pass by 'out' is a 'must def'. Good enough for a move optimizer.
+      genDef(c, n[i])
   # every call can potentially raise:
-  if c.inTryStmt > 0 and canRaiseConservative(n[0]):
+  if false: # c.inTryStmt > 0 and canRaiseConservative(n[0]):
     # we generate the instruction sequence:
     # fork lab1
     # goto exceptionHandler (except or finally)
     # lab1:
     # join F1
-    forkT(n):
+    forkT:
       for i in countdown(c.blocks.high, 0):
         if c.blocks[i].isTryBlock:
           genBreakOrRaiseAux(c, i, n)
@@ -763,12 +429,18 @@ proc gen(c: var Con; n: PNode) =
       else:
         genCall(c, n)
       if sfNoReturn in n[0].sym.flags:
-        genNoReturn(c, n)
+        genNoReturn(c)
     else:
       genCall(c, n)
   of nkCharLit..nkNilLit: discard
-  of nkAsgn, nkFastAsgn:
+  of nkAsgn, nkFastAsgn, nkSinkAsgn:
     gen(c, n[1])
+
+    if n[0].kind in PathKinds0:
+      let a = c.skipTrivials(n[0])
+      if a.kind in nkCallKinds:
+        gen(c, a)
+
     # watch out: 'obj[i].f2 = value' sets 'f2' but
     # "uses" 'i'. But we are only talking about builtin array indexing so
     # it doesn't matter and 'x = 34' is NOT a usage of 'x'.
@@ -798,13 +470,32 @@ proc gen(c: var Con; n: PNode) =
   of nkDefer: doAssert false, "dfa construction pass requires the elimination of 'defer'"
   else: discard
 
-proc constructCfg*(s: PSym; body: PNode): ControlFlowGraph =
+when false:
+  proc optimizeJumps(c: var ControlFlowGraph) =
+    for i in 0..<c.len:
+      case c[i].kind
+      of goto, fork:
+        var pc = i + c[i].dest
+        if pc < c.len and c[pc].kind == goto:
+          while pc < c.len and c[pc].kind == goto:
+            let newPc = pc + c[pc].dest
+            if newPc > pc:
+              pc = newPc
+            else:
+              break
+          c[i].dest = pc - i
+      of loop, def, use: discard
+
+proc constructCfg*(s: PSym; body: PNode; root: PSym): ControlFlowGraph =
   ## constructs a control flow graph for ``body``.
-  var c = Con(code: @[], blocks: @[], owner: s)
+  var c = Con(code: @[], blocks: @[], owner: s, root: root)
   withBlock(s):
     gen(c, body)
-    genImplicitReturn(c)
+    if root.kind == skResult:
+      genImplicitReturn(c)
   when defined(gcArc) or defined(gcOrc):
     result = c.code # will move
   else:
     shallowCopy(result, c.code)
+  when false:
+    optimizeJumps result

compiler/docgen2.nim

@@ -11,7 +11,7 @@
 # semantic checking.
 
 import
-  options, ast, msgs, passes, docgen, lineinfos, pathutils
+  options, ast, msgs, docgen, lineinfos, pathutils, packages
 
 from modulegraphs import ModuleGraph, PPassContext
 
@@ -23,7 +23,7 @@ type
   PGen = ref TGen
 
 proc shouldProcess(g: PGen): bool =
-  (optWholeProject in g.doc.conf.globalOptions and g.module.getnimblePkgId == g.doc.conf.mainPackageId) or
+  (optWholeProject in g.doc.conf.globalOptions and g.doc.conf.belongsToProjectPackage(g.module)) or
       sfMainModule in g.module.flags or g.config.projectMainIdx == g.module.info.fileIndex
 
 template closeImpl(body: untyped) {.dirty.} =
@@ -31,27 +31,28 @@ template closeImpl(body: untyped) {.dirty.} =
   let useWarning = sfMainModule notin g.module.flags
   let groupedToc = true
   if shouldProcess(g):
+    finishGenerateDoc(g.doc)
     body
     try:
       generateIndex(g.doc)
     except IOError:
       discard
 
-proc close(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
+proc closeDoc*(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
   closeImpl:
     writeOutput(g.doc, useWarning, groupedToc)
 
-proc closeJson(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
+proc closeJson*(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
   closeImpl:
     writeOutputJson(g.doc, useWarning)
 
-proc processNode(c: PPassContext, n: PNode): PNode =
+proc processNode*(c: PPassContext, n: PNode): PNode =
   result = n
   var g = PGen(c)
   if shouldProcess(g):
-    generateDoc(g.doc, n, n)
+    generateDoc(g.doc, n, n, g.config)
 
-proc processNodeJson(c: PPassContext, n: PNode): PNode =
+proc processNodeJson*(c: PPassContext, n: PNode): PNode =
   result = n
   var g = PGen(c)
   if shouldProcess(g):
@@ -63,25 +64,15 @@ template myOpenImpl(ext: untyped) {.dirty.} =
   g.module = module
   g.config = graph.config
   var d = newDocumentor(AbsoluteFile toFullPath(graph.config, FileIndex module.position),
-      graph.cache, graph.config, ext, module)
-  d.hasToc = true
+      graph.cache, graph.config, ext, module, hasToc = true)
   g.doc = d
   result = g
 
-proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
+proc openHtml*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
   myOpenImpl(HtmlExt)
 
-proc myOpenTex(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
+proc openTex*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
   myOpenImpl(TexExt)
 
-proc myOpenJson(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
+proc openJson*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
   myOpenImpl(JsonExt)
-
-const docgen2Pass* = makePass(open = myOpen, process = processNode, close = close)
-const docgen2TexPass* = makePass(open = myOpenTex, process = processNode,
-                                 close = close)
-const docgen2JsonPass* = makePass(open = myOpenJson, process = processNodeJson,
-                                  close = closeJson)
-
-proc finishDoc2Pass*(project: string) =
-  discard

compiler/enumtostr.nim

@@ -1,6 +1,10 @@
 
 import ast, idents, lineinfos, modulegraphs, magicsys
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
+
 proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph; idgen: IdGenerator): PSym =
   result = newSym(skProc, getIdent(g.cache, "$"), nextSymId idgen, t.owner, info)
 
@@ -26,7 +30,7 @@ proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph; idgen: IdGener
     assert(t.n[i].kind == nkSym)
     var field = t.n[i].sym
     let val = if field.ast == nil: field.name.s else: field.ast.strVal
-    caseStmt.add newTree(nkOfBranch, newSymNode(field),
+    caseStmt.add newTree(nkOfBranch, newIntTypeNode(field.position, t),
       newTree(nkStmtList, newTree(nkFastAsgn, newSymNode(res), newStrNode(val, info))))
     #newIntTypeNode(nkIntLit, field.position, t)
 

compiler/errorhandling.nim

@@ -12,6 +12,9 @@
 
 import ast, renderer, options, strutils, types
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 type
   ErrorKind* = enum ## expand as you need.
     RawTypeMismatchError

compiler/evalffi.nim

@@ -65,19 +65,19 @@ proc importcSymbol*(conf: ConfigRef, sym: PSym): PNode =
     if (lib.isNil or lib.kind == libHeader) and not gExeHandle.isNil:
       libPathMsg = "current exe: " & getAppFilename() & " nor libc: " & libcDll
       # first try this exe itself:
-      theAddr = gExeHandle.symAddr(name)
+      theAddr = gExeHandle.symAddr(name.cstring)
       # then try libc:
       if theAddr.isNil:
         let dllhandle = getDll(conf, gDllCache, libcDll, sym.info)
-        theAddr = dllhandle.symAddr(name)
+        theAddr = dllhandle.symAddr(name.cstring)
     elif not lib.isNil:
       let dll = if lib.kind == libHeader: libcDll else: lib.path.strVal
       libPathMsg = dll
       let dllhandle = getDll(conf, gDllCache, dll, sym.info)
-      theAddr = dllhandle.symAddr(name)
+      theAddr = dllhandle.symAddr(name.cstring)
     if theAddr.isNil: globalError(conf, sym.info,
       "cannot import symbol: " & name & " from " & libPathMsg)
-    result.intVal = cast[ByteAddress](theAddr)
+    result.intVal = cast[int](theAddr)
 
 proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.Type =
   if t == nil: return addr libffi.type_void
@@ -110,10 +110,10 @@ proc mapCallConv(conf: ConfigRef, cc: TCallingConvention, info: TLineInfo): TABI
   else:
     globalError(conf, info, "cannot map calling convention to FFI")
 
-template rd(T, p: untyped): untyped = (cast[ptr T](p))[]
-template wr(T, p, v: untyped): untyped = (cast[ptr T](p))[] = v
+template rd(typ, p: untyped): untyped = (cast[ptr typ](p))[]
+template wr(typ, p, v: untyped): untyped = (cast[ptr typ](p))[] = v
 template `+!`(x, y: untyped): untyped =
-  cast[pointer](cast[ByteAddress](x) + y)
+  cast[pointer](cast[int](x) + y)
 
 proc packSize(conf: ConfigRef, v: PNode, typ: PType): int =
   ## computes the size of the blob
@@ -177,8 +177,8 @@ const maxPackDepth = 20
 var packRecCheck = 0
 
 proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) =
-  template awr(T, v: untyped): untyped =
-    wr(T, res, v)
+  template awr(typ, v: untyped): untyped =
+    wr(typ, res, v)
 
   case typ.kind
   of tyBool: awr(bool, v.intVal != 0)
@@ -369,13 +369,13 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
       # in their unboxed representation so nothing it to be unpacked:
       result = n
     else:
-      awi(nkPtrLit, cast[ByteAddress](p))
+      awi(nkPtrLit, cast[int](p))
   of tyPtr, tyRef, tyVar, tyLent:
     let p = rd(pointer, x)
     if p.isNil:
       setNil()
     elif n == nil or n.kind == nkPtrLit:
-      awi(nkPtrLit, cast[ByteAddress](p))
+      awi(nkPtrLit, cast[int](p))
     elif n != nil and n.len == 1:
       internalAssert(conf, n.kind == nkRefTy)
       n[0] = unpack(conf, p, typ.lastSon, n[0])

compiler/evaltempl.nim

@@ -26,7 +26,7 @@ type
 
 proc copyNode(ctx: TemplCtx, a, b: PNode): PNode =
   result = copyNode(a)
-  if ctx.instLines: result.info = b.info
+  if ctx.instLines: setInfoRecursive(result, b.info)
 
 proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
   template handleParam(param) =
@@ -83,10 +83,14 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
         not c.isDeclarative:
       c.isDeclarative = true
       isDeclarative = true
-    var res = copyNode(c, templ, actual)
-    for i in 0..<templ.len:
-      evalTemplateAux(templ[i], actual, c, res)
-    result.add res
+    if (not c.isDeclarative) and templ.kind in nkCallKinds and isRunnableExamples(templ[0]):
+      # fixes bug #16993, bug #18054
+      discard
+    else:
+      var res = copyNode(c, templ, actual)
+      for i in 0..<templ.len:
+        evalTemplateAux(templ[i], actual, c, res)
+      result.add res
     if isDeclarative: c.isDeclarative = false
 
 const
@@ -200,7 +204,7 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
     result = copyNode(body)
     ctx.instLines = sfCallsite in tmpl.flags
     if ctx.instLines:
-      result.info = n.info
+      setInfoRecursive(result, n.info)
     for i in 0..<body.safeLen:
       evalTemplateAux(body[i], args, ctx, result)
   result.flags.incl nfFromTemplate

compiler/extccomp.nim

@@ -12,9 +12,14 @@
 # from a lineinfos file, to provide generalized procedures to compile
 # nim files.
 
-import ropes, platform, condsyms, options, msgs, lineinfos, pathutils
+import ropes, platform, condsyms, options, msgs, lineinfos, pathutils, modulepaths
 
-import os, strutils, osproc, std/sha1, streams, sequtils, times, strtabs, json
+import std/[os, osproc, sha1, streams, sequtils, times, strtabs, json, jsonutils, sugar, parseutils]
+
+import std / strutils except addf
+
+when defined(nimPreviewSlimSystem):
+  import std/syncio
 
 type
   TInfoCCProp* = enum         # properties of the C compiler:
@@ -83,10 +88,10 @@ compiler gcc:
     linkLibCmd: " -l$1",
     debug: "",
     pic: "-fPIC",
-    asmStmtFrmt: "asm($1);$n",
+    asmStmtFrmt: "__asm__($1);$n",
     structStmtFmt: "$1 $3 $2 ", # struct|union [packed] $name
     produceAsm: gnuAsmListing,
-    cppXsupport: "-std=gnu++14 -funsigned-char",
+    cppXsupport: "-std=gnu++17 -funsigned-char",
     props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard, hasGnuAsm,
             hasAttribute})
 
@@ -113,7 +118,7 @@ compiler nintendoSwitchGCC:
     asmStmtFrmt: "asm($1);$n",
     structStmtFmt: "$1 $3 $2 ", # struct|union [packed] $name
     produceAsm: gnuAsmListing,
-    cppXsupport: "-std=gnu++14 -funsigned-char",
+    cppXsupport: "-std=gnu++17 -funsigned-char",
     props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard, hasGnuAsm,
             hasAttribute})
 
@@ -150,7 +155,7 @@ compiler vcc:
     compileTmpl: "/c$vccplatform $options $include /nologo /Fo$objfile $file",
     buildGui: " /SUBSYSTEM:WINDOWS user32.lib ",
     buildDll: " /LD",
-    buildLib: "lib /OUT:$libfile $objfiles",
+    buildLib: "vccexe --command:lib$vccplatform /nologo /OUT:$libfile $objfiles",
     linkerExe: "cl",
     linkTmpl: "$builddll$vccplatform /Fe$exefile $objfiles $buildgui /nologo $options",
     includeCmd: " /I",
@@ -252,7 +257,7 @@ compiler envcc:
     buildGui: "",
     buildDll: " -shared ",
     buildLib: "", # XXX: not supported yet
-    linkerExe: "cc",
+    linkerExe: "",
     linkTmpl: "-o $exefile $buildgui $builddll $objfiles $options",
     includeCmd: " -I",
     linkDirCmd: "", # XXX: not supported yet
@@ -323,7 +328,9 @@ proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string =
                      platform.OS[conf.target.targetOS].name & '.' &
                      CC[c].name & fullSuffix
     result = getConfigVar(conf, fullCCname)
-    if result.len == 0:
+    if existsConfigVar(conf, fullCCname):
+      result = getConfigVar(conf, fullCCname)
+    else:
       # not overridden for this cross compilation setting?
       result = getConfigVar(conf, CC[c].name & fullSuffix)
   else:
@@ -367,6 +374,7 @@ proc initVars*(conf: ConfigRef) =
 
 proc completeCfilePath*(conf: ConfigRef; cfile: AbsoluteFile,
                         createSubDir: bool = true): AbsoluteFile =
+  ## Generate the absolute file path to the generated modules.
   result = completeGeneratedFilePath(conf, cfile, createSubDir)
 
 proc toObjFile*(conf: ConfigRef; filename: AbsoluteFile): AbsoluteFile =
@@ -377,7 +385,7 @@ proc addFileToCompile*(conf: ConfigRef; cf: Cfile) =
   conf.toCompile.add(cf)
 
 proc addLocalCompileOption*(conf: ConfigRef; option: string; nimfile: AbsoluteFile) =
-  let key = completeCfilePath(conf, withPackageName(conf, nimfile)).string
+  let key = completeCfilePath(conf, mangleModuleName(conf, nimfile).AbsoluteFile).string
   var value = conf.cfileSpecificOptions.getOrDefault(key)
   if strutils.find(value, option, 0) < 0:
     addOpt(value, option)
@@ -430,7 +438,13 @@ proc noAbsolutePaths(conf: ConfigRef): bool {.inline.} =
   # really: Cross compilation from Linux to Linux for example is entirely
   # reasonable.
   # `optGenMapping` is included here for niminst.
-  result = conf.globalOptions * {optGenScript, optGenMapping} != {}
+  # We use absolute paths for vcc / cl, see issue #19883.
+  let options =
+    if conf.cCompiler == ccVcc:
+      {optGenMapping}
+    else:
+      {optGenScript, optGenMapping}
+  result = conf.globalOptions * options != {}
 
 proc cFileSpecificOptions(conf: ConfigRef; nimname, fullNimFile: string): string =
   result = conf.compileOptions
@@ -483,7 +497,10 @@ proc needsExeExt(conf: ConfigRef): bool {.inline.} =
            (conf.target.hostOS == osWindows)
 
 proc useCpp(conf: ConfigRef; cfile: AbsoluteFile): bool =
-  conf.backend == backendCpp and not cfile.string.endsWith(".c")
+  # List of possible file extensions taken from gcc
+  for ext in [".C", ".cc", ".cpp", ".CPP", ".c++", ".cp", ".cxx"]:
+    if cfile.string.endsWith(ext): return true
+  false
 
 proc envFlags(conf: ConfigRef): string =
   result = if conf.backend == backendCpp:
@@ -491,14 +508,14 @@ proc envFlags(conf: ConfigRef): string =
           else:
             getEnv("CFLAGS")
 
-proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; cfile: AbsoluteFile): string =
+proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; isCpp: bool): string =
   if compiler == ccEnv:
-    result = if useCpp(conf, cfile):
+    result = if isCpp:
               getEnv("CXX")
             else:
               getEnv("CC")
   else:
-    result = if useCpp(conf, cfile):
+    result = if isCpp:
               CC[compiler].cppCompiler
             else:
               CC[compiler].compilerExe
@@ -512,47 +529,36 @@ proc ccHasSaneOverflow*(conf: ConfigRef): bool =
     result = false # assume an old or crappy GCC
     var exe = getConfigVar(conf, conf.cCompiler, ".exe")
     if exe.len == 0: exe = CC[conf.cCompiler].compilerExe
-    let (s, exitCode) = try: execCmdEx(exe & " --version") except: ("", 1)
+    # NOTE: should we need the full version, use -dumpfullversion
+    let (s, exitCode) = try: execCmdEx(exe & " -dumpversion") except IOError, OSError, ValueError: ("", 1)
     if exitCode == 0:
-      var i = 0
-      var j = 0
-      # the version is the last part of the first line:
-      while i < s.len and s[i] != '\n':
-        if s[i] in {' ', '\t'}: j = i+1
-        inc i
-      if j > 0:
-        var major = 0
-        while j < s.len and s[j] in {'0'..'9'}:
-          major = major * 10 + (ord(s[j]) - ord('0'))
-          inc j
-        if i < s.len and s[j] == '.': inc j
-        while j < s.len and s[j] in {'0'..'9'}:
-          inc j
-        if j+1 < s.len and s[j] == '.' and s[j+1] in {'0'..'9'}:
-          # we found a third version number, chances are high
-          # we really parsed the version:
-          result = major >= 5
+      var major: int
+      discard parseInt(s, major)
+      result = major >= 5
   else:
     result = conf.cCompiler == ccCLang
 
 proc getLinkerExe(conf: ConfigRef; compiler: TSystemCC): string =
   result = if CC[compiler].linkerExe.len > 0: CC[compiler].linkerExe
-           elif optMixedMode in conf.globalOptions and conf.backend != backendCpp: CC[compiler].cppCompiler
-           else: getCompilerExe(conf, compiler, AbsoluteFile"")
+           else: getCompilerExe(conf, compiler, optMixedMode in conf.globalOptions or conf.backend == backendCpp)
 
 proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
                          isMainFile = false; produceOutput = false): string =
-  let c = conf.cCompiler
+  let
+    c = conf.cCompiler
+    isCpp = useCpp(conf, cfile.cname)
   # We produce files like module.nim.cpp, so the absolute Nim filename is not
   # cfile.name but `cfile.cname.changeFileExt("")`:
   var options = cFileSpecificOptions(conf, cfile.nimname, cfile.cname.changeFileExt("").string)
-  if useCpp(conf, cfile.cname):
+  if isCpp:
     # needs to be prepended so that --passc:-std=c++17 can override default.
     # we could avoid allocation by making cFileSpecificOptions inplace
     options = CC[c].cppXsupport & ' ' & options
+    # If any C++ file was compiled, we need to use C++ driver for linking as well
+    incl conf.globalOptions, optMixedMode
 
   var exe = getConfigVar(conf, c, ".exe")
-  if exe.len == 0: exe = getCompilerExe(conf, c, cfile.cname)
+  if exe.len == 0: exe = getCompilerExe(conf, c, isCpp)
 
   if needsExeExt(conf): exe = addFileExt(exe, "exe")
   if (optGenDynLib in conf.globalOptions or (conf.hcrOn and not isMainFile)) and
@@ -572,7 +578,7 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
 
     compilePattern = joinPath(conf.cCompilerPath, exe)
   else:
-    compilePattern = getCompilerExe(conf, c, cfile.cname)
+    compilePattern = getCompilerExe(conf, c, isCpp)
 
   includeCmd.add(join([CC[c].includeCmd, quoteShell(conf.projectPath.string)]))
 
@@ -614,7 +620,7 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
           "for the selected C compiler: " & CC[conf.cCompiler].name)
 
   result.add(' ')
-  result.addf(CC[c].compileTmpl, [
+  strutils.addf(result, CC[c].compileTmpl, [
     "dfile", dfile,
     "file", cfsh, "objfile", quoteShell(objfile),
     "options", options, "include", includeCmd,
@@ -634,7 +640,7 @@ proc footprint(conf: ConfigRef; cfile: Cfile): SecureHash =
 proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool =
   if conf.backend == backendJs: return false # pre-existing behavior, but not sure it's good
 
-  let hashFile = toGeneratedFile(conf, conf.withPackageName(cfile.cname), "sha1")
+  let hashFile = toGeneratedFile(conf, conf.mangleModuleName(cfile.cname).AbsoluteFile, "sha1")
   let currentHash = footprint(conf, cfile)
   var f: File
   if open(f, hashFile.string, fmRead):
@@ -649,8 +655,10 @@ proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool =
       close(f)
 
 proc addExternalFileToCompile*(conf: ConfigRef; c: var Cfile) =
+  # we want to generate the hash file unconditionally
+  let extFileChanged = externalFileChanged(conf, c)
   if optForceFullMake notin conf.globalOptions and fileExists(c.obj) and
-      not externalFileChanged(conf, c):
+      not extFileChanged:
     c.flags.incl CfileFlag.Cached
   else:
     # make sure Nim keeps recompiling the external file on reruns
@@ -665,11 +673,13 @@ proc addExternalFileToCompile*(conf: ConfigRef; filename: AbsoluteFile) =
   addExternalFileToCompile(conf, c)
 
 proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
-                objfiles: string, isDllBuild: bool): string =
+                objfiles: string, isDllBuild: bool, removeStaticFile: bool): string =
   if optGenStaticLib in conf.globalOptions:
-    removeFile output # fixes: bug #16947
+    if removeStaticFile:
+      removeFile output # fixes: bug #16947
     result = CC[conf.cCompiler].buildLib % ["libfile", quoteShell(output),
-                                            "objfiles", objfiles]
+                                            "objfiles", objfiles,
+                                            "vccplatform", vccplatform(conf)]
   else:
     var linkerExe = getConfigVar(conf, conf.cCompiler, ".linkerexe")
     if linkerExe.len == 0: linkerExe = getLinkerExe(conf, conf.cCompiler)
@@ -702,7 +712,7 @@ proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
         "buildgui", buildgui, "options", linkOptions, "objfiles", objfiles,
         "exefile", exefile, "nim", getPrefixDir(conf).string, "lib", conf.libpath.string])
     result.add ' '
-    result.addf(linkTmpl, ["builddll", builddll,
+    strutils.addf(result, linkTmpl, ["builddll", builddll,
         "mapfile", mapfile,
         "buildgui", buildgui, "options", linkOptions,
         "objfiles", objfiles, "exefile", exefile,
@@ -750,8 +760,9 @@ proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
   if optCDebug in conf.globalOptions and conf.cCompiler == ccVcc:
     result.add " /Zi /FS /Od"
 
-template getLinkCmd(conf: ConfigRef; output: AbsoluteFile, objfiles: string): string =
-  getLinkCmd(conf, output, objfiles, optGenDynLib in conf.globalOptions)
+template getLinkCmd(conf: ConfigRef; output: AbsoluteFile, objfiles: string,
+                    removeStaticFile = false): string =
+  getLinkCmd(conf, output, objfiles, optGenDynLib in conf.globalOptions, removeStaticFile)
 
 template tryExceptOSErrorMessage(conf: ConfigRef; errorPrefix: string = "", body: untyped) =
   try:
@@ -825,6 +836,15 @@ proc linkViaResponseFile(conf: ConfigRef; cmd: string) =
   finally:
     removeFile(linkerArgs)
 
+proc linkViaShellScript(conf: ConfigRef; cmd: string) =
+  let linkerScript = conf.projectName & "_" & "linkerScript.sh"
+  writeFile(linkerScript, cmd)
+  let shell = getEnv("SHELL")
+  try:
+    execLinkCmd(conf, shell & " " & linkerScript)
+  finally:
+    removeFile(linkerScript)
+
 proc getObjFilePath(conf: ConfigRef, f: Cfile): string =
   if noAbsolutePaths(conf): f.obj.extractFilename
   else: f.obj.string
@@ -838,9 +858,23 @@ proc hcrLinkTargetName(conf: ConfigRef, objFile: string, isMain = false): Absolu
 proc displayProgressCC(conf: ConfigRef, path, compileCmd: string): string =
   if conf.hasHint(hintCC):
     if optListCmd in conf.globalOptions or conf.verbosity > 1:
-      result = MsgKindToStr[hintCC] % (demanglePackageName(path.splitFile.name) & ": " & compileCmd)
+      result = MsgKindToStr[hintCC] % (demangleModuleName(path.splitFile.name) & ": " & compileCmd)
     else:
-      result = MsgKindToStr[hintCC] % demanglePackageName(path.splitFile.name)
+      result = MsgKindToStr[hintCC] % demangleModuleName(path.splitFile.name)
+
+proc preventLinkCmdMaxCmdLen(conf: ConfigRef, linkCmd: string) =
+  # Prevent linkcmd from exceeding the maximum command line length.
+  # Windows's command line limit is about 8K (8191 characters) so C compilers on
+  # Windows support a feature where the command line can be passed via ``@linkcmd``
+  # to them.
+  const MaxCmdLen = when defined(windows): 8_000 elif defined(macosx): 260_000 else: 32_000
+  if linkCmd.len > MaxCmdLen:
+    when defined(macosx):
+      linkViaShellScript(conf, linkCmd)
+    else:
+      linkViaResponseFile(conf, linkCmd)
+  else:
+    execLinkCmd(conf, linkCmd)
 
 proc callCCompiler*(conf: ConfigRef) =
   var
@@ -850,7 +884,7 @@ proc callCCompiler*(conf: ConfigRef) =
     return # speed up that call if only compiling and no script shall be
            # generated
   #var c = cCompiler
-  var script: Rope = nil
+  var script: Rope = ""
   var cmds: TStringSeq
   var prettyCmds: TStringSeq
   let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx])
@@ -891,7 +925,7 @@ proc callCCompiler*(conf: ConfigRef) =
         let objFile = conf.getObjFilePath(x)
         let buildDll = idx != mainFileIdx
         let linkTarget = conf.hcrLinkTargetName(objFile, not buildDll)
-        cmds.add(getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll))
+        cmds.add(getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll, removeStaticFile = true))
         # try to remove all .pdb files for the current binary so they don't accumulate endlessly in the nimcache
         # for more info check the comment inside of getLinkCmd() where the /PDB:<filename> MSVC flag is used
         if isVSCompatible(conf):
@@ -913,18 +947,12 @@ proc callCCompiler*(conf: ConfigRef) =
         objfiles.add(' ')
         objfiles.add(quoteShell(objFile))
       let mainOutput = if optGenScript notin conf.globalOptions: conf.prepareToWriteOutput
-                       else: AbsoluteFile(conf.projectName)
-      linkCmd = getLinkCmd(conf, mainOutput, objfiles)
+                       else: AbsoluteFile(conf.outFile)
+
+      linkCmd = getLinkCmd(conf, mainOutput, objfiles, removeStaticFile = true)
       extraCmds = getExtraCmds(conf, mainOutput)
       if optCompileOnly notin conf.globalOptions:
-        const MaxCmdLen = when defined(windows): 8_000 else: 32_000
-        if linkCmd.len > MaxCmdLen:
-          # Windows's command line limit is about 8K (don't laugh...) so C compilers on
-          # Windows support a feature where the command line can be passed via ``@linkcmd``
-          # to them.
-          linkViaResponseFile(conf, linkCmd)
-        else:
-          execLinkCmd(conf, linkCmd)
+        preventLinkCmdMaxCmdLen(conf, linkCmd)
         for cmd in extraCmds:
           execExternalProgram(conf, cmd, hintExecuting)
   else:
@@ -942,195 +970,91 @@ proc jsonBuildInstructionsFile*(conf: ConfigRef): AbsoluteFile =
   # works out of the box with `hashMainCompilationParams`.
   result = getNimcacheDir(conf) / conf.outFile.changeFileExt("json")
 
+const cacheVersion = "D20210525T193831" # update when `BuildCache` spec changes
+type BuildCache = object
+  cacheVersion: string
+  outputFile: string
+  compile: seq[(string, string)]
+  link: seq[string]
+  linkcmd: string
+  extraCmds: seq[string]
+  configFiles: seq[string] # the hash shouldn't be needed
+  stdinInput: bool
+  projectIsCmd: bool
+  cmdInput: string
+  currentDir: string
+  cmdline: string
+  depfiles: seq[(string, string)]
+  nimexe: string
+
 proc writeJsonBuildInstructions*(conf: ConfigRef) =
-  # xxx use std/json instead, will result in simpler, more maintainable code.
-  template lit(x: string) = f.write x
-  template str(x: string) =
-    buf.setLen 0
-    escapeJson(x, buf)
-    f.write buf
-
-  proc cfiles(conf: ConfigRef; f: File; buf: var string; clist: CfileList, isExternal: bool) =
-    var comma = false
-    for i, it in clist:
-      if CfileFlag.Cached in it.flags: continue
-      let compileCmd = getCompileCFileCmd(conf, it)
-      if comma: lit ",\L" else: comma = true
-      lit "["
-      str it.cname.string
-      lit ", "
-      str compileCmd
-      lit "]"
-
-  proc linkfiles(conf: ConfigRef; f: File; buf, objfiles: var string; clist: CfileList;
-                 llist: seq[string]) =
-    var pastStart = false
-    template impl(path) =
-      let path2 = quoteShell(path)
-      objfiles.add(' ')
-      objfiles.add(path2)
-      if pastStart: lit ",\L"
-      str path2
-      pastStart = true
-    for it in llist:
-      let objfile = if noAbsolutePaths(conf): it.extractFilename else: it
-      impl(addFileExt(objfile, CC[conf.cCompiler].objExt))
-    for it in clist:
-      impl(it.obj)
-    lit "\L"
-
-  proc depfiles(conf: ConfigRef; f: File; buf: var string) =
-    var i = 0
-    for it in conf.m.fileInfos:
+  var linkFiles = collect(for it in conf.externalToLink:
+    var it = it
+    if conf.noAbsolutePaths: it = it.extractFilename
+    it.addFileExt(CC[conf.cCompiler].objExt))
+  for it in conf.toCompile: linkFiles.add it.obj.string
+  var bcache = BuildCache(
+    cacheVersion: cacheVersion,
+    outputFile: conf.absOutFile.string,
+    compile: collect(for i, it in conf.toCompile:
+      if CfileFlag.Cached notin it.flags: (it.cname.string, getCompileCFileCmd(conf, it))),
+    link: linkFiles,
+    linkcmd: getLinkCmd(conf, conf.absOutFile, linkFiles.quoteShellCommand),
+    extraCmds: getExtraCmds(conf, conf.absOutFile),
+    stdinInput: conf.projectIsStdin,
+    projectIsCmd: conf.projectIsCmd,
+    cmdInput: conf.cmdInput,
+    configFiles: conf.configFiles.mapIt(it.string),
+    currentDir: getCurrentDir())
+  if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"):
+    bcache.cmdline = conf.commandLine
+    bcache.depfiles = collect(for it in conf.m.fileInfos:
       let path = it.fullPath.string
       if isAbsolute(path): # TODO: else?
-        if i > 0: lit "],\L"
-        lit "["
-        str path
-        lit ", "
-        str $secureHashFile(path)
-        inc i
-    lit "]\L"
-
-
-  var buf = newStringOfCap(50)
-  let jsonFile = conf.jsonBuildInstructionsFile
-  conf.jsonBuildFile = jsonFile
-  let output = conf.absOutFile
-
-  var f: File
-  if open(f, jsonFile.string, fmWrite):
-    lit "{\L"
-    lit "\"outputFile\": "
-    str $output
-
-    lit ",\L\"compile\":[\L"
-    cfiles(conf, f, buf, conf.toCompile, false)
-    lit "],\L\"link\":[\L"
-    var objfiles = ""
-    # XXX add every file here that is to link
-    linkfiles(conf, f, buf, objfiles, conf.toCompile, conf.externalToLink)
-
-    lit "],\L\"linkcmd\": "
-    str getLinkCmd(conf, output, objfiles)
-
-    lit ",\L\"extraCmds\": "
-    lit $(%* getExtraCmds(conf, conf.absOutFile))
-
-    lit ",\L\"stdinInput\": "
-    lit $(%* conf.projectIsStdin)
-    lit ",\L\"projectIsCmd\": "
-    lit $(%* conf.projectIsCmd)
-    lit ",\L\"cmdInput\": "
-    lit $(%* conf.cmdInput)
-    lit ",\L\"currentDir\": "
-    lit $(%* getCurrentDir())
-
-    if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"):
-      lit ",\L\"cmdline\": "
-      str conf.commandLine
-      lit ",\L\"depfiles\":[\L"
-      depfiles(conf, f, buf)
-      lit "],\L\"nimexe\": \L"
-      str hashNimExe()
-      lit "\L"
-
-    lit "\L}\L"
-    close(f)
+        (path, $secureHashFile(path)))
+    bcache.nimexe = hashNimExe()
+  conf.jsonBuildFile = conf.jsonBuildInstructionsFile
+  conf.jsonBuildFile.string.writeFile(bcache.toJson.pretty)
 
 proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile): bool =
-  if not fileExists(jsonFile): return true
-  if not fileExists(conf.absOutFile): return true
-  result = false
-  try:
-    let data = json.parseFile(jsonFile.string)
-    for key in "depfiles cmdline stdinInput currentDir".split:
-      if not data.hasKey(key): return true
-    if getCurrentDir() != data["currentDir"].getStr:
-      # fixes bug #16271
-      # Note that simply comparing `expandFilename(projectFile)` would
-      # not be sufficient in case other flags depend implicitly on `getCurrentDir`,
-      # and would require much more care. Simply re-compiling is safer for now.
-      # A better strategy for future work would be to cache (with an LRU cache)
-      # the N most recent unique build instructions, as done with `rdmd`,
-      # which is both robust and avoids recompilation when switching back and forth
-      # between projects, see https://github.com/timotheecour/Nim/issues/199
-      return true
-    let oldCmdLine = data["cmdline"].getStr
-    if conf.commandLine != oldCmdLine:
-      return true
-    if hashNimExe() != data["nimexe"].getStr:
-      return true
-    let stdinInput = data["stdinInput"].getBool
-    let projectIsCmd = data["projectIsCmd"].getBool
-    if conf.projectIsStdin or stdinInput:
-      # could optimize by returning false if stdin input was the same,
-      # but I'm not sure how to get full stding input
-      return true
-
-    if conf.projectIsCmd or projectIsCmd:
-      if not (conf.projectIsCmd and projectIsCmd): return true
-      if not data.hasKey("cmdInput"): return true
-      let cmdInput = data["cmdInput"].getStr
-      if cmdInput != conf.cmdInput: return true
-
-    let depfilesPairs = data["depfiles"]
-    doAssert depfilesPairs.kind == JArray
-    for p in depfilesPairs:
-      doAssert p.kind == JArray
-      # >= 2 for forwards compatibility with potential later .json files:
-      doAssert p.len >= 2
-      let depFilename = p[0].getStr
-      let oldHashValue = p[1].getStr
-      let newHashValue = $secureHashFile(depFilename)
-      if oldHashValue != newHashValue:
-        return true
+  if not fileExists(jsonFile) or not fileExists(conf.absOutFile): return true
+  var bcache: BuildCache
+  try: bcache.fromJson(jsonFile.string.parseFile)
   except IOError, OSError, ValueError:
-    echo "Warning: JSON processing failed: ", getCurrentExceptionMsg()
-    result = true
+    stderr.write "Warning: JSON processing failed for: $#\n" % jsonFile.string
+    return true
+  if bcache.currentDir != getCurrentDir() or # fixes bug #16271
+     bcache.configFiles != conf.configFiles.mapIt(it.string) or
+     bcache.cacheVersion != cacheVersion or bcache.outputFile != conf.absOutFile.string or
+     bcache.cmdline != conf.commandLine or bcache.nimexe != hashNimExe() or
+     bcache.projectIsCmd != conf.projectIsCmd or conf.cmdInput != bcache.cmdInput: return true
+  if bcache.stdinInput or conf.projectIsStdin: return true
+    # xxx optimize by returning false if stdin input was the same
+  for (file, hash) in bcache.depfiles:
+    if $secureHashFile(file) != hash: return true
 
 proc runJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile) =
-  try:
-    let data = json.parseFile(jsonFile.string)
-    let output = data["outputFile"].getStr
-    createDir output.parentDir
-    let outputCurrent = $conf.absOutFile
-    if output != outputCurrent:
-      # previously, any specified output file would be silently ignored;
-      # simply copying won't work in some cases, for example with `extraCmds`,
-      # so we just make it an error, user should use same command for jsonscript
-      # as was used with --compileOnly.
-      globalError(conf, gCmdLineInfo, "jsonscript command outputFile '$1' must match '$2' which was specified during --compileOnly, see \"outputFile\" entry in '$3' " % [outputCurrent, output, jsonFile.string])
-
-    let toCompile = data["compile"]
-    doAssert toCompile.kind == JArray
-    var cmds: TStringSeq
-    var prettyCmds: TStringSeq
-    let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx])
-    for c in toCompile:
-      doAssert c.kind == JArray
-      doAssert c.len >= 2
-
-      cmds.add(c[1].getStr)
-      prettyCmds.add displayProgressCC(conf, c[0].getStr, c[1].getStr)
-
-    execCmdsInParallel(conf, cmds, prettyCb)
-
-    let linkCmd = data["linkcmd"]
-    doAssert linkCmd.kind == JString
-    execLinkCmd(conf, linkCmd.getStr)
-    if data.hasKey("extraCmds"):
-      let extraCmds = data["extraCmds"]
-      doAssert extraCmds.kind == JArray
-      for cmd in extraCmds:
-        doAssert cmd.kind == JString, $cmd.kind
-        let cmd2 = cmd.getStr
-        execExternalProgram(conf, cmd2, hintExecuting)
-
-  except:
+  var bcache: BuildCache
+  try: bcache.fromJson(jsonFile.string.parseFile)
+  except ValueError, KeyError, JsonKindError:
     let e = getCurrentException()
-    conf.quitOrRaise "\ncaught exception:\n" & e.msg & "\nstacktrace:\n" & e.getStackTrace() &
-         "error evaluating JSON file: " & jsonFile.string
+    conf.quitOrRaise "\ncaught exception:\n$#\nstacktrace:\n$#error evaluating JSON file: $#" %
+      [e.msg, e.getStackTrace(), jsonFile.string]
+  let output = bcache.outputFile
+  createDir output.parentDir
+  let outputCurrent = $conf.absOutFile
+  if output != outputCurrent or bcache.cacheVersion != cacheVersion:
+    globalError(conf, gCmdLineInfo,
+      "jsonscript command outputFile '$1' must match '$2' which was specified during --compileOnly, see \"outputFile\" entry in '$3' " %
+      [outputCurrent, output, jsonFile.string])
+  var cmds, prettyCmds: TStringSeq
+  let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx])
+  for (name, cmd) in bcache.compile:
+    cmds.add cmd
+    prettyCmds.add displayProgressCC(conf, name, cmd)
+  execCmdsInParallel(conf, cmds, prettyCb)
+  preventLinkCmdMaxCmdLen(conf, bcache.linkcmd)
+  for cmd in bcache.extraCmds: execExternalProgram(conf, cmd, hintExecuting)
 
 proc genMappingFiles(conf: ConfigRef; list: CfileList): Rope =
   for it in list:

compiler/filter_tmpl.nim

@@ -22,7 +22,7 @@ type
     info: TLineInfo
     indent, emitPar: int
     x: string                # the current input line
-    outp: PLLStream          # the output will be parsed by pnimsyn
+    outp: PLLStream          # the output will be parsed by parser
     subsChar, nimDirective: char
     emit, conc, toStr: string
     curly, bracket, par: int

compiler/gorgeimpl.nim

@@ -12,6 +12,9 @@
 import msgs, std / sha1, os, osproc, streams, options,
   lineinfos, pathutils
 
+when defined(nimPreviewSlimSystem):
+  import std/syncio
+
 proc readOutput(p: Process): (string, int) =
   result[0] = ""
   var output = p.outputStream
@@ -46,7 +49,11 @@ proc opGorge*(cmd, input, cache: string, info: TLineInfo; conf: ConfigRef): (str
       if result[1] == 0:
         writeFile(filename, result[0])
     except IOError, OSError:
-      if not readSuccessful: result = ("", -1)
+      if not readSuccessful:
+        when defined(nimLegacyGorgeErrors):
+          result = ("", -1)
+        else:
+          result = ("Error running startProcess: " & getCurrentExceptionMsg(), -1)
   else:
     try:
       var p = startProcess(cmd, workingDir,
@@ -57,4 +64,7 @@ proc opGorge*(cmd, input, cache: string, info: TLineInfo; conf: ConfigRef): (str
       result = p.readOutput
       p.close()
     except IOError, OSError:
-      result = ("", -1)
+      when defined(nimLegacyGorgeErrors):
+        result = ("", -1)
+      else:
+        result = ("Error running startProcess: " & getCurrentExceptionMsg(), -1)

compiler/guards.nim

@@ -12,6 +12,9 @@
 import ast, astalgo, msgs, magicsys, nimsets, trees, types, renderer, idents,
   saturate, modulegraphs, options, lineinfos, int128
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 const
   someEq = {mEqI, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc,
     mEqStr, mEqSet, mEqCString}
@@ -65,9 +68,12 @@ proc isLetLocation(m: PNode, isApprox: bool): bool =
     case n.kind
     of nkDotExpr, nkCheckedFieldExpr, nkObjUpConv, nkObjDownConv:
       n = n[0]
-    of nkDerefExpr, nkHiddenDeref:
+    of nkDerefExpr:
       n = n[0]
       inc derefs
+    of nkHiddenDeref:
+      n = n[0]
+      if not isApprox: inc derefs
     of nkBracketExpr:
       if isConstExpr(n[1]) or isLet(n[1]) or isConstExpr(n[1].skipConv):
         n = n[0]
@@ -197,7 +203,7 @@ proc highBound*(conf: ConfigRef; x: PNode; o: Operators): PNode =
              nkIntLit.newIntNode(lastOrd(conf, typ))
            elif typ.kind == tySequence and x.kind == nkSym and
                x.sym.kind == skConst:
-             nkIntLit.newIntNode(x.sym.ast.len-1)
+             nkIntLit.newIntNode(x.sym.astdef.len-1)
            else:
              o.opAdd.buildCall(o.opLen.buildCall(x), minusOne())
   result.info = x.info
@@ -441,8 +447,14 @@ proc sameTree*(a, b: PNode): bool =
 proc hasSubTree(n, x: PNode): bool =
   if n.sameTree(x): result = true
   else:
-    for i in 0..n.safeLen-1:
-      if hasSubTree(n[i], x): return true
+    case n.kind
+    of nkEmpty..nkNilLit:
+      result = n.sameTree(x)
+    of nkFormalParams:
+      discard
+    else:
+      for i in 0..<n.len:
+        if hasSubTree(n[i], x): return true
 
 proc invalidateFacts*(s: var seq[PNode], n: PNode) =
   # We are able to guard local vars (as opposed to 'let' variables)!
@@ -1045,8 +1057,12 @@ proc buildProperFieldCheck(access, check: PNode; o: Operators): PNode =
     assert check.getMagic == mNot
     result = buildProperFieldCheck(access, check[1], o).neg(o)
 
-proc checkFieldAccess*(m: TModel, n: PNode; conf: ConfigRef) =
+proc checkFieldAccess*(m: TModel, n: PNode; conf: ConfigRef; produceError: bool) =
   for i in 1..<n.len:
     let check = buildProperFieldCheck(n[0], n[i], m.g.operators)
     if check != nil and m.doesImply(check) != impYes:
-      message(conf, n.info, warnProveField, renderTree(n[0])); break
+      if produceError:
+        localError(conf, n.info, "field access outside of valid case branch: " & renderTree(n[0]))
+      else:
+        message(conf, n.info, warnProveField, renderTree(n[0]))
+      break

compiler/hlo.nim

@@ -8,6 +8,10 @@
 #
 
 # This include implements the high level optimization pass.
+# included from sem.nim
+
+when defined(nimPreviewSlimSystem):
+  import std/assertions
 
 proc hlo(c: PContext, n: PNode): PNode
 
@@ -67,7 +71,7 @@ proc hlo(c: PContext, n: PNode): PNode =
     # already processed (special cases in semstmts.nim)
     result = n
   else:
-    if n.kind in {nkFastAsgn, nkAsgn, nkIdentDefs, nkVarTuple} and
+    if n.kind in {nkFastAsgn, nkAsgn, nkSinkAsgn, nkIdentDefs, nkVarTuple} and
         n[0].kind == nkSym and
         {sfGlobal, sfPure} * n[0].sym.flags == {sfGlobal, sfPure}:
       # do not optimize 'var g {.global} = re(...)' again!

compiler/ic/bitabs.nim

@@ -1,8 +1,11 @@
 ## A BiTable is a table that can be seen as an optimized pair
-## of (Table[LitId, Val], Table[Val, LitId]).
+## of `(Table[LitId, Val], Table[Val, LitId])`.
 
 import hashes, rodfiles
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 type
   LitId* = distinct uint32
 
@@ -90,13 +93,13 @@ proc getOrIncl*[T](t: var BiTable[T]; v: T): LitId =
   t.vals.add v
 
 
-proc `[]`*[T](t: var BiTable[T]; LitId: LitId): var T {.inline.} =
-  let idx = idToIdx LitId
+proc `[]`*[T](t: var BiTable[T]; litId: LitId): var T {.inline.} =
+  let idx = idToIdx litId
   assert idx < t.vals.len
   result = t.vals[idx]
 
-proc `[]`*[T](t: BiTable[T]; LitId: LitId): lent T {.inline.} =
-  let idx = idToIdx LitId
+proc `[]`*[T](t: BiTable[T]; litId: LitId): lent T {.inline.} =
+  let idx = idToIdx litId
   assert idx < t.vals.len
   result = t.vals[idx]
 

compiler/ic/cbackend.nim

@@ -20,8 +20,11 @@
 
 import std/packedsets, algorithm, tables
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 import ".."/[ast, options, lineinfos, modulegraphs, cgendata, cgen,
-  pathutils, extccomp, msgs]
+  pathutils, extccomp, msgs, modulepaths]
 
 import packed_ast, ic, dce, rodfiles
 
@@ -61,7 +64,8 @@ proc addFileToLink(config: ConfigRef; m: PSym) =
       if config.backend == backendCpp: ".nim.cpp"
       elif config.backend == backendObjc: ".nim.m"
       else: ".nim.c"
-  let cfile = changeFileExt(completeCfilePath(config, withPackageName(config, filename)), ext)
+  let cfile = changeFileExt(completeCfilePath(config,
+                            mangleModuleName(config, filename).AbsoluteFile), ext)
   let objFile = completeCfilePath(config, toObjFile(config, cfile))
   if fileExists(objFile):
     var cf = Cfile(nimname: m.name.s, cname: cfile,

compiler/ic/dce.nim

@@ -10,6 +10,10 @@
 ## Dead code elimination (=DCE) for IC.
 
 import std/[intsets, tables]
+
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 import ".." / [ast, options, lineinfos, types]
 
 import packed_ast, ic, bitabs

compiler/ic/ic.nim

@@ -10,10 +10,13 @@
 import hashes, tables, intsets, std/sha1
 import packed_ast, bitabs, rodfiles
 import ".." / [ast, idents, lineinfos, msgs, ropes, options,
-  pathutils, condsyms]
+  pathutils, condsyms, packages, modulepaths]
 #import ".." / [renderer, astalgo]
 from os import removeFile, isAbsolute
 
+when defined(nimPreviewSlimSystem):
+  import std/[syncio, assertions, formatfloat]
+
 type
   PackedConfig* = object
     backend: TBackend
@@ -40,11 +43,10 @@ type
     reexports*: seq[(LitId, PackedItemId)]
     compilerProcs*: seq[(LitId, int32)]
     converters*, methods*, trmacros*, pureEnums*: seq[int32]
-    macroUsages*: seq[(PackedItemId, PackedLineInfo)]
 
     typeInstCache*: seq[(PackedItemId, PackedItemId)]
     procInstCache*: seq[PackedInstantiation]
-    attachedOps*: seq[(TTypeAttachedOp, PackedItemId, PackedItemId)]
+    attachedOps*: seq[(PackedItemId, TTypeAttachedOp, PackedItemId)]
     methodsPerType*: seq[(PackedItemId, int, PackedItemId)]
     enumToStringProcs*: seq[(PackedItemId, PackedItemId)]
 
@@ -94,10 +96,10 @@ proc toString*(tree: PackedTree; n: NodePos; m: PackedModule; nesting: int;
     result.addInt m.numbers[LitId tree.nodes[pos].operand]
   of externUIntLit:
     result.add " "
-    result.add $cast[uint64](m.numbers[LitId tree.nodes[pos].operand])
+    result.addInt cast[uint64](m.numbers[LitId tree.nodes[pos].operand])
   of nkFloatLit..nkFloat128Lit:
     result.add " "
-    result.add $cast[BiggestFloat](m.numbers[LitId tree.nodes[pos].operand])
+    result.addFloat cast[BiggestFloat](m.numbers[LitId tree.nodes[pos].operand])
   else:
     result.add "(\n"
     for i in 1..(nesting+1)*2: result.add ' '
@@ -352,7 +354,7 @@ proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemI
 
     var p = PackedType(kind: t.kind, flags: t.flags, callConv: t.callConv,
       size: t.size, align: t.align, nonUniqueId: t.itemId.item,
-      paddingAtEnd: t.paddingAtEnd, lockLevel: t.lockLevel)
+      paddingAtEnd: t.paddingAtEnd)
     storeNode(p, t, n)
     p.typeInst = t.typeInst.storeType(c, m)
     for kid in items t.sons:
@@ -400,7 +402,7 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId
       p.bitsize = s.bitsize
       p.alignment = s.alignment
 
-    p.externalName = toLitId(if s.loc.r.isNil: "" else: $s.loc.r, m)
+    p.externalName = toLitId(s.loc.r, m)
     p.locFlags = s.loc.flags
     c.addMissing s.typ
     p.typ = s.typ.storeType(c, m)
@@ -507,6 +509,9 @@ proc toPackedNodeIgnoreProcDefs(n: PNode, encoder: var PackedEncoder; m: var Pac
   of nkStmtList, nkStmtListExpr:
     for it in n:
       toPackedNodeIgnoreProcDefs(it, encoder, m)
+  of nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt,
+     nkFromStmt, nkIncludeStmt:
+    discard "nothing to do"
   else:
     toPackedNode(n, m.topLevel, encoder, m)
 
@@ -523,6 +528,15 @@ proc toPackedGeneratedProcDef*(s: PSym, encoder: var PackedEncoder; m: var Packe
   toPackedProcDef(s.ast, m.topLevel, encoder, m)
   #flush encoder, m
 
+proc storeAttachedProcDef*(t: PType; op: TTypeAttachedOp; s: PSym,
+                           encoder: var PackedEncoder; m: var PackedModule) =
+  assert s.kind in routineKinds
+  assert isActive(encoder)
+  let tid = storeTypeLater(t, encoder, m)
+  let sid = storeSymLater(s, encoder, m)
+  m.attachedOps.add (tid, op, sid)
+  toPackedGeneratedProcDef(s, encoder, m)
+
 proc storeInstantiation*(c: var PackedEncoder; m: var PackedModule; s: PSym; i: PInstantiation) =
   var t = newSeq[PackedItemId](i.concreteTypes.len)
   for j in 0..high(i.concreteTypes):
@@ -545,6 +559,10 @@ proc loadError(err: RodFileError; filename: AbsoluteFile; config: ConfigRef;) =
     rawMessage(config, warnCannotOpenFile, filename.string & " reason: " & $err)
     #echo "Error: ", $err, " loading file: ", filename.string
 
+proc toRodFile*(conf: ConfigRef; f: AbsoluteFile; ext = RodExt): AbsoluteFile =
+  result = changeFileExt(completeGeneratedFilePath(conf,
+    mangleModuleName(conf, f).AbsoluteFile), ext)
+
 proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef;
                   ignoreConfig = false): RodFileError =
   var f = rodfiles.open(filename.string)
@@ -587,7 +605,6 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
   loadSeqSection convertersSection, m.converters
   loadSeqSection methodsSection, m.methods
   loadSeqSection pureEnumsSection, m.pureEnums
-  loadSeqSection macroUsagesSection, m.macroUsages
 
   loadSeqSection toReplaySection, m.toReplay.nodes
   loadSeqSection topLevelSection, m.topLevel.nodes
@@ -651,7 +668,6 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
   storeSeqSection convertersSection, m.converters
   storeSeqSection methodsSection, m.methods
   storeSeqSection pureEnumsSection, m.pureEnums
-  storeSeqSection macroUsagesSection, m.macroUsages
 
   storeSeqSection toReplaySection, m.toReplay.nodes
   storeSeqSection topLevelSection, m.topLevel.nodes
@@ -884,7 +900,7 @@ proc typeHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
                           t: PackedType; si, item: int32): PType =
   result = PType(itemId: ItemId(module: si, item: t.nonUniqueId), kind: t.kind,
                 flags: t.flags, size: t.size, align: t.align,
-                paddingAtEnd: t.paddingAtEnd, lockLevel: t.lockLevel,
+                paddingAtEnd: t.paddingAtEnd,
                 uniqueId: ItemId(module: si, item: item),
                 callConv: t.callConv)
 
@@ -924,17 +940,6 @@ proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t
       result = g[si].types[t.item]
     assert result.itemId.item > 0
 
-proc newPackage(config: ConfigRef; cache: IdentCache; fileIdx: FileIndex): PSym =
-  let filename = AbsoluteFile toFullPath(config, fileIdx)
-  let name = getIdent(cache, splitFile(filename).name)
-  let info = newLineInfo(fileIdx, 1, 1)
-  let
-    pck = getPackageName(config, filename.string)
-    pck2 = if pck.len > 0: pck else: "unknown"
-    pack = getIdent(cache, pck2)
-  result = newSym(skPackage, getIdent(cache, pck2),
-    ItemId(module: PackageModuleId, item: int32(fileIdx)), nil, info)
-
 proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
                        fileIdx: FileIndex; m: var LoadedModule) =
   m.iface = initTable[PIdent, seq[PackedItemId]]()
@@ -962,7 +967,7 @@ proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCa
                   name: getIdent(cache, splitFile(filename).name),
                   info: newLineInfo(fileIdx, 1, 1),
                   position: int(fileIdx))
-  m.module.owner = newPackage(conf, cache, fileIdx)
+  m.module.owner = getPackage(conf, cache, fileIdx)
   m.module.flags = m.fromDisk.moduleFlags
 
 proc loadToReplayNodes(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;

compiler/ic/integrity.nim

@@ -11,6 +11,10 @@
 ## The set must cover a complete Nim project.
 
 import sets
+
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 import ".." / [ast, modulegraphs]
 import packed_ast, bitabs, ic
 

compiler/ic/navigator.nim

@@ -16,6 +16,9 @@ import sets
 from os import nil
 from std/private/miscdollars import toLocation
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 import ".." / [ast, modulegraphs, msgs, options]
 import packed_ast, bitabs, ic
 

compiler/ic/packed_ast.nim

@@ -16,6 +16,9 @@ import hashes, tables, strtabs
 import bitabs
 import ".." / [ast, options]
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 type
   SymId* = distinct int32
   ModuleId* = distinct int32
@@ -81,7 +84,6 @@ type
     size*: BiggestInt
     align*: int16
     paddingAtEnd*: int16
-    lockLevel*: TLockLevel # lock level as required for deadlock checking
     # not serialized: loc*: TLoc because it is backend-specific
     typeInst*: PackedItemId
     nonUniqueId*: int32

compiler/ic/replayer.nim

@@ -16,6 +16,9 @@ import ".." / [ast, modulegraphs, trees, extccomp, btrees,
 
 import tables
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 import packed_ast, ic, bitabs
 
 proc replayStateChanges*(module: PSym; g: ModuleGraph) =
@@ -86,6 +89,20 @@ proc replayStateChanges*(module: PSym; g: ModuleGraph) =
       else:
         internalAssert g.config, false
 
+proc replayBackendProcs*(g: ModuleGraph; module: int) =
+  for it in mitems(g.packed[module].fromDisk.attachedOps):
+    let key = translateId(it[0], g.packed, module, g.config)
+    let op = it[1]
+    let tmp = translateId(it[2], g.packed, module, g.config)
+    let symId = FullId(module: tmp.module, packed: it[2])
+    g.attachedOps[op][key] = LazySym(id: symId, sym: nil)
+
+  for it in mitems(g.packed[module].fromDisk.enumToStringProcs):
+    let key = translateId(it[0], g.packed, module, g.config)
+    let tmp = translateId(it[1], g.packed, module, g.config)
+    let symId = FullId(module: tmp.module, packed: it[1])
+    g.enumToStringProcs[key] = LazySym(id: symId, sym: nil)
+
 proc replayGenericCacheInformation*(g: ModuleGraph; module: int) =
   ## We remember the generic instantiations a module performed
   ## in order to to avoid the code bloat that generic code tends
@@ -117,11 +134,7 @@ proc replayGenericCacheInformation*(g: ModuleGraph; module: int) =
     let symId = FullId(module: tmp.module, packed: it[2])
     g.methodsPerType.mgetOrPut(key, @[]).add (col, LazySym(id: symId, sym: nil))
 
-  for it in mitems(g.packed[module].fromDisk.enumToStringProcs):
-    let key = translateId(it[0], g.packed, module, g.config)
-    let tmp = translateId(it[1], g.packed, module, g.config)
-    let symId = FullId(module: tmp.module, packed: it[1])
-    g.enumToStringProcs[key] = LazySym(id: symId, sym: nil)
+  replayBackendProcs(g, module)
 
   for it in mitems(g.packed[module].fromDisk.methods):
     let sym = loadSymFromId(g.config, g.cache, g.packed, module,

compiler/ic/rodfiles.nim

@@ -16,12 +16,15 @@
 
 from typetraits import supportsCopyMem
 
+when defined(nimPreviewSlimSystem):
+  import std/[syncio, assertions]
+
 ## Overview
 ## ========
 ## `RodFile` represents a Rod File (versioned binary format), and the
 ## associated data for common interactions such as IO and error tracking
 ## (`RodFileError`). The file format broken up into sections (`RodSection`)
-## and preceeded by a header (see: `cookie`). The precise layout, section
+## and preceded by a header (see: `cookie`). The precise layout, section
 ## ordering and data following the section are determined by the user. See
 ## `ic.loadRodFile`.
 ##
@@ -34,8 +37,7 @@ from typetraits import supportsCopyMem
 ##
 ## Now read the bits below to understand what's missing.
 ##
-## Issues with the Example
-## ```````````````````````
+## ### Issues with the Example
 ## Missing Sections:
 ## This is a low level API, so headers and sections need to be stored and
 ## loaded by the user, see `storeHeader` & `loadHeader` and `storeSection` &
@@ -82,7 +84,6 @@ type
     convertersSection
     methodsSection
     pureEnumsSection
-    macroUsagesSection
     toReplaySection
     topLevelSection
     bodiesSection

compiler/idents.nim

@@ -14,6 +14,9 @@
 import
   hashes, wordrecg
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 type
   PIdent* = ref TIdent
   TIdent*{.acyclic.} = object

compiler/importer.nim

@@ -12,7 +12,11 @@
 import
   intsets, ast, astalgo, msgs, options, idents, lookups,
   semdata, modulepaths, sigmatch, lineinfos, sets,
-  modulegraphs, wordrecg
+  modulegraphs, wordrecg, tables
+from strutils import `%`
+
+when defined(nimPreviewSlimSystem):
+  import std/assertions
 
 proc readExceptSet*(c: PContext, n: PNode): IntSet =
   assert n.kind in {nkImportExceptStmt, nkExportExceptStmt}
@@ -184,11 +188,13 @@ template addUnnamedIt(c: PContext, fromMod: PSym; filter: untyped) {.dirty.} =
   for it in mitems c.graph.ifaces[fromMod.position].converters:
     if filter:
       loadPackedSym(c.graph, it)
-      addConverter(c, it)
+      if sfExported in it.sym.flags:
+        addConverter(c, it)
   for it in mitems c.graph.ifaces[fromMod.position].patterns:
     if filter:
       loadPackedSym(c.graph, it)
-      addPattern(c, it)
+      if sfExported in it.sym.flags:
+        addPattern(c, it)
   for it in mitems c.graph.ifaces[fromMod.position].pureEnums:
     if filter:
       loadPackedSym(c.graph, it)
@@ -196,7 +202,7 @@ template addUnnamedIt(c: PContext, fromMod: PSym; filter: untyped) {.dirty.} =
 
 proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) =
   c.addImport ImportedModule(m: fromMod, mode: importExcept, exceptSet: exceptSet)
-  addUnnamedIt(c, fromMod, it.sym.id notin exceptSet)
+  addUnnamedIt(c, fromMod, it.sym.name.id notin exceptSet)
 
 proc importAllSymbols*(c: PContext, fromMod: PSym) =
   c.addImport ImportedModule(m: fromMod, mode: importAll)
@@ -224,19 +230,21 @@ proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym; im
 
 proc importModuleAs(c: PContext; n: PNode, realModule: PSym, importHidden: bool): PSym =
   result = realModule
-  c.unusedImports.add((realModule, n.info))
   template createModuleAliasImpl(ident): untyped =
-    createModuleAlias(realModule, nextSymId c.idgen, ident, realModule.info, c.config.options)
+    createModuleAlias(realModule, nextSymId c.idgen, ident, n.info, c.config.options)
   if n.kind != nkImportAs: discard
   elif n.len != 2 or n[1].kind != nkIdent:
     localError(c.config, n.info, "module alias must be an identifier")
   elif n[1].ident.id != realModule.name.id:
     # some misguided guy will write 'import abc.foo as foo' ...
     result = createModuleAliasImpl(n[1].ident)
+  if result == realModule:
+    # avoids modifying `realModule`, see D20201209T194412 for `import {.all.}`
+    result = createModuleAliasImpl(realModule.name)
   if importHidden:
-    if result == realModule: # avoids modifying `realModule`, see D20201209T194412.
-      result = createModuleAliasImpl(realModule.name)
     result.options.incl optImportHidden
+  c.unusedImports.add((result, n.info))
+  c.importModuleMap[result.id] = realModule.id
 
 proc transformImportAs(c: PContext; n: PNode): tuple[node: PNode, importHidden: bool] =
   var ret: typeof(result)
@@ -274,27 +282,33 @@ proc myImportModule(c: PContext, n: var PNode, importStmtResult: PNode): PSym =
                 toFullPath(c.config, c.graph.importStack[i+1])
       c.recursiveDep = err
 
+    var realModule: PSym
     discard pushOptionEntry(c)
-    result = importModuleAs(c, n, c.graph.importModuleCallback(c.graph, c.module, f), transf.importHidden)
+    realModule = c.graph.importModuleCallback(c.graph, c.module, f)
+    result = importModuleAs(c, n, realModule, transf.importHidden)
     popOptionEntry(c)
 
     #echo "set back to ", L
     c.graph.importStack.setLen(L)
     # we cannot perform this check reliably because of
-    # test: modules/import_in_config)
-    when true:
-      if result.info.fileIndex == c.module.info.fileIndex and
-          result.info.fileIndex == n.info.fileIndex:
-        localError(c.config, n.info, "A module cannot import itself")
-    if sfDeprecated in result.flags:
-      if result.constraint != nil:
-        message(c.config, n.info, warnDeprecated, result.constraint.strVal & "; " & result.name.s & " is deprecated")
-      else:
-        message(c.config, n.info, warnDeprecated, result.name.s & " is deprecated")
+    # test: modules/import_in_config) # xxx is that still true?
+    if realModule == c.module:
+      localError(c.config, n.info, "module '$1' cannot import itself" % realModule.name.s)
+    if sfDeprecated in realModule.flags:
+      var prefix = ""
+      if realModule.constraint != nil: prefix = realModule.constraint.strVal & "; "
+      message(c.config, n.info, warnDeprecated, prefix & realModule.name.s & " is deprecated")
     suggestSym(c.graph, n.info, result, c.graph.usageSym, false)
     importStmtResult.add newSymNode(result, n.info)
     #newStrNode(toFullPath(c.config, f), n.info)
 
+proc afterImport(c: PContext, m: PSym) =
+  # fixes bug #17510, for re-exported symbols
+  let realModuleId = c.importModuleMap[m.id]
+  for s in allSyms(c.graph, m):
+    if s.owner.id != realModuleId:
+      c.exportIndirections.incl((m.id, s.id))
+
 proc impMod(c: PContext; it: PNode; importStmtResult: PNode) =
   var it = it
   let m = myImportModule(c, it, importStmtResult)
@@ -303,6 +317,7 @@ proc impMod(c: PContext; it: PNode; importStmtResult: PNode) =
     addDecl(c, m, it.info) # add symbol to symbol table of module
     importAllSymbols(c, m)
     #importForwarded(c, m.ast, emptySet, m)
+    afterImport(c, m)
 
 proc evalImport*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
@@ -341,6 +356,7 @@ proc evalFrom*(c: PContext, n: PNode): PNode =
       if n[i].kind != nkNilLit:
         importSymbol(c, n[i], m, im.imported)
     c.addImport im
+    afterImport(c, m)
 
 proc evalImportExcept*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
@@ -351,3 +367,4 @@ proc evalImportExcept*(c: PContext, n: PNode): PNode =
     addDecl(c, m, n.info)               # add symbol to symbol table of module
     importAllSymbolsExcept(c, m, readExceptSet(c, n))
     #importForwarded(c, m.ast, exceptSet, m)
+    afterImport(c, m)

compiler/injectdestructors.nim

@@ -15,21 +15,26 @@
 
 import
   intsets, strtabs, ast, astalgo, msgs, renderer, magicsys, types, idents,
-  strutils, options, dfa, lowerings, tables, modulegraphs, msgs,
+  strutils, options, lowerings, tables, modulegraphs,
   lineinfos, parampatterns, sighashes, liftdestructors, optimizer,
-  varpartitions
+  varpartitions, aliasanalysis, dfa
+
+when defined(nimPreviewSlimSystem):
+  import std/assertions
 
 from trees import exprStructuralEquivalent, getRoot
 
 type
   Con = object
     owner: PSym
-    g: ControlFlowGraph
+    when true:
+      g: ControlFlowGraph
     graph: ModuleGraph
     inLoop, inSpawn, inLoopCond: int
     uninit: IntSet # set of uninit'ed vars
-    uninitComputed: bool
     idgen: IdGenerator
+    body: PNode
+    otherUsage: TLineInfo
 
   Scope = object # we do scope-based memory management.
     # a scope is comparable to an nkStmtListExpr like
@@ -37,6 +42,8 @@ type
     vars: seq[PSym]
     wasMoved: seq[PNode]
     final: seq[PNode] # finally section
+    locals: seq[PSym]
+    body: PNode
     needsTry: bool
     parent: ptr Scope
 
@@ -67,167 +74,100 @@ proc getTemp(c: var Con; s: var Scope; typ: PType; info: TLineInfo): PNode =
   s.vars.add(sym)
   result = newSymNode(sym)
 
-proc nestedScope(parent: var Scope): Scope =
-  Scope(vars: @[], wasMoved: @[], final: @[], needsTry: false, parent: addr(parent))
+proc nestedScope(parent: var Scope; body: PNode): Scope =
+  Scope(vars: @[], locals: @[], wasMoved: @[], final: @[], body: body, needsTry: false, parent: addr(parent))
 
-proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode
-proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope; isDecl = false): PNode
-
-import sets, hashes, tables
-
-proc hash(n: PNode): Hash = hash(cast[pointer](n))
-
-proc aliasesCached(cache: var Table[(PNode, PNode), AliasKind], obj, field: PNode): AliasKind =
-  let key = (obj, field)
-  if not cache.hasKey(key):
-    cache[key] = aliases(obj, field)
-  cache[key]
+proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode; tmpFlags = {sfSingleUsedTemp}): PNode
 
 type
-  State = ref object
-    lastReads: IntSet
-    potentialLastReads: IntSet
-    notLastReads: IntSet
-    alreadySeen: HashSet[PNode]
+  MoveOrCopyFlag = enum
+    IsDecl, IsExplicitSink
 
-proc preprocessCfg(cfg: var ControlFlowGraph) =
-  for i in 0..<cfg.len:
-    if cfg[i].kind in {goto, fork} and i + cfg[i].dest > cfg.len:
-      cfg[i].dest = cfg.len - i
+proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope; flags: set[MoveOrCopyFlag] = {}): PNode
 
-proc mergeStates(a: var State, b: sink State) =
-  # Inplace for performance:
-  #   lastReads = a.lastReads + b.lastReads
-  #   potentialLastReads = (a.potentialLastReads + b.potentialLastReads) - (a.notLastReads + b.notLastReads)
-  #   notLastReads = a.notLastReads + b.notLastReads
-  #   alreadySeen = a.alreadySeen + b.alreadySeen
-  # b is never nil
-  if a == nil:
-    a = b
+when false:
+  var
+    perfCounters: array[InstrKind, int]
+
+  proc showCounters*() =
+    for i in low(InstrKind)..high(InstrKind):
+      echo "INSTR ", i, " ", perfCounters[i]
+
+proc isLastReadImpl(n: PNode; c: var Con; scope: var Scope): bool =
+  let root = parampatterns.exprRoot(n, allowCalls=false)
+  if root == nil: return false
+
+  var s = addr(scope)
+  while s != nil:
+    if s.locals.contains(root): break
+    s = s.parent
+
+  c.g = constructCfg(c.owner, if s != nil: s.body else: c.body, root)
+  dbg:
+    echo "\n### ", c.owner.name.s, ":\nCFG:"
+    echoCfg(c.g)
+    #echo c.body
+
+  var j = 0
+  while j < c.g.len:
+    if c.g[j].kind == use and c.g[j].n == n: break
+    inc j
+  c.otherUsage = unknownLineInfo
+  if j < c.g.len:
+    var pcs = @[j+1]
+    var marked = initIntSet()
+    result = true
+    while pcs.len > 0:
+      var pc = pcs.pop()
+      if not marked.contains(pc):
+        let oldPc = pc
+        while pc < c.g.len:
+          dbg:
+            echo "EXEC ", c.g[pc].kind, " ", pc, " ", n
+          when false:
+            inc perfCounters[c.g[pc].kind]
+          case c.g[pc].kind
+          of loop:
+            let back = pc + c.g[pc].dest
+            if not marked.containsOrIncl(back):
+              pc = back
+            else:
+              break
+          of goto:
+            pc = pc + c.g[pc].dest
+          of fork:
+            if not marked.contains(pc+1):
+              pcs.add pc + 1
+            pc = pc + c.g[pc].dest
+          of use:
+            if c.g[pc].n.aliases(n) != no or n.aliases(c.g[pc].n) != no:
+              c.otherUsage = c.g[pc].n.info
+              return false
+            inc pc
+          of def:
+            if c.g[pc].n.aliases(n) == yes:
+              # the path leads to a redefinition of 's' --> sink 's'.
+              break
+            elif n.aliases(c.g[pc].n) != no:
+              # only partially writes to 's' --> can't sink 's', so this def reads 's'
+              # or maybe writes to 's' --> can't sink 's'
+              c.otherUsage = c.g[pc].n.info
+              return false
+            inc pc
+        marked.incl oldPc
   else:
-    a.lastReads.incl b.lastReads
-    a.potentialLastReads.incl b.potentialLastReads
-    a.potentialLastReads.excl a.notLastReads
-    a.potentialLastReads.excl b.notLastReads
-    a.notLastReads.incl b.notLastReads
-    a.alreadySeen.incl b.alreadySeen
+    result = false
 
-proc computeLastReadsAndFirstWrites(cfg: ControlFlowGraph) =
-  var cache = initTable[(PNode, PNode), AliasKind]()
-  template aliasesCached(obj, field: PNode): AliasKind =
-    aliasesCached(cache, obj, field)
+proc isLastRead(n: PNode; c: var Con; s: var Scope): bool =
+  if not hasDestructor(c, n.typ): return true
 
-  var cfg = cfg
-  preprocessCfg(cfg)
-
-  var states = newSeq[State](cfg.len + 1)
-  states[0] = State()
-
-  for pc in 0..<cfg.len:
-    template state: State = states[pc]
-    if state != nil:
-      case cfg[pc].kind
-      of def:
-        var potentialLastReadsCopy = state.potentialLastReads
-        for r in potentialLastReadsCopy:
-          if cfg[pc].n.aliasesCached(cfg[r].n) == yes:
-            # the path leads to a redefinition of 's' --> sink 's'.
-            state.lastReads.incl r
-            state.potentialLastReads.excl r
-          elif cfg[r].n.aliasesCached(cfg[pc].n) != no:
-            # only partially writes to 's' --> can't sink 's', so this def reads 's'
-            # or maybe writes to 's' --> can't sink 's'
-            cfg[r].n.comment = '\n' & $pc
-            state.potentialLastReads.excl r
-            state.notLastReads.incl r
-
-        var alreadySeenThisNode = false
-        for s in state.alreadySeen:
-          if cfg[pc].n.aliasesCached(s) != no or s.aliasesCached(cfg[pc].n) != no:
-            alreadySeenThisNode = true; break
-        if alreadySeenThisNode: cfg[pc].n.flags.excl nfFirstWrite
-        else: cfg[pc].n.flags.incl nfFirstWrite
-
-        state.alreadySeen.incl cfg[pc].n
-
-        mergeStates(states[pc + 1], move(states[pc]))
-      of use:
-        var potentialLastReadsCopy = state.potentialLastReads
-        for r in potentialLastReadsCopy:
-          if cfg[pc].n.aliasesCached(cfg[r].n) != no or cfg[r].n.aliasesCached(cfg[pc].n) != no:
-            cfg[r].n.comment = '\n' & $pc
-            state.potentialLastReads.excl r
-            state.notLastReads.incl r
-
-        state.potentialLastReads.incl pc
-
-        state.alreadySeen.incl cfg[pc].n
-
-        mergeStates(states[pc + 1], move(states[pc]))
-      of goto:
-        mergeStates(states[pc + cfg[pc].dest], move(states[pc]))
-      of fork:
-        var copy = State()
-        copy[] = states[pc][]
-        mergeStates(states[pc + cfg[pc].dest], copy)
-        mergeStates(states[pc + 1], move(states[pc]))
-
-  let lastReads = (states[^1].lastReads + states[^1].potentialLastReads) - states[^1].notLastReads
-  var lastReadTable: Table[PNode, seq[int]]
-  for position, node in cfg:
-    if node.kind == use:
-      lastReadTable.mgetOrPut(node.n, @[]).add position
-  for node, positions in lastReadTable:
-    block checkIfAllPosLastRead:
-      for p in positions:
-        if p notin lastReads: break checkIfAllPosLastRead
-      node.flags.incl nfLastRead
-
-proc isLastRead(n: PNode; c: var Con): bool =
-  let m = dfa.skipConvDfa(n)
-  (m.kind == nkSym and sfSingleUsedTemp in m.sym.flags) or nfLastRead in m.flags
+  let m = skipConvDfa(n)
+  result = (m.kind == nkSym and sfSingleUsedTemp in m.sym.flags) or
+      isLastReadImpl(n, c, s)
 
 proc isFirstWrite(n: PNode; c: var Con): bool =
-  let m = dfa.skipConvDfa(n)
-  nfFirstWrite in m.flags
-
-proc initialized(code: ControlFlowGraph; pc: int,
-                 init, uninit: var IntSet; until: int): int =
-  ## Computes the set of definitely initialized variables across all code paths
-  ## as an IntSet of IDs.
-  var pc = pc
-  while pc < code.len:
-    case code[pc].kind
-    of goto:
-      pc += code[pc].dest
-    of fork:
-      var initA = initIntSet()
-      var initB = initIntSet()
-      var variantA = pc + 1
-      var variantB = pc + code[pc].dest
-      while variantA != variantB:
-        if max(variantA, variantB) > until:
-          break
-        if variantA < variantB:
-          variantA = initialized(code, variantA, initA, uninit, min(variantB, until))
-        else:
-          variantB = initialized(code, variantB, initB, uninit, min(variantA, until))
-      pc = min(variantA, variantB)
-      # we add vars if they are in both branches:
-      for v in initA:
-        if v in initB:
-          init.incl v
-    of use:
-      let v = code[pc].n.sym
-      if v.kind != skParam and v.id notin init:
-        # attempt to read an uninit'ed variable
-        uninit.incl v.id
-      inc pc
-    of def:
-      let v = code[pc].n.sym
-      init.incl v.id
-      inc pc
-  return pc
+  let m = skipConvDfa(n)
+  result = nfFirstWrite in m.flags
 
 proc isCursor(n: PNode): bool =
   case n.kind
@@ -245,17 +185,17 @@ template isUnpackedTuple(n: PNode): bool =
   ## hence unpacked tuples themselves don't need to be destroyed
   (n.kind == nkSym and n.sym.kind == skTemp and n.sym.typ.kind == tyTuple)
 
-from strutils import parseInt
-
 proc checkForErrorPragma(c: Con; t: PType; ri: PNode; opname: string) =
   var m = "'" & opname & "' is not available for type <" & typeToString(t) & ">"
   if (opname == "=" or opname == "=copy") and ri != nil:
     m.add "; requires a copy because it's not the last read of '"
     m.add renderTree(ri)
     m.add '\''
-    if ri.comment.startsWith('\n'):
+    if c.otherUsage != unknownLineInfo:
+       # ri.comment.startsWith('\n'):
       m.add "; another read is done here: "
-      m.add c.graph.config $ c.g[parseInt(ri.comment[1..^1])].n.info
+      m.add c.graph.config $ c.otherUsage
+      #m.add c.graph.config $ c.g[parseInt(ri.comment[1..^1])].n.info
     elif ri.kind == nkSym and ri.sym.kind == skParam and not isSinkType(ri.sym.typ):
       m.add "; try to make "
       m.add renderTree(ri)
@@ -277,7 +217,7 @@ proc genOp(c: var Con; t: PType; kind: TTypeAttachedOp; dest, ri: PNode): PNode
   var op = getAttachedOp(c.graph, t, kind)
   if op == nil or op.ast.isGenericRoutine:
     # give up and find the canonical type instead:
-    let h = sighashes.hashType(t, {CoType, CoConsiderOwned, CoDistinct})
+    let h = sighashes.hashType(t, c.graph.config, {CoType, CoConsiderOwned, CoDistinct})
     let canon = c.graph.canonTypes.getOrDefault(h)
     if canon != nil:
       op = getAttachedOp(c.graph, canon, kind)
@@ -308,8 +248,8 @@ proc canBeMoved(c: Con; t: PType): bool {.inline.} =
 proc isNoInit(dest: PNode): bool {.inline.} =
   result = dest.kind == nkSym and sfNoInit in dest.sym.flags
 
-proc genSink(c: var Con; dest, ri: PNode, isDecl = false): PNode =
-  if (c.inLoopCond == 0 and (isUnpackedTuple(dest) or isDecl or
+proc genSink(c: var Con; dest, ri: PNode; flags: set[MoveOrCopyFlag] = {}): PNode =
+  if (c.inLoopCond == 0 and (isUnpackedTuple(dest) or IsDecl in flags or
       (isAnalysableFieldAccess(dest, c.owner) and isFirstWrite(dest, c)))) or
       isNoInit(dest):
     # optimize sink call into a bitwise memcopy
@@ -361,16 +301,22 @@ proc genMarkCyclic(c: var Con; result, dest: PNode) =
         xenv.typ = getSysType(c.graph, dest.info, tyPointer)
         result.add callCodegenProc(c.graph, "nimMarkCyclic", dest.info, xenv)
 
-proc genCopyNoCheck(c: var Con; dest, ri: PNode): PNode =
+proc genCopyNoCheck(c: var Con; dest, ri: PNode; a: TTypeAttachedOp): PNode =
   let t = dest.typ.skipTypes({tyGenericInst, tyAlias, tySink})
-  result = c.genOp(t, attachedAsgn, dest, ri)
+  result = c.genOp(t, a, dest, ri)
+  assert ri.typ != nil
 
-proc genCopy(c: var Con; dest, ri: PNode): PNode =
+proc genCopy(c: var Con; dest, ri: PNode; flags: set[MoveOrCopyFlag]): PNode =
   let t = dest.typ
   if tfHasOwned in t.flags and ri.kind != nkNilLit:
     # try to improve the error message here:
-    c.checkForErrorPragma(t, ri, "=copy")
-  result = c.genCopyNoCheck(dest, ri)
+    if IsExplicitSink in flags:
+      c.checkForErrorPragma(t, ri, "=sink")
+    else:
+      c.checkForErrorPragma(t, ri, "=copy")
+  let a = if IsExplicitSink in flags: attachedSink else: attachedAsgn
+  result = c.genCopyNoCheck(dest, ri, a)
+  assert ri.typ != nil
 
 proc genDiscriminantAsgn(c: var Con; s: var Scope; n: PNode): PNode =
   # discriminator is ordinal value that doesn't need sink destroy
@@ -408,11 +354,18 @@ It is best to factor out piece of object that needs custom destructor into separ
   result.add newTree(nkFastAsgn, le, tmp)
 
 proc genWasMoved(c: var Con, n: PNode): PNode =
-  result = newNodeI(nkCall, n.info)
-  result.add(newSymNode(createMagic(c.graph, c.idgen, "wasMoved", mWasMoved)))
-  result.add copyTree(n) #mWasMoved does not take the address
-  #if n.kind != nkSym:
-  #  message(c.graph.config, n.info, warnUser, "wasMoved(" & $n & ")")
+  let typ = n.typ.skipTypes({tyGenericInst, tyAlias, tySink})
+  let op = getAttachedOp(c.graph, n.typ, attachedWasMoved)
+  if op != nil:
+    if sfError in op.flags:
+      c.checkForErrorPragma(n.typ, n, "=wasMoved")
+    result = genOp(c, op, n)
+  else:
+    result = newNodeI(nkCall, n.info)
+    result.add(newSymNode(createMagic(c.graph, c.idgen, "wasMoved", mWasMoved)))
+    result.add copyTree(n) #mWasMoved does not take the address
+    #if n.kind != nkSym:
+    #  message(c.graph.config, n.info, warnUser, "wasMoved(" & $n & ")")
 
 proc genDefaultCall(t: PType; c: Con; info: TLineInfo): PNode =
   result = newNodeI(nkCall, info)
@@ -454,7 +407,7 @@ proc passCopyToSink(n: PNode; c: var Con; s: var Scope): PNode =
   let tmp = c.getTemp(s, n.typ, n.info)
   if hasDestructor(c, n.typ):
     result.add c.genWasMoved(tmp)
-    var m = c.genCopy(tmp, n)
+    var m = c.genCopy(tmp, n, {})
     m.add p(n, c, s, normal)
     c.finishCopy(m, n, isFromSink = true)
     result.add m
@@ -499,7 +452,7 @@ proc ensureDestruction(arg, orig: PNode; c: var Con; s: var Scope): PNode =
     # This was already done in the sink parameter handling logic.
     result = newNodeIT(nkStmtListExpr, arg.info, arg.typ)
     let tmp = c.getTemp(s, arg.typ, arg.info)
-    result.add c.genSink(tmp, arg, isDecl = true)
+    result.add c.genSink(tmp, arg, {IsDecl})
     result.add tmp
     s.final.add c.genDestroy(tmp)
   else:
@@ -544,7 +497,7 @@ proc pVarTopLevel(v: PNode; c: var Con; s: var Scope; res: PNode) =
       res.add newTree(nkFastAsgn, v, genDefaultCall(v.typ, c, v.info))
   elif sfThread notin v.sym.flags and sfCursor notin v.sym.flags:
     # do not destroy thread vars for now at all for consistency.
-    if sfGlobal in v.sym.flags and s.parent == nil: #XXX: Rethink this logic (see tarcmisc.test2)
+    if {sfGlobal, sfPure} <= v.sym.flags or sfGlobal in v.sym.flags and s.parent == nil:
       c.graph.globalDestructors.add c.genDestroy(v)
     else:
       s.final.add c.genDestroy(v)
@@ -571,17 +524,17 @@ proc processScope(c: var Con; s: var Scope; ret: PNode): PNode =
 
   if s.parent != nil: s.parent[].needsTry = s.parent[].needsTry or s.needsTry
 
-template processScopeExpr(c: var Con; s: var Scope; ret: PNode, processCall: untyped): PNode =
+template processScopeExpr(c: var Con; s: var Scope; ret: PNode, processCall: untyped, tmpFlags: TSymFlags): PNode =
   assert not ret.typ.isEmptyType
-  var result = newNodeI(nkStmtListExpr, ret.info)
+  var result = newNodeIT(nkStmtListExpr, ret.info, ret.typ)
   # There is a possibility to do this check: s.wasMoved.len > 0 or s.final.len > 0
   # later and use it to eliminate the temporary when theres no need for it, but its
   # tricky because you would have to intercept moveOrCopy at a certain point
   let tmp = c.getTemp(s.parent[], ret.typ, ret.info)
-  tmp.sym.flags.incl sfSingleUsedTemp
+  tmp.sym.flags = tmpFlags
   let cpy = if hasDestructor(c, ret.typ):
               s.parent[].final.add c.genDestroy(tmp)
-              moveOrCopy(tmp, ret, c, s, isDecl = true)
+              moveOrCopy(tmp, ret, c, s, {IsDecl})
             else:
               newTree(nkFastAsgn, tmp, p(ret, c, s, normal))
 
@@ -605,7 +558,8 @@ template processScopeExpr(c: var Con; s: var Scope; ret: PNode, processCall: unt
 
   result
 
-template handleNestedTempl(n, processCall: untyped, willProduceStmt = false) =
+template handleNestedTempl(n, processCall: untyped, willProduceStmt = false,
+                           tmpFlags = {sfSingleUsedTemp}) =
   template maybeVoid(child, s): untyped =
     if isEmptyType(child.typ): p(child, c, s, normal)
     else: processCall(child, s)
@@ -629,11 +583,11 @@ template handleNestedTempl(n, processCall: untyped, willProduceStmt = false) =
       var branch = shallowCopy(it)
       for j in 0 ..< it.len-1:
         branch[j] = copyTree(it[j])
-      var ofScope = nestedScope(s)
+      var ofScope = nestedScope(s, it.lastSon)
       branch[^1] = if it[^1].typ.isEmptyType or willProduceStmt:
                      processScope(c, ofScope, maybeVoid(it[^1], ofScope))
                    else:
-                     processScopeExpr(c, ofScope, it[^1], processCall)
+                     processScopeExpr(c, ofScope, it[^1], processCall, tmpFlags)
       result.add branch
 
   of nkWhileStmt:
@@ -642,7 +596,7 @@ template handleNestedTempl(n, processCall: untyped, willProduceStmt = false) =
     result = copyNode(n)
     result.add p(n[0], c, s, normal)
     dec c.inLoopCond
-    var bodyScope = nestedScope(s)
+    var bodyScope = nestedScope(s, n[1])
     let bodyResult = p(n[1], c, bodyScope, normal)
     result.add processScope(c, bodyScope, bodyResult)
     dec c.inLoop
@@ -654,7 +608,7 @@ template handleNestedTempl(n, processCall: untyped, willProduceStmt = false) =
     for i in 0..<last-1:
       result[i] = n[i]
     result[last-1] = p(n[last-1], c, s, normal)
-    var bodyScope = nestedScope(s)
+    var bodyScope = nestedScope(s, n[1])
     let bodyResult = p(n[last], c, bodyScope, normal)
     result[last] = processScope(c, bodyScope, bodyResult)
     dec c.inLoop
@@ -662,18 +616,18 @@ template handleNestedTempl(n, processCall: untyped, willProduceStmt = false) =
   of nkBlockStmt, nkBlockExpr:
     result = copyNode(n)
     result.add n[0]
-    var bodyScope = nestedScope(s)
+    var bodyScope = nestedScope(s, n[1])
     result.add if n[1].typ.isEmptyType or willProduceStmt:
                  processScope(c, bodyScope, processCall(n[1], bodyScope))
                else:
-                 processScopeExpr(c, bodyScope, n[1], processCall)
+                 processScopeExpr(c, bodyScope, n[1], processCall, tmpFlags)
 
   of nkIfStmt, nkIfExpr:
     result = copyNode(n)
     for i in 0..<n.len:
       let it = n[i]
       var branch = shallowCopy(it)
-      var branchScope = nestedScope(s)
+      var branchScope = nestedScope(s, it.lastSon)
       if it.kind in {nkElifBranch, nkElifExpr}:
         #Condition needs to be destroyed outside of the condition/branch scope
         branch[0] = p(it[0], c, s, normal)
@@ -681,26 +635,26 @@ template handleNestedTempl(n, processCall: untyped, willProduceStmt = false) =
       branch[^1] = if it[^1].typ.isEmptyType or willProduceStmt:
                      processScope(c, branchScope, maybeVoid(it[^1], branchScope))
                    else:
-                     processScopeExpr(c, branchScope, it[^1], processCall)
+                     processScopeExpr(c, branchScope, it[^1], processCall, tmpFlags)
       result.add branch
 
   of nkTryStmt:
     result = copyNode(n)
-    var tryScope = nestedScope(s)
+    var tryScope = nestedScope(s, n[0])
     result.add if n[0].typ.isEmptyType or willProduceStmt:
                  processScope(c, tryScope, maybeVoid(n[0], tryScope))
                else:
-                 processScopeExpr(c, tryScope, n[0], maybeVoid)
+                 processScopeExpr(c, tryScope, n[0], maybeVoid, tmpFlags)
 
     for i in 1..<n.len:
       let it = n[i]
       var branch = copyTree(it)
-      var branchScope = nestedScope(s)
+      var branchScope = nestedScope(s, it[^1])
       branch[^1] = if it[^1].typ.isEmptyType or willProduceStmt or it.kind == nkFinally:
                      processScope(c, branchScope, if it.kind == nkFinally: p(it[^1], c, branchScope, normal)
                                                   else: maybeVoid(it[^1], branchScope))
                    else:
-                     processScopeExpr(c, branchScope, it[^1], processCall)
+                     processScopeExpr(c, branchScope, it[^1], processCall, tmpFlags)
       result.add branch
 
   of nkWhen: # This should be a "when nimvm" node.
@@ -716,7 +670,7 @@ proc pRaiseStmt(n: PNode, c: var Con; s: var Scope): PNode =
       result.add call
     else:
       let tmp = c.getTemp(s, n[0].typ, n.info)
-      var m = c.genCopyNoCheck(tmp, n[0])
+      var m = c.genCopyNoCheck(tmp, n[0], attachedAsgn)
       m.add p(n[0], c, s, normal)
       c.finishCopy(m, n[0], isFromSink = false)
       result = newTree(nkStmtList, c.genWasMoved(tmp), m)
@@ -733,11 +687,11 @@ proc pRaiseStmt(n: PNode, c: var Con; s: var Scope): PNode =
       result.add copyNode(n[0])
   s.needsTry = true
 
-proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
+proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode; tmpFlags = {sfSingleUsedTemp}): PNode =
   if n.kind in {nkStmtList, nkStmtListExpr, nkBlockStmt, nkBlockExpr, nkIfStmt,
                 nkIfExpr, nkCaseStmt, nkWhen, nkWhileStmt, nkParForStmt, nkTryStmt}:
     template process(child, s): untyped = p(child, c, s, mode)
-    handleNestedTempl(n, process)
+    handleNestedTempl(n, process, tmpFlags = tmpFlags)
   elif mode == sinkArg:
     if n.containsConstSeq:
       # const sequences are not mutable and so we need to pass a copy to the
@@ -748,7 +702,7 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
          nkCallKinds + nkLiterals:
       result = p(n, c, s, consumed)
     elif ((n.kind == nkSym and isSinkParam(n.sym)) or isAnalysableFieldAccess(n, c.owner)) and
-        isLastRead(n, c) and not (n.kind == nkSym and isCursor(n)):
+        isLastRead(n, c, s) and not (n.kind == nkSym and isCursor(n)):
       # Sinked params can be consumed only once. We need to reset the memory
       # to disable the destructor which we have not elided
       result = destructiveMoveVar(n, c, s)
@@ -774,7 +728,7 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
       result = passCopyToSink(n, c, s)
   else:
     case n.kind
-    of nkBracket, nkObjConstr, nkTupleConstr, nkClosure, nkCurly:
+    of nkBracket, nkTupleConstr, nkClosure, nkCurly:
       # Let C(x) be the construction, 'x' the vector of arguments.
       # C(x) either owns 'x' or it doesn't.
       # If C(x) owns its data, we must consume C(x).
@@ -785,13 +739,11 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
       # don't destroy it"
       # but if C(x) is a ref it MUST own its data since we must destroy it
       # so then we have no choice but to use 'sinkArg'.
-      let isRefConstr = n.kind == nkObjConstr and n.typ.skipTypes(abstractInst).kind == tyRef
-      let m = if isRefConstr: sinkArg
-              elif mode == normal: normal
+      let m = if mode == normal: normal
               else: sinkArg
 
       result = copyTree(n)
-      for i in ord(n.kind in {nkObjConstr, nkClosure})..<n.len:
+      for i in ord(n.kind == nkClosure)..<n.len:
         if n[i].kind == nkExprColonExpr:
           result[i][1] = p(n[i][1], c, s, m)
         elif n[i].kind == nkRange:
@@ -799,6 +751,24 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
           result[i][1] = p(n[i][1], c, s, m)
         else:
           result[i] = p(n[i], c, s, m)
+    of nkObjConstr:
+      # see also the remark about `nkTupleConstr`.
+      let t = n.typ.skipTypes(abstractInst)
+      let isRefConstr = t.kind == tyRef
+      let m = if isRefConstr: sinkArg
+              elif mode == normal: normal
+              else: sinkArg
+
+      result = copyTree(n)
+      for i in 1..<n.len:
+        if n[i].kind == nkExprColonExpr:
+          let field = lookupFieldAgain(t, n[i][0].sym)
+          if field != nil and sfCursor in field.flags:
+            result[i][1] = p(n[i][1], c, s, normal)
+          else:
+            result[i][1] = p(n[i][1], c, s, m)
+        else:
+          result[i] = p(n[i], c, s, m)
       if mode == normal and isRefConstr:
         result = ensureDestruction(result, n, c, s)
     of nkCallKinds:
@@ -852,32 +822,43 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
       for it in n:
         var ri = it[^1]
         if it.kind == nkVarTuple and hasDestructor(c, ri.typ):
+          for i in 0..<it.len-2:
+            if it[i].kind == nkSym: s.locals.add it[i].sym
           let x = lowerTupleUnpacking(c.graph, it, c.idgen, c.owner)
           result.add p(x, c, s, consumed)
-        elif it.kind == nkIdentDefs and hasDestructor(c, it[0].typ):
+        elif it.kind == nkIdentDefs and hasDestructor(c, skipPragmaExpr(it[0]).typ):
           for j in 0..<it.len-2:
-            let v = it[j]
+            let v = skipPragmaExpr(it[j])
             if v.kind == nkSym:
               if sfCompileTime in v.sym.flags: continue
+              s.locals.add v.sym
               pVarTopLevel(v, c, s, result)
             if ri.kind != nkEmpty:
-              result.add moveOrCopy(v, ri, c, s, isDecl = v.kind == nkSym)
+              result.add moveOrCopy(v, ri, c, s, if v.kind == nkSym: {IsDecl} else: {})
             elif ri.kind == nkEmpty and c.inLoop > 0:
-              result.add moveOrCopy(v, genDefaultCall(v.typ, c, v.info), c, s, isDecl = v.kind == nkSym)
+              let skipInit = v.kind == nkDotExpr and # Closure var
+                             sfNoInit in v[1].sym.flags
+              if not skipInit:
+                result.add moveOrCopy(v, genDefaultCall(v.typ, c, v.info), c, s, if v.kind == nkSym: {IsDecl} else: {})
         else: # keep the var but transform 'ri':
           var v = copyNode(n)
           var itCopy = copyNode(it)
           for j in 0..<it.len-1:
             itCopy.add it[j]
-          itCopy.add p(it[^1], c, s, normal)
+          var flags = {sfSingleUsedTemp}
+          if it.kind == nkIdentDefs and it.len == 3 and it[0].kind == nkSym and
+                                        sfGlobal in it[0].sym.flags:
+            flags.incl sfGlobal
+          itCopy.add p(it[^1], c, s, normal, tmpFlags = flags)
           v.add itCopy
           result.add v
-    of nkAsgn, nkFastAsgn:
+    of nkAsgn, nkFastAsgn, nkSinkAsgn:
       if hasDestructor(c, n[0].typ) and n[1].kind notin {nkProcDef, nkDo, nkLambda}:
         if n[0].kind in {nkDotExpr, nkCheckedFieldExpr}:
           cycleCheck(n, c)
-        assert n[1].kind notin {nkAsgn, nkFastAsgn}
-        result = moveOrCopy(p(n[0], c, s, mode), n[1], c, s)
+        assert n[1].kind notin {nkAsgn, nkFastAsgn, nkSinkAsgn}
+        let flags = if n.kind == nkSinkAsgn: {IsExplicitSink} else: {}
+        result = moveOrCopy(p(n[0], c, s, mode), n[1], c, s, flags)
       elif isDiscriminantField(n[0]):
         result = c.genDiscriminantAsgn(s, n)
       else:
@@ -931,7 +912,7 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
       for i in 1 ..< n.len:
         result[i] = n[i]
       if mode == sinkArg and hasDestructor(c, n.typ):
-        if isAnalysableFieldAccess(n, c.owner) and isLastRead(n, c):
+        if isAnalysableFieldAccess(n, c.owner) and isLastRead(n, c, s):
           s.wasMoved.add c.genWasMoved(n)
         else:
           result = passCopyToSink(result, c, s)
@@ -941,7 +922,7 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
       for i in 0 ..< n.len:
         result[i] = p(n[i], c, s, normal)
       if mode == sinkArg and hasDestructor(c, n.typ):
-        if isAnalysableFieldAccess(n, c.owner) and isLastRead(n, c):
+        if isAnalysableFieldAccess(n, c.owner) and isLastRead(n, c, s):
           # consider 'a[(g; destroy(g); 3)]', we want to say 'wasMoved(a[3])'
           # without the junk, hence 'c.genWasMoved(n)'
           # and not 'c.genWasMoved(result)':
@@ -1002,14 +983,34 @@ proc sameLocation*(a, b: PNode): bool =
     of nkHiddenStdConv, nkHiddenSubConv: sameLocation(a[1], b)
     else: false
 
-proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope, isDecl = false): PNode =
+proc genFieldAccessSideEffects(c: var Con; dest, ri: PNode; flags: set[MoveOrCopyFlag] = {}): PNode =
+  # with side effects
+  var temp = newSym(skLet, getIdent(c.graph.cache, "bracketTmp"), nextSymId c.idgen, c.owner, ri[1].info)
+  temp.typ = ri[1].typ
+  var v = newNodeI(nkLetSection, ri[1].info)
+  let tempAsNode = newSymNode(temp)
+
+  var vpart = newNodeI(nkIdentDefs, tempAsNode.info, 3)
+  vpart[0] = tempAsNode
+  vpart[1] = newNodeI(nkEmpty, tempAsNode.info)
+  vpart[2] = ri[1]
+  v.add(vpart)
+
+  var newAccess = copyNode(ri)
+  newAccess.add ri[0]
+  newAccess.add tempAsNode
+
+  var snk = c.genSink(dest, newAccess, flags)
+  result = newTree(nkStmtList, v, snk, c.genWasMoved(newAccess))
+
+proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope, flags: set[MoveOrCopyFlag] = {}): PNode =
   if sameLocation(dest, ri):
     # rule (self-assignment-removal):
     result = newNodeI(nkEmpty, dest.info)
   elif isCursor(dest):
     case ri.kind:
     of nkStmtListExpr, nkBlockExpr, nkIfExpr, nkCaseStmt, nkTryStmt:
-      template process(child, s): untyped = moveOrCopy(dest, child, c, s, isDecl)
+      template process(child, s): untyped = moveOrCopy(dest, child, c, s, flags)
       # We know the result will be a stmt so we use that fact to optimize
       handleNestedTempl(ri, process, willProduceStmt = true)
     else:
@@ -1017,108 +1018,106 @@ proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope, isDecl = false): PNod
   else:
     case ri.kind
     of nkCallKinds:
-      result = c.genSink(dest, p(ri, c, s, consumed), isDecl)
+      result = c.genSink(dest, p(ri, c, s, consumed), flags)
     of nkBracketExpr:
       if isUnpackedTuple(ri[0]):
         # unpacking of tuple: take over the elements
-        result = c.genSink(dest, p(ri, c, s, consumed), isDecl)
-      elif isAnalysableFieldAccess(ri, c.owner) and isLastRead(ri, c):
+        result = c.genSink(dest, p(ri, c, s, consumed), flags)
+      elif isAnalysableFieldAccess(ri, c.owner) and isLastRead(ri, c, s):
         if aliases(dest, ri) == no:
           # Rule 3: `=sink`(x, z); wasMoved(z)
-          var snk = c.genSink(dest, ri, isDecl)
-          result = newTree(nkStmtList, snk, c.genWasMoved(ri))
+          if isAtom(ri[1]):
+            var snk = c.genSink(dest, ri, flags)
+            result = newTree(nkStmtList, snk, c.genWasMoved(ri))
+          else:
+            result = genFieldAccessSideEffects(c, dest, ri, flags)
         else:
-          result = c.genSink(dest, destructiveMoveVar(ri, c, s), isDecl)
+          result = c.genSink(dest, destructiveMoveVar(ri, c, s), flags)
       else:
-        result = c.genCopy(dest, ri)
+        result = c.genCopy(dest, ri, flags)
         result.add p(ri, c, s, consumed)
         c.finishCopy(result, dest, isFromSink = false)
     of nkBracket:
       # array constructor
       if ri.len > 0 and isDangerousSeq(ri.typ):
-        result = c.genCopy(dest, ri)
+        result = c.genCopy(dest, ri, flags)
         result.add p(ri, c, s, consumed)
         c.finishCopy(result, dest, isFromSink = false)
       else:
-        result = c.genSink(dest, p(ri, c, s, consumed), isDecl)
+        result = c.genSink(dest, p(ri, c, s, consumed), flags)
     of nkObjConstr, nkTupleConstr, nkClosure, nkCharLit..nkNilLit:
-      result = c.genSink(dest, p(ri, c, s, consumed), isDecl)
+      result = c.genSink(dest, p(ri, c, s, consumed), flags)
     of nkSym:
-      if isSinkParam(ri.sym) and isLastRead(ri, c):
+      if isSinkParam(ri.sym) and isLastRead(ri, c, s):
         # Rule 3: `=sink`(x, z); wasMoved(z)
-        let snk = c.genSink(dest, ri, isDecl)
+        let snk = c.genSink(dest, ri, flags)
         result = newTree(nkStmtList, snk, c.genWasMoved(ri))
       elif ri.sym.kind != skParam and ri.sym.owner == c.owner and
-          isLastRead(ri, c) and canBeMoved(c, dest.typ) and not isCursor(ri):
+          isLastRead(ri, c, s) and canBeMoved(c, dest.typ) and not isCursor(ri):
         # Rule 3: `=sink`(x, z); wasMoved(z)
-        let snk = c.genSink(dest, ri, isDecl)
+        let snk = c.genSink(dest, ri, flags)
         result = newTree(nkStmtList, snk, c.genWasMoved(ri))
       else:
-        result = c.genCopy(dest, ri)
+        result = c.genCopy(dest, ri, flags)
         result.add p(ri, c, s, consumed)
         c.finishCopy(result, dest, isFromSink = false)
-    of nkHiddenSubConv, nkHiddenStdConv, nkConv, nkObjDownConv, nkObjUpConv:
-      result = c.genSink(dest, p(ri, c, s, sinkArg), isDecl)
+    of nkHiddenSubConv, nkHiddenStdConv, nkConv, nkObjDownConv, nkObjUpConv, nkCast:
+      result = c.genSink(dest, p(ri, c, s, sinkArg), flags)
     of nkStmtListExpr, nkBlockExpr, nkIfExpr, nkCaseStmt, nkTryStmt:
-      template process(child, s): untyped = moveOrCopy(dest, child, c, s, isDecl)
+      template process(child, s): untyped = moveOrCopy(dest, child, c, s, flags)
       # We know the result will be a stmt so we use that fact to optimize
       handleNestedTempl(ri, process, willProduceStmt = true)
     of nkRaiseStmt:
       result = pRaiseStmt(ri, c, s)
     else:
-      if isAnalysableFieldAccess(ri, c.owner) and isLastRead(ri, c) and
+      if isAnalysableFieldAccess(ri, c.owner) and isLastRead(ri, c, s) and
           canBeMoved(c, dest.typ):
         # Rule 3: `=sink`(x, z); wasMoved(z)
-        let snk = c.genSink(dest, ri, isDecl)
+        let snk = c.genSink(dest, ri, flags)
         result = newTree(nkStmtList, snk, c.genWasMoved(ri))
       else:
-        result = c.genCopy(dest, ri)
+        result = c.genCopy(dest, ri, flags)
         result.add p(ri, c, s, consumed)
         c.finishCopy(result, dest, isFromSink = false)
 
-proc computeUninit(c: var Con) =
-  if not c.uninitComputed:
-    c.uninitComputed = true
-    c.uninit = initIntSet()
-    var init = initIntSet()
-    discard initialized(c.g, pc = 0, init, c.uninit, int.high)
+when false:
+  proc computeUninit(c: var Con) =
+    if not c.uninitComputed:
+      c.uninitComputed = true
+      c.uninit = initIntSet()
+      var init = initIntSet()
+      discard initialized(c.g, pc = 0, init, c.uninit, int.high)
 
-proc injectDefaultCalls(n: PNode, c: var Con) =
-  case n.kind
-  of nkVarSection, nkLetSection:
-    for it in n:
-      if it.kind == nkIdentDefs and it[^1].kind == nkEmpty:
-        computeUninit(c)
-        for j in 0..<it.len-2:
-          let v = it[j]
-          doAssert v.kind == nkSym
-          if c.uninit.contains(v.sym.id):
-            it[^1] = genDefaultCall(v.sym.typ, c, v.info)
-            break
-  of nkNone..nkNilLit, nkTypeSection, nkProcDef, nkConverterDef, nkMethodDef,
-      nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo, nkFuncDef:
-    discard
-  else:
-    for i in 0..<n.safeLen:
-      injectDefaultCalls(n[i], c)
+  proc injectDefaultCalls(n: PNode, c: var Con) =
+    case n.kind
+    of nkVarSection, nkLetSection:
+      for it in n:
+        if it.kind == nkIdentDefs and it[^1].kind == nkEmpty:
+          computeUninit(c)
+          for j in 0..<it.len-2:
+            let v = skipPragmaExpr(it[j])
+            doAssert v.kind == nkSym
+            if c.uninit.contains(v.sym.id):
+              it[^1] = genDefaultCall(v.sym.typ, c, v.info)
+              break
+    of nkNone..nkNilLit, nkTypeSection, nkProcDef, nkConverterDef, nkMethodDef,
+        nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo, nkFuncDef:
+      discard
+    else:
+      for i in 0..<n.safeLen:
+        injectDefaultCalls(n[i], c)
 
 proc injectDestructorCalls*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; n: PNode): PNode =
   when toDebug.len > 0:
     shouldDebug = toDebug == owner.name.s or toDebug == "always"
   if sfGeneratedOp in owner.flags or (owner.kind == skIterator and isInlineIterator(owner.typ)):
     return n
-  var c = Con(owner: owner, graph: g, g: constructCfg(owner, n), idgen: idgen)
-  dbg:
-    echo "\n### ", owner.name.s, ":\nCFG:"
-    echoCfg(c.g)
-    echo n
+  var c = Con(owner: owner, graph: g, idgen: idgen, body: n, otherUsage: unknownLineInfo)
 
   if optCursorInference in g.config.options:
     computeCursors(owner, n, g)
 
-  computeLastReadsAndFirstWrites(c.g)
-
-  var scope: Scope
+  var scope = Scope(body: n)
   let body = p(n, c, scope, normal)
 
   if owner.kind in {skProc, skFunc, skMethod, skIterator, skConverter}:

compiler/installer.ini

@@ -13,6 +13,7 @@ Platforms: """
   netbsd: i386;amd64
   openbsd: i386;amd64;arm;arm64
   dragonfly: i386;amd64
+  crossos: amd64
   haiku: i386;amd64
   android: i386;arm;arm64
   nintendoswitch: arm64
@@ -64,11 +65,12 @@ Files: "compiler"
 Files: "doc"
 Files: "doc/html"
 Files: "tools"
-Files: "tools/nim-gdb.py"
+Files: "tools/debug/nim-gdb.py"
 Files: "nimpretty"
 Files: "testament"
 Files: "nimsuggest"
 Files: "nimsuggest/tests/*.nim"
+Files: "changelogs/*.md"
 
 [Lib]
 Files: "lib"
@@ -119,7 +121,7 @@ Files: "bin/nim"
 InstallScript: "yes"
 UninstallScript: "yes"
 Files: "bin/nim-gdb"
-Files: "bin/nim-gdb.bash"
+Files: "build_all.sh"
 
 
 [InnoSetup]
@@ -141,5 +143,5 @@ shortDesc: "The Nim Compiler"
 licenses: "bin/nim,MIT;lib/*,MIT;"
 
 [nimble]
-pkgName: "compiler"
+pkgName: "nim"
 pkgFiles: "compiler/*;doc/basicopt.txt;doc/advopt.txt;doc/nimdoc.css"

compiler/int128.nim

@@ -5,6 +5,9 @@
 
 from math import trunc
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 type
   Int128* = object
     udata: array[4, uint32]
@@ -30,39 +33,34 @@ template high*(t: typedesc[Int128]): Int128 = Max
 proc `$`*(a: Int128): string
 
 proc toInt128*[T: SomeInteger | bool](arg: T): Int128 =
-  when T is bool: result.sdata(0) = int32(arg)
-  elif T is SomeUnsignedInt:
-    when sizeof(arg) <= 4:
-      result.udata[0] = uint32(arg)
+  {.noSideEffect.}:
+    when T is bool: result.sdata(0) = int32(arg)
+    elif T is SomeUnsignedInt:
+      when sizeof(arg) <= 4:
+        result.udata[0] = uint32(arg)
+      else:
+        result.udata[0] = uint32(arg and T(0xffffffff))
+        result.udata[1] = uint32(arg shr 32)
+    elif sizeof(arg) <= 4:
+      result.sdata(0) = int32(arg)
+      if arg < 0: # sign extend
+        result.sdata(1) = -1
+        result.sdata(2) = -1
+        result.sdata(3) = -1
     else:
-      result.udata[0] = uint32(arg and T(0xffffffff))
-      result.udata[1] = uint32(arg shr 32)
-  elif sizeof(arg) <= 4:
-    result.sdata(0) = int32(arg)
-    if arg < 0: # sign extend
-      result.sdata(1) = -1
-      result.sdata(2) = -1
-      result.sdata(3) = -1
-  else:
-    let tmp = int64(arg)
-    result.udata[0] = uint32(tmp and 0xffffffff)
-    result.sdata(1) = int32(tmp shr 32)
-    if arg < 0: # sign extend
-      result.sdata(2) = -1
-      result.sdata(3) = -1
+      let tmp = int64(arg)
+      result.udata[0] = uint32(tmp and 0xffffffff)
+      result.sdata(1) = int32(tmp shr 32)
+      if arg < 0: # sign extend
+        result.sdata(2) = -1
+        result.sdata(3) = -1
 
 template isNegative(arg: Int128): bool =
   arg.sdata(3) < 0
 
-template isNegative(arg: int32): bool =
-  arg < 0
-
 proc bitconcat(a, b: uint32): uint64 =
   (uint64(a) shl 32) or uint64(b)
 
-proc bitsplit(a: uint64): (uint32, uint32) =
-  (cast[uint32](a shr 32), cast[uint32](a))
-
 proc toInt64*(arg: Int128): int64 =
   if isNegative(arg):
     assert(arg.sdata(3) == -1, "out of range")
@@ -208,12 +206,6 @@ proc `==`*(a, b: Int128): bool =
   if a.udata[3] != b.udata[3]: return false
   return true
 
-proc inplaceBitnot(a: var Int128) =
-  a.udata[0] = not a.udata[0]
-  a.udata[1] = not a.udata[1]
-  a.udata[2] = not a.udata[2]
-  a.udata[3] = not a.udata[3]
-
 proc bitnot*(a: Int128): Int128 =
   result.udata[0] = not a.udata[0]
   result.udata[1] = not a.udata[1]
@@ -354,23 +346,10 @@ proc low64(a: Int128): uint64 =
   bitconcat(a.udata[1], a.udata[0])
 
 proc `*`*(lhs, rhs: Int128): Int128 =
-  let
-    a = cast[uint64](lhs.udata[0])
-    b = cast[uint64](lhs.udata[1])
-    c = cast[uint64](lhs.udata[2])
-    d = cast[uint64](lhs.udata[3])
-
-    e = cast[uint64](rhs.udata[0])
-    f = cast[uint64](rhs.udata[1])
-    g = cast[uint64](rhs.udata[2])
-    h = cast[uint64](rhs.udata[3])
-
-
-  let a32 = cast[uint64](lhs.udata[1])
-  let a00 = cast[uint64](lhs.udata[0])
-  let b32 = cast[uint64](rhs.udata[1])
-  let b00 = cast[uint64](rhs.udata[0])
-
+  let a32 = uint64(lhs.udata[1])
+  let a00 = uint64(lhs.udata[0])
+  let b32 = uint64(rhs.udata[1])
+  let b00 = uint64(rhs.udata[0])
   result = makeInt128(high64(lhs) * low64(rhs) + low64(lhs) * high64(rhs) + a32 * b32, a00 * b00)
   result += toInt128(a32 * b00) shl 32
   result += toInt128(a00 * b32) shl 32
@@ -441,17 +420,17 @@ proc divMod*(dividend, divisor: Int128): tuple[quotient, remainder: Int128] =
     result.remainder = dividend
 
 proc `div`*(a, b: Int128): Int128 =
-  let (a, b) = divMod(a, b)
+  let (a, _) = divMod(a, b)
   return a
 
 proc `mod`*(a, b: Int128): Int128 =
-  let (a, b) = divMod(a, b)
+  let (_, b) = divMod(a, b)
   return b
 
 proc addInt128*(result: var string; value: Int128) =
   let initialSize = result.len
   if value == Zero:
-    result.add "0"
+    result.add '0'
   elif value == low(Int128):
     result.add "-170141183460469231731687303715884105728"
   else:
@@ -472,6 +451,8 @@ proc addInt128*(result: var string; value: Int128) =
       j -= 1
 
 proc `$`*(a: Int128): string =
+  # "-170141183460469231731687303715884105728".len == 41
+  result = newStringOfCap(41)
   result.addInt128(a)
 
 proc parseDecimalInt128*(arg: string, pos: int = 0): Int128 =

compiler/isolation_check.nim

@@ -13,6 +13,9 @@
 import
   ast, types, renderer, intsets
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 proc canAlias(arg, ret: PType; marker: var IntSet): bool
 
 proc canAliasN(arg: PType; n: PNode; marker: var IntSet): bool =
@@ -64,19 +67,30 @@ proc canAlias(arg, ret: PType; marker: var IntSet): bool =
   else:
     result = false
 
-proc isValueOnlyType(t: PType): bool = 
+proc isValueOnlyType(t: PType): bool =
   # t doesn't contain pointers and references
   proc wrap(t: PType): bool {.nimcall.} = t.kind in {tyRef, tyPtr, tyVar, tyLent}
   result = not types.searchTypeFor(t, wrap)
 
 proc canAlias*(arg, ret: PType): bool =
   if isValueOnlyType(arg):
-    # can alias only with unsafeAddr(arg.x) and we don't care if it is not safe
+    # can alias only with addr(arg.x) and we don't care if it is not safe
     result = false
   else:
     var marker = initIntSet()
     result = canAlias(arg, ret, marker)
 
+proc containsVariable(n: PNode): bool =
+  case n.kind
+  of nodesToIgnoreSet:
+    result = false
+  of nkSym:
+    result = n.sym.kind in {skForVar, skParam, skVar, skLet, skConst, skResult, skTemp}
+  else:
+    for ch in n:
+      if containsVariable(ch): return true
+    result = false
+
 proc checkIsolate*(n: PNode): bool =
   if types.containsTyRef(n.typ):
     # XXX Maybe require that 'n.typ' is acyclic. This is not much
@@ -88,7 +102,7 @@ proc checkIsolate*(n: PNode): bool =
       # XXX: as long as we don't update the analysis while examining arguments
       #      we can do an early check of the return type, otherwise this is a
       #      bug and needs to be moved below
-      if n[0].typ.flags * {tfGcSafe, tfNoSideEffect} == {}:
+      if tfNoSideEffect notin n[0].typ.flags:
         return false
       for i in 1..<n.len:
         if checkIsolate(n[i]):
@@ -96,7 +110,11 @@ proc checkIsolate*(n: PNode): bool =
         else:
           let argType = n[i].typ
           if argType != nil and not isCompileTimeOnly(argType) and containsTyRef(argType):
-            if argType.canAlias(n.typ):
+            if argType.canAlias(n.typ) or containsVariable(n[i]):
+              # bug #19013: Alias information is not enough, we need to check for potential
+              # "overlaps". I claim the problem can only happen by reading again from a location
+              # that materialized which is only possible if a variable that contains a `ref`
+              # is involved.
               return false
       result = true
     of nkIfStmt, nkIfExpr:

compiler/jstypes.nim

@@ -19,13 +19,13 @@ proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
     s, u: Rope
     field: PSym
     b: PNode
-  result = nil
+  result = ""
   case n.kind
   of nkRecList:
     if n.len == 1:
       result = genObjectFields(p, typ, n[0])
     else:
-      s = nil
+      s = ""
       for i in 0..<n.len:
         if i > 0: s.add(", \L")
         s.add(genObjectFields(p, typ, n[i]))
@@ -44,13 +44,13 @@ proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
     s = genTypeInfo(p, field.typ)
     for i in 1..<n.len:
       b = n[i]           # branch
-      u = nil
+      u = ""
       case b.kind
       of nkOfBranch:
         if b.len < 2:
           internalError(p.config, b.info, "genObjectFields; nkOfBranch broken")
         for j in 0..<b.len - 1:
-          if u != nil: u.add(", ")
+          if u != "": u.add(", ")
           if b[j].kind == nkRange:
             u.addf("[$1, $2]", [rope(getOrdValue(b[j][0])),
                                  rope(getOrdValue(b[j][1]))])
@@ -59,7 +59,7 @@ proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
       of nkElse:
         u = rope(lengthOrd(p.config, field.typ))
       else: internalError(p.config, n.info, "genObjectFields(nkRecCase)")
-      if result != nil: result.add(", \L")
+      if result != "": result.add(", \L")
       result.addf("[setConstr($1), $2]",
            [u, genObjectFields(p, typ, lastSon(b))])
     result = ("{kind: 3, offset: \"$1\", len: $3, " &
@@ -84,7 +84,7 @@ proc genObjectInfo(p: PProc, typ: PType, name: Rope) =
          [name, genTypeInfo(p, typ[0].skipTypes(skipPtrs))])
 
 proc genTupleFields(p: PProc, typ: PType): Rope =
-  var s: Rope = nil
+  var s: Rope = ""
   for i in 0..<typ.len:
     if i > 0: s.add(", \L")
     s.addf("{kind: 1, offset: \"Field$1\", len: 0, " &
@@ -102,7 +102,7 @@ proc genTupleInfo(p: PProc, typ: PType, name: Rope) =
   p.g.typeInfo.addf("$1.node = NNI$2;$n", [name, rope(typ.id)])
 
 proc genEnumInfo(p: PProc, typ: PType, name: Rope) =
-  var s: Rope = nil
+  var s: Rope = ""
   for i in 0..<typ.n.len:
     if (typ.n[i].kind != nkSym): internalError(p.config, typ.n.info, "genEnumInfo")
     let field = typ.n[i].sym
@@ -133,16 +133,16 @@ proc genTypeInfo(p: PProc, typ: PType): Rope =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
       [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
-  of tyVar, tyLent, tyRef, tyPtr, tySequence, tyRange, tySet:
+  of tyVar, tyLent, tyRef, tyPtr, tySequence, tyRange, tySet, tyOpenArray:
     var s =
-      "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
+      "var $1 = {size: 0, kind: $2, base: null, node: null, finalizer: null};$n" %
               [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
     p.g.typeInfo.addf("$1.base = $2;$n",
          [result, genTypeInfo(p, t.lastSon)])
   of tyArray:
     var s =
-      "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
+      "var $1 = {size: 0, kind: $2, base: null, node: null, finalizer: null};$n" %
               [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
     p.g.typeInfo.addf("$1.base = $2;$n",

compiler/lambdalifting.nim

@@ -12,7 +12,10 @@
 import
   intsets, strutils, options, ast, astalgo, msgs,
   idents, renderer, types, magicsys, lowerings, tables, modulegraphs, lineinfos,
-  transf, liftdestructors
+  transf, liftdestructors, typeallowed
+
+when defined(nimPreviewSlimSystem):
+  import std/assertions
 
 discard """
   The basic approach is that captured vars need to be put on the heap and
@@ -191,9 +194,7 @@ proc interestingVar(s: PSym): bool {.inline.} =
     s.typ.kind notin {tyStatic, tyTypeDesc}
 
 proc illegalCapture(s: PSym): bool {.inline.} =
-  result = skipTypes(s.typ, abstractInst).kind in
-                   {tyVar, tyOpenArray, tyVarargs, tyLent} or
-      s.kind == skResult
+  result = classifyViewType(s.typ) != noView or s.kind == skResult
 
 proc isInnerProc(s: PSym): bool =
   if s.kind in {skProc, skFunc, skMethod, skConverter, skIterator} and s.magic == mNone:
@@ -244,6 +245,13 @@ proc createTypeBoundOpsLL(g: ModuleGraph; refType: PType; info: TLineInfo; idgen
     if tfHasAsgn in refType.flags or optSeqDestructors in g.config.globalOptions:
       owner.flags.incl sfInjectDestructors
 
+proc genCreateEnv(env: PNode): PNode =
+  var c = newNodeIT(nkObjConstr, env.info, env.typ)
+  c.add newNodeIT(nkType, env.info, env.typ)
+  let e = copyTree(env)
+  e.flags.incl nfFirstWrite
+  result = newAsgnStmt(e, c)
+
 proc liftIterSym*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PNode =
   # transforms  (iter)  to  (let env = newClosure[iter](); (iter, env))
   if liftingHarmful(g.config, owner): return n
@@ -267,23 +275,19 @@ proc liftIterSym*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PN
     addVar(v, env)
     result.add(v)
   # add 'new' statement:
-  result.add newCall(getSysSym(g, n.info, "internalNew"), env)
+  #result.add newCall(getSysSym(g, n.info, "internalNew"), env)
+  result.add genCreateEnv(env)
   createTypeBoundOpsLL(g, env.typ, n.info, idgen, owner)
   result.add makeClosure(g, idgen, iter, env, n.info)
 
 proc freshVarForClosureIter*(g: ModuleGraph; s: PSym; idgen: IdGenerator; owner: PSym): PNode =
   let envParam = getHiddenParam(g, owner)
   let obj = envParam.typ.skipTypes({tyOwned, tyRef, tyPtr})
-  addField(obj, s, g.cache, idgen)
+  let field = addField(obj, s, g.cache, idgen)
 
   var access = newSymNode(envParam)
   assert obj.kind == tyObject
-  let field = getFieldFromObj(obj, s)
-  if field != nil:
-    result = rawIndirectAccess(access, field, s.info)
-  else:
-    localError(g.config, s.info, "internal error: cannot generate fresh variable")
-    result = access
+  result = rawIndirectAccess(access, field, s.info)
 
 # ------------------ new stuff -------------------------------------------
 
@@ -292,7 +296,8 @@ proc markAsClosure(g: ModuleGraph; owner: PSym; n: PNode) =
   if illegalCapture(s):
     localError(g.config, n.info,
       ("'$1' is of type <$2> which cannot be captured as it would violate memory" &
-       " safety, declared here: $3; using '-d:nimWorkaround14447' helps in some cases") %
+       " safety, declared here: $3; using '-d:nimNoLentIterators' helps in some cases." &
+       " Consider using a <ref $2> which can be captured.") %
       [s.name.s, typeToString(s.typ), g.config$s.info])
   elif not (owner.typ.callConv == ccClosure or owner.typ.callConv == ccNimCall and tfExplicitCallConv notin owner.typ.flags):
     localError(g.config, n.info, "illegal capture '$1' because '$2' has the calling convention: <$3>" %
@@ -436,7 +441,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
     if innerProc:
       if s.isIterator: c.somethingToDo = true
       if not c.processed.containsOrIncl(s.id):
-        let body = transformBody(c.graph, c.idgen, s, cache = true)
+        let body = transformBody(c.graph, c.idgen, s, useCache)
         detectCapturedVars(body, s, c)
     let ow = s.skipGenericOwner
     if ow == owner:
@@ -451,9 +456,9 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
             if s.name.id == getIdent(c.graph.cache, ":state").id:
               obj.n[0].sym.itemId = ItemId(module: s.itemId.module, item: -s.itemId.item)
             else:
-              addField(obj, s, c.graph.cache, c.idgen)
+              discard addField(obj, s, c.graph.cache, c.idgen)
     # direct or indirect dependency:
-    elif (innerProc and s.typ.callConv == ccClosure) or interestingVar(s):
+    elif (innerProc and not s.isIterator and s.typ.callConv == ccClosure) or interestingVar(s):
       discard """
         proc outer() =
           var x: int
@@ -473,7 +478,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
       if interestingVar(s) and not c.capturedVars.containsOrIncl(s.id):
         let obj = c.getEnvTypeForOwner(ow, n.info).skipTypes({tyOwned, tyRef, tyPtr})
         #getHiddenParam(owner).typ.skipTypes({tyOwned, tyRef, tyPtr})
-        addField(obj, s, c.graph.cache, c.idgen)
+        discard addField(obj, s, c.graph.cache, c.idgen)
       # create required upFields:
       var w = owner.skipGenericOwner
       if isInnerProc(w) or owner.isIterator:
@@ -599,7 +604,7 @@ proc rawClosureCreation(owner: PSym;
         addVar(v, unowned)
 
     # add 'new' statement:
-    result.add(newCall(getSysSym(d.graph, env.info, "internalNew"), env))
+    result.add genCreateEnv(env)
     if optOwnedRefs in d.graph.config.globalOptions:
       let unowned = c.unownedEnvVars[owner.id]
       assert unowned != nil
@@ -661,7 +666,7 @@ proc closureCreationForIter(iter: PNode;
     var vs = newNodeI(nkVarSection, iter.info)
     addVar(vs, vnode)
     result.add(vs)
-  result.add(newCall(getSysSym(d.graph, iter.info, "internalNew"), vnode))
+  result.add genCreateEnv(vnode)
   createTypeBoundOpsLL(d.graph, vnode.typ, iter.info, d.idgen, owner)
 
   let upField = lookupInRecord(v.typ.skipTypes({tyOwned, tyRef, tyPtr}).n, getIdent(d.graph.cache, upName))
@@ -732,7 +737,7 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: var DetectionPass;
         #  echo renderTree(s.getBody, {renderIds})
         let oldInContainer = c.inContainer
         c.inContainer = 0
-        var body = transformBody(d.graph, d.idgen, s, cache = false)
+        var body = transformBody(d.graph, d.idgen, s, dontUseCache)
         body = liftCapturedVars(body, s, d, c)
         if c.envVars.getOrDefault(s.id).isNil:
           s.transformedBody = body
@@ -777,7 +782,7 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: var DetectionPass;
       n[1] = liftCapturedVars(n[1], owner, d, c)
       if n[1].kind == nkClosure: result = n[1]
   of nkReturnStmt:
-    if n[0].kind in {nkAsgn, nkFastAsgn}:
+    if n[0].kind in {nkAsgn, nkFastAsgn, nkSinkAsgn}:
       # we have a `result = result` expression produced by the closure
       # transform, let's not touch the LHS in order to make the lifting pass
       # correct when `result` is lifted
@@ -854,7 +859,7 @@ proc liftIterToProc*(g: ModuleGraph; fn: PSym; body: PNode; ptrType: PType;
   fn.typ.callConv = oldCC
 
 proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool;
-                  idgen: IdGenerator): PNode =
+                  idgen: IdGenerator, force: bool): PNode =
   # XXX backend == backendJs does not suffice! The compiletime stuff needs
   # the transformation even when compiling to JS ...
 
@@ -863,7 +868,7 @@ proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool;
 
   if body.kind == nkEmpty or (
       g.config.backend == backendJs and not isCompileTime) or
-      fn.skipGenericOwner.kind != skModule:
+      (fn.skipGenericOwner.kind != skModule and not force):
 
     # ignore forward declaration:
     result = body
@@ -946,7 +951,7 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; idgen: IdGenerator; owner: PSym):
     addVar(v, newSymNode(env))
     result.add(v)
     # add 'new' statement:
-    result.add(newCall(getSysSym(g, env.info, "internalNew"), env.newSymNode))
+    result.add genCreateEnv(env.newSymNode)
     createTypeBoundOpsLL(g, env.typ, body.info, idgen, owner)
 
   elif op.kind == nkStmtListExpr:

compiler/layouter.nim

@@ -510,7 +510,7 @@ proc emitTok*(em: var Emitter; L: Lexer; tok: Token) =
     rememberSplit(splitComma)
     wrSpace em
   of openPars:
-    if tok.strongSpaceA > 0 and not em.endsInWhite and
+    if tok.strongSpaceA and not em.endsInWhite and
         (not em.wasExportMarker or tok.tokType == tkCurlyDotLe):
       wrSpace em
     wr(em, $tok.tokType, ltSomeParLe)
@@ -528,7 +528,7 @@ proc emitTok*(em: var Emitter; L: Lexer; tok: Token) =
     wr(em, $tok.tokType, ltOther)
     if not em.inquote: wrSpace(em)
   of tkOpr, tkDotDot:
-    if em.inquote or ((tok.strongSpaceA == 0 and tok.strongSpaceB == 0) and
+    if em.inquote or (((not tok.strongSpaceA) and tok.strongSpaceB == tsNone) and
         tok.ident.s notin ["<", ">", "<=", ">=", "==", "!="]):
       # bug #9504: remember to not spacify a keyword:
       lastTokWasTerse = true
@@ -538,7 +538,7 @@ proc emitTok*(em: var Emitter; L: Lexer; tok: Token) =
       if not em.endsInWhite: wrSpace(em)
       wr(em, tok.ident.s, ltOpr)
       template isUnary(tok): bool =
-        tok.strongSpaceB == 0 and tok.strongSpaceA > 0
+        tok.strongSpaceB == tsNone and tok.strongSpaceA
 
       if not isUnary(tok):
         rememberSplit(splitBinary)
@@ -551,11 +551,15 @@ proc emitTok*(em: var Emitter; L: Lexer; tok: Token) =
     if not preventComment:
       emitComment(em, tok, dontIndent = false)
   of tkIntLit..tkStrLit, tkRStrLit, tkTripleStrLit, tkGStrLit, tkGTripleStrLit, tkCharLit:
-    let lit = fileSection(em.config, em.fid, tok.offsetA, tok.offsetB)
-    if endsInAlpha(em) and tok.tokType notin {tkGStrLit, tkGTripleStrLit}: wrSpace(em)
-    em.lineSpan = countNewlines(lit)
-    if em.lineSpan > 0: calcCol(em, lit)
-    wr em, lit, ltLit
+    if not em.inquote:
+      let lit = fileSection(em.config, em.fid, tok.offsetA, tok.offsetB)
+      if endsInAlpha(em) and tok.tokType notin {tkGStrLit, tkGTripleStrLit}: wrSpace(em)
+      em.lineSpan = countNewlines(lit)
+      if em.lineSpan > 0: calcCol(em, lit)
+      wr em, lit, ltLit
+    else:
+      if endsInAlpha(em): wrSpace(em)
+      wr em, tok.literal, ltLit
   of tkEof: discard
   else:
     let lit = if tok.ident != nil: tok.ident.s else: tok.literal

compiler/lexer.nim

@@ -7,18 +7,21 @@
 #    distribution, for details about the copyright.
 #
 
-# This scanner is handwritten for efficiency. I used an elegant buffering
+# This lexer is handwritten for efficiency. I used an elegant buffering
 # scheme which I have not seen anywhere else:
 # We guarantee that a whole line is in the buffer. Thus only when scanning
 # the \n or \r character we have to check whether we need to read in the next
 # chunk. (\n or \r already need special handling for incrementing the line
-# counter; choosing both \n and \r allows the scanner to properly read Unix,
+# counter; choosing both \n and \r allows the lexer to properly read Unix,
 # DOS or Macintosh text files, even when it is not the native format.
 
 import
   hashes, options, msgs, strutils, platform, idents, nimlexbase, llstream,
   wordrecg, lineinfos, pathutils, parseutils
 
+when defined(nimPreviewSlimSystem):
+  import std/[assertions, formatfloat]
+
 const
   MaxLineLength* = 80         # lines longer than this lead to a warning
   numChars*: set[char] = {'0'..'9', 'a'..'z', 'A'..'Z'}
@@ -90,19 +93,22 @@ type
                               # so that it is the correct default value
     base2, base8, base16
 
-  Token* = object             # a Nim token
-    tokType*: TokType         # the type of the token
-    indent*: int              # the indentation; != -1 if the token has been
-                              # preceded with indentation
-    ident*: PIdent            # the parsed identifier
-    iNumber*: BiggestInt      # the parsed integer literal
-    fNumber*: BiggestFloat    # the parsed floating point literal
-    base*: NumericalBase      # the numerical base; only valid for int
-                              # or float literals
-    strongSpaceA*: int8       # leading spaces of an operator
-    strongSpaceB*: int8       # trailing spaces of an operator
-    literal*: string          # the parsed (string) literal; and
-                              # documentation comments are here too
+  TokenSpacing* = enum
+    tsNone, tsTrailing, tsEof
+
+  Token* = object                # a Nim token
+    tokType*: TokType            # the type of the token
+    indent*: int                 # the indentation; != -1 if the token has been
+                                 # preceded with indentation
+    ident*: PIdent               # the parsed identifier
+    iNumber*: BiggestInt         # the parsed integer literal
+    fNumber*: BiggestFloat       # the parsed floating point literal
+    base*: NumericalBase         # the numerical base; only valid for int
+                                 # or float literals
+    strongSpaceA*: bool          # leading spaces of an operator
+    strongSpaceB*: TokenSpacing  # trailing spaces of an operator
+    literal*: string             # the parsed (string) literal; and
+                                 # documentation comments are here too
     line*, col*: int
     when defined(nimpretty):
       offsetA*, offsetB*: int # used for pretty printing so that literals
@@ -121,6 +127,8 @@ type
     cache*: IdentCache
     when defined(nimsuggest):
       previousToken: TLineInfo
+      tokenEnd*: TLineInfo
+      previousTokenEnd*: TLineInfo
     config*: ConfigRef
 
 proc getLineInfo*(L: Lexer, tok: Token): TLineInfo {.inline.} =
@@ -170,7 +178,7 @@ proc initToken*(L: var Token) =
   L.tokType = tkInvalid
   L.iNumber = 0
   L.indent = 0
-  L.strongSpaceA = 0
+  L.strongSpaceA = false
   L.literal = ""
   L.fNumber = 0.0
   L.base = base10
@@ -183,7 +191,7 @@ proc fillToken(L: var Token) =
   L.tokType = tkInvalid
   L.iNumber = 0
   L.indent = 0
-  L.strongSpaceA = 0
+  L.strongSpaceA = false
   setLen(L.literal, 0)
   L.fNumber = 0.0
   L.base = base10
@@ -502,11 +510,11 @@ proc getNumber(L: var Lexer, result: var Token) =
         of tkUInt16Lit: setNumber result.iNumber, xi and 0xffff
         of tkUInt32Lit: setNumber result.iNumber, xi and 0xffffffff
         of tkFloat32Lit:
-          setNumber result.fNumber, (cast[PFloat32](addr(xi)))[]
+          setNumber result.fNumber, (cast[ptr float32](addr(xi)))[]
           # note: this code is endian neutral!
           # XXX: Test this on big endian machine!
         of tkFloat64Lit, tkFloatLit:
-          setNumber result.fNumber, (cast[PFloat64](addr(xi)))[]
+          setNumber result.fNumber, (cast[ptr float64](addr(xi)))[]
         else: internalError(L.config, getLineInfo(L), "getNumber")
 
         # Bounds checks. Non decimal literals are allowed to overflow the range of
@@ -536,9 +544,9 @@ proc getNumber(L: var Lexer, result: var Token) =
           try:
             len = parseBiggestUInt(result.literal, iNumber)
           except ValueError:
-            raise newException(OverflowDefect, "number out of range: " & $result.literal)
+            raise newException(OverflowDefect, "number out of range: " & result.literal)
           if len != result.literal.len:
-            raise newException(ValueError, "invalid integer: " & $result.literal)
+            raise newException(ValueError, "invalid integer: " & result.literal)
           result.iNumber = cast[int64](iNumber)
         else:
           var iNumber: int64
@@ -546,9 +554,9 @@ proc getNumber(L: var Lexer, result: var Token) =
           try:
             len = parseBiggestInt(result.literal, iNumber)
           except ValueError:
-            raise newException(OverflowDefect, "number out of range: " & $result.literal)
+            raise newException(OverflowDefect, "number out of range: " & result.literal)
           if len != result.literal.len:
-            raise newException(ValueError, "invalid integer: " & $result.literal)
+            raise newException(ValueError, "invalid integer: " & result.literal)
           result.iNumber = iNumber
 
         # Explicit bounds checks.
@@ -838,6 +846,49 @@ proc getCharacter(L: var Lexer; tok: var Token) =
       lexMessage(L, errGenerated, "missing closing ' for character literal")
     tokenEndIgnore(tok, L.bufpos)
 
+const
+  UnicodeOperatorStartChars = {'\226', '\194', '\195'}
+    # the allowed unicode characters ("∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔")
+    # all start with one of these.
+
+type
+  UnicodeOprPred = enum
+    Mul, Add
+
+proc unicodeOprLen(buf: cstring; pos: int): (int8, UnicodeOprPred) =
+  template m(len): untyped = (int8(len), Mul)
+  template a(len): untyped = (int8(len), Add)
+  result = 0.m
+  case buf[pos]
+  of '\226':
+    if buf[pos+1] == '\136':
+      if buf[pos+2] == '\152': result = 3.m # ∘
+      elif buf[pos+2] == '\153': result = 3.m # ∙
+      elif buf[pos+2] == '\167': result = 3.m # ∧
+      elif buf[pos+2] == '\168': result = 3.a # ∨
+      elif buf[pos+2] == '\169': result = 3.m # ∩
+      elif buf[pos+2] == '\170': result = 3.a # ∪
+    elif buf[pos+1] == '\138':
+      if buf[pos+2] == '\147': result = 3.m # ⊓
+      elif buf[pos+2] == '\148': result = 3.a # ⊔
+      elif buf[pos+2] == '\149': result = 3.a # ⊕
+      elif buf[pos+2] == '\150': result = 3.a # ⊖
+      elif buf[pos+2] == '\151': result = 3.m # ⊗
+      elif buf[pos+2] == '\152': result = 3.m # ⊘
+      elif buf[pos+2] == '\153': result = 3.m # ⊙
+      elif buf[pos+2] == '\155': result = 3.m # ⊛
+      elif buf[pos+2] == '\158': result = 3.a # ⊞
+      elif buf[pos+2] == '\159': result = 3.a # ⊟
+      elif buf[pos+2] == '\160': result = 3.m # ⊠
+      elif buf[pos+2] == '\161': result = 3.m # ⊡
+    elif buf[pos+1] == '\152' and buf[pos+2] == '\133': result = 3.m # ★
+  of '\194':
+    if buf[pos+1] == '\177': result = 2.a # ±
+  of '\195':
+    if buf[pos+1] == '\151': result = 2.m # ×
+  else:
+    discard
+
 proc getSymbol(L: var Lexer, tok: var Token) =
   var h: Hash = 0
   var pos = L.bufpos
@@ -846,7 +897,7 @@ proc getSymbol(L: var Lexer, tok: var Token) =
   while true:
     var c = L.buf[pos]
     case c
-    of 'a'..'z', '0'..'9', '\x80'..'\xFF':
+    of 'a'..'z', '0'..'9':
       h = h !& ord(c)
       inc(pos)
     of 'A'..'Z':
@@ -860,10 +911,16 @@ proc getSymbol(L: var Lexer, tok: var Token) =
         break
       inc(pos)
       suspicious = true
+    of '\x80'..'\xFF':
+      if c in UnicodeOperatorStartChars and unicodeOprLen(L.buf, pos)[0] != 0:
+        break
+      else:
+        h = h !& ord(c)
+        inc(pos)
     else: break
   tokenEnd(tok, pos-1)
   h = !$h
-  tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
+  tok.ident = L.cache.getIdent(cast[cstring](addr(L.buf[L.bufpos])), pos - L.bufpos, h)
   if (tok.ident.id < ord(tokKeywordLow) - ord(tkSymbol)) or
       (tok.ident.id > ord(tokKeywordHigh) - ord(tkSymbol)):
     tok.tokType = tkSymbol
@@ -877,7 +934,7 @@ proc getSymbol(L: var Lexer, tok: var Token) =
 proc endOperator(L: var Lexer, tok: var Token, pos: int,
                  hash: Hash) {.inline.} =
   var h = !$hash
-  tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
+  tok.ident = L.cache.getIdent(cast[cstring](addr(L.buf[L.bufpos])), pos - L.bufpos, h)
   if (tok.ident.id < oprLow) or (tok.ident.id > oprHigh): tok.tokType = tkOpr
   else: tok.tokType = TokType(tok.ident.id - oprLow + ord(tkColon))
   L.bufpos = pos
@@ -888,42 +945,63 @@ proc getOperator(L: var Lexer, tok: var Token) =
   var h: Hash = 0
   while true:
     var c = L.buf[pos]
-    if c notin OpChars: break
-    h = h !& ord(c)
-    inc(pos)
+    if c in OpChars:
+      h = h !& ord(c)
+      inc(pos)
+    elif c in UnicodeOperatorStartChars:
+      let oprLen = unicodeOprLen(L.buf, pos)[0]
+      if oprLen == 0: break
+      for i in 0..<oprLen:
+        h = h !& ord(L.buf[pos])
+        inc pos
+    else:
+      break
   endOperator(L, tok, pos, h)
   tokenEnd(tok, pos-1)
   # advance pos but don't store it in L.bufpos so the next token (which might
   # be an operator too) gets the preceding spaces:
-  tok.strongSpaceB = 0
+  tok.strongSpaceB = tsNone
   while L.buf[pos] == ' ':
     inc pos
-    inc tok.strongSpaceB
+    if tok.strongSpaceB != tsTrailing:
+      tok.strongSpaceB = tsTrailing
   if L.buf[pos] in {CR, LF, nimlexbase.EndOfFile}:
-    tok.strongSpaceB = -1
+    tok.strongSpaceB = tsEof
 
 proc getPrecedence*(tok: Token): int =
   ## Calculates the precedence of the given token.
+  const
+    MulPred = 9
+    PlusPred = 8
   case tok.tokType
   of tkOpr:
     let relevantChar = tok.ident.s[0]
 
     # arrow like?
     if tok.ident.s.len > 1 and tok.ident.s[^1] == '>' and
-      tok.ident.s[^2] in {'-', '~', '='}: return 1
+      tok.ident.s[^2] in {'-', '~', '='}: return 0
 
     template considerAsgn(value: untyped) =
       result = if tok.ident.s[^1] == '=': 1 else: value
 
     case relevantChar
     of '$', '^': considerAsgn(10)
-    of '*', '%', '/', '\\': considerAsgn(9)
+    of '*', '%', '/', '\\': considerAsgn(MulPred)
     of '~': result = 8
-    of '+', '-', '|': considerAsgn(8)
+    of '+', '-', '|': considerAsgn(PlusPred)
     of '&': considerAsgn(7)
     of '=', '<', '>', '!': result = 5
     of '.': considerAsgn(6)
     of '?': result = 2
+    of UnicodeOperatorStartChars:
+      if tok.ident.s[^1] == '=':
+        result = 1
+      else:
+        let (len, pred) = unicodeOprLen(cstring(tok.ident.s), 0)
+        if len != 0:
+          result = if pred == Mul: MulPred else: PlusPred
+        else:
+          result = 2
     else: considerAsgn(2)
   of tkDiv, tkMod, tkShl, tkShr: result = 9
   of tkDotDot: result = 6
@@ -1075,7 +1153,7 @@ proc scanComment(L: var Lexer, tok: var Token) =
 proc skip(L: var Lexer, tok: var Token) =
   var pos = L.bufpos
   tokenBegin(tok, pos)
-  tok.strongSpaceA = 0
+  tok.strongSpaceA = false
   when defined(nimpretty):
     var hasComment = false
     var commentIndent = L.currLineIndent
@@ -1086,7 +1164,8 @@ proc skip(L: var Lexer, tok: var Token) =
     case L.buf[pos]
     of ' ':
       inc(pos)
-      inc(tok.strongSpaceA)
+      if not tok.strongSpaceA:
+        tok.strongSpaceA = true
     of '\t':
       if not L.allowTabs: lexMessagePos(L, errGenerated, pos, "tabs are not allowed, use spaces instead")
       inc(pos)
@@ -1108,7 +1187,7 @@ proc skip(L: var Lexer, tok: var Token) =
           pos = L.bufpos
         else:
           break
-      tok.strongSpaceA = 0
+      tok.strongSpaceA = false
       when defined(nimpretty):
         if L.buf[pos] == '#' and tok.line < 0: commentIndent = indent
       if L.buf[pos] > ' ' and (L.buf[pos] != '#' or L.buf[pos+1] == '#'):
@@ -1147,6 +1226,10 @@ proc skip(L: var Lexer, tok: var Token) =
 proc rawGetTok*(L: var Lexer, tok: var Token) =
   template atTokenEnd() {.dirty.} =
     when defined(nimsuggest):
+      L.previousTokenEnd.line = L.tokenEnd.line
+      L.previousTokenEnd.col = L.tokenEnd.col
+      L.tokenEnd.line = tok.line.uint16
+      L.tokenEnd.col = getColNumber(L, L.bufpos).int16
       # we attach the cursor to the last *strong* token
       if tok.tokType notin weakTokens:
         L.previousToken.line = tok.line.uint16
@@ -1167,10 +1250,15 @@ proc rawGetTok*(L: var Lexer, tok: var Token) =
   var c = L.buf[L.bufpos]
   tok.line = L.lineNumber
   tok.col = getColNumber(L, L.bufpos)
-  if c in SymStartChars - {'r', 'R'}:
+  if c in SymStartChars - {'r', 'R'} - UnicodeOperatorStartChars:
     getSymbol(L, tok)
   else:
     case c
+    of UnicodeOperatorStartChars:
+      if unicodeOprLen(L.buf, L.bufpos)[0] != 0:
+        getOperator(L, tok)
+      else:
+        getSymbol(L, tok)
     of '#':
       scanComment(L, tok)
     of '*':
@@ -1278,7 +1366,11 @@ proc rawGetTok*(L: var Lexer, tok: var Token) =
       getNumber(L, tok)
       let c = L.buf[L.bufpos]
       if c in SymChars+{'_'}:
-        lexMessage(L, errGenerated, "invalid token: no whitespace between number and identifier")
+        if c in UnicodeOperatorStartChars and
+            unicodeOprLen(L.buf, L.bufpos)[0] != 0:
+          discard
+        else:
+          lexMessage(L, errGenerated, "invalid token: no whitespace between number and identifier")
     of '-':
       if L.buf[L.bufpos+1] in {'0'..'9'} and
           (L.bufpos-1 == 0 or L.buf[L.bufpos-1] in UnaryMinusWhitelist):
@@ -1289,7 +1381,11 @@ proc rawGetTok*(L: var Lexer, tok: var Token) =
         getNumber(L, tok)
         let c = L.buf[L.bufpos]
         if c in SymChars+{'_'}:
-          lexMessage(L, errGenerated, "invalid token: no whitespace between number and identifier")
+          if c in UnicodeOperatorStartChars and
+              unicodeOprLen(L.buf, L.bufpos)[0] != 0:
+            discard
+          else:
+            lexMessage(L, errGenerated, "invalid token: no whitespace between number and identifier")
       else:
         getOperator(L, tok)
     else:

compiler/liftdestructors.nim

@@ -8,13 +8,16 @@
 #
 
 ## This module implements lifting for type-bound operations
-## (``=sink``, ``=``, ``=destroy``, ``=deepCopy``).
+## (``=sink``, ``=copy``, ``=destroy``, ``=deepCopy``).
 
 import modulegraphs, lineinfos, idents, ast, renderer, semdata,
   sighashes, lowerings, options, types, msgs, magicsys, tables, ccgutils
 
 from trees import isCaseObj
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 type
   TLiftCtx = object
     g: ModuleGraph
@@ -85,6 +88,8 @@ proc defaultOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     let call = genBuiltin(c, mDefault, "default", x)
     call.typ = t
     body.add newAsgnStmt(x, call)
+  elif c.kind == attachedWasMoved:
+    body.add genBuiltin(c, mWasMoved, "wasMoved", x)
 
 proc genAddr(c: var TLiftCtx; x: PNode): PNode =
   if x.kind == nkHiddenDeref:
@@ -142,6 +147,11 @@ proc destructorCall(c: var TLiftCtx; op: PSym; x: PNode): PNode =
   else:
     result = destroy
 
+proc genWasMovedCall(c: var TLiftCtx; op: PSym; x: PNode): PNode =
+  result = newNodeIT(nkCall, x.info, op.typ[0])
+  result.add(newSymNode(op))
+  result.add genAddr(c, x)
+
 proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool) =
   case n.kind
   of nkSym:
@@ -162,9 +172,12 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool)
       # the value needs to be destroyed before we assign the selector
       # or the value is lost
       let prevKind = c.kind
+      let prevAddMemReset = c.addMemReset
       c.kind = attachedDestructor
+      c.addMemReset = true
       fillBodyObj(c, n, body, x, y, enforceDefaultOp = false)
       c.kind = prevKind
+      c.addMemReset = prevAddMemReset
       localEnforceDefaultOp = true
 
     if c.kind != attachedDestructor:
@@ -418,14 +431,16 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
     #result = addDestructorCall(c, t, body, x)
   of attachedAsgn, attachedSink, attachedTrace:
     var op = getAttachedOp(c.g, t, c.kind)
+    if op != nil and sfOverriden in op.flags:
+      if op.ast.isGenericRoutine:
+        # patch generic =trace:
+        op = instantiateGeneric(c, op, t, t.typeInst)
+        setAttachedOp(c.g, c.idgen.module, t, c.kind, op)
+
     result = considerAsgnOrSink(c, t, body, x, y, op)
     if op != nil:
       setAttachedOp(c.g, c.idgen.module, t, c.kind, op)
-  of attachedDispose:
-    var op = getAttachedOp(c.g, t, c.kind)
-    result = considerAsgnOrSink(c, t, body, x, nil, op)
-    if op != nil:
-      setAttachedOp(c.g, c.idgen.module, t, c.kind, op)
+
   of attachedDeepCopy:
     let op = getAttachedOp(c.g, t, attachedDeepCopy)
     if op != nil:
@@ -434,6 +449,20 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
       body.add newDeepCopyCall(c, op, x, y)
       result = true
 
+  of attachedWasMoved:
+    var op = getAttachedOp(c.g, t, attachedWasMoved)
+    if op != nil and sfOverriden in op.flags:
+
+      if op.ast.isGenericRoutine:
+        # patch generic destructor:
+        op = instantiateGeneric(c, op, t, t.typeInst)
+        setAttachedOp(c.g, c.idgen.module, t, attachedWasMoved, op)
+
+      #markUsed(c.g.config, c.info, op, c.g.usageSym)
+      onUse(c.info, op)
+      body.add genWasMovedCall(c, op, x)
+      result = true
+
 proc declareCounter(c: var TLiftCtx; body: PNode; first: BiggestInt): PNode =
   var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId(c.idgen), c.fn, c.info)
   temp.typ = getSysType(c.g, body.info, tyInt)
@@ -516,9 +545,7 @@ proc fillSeqOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     if canFormAcycle(t.elemType):
       # follow all elements:
       forallElements(c, t, body, x, y)
-  of attachedDispose:
-    forallElements(c, t, body, x, y)
-    body.add genBuiltin(c, mDestroy, "destroy", x)
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x)
 
 proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   createTypeBoundOps(c.g, c.c, t, body.info, c.idgen)
@@ -526,7 +553,7 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   # operation here:
   var t = t
   if t.assignment == nil or t.destructor == nil:
-    let h = sighashes.hashType(t, {CoType, CoConsiderOwned, CoDistinct})
+    let h = sighashes.hashType(t,c.g.config, {CoType, CoConsiderOwned, CoDistinct})
     let canon = c.g.canonTypes.getOrDefault(h)
     if canon != nil: t = canon
 
@@ -556,11 +583,7 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       if op == nil:
         return # protect from recursion
       body.add newHookCall(c, op, x, y)
-  of attachedDispose:
-    let op = getAttachedOp(c.g, t, c.kind)
-    if op == nil:
-      return # protect from recursion
-    body.add newHookCall(c, op, x, nil)
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x)
 
 proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   case c.kind
@@ -572,10 +595,11 @@ proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     doAssert t.destructor != nil
     moveCall.add destructorCall(c, t.destructor, x)
     body.add moveCall
-  of attachedDestructor, attachedDispose:
+  of attachedDestructor:
     body.add genBuiltin(c, mDestroy, "destroy", x)
   of attachedTrace:
     discard "strings are atomic and have no inner elements that are to trace"
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x)
 
 proc cyclicType*(t: PType): bool =
   case t.kind
@@ -609,6 +633,11 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   createTypeBoundOps(c.g, c.c, elemType, c.info, c.idgen)
   let isCyclic = c.g.config.selectedGC == gcOrc and types.canFormAcycle(elemType)
 
+  let isInheritableAcyclicRef = c.g.config.selectedGC == gcOrc and
+                      (not isPureObject(elemType)) and
+                      tfAcyclic in skipTypes(elemType, abstractInst+{tyOwned}-{tyTypeDesc}).flags
+  # dynamic Acyclic refs need to use dyn decRef
+
   let tmp =
     if isCyclic and c.kind in {attachedAsgn, attachedSink}:
       declareTempOf(c, body, x)
@@ -632,6 +661,8 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicStatic", c.info, tmp, typInfo)
     else:
       cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicDyn", c.info, tmp)
+  elif isInheritableAcyclicRef:
+    cond = callCodegenProc(c.g, "nimDecRefIsLastDyn", c.info, x)
   else:
     cond = callCodegenProc(c.g, "nimDecRefIsLast", c.info, x)
   cond.typ = getSysType(c.g, x.info, tyBool)
@@ -667,16 +698,8 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
         # If the ref is polymorphic we have to account for this
         body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(x, c.idgen), y)
       #echo "can follow ", elemType, " static ", isFinal(elemType)
-  of attachedDispose:
-    # this is crucial! dispose is like =destroy but we don't follow refs
-    # as that is dealt within the cycle collector.
-    if not isCyclic:
-      body.add genIf(c, cond, actions)
-    when false:
-      let cond = copyTree(x)
-      cond.typ = getSysType(c.g, x.info, tyBool)
-      actions.add callCodegenProc(c.g, "nimRawDispose", c.info, x)
-      body.add genIf(c, cond, actions)
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x)
+
 
 proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   ## Closures are really like refs except they always use a virtual destructor
@@ -725,14 +748,7 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   of attachedDeepCopy: assert(false, "cannot happen")
   of attachedTrace:
     body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(xenv, c.idgen), y)
-  of attachedDispose:
-    # this is crucial! dispose is like =destroy but we don't follow refs
-    # as that is dealt within the cycle collector.
-    when false:
-      let cond = copyTree(xenv)
-      cond.typ = getSysType(c.g, xenv.info, tyBool)
-      actions.add callCodegenProc(c.g, "nimRawDispose", c.info, xenv)
-      body.add genIf(c, cond, actions)
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x)
 
 proc weakrefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   case c.kind
@@ -756,7 +772,8 @@ proc weakrefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     else:
       body.sons.insert(des, 0)
   of attachedDeepCopy: assert(false, "cannot happen")
-  of attachedTrace, attachedDispose: discard
+  of attachedTrace: discard
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x)
 
 proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   var actions = newNodeI(nkStmtList, c.info)
@@ -781,7 +798,8 @@ proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   of attachedDestructor:
     body.add genIf(c, x, actions)
   of attachedDeepCopy: assert(false, "cannot happen")
-  of attachedTrace, attachedDispose: discard
+  of attachedTrace: discard
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x)
 
 proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   if c.kind == attachedDeepCopy:
@@ -815,7 +833,8 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       else:
         body.sons.insert(des, 0)
     of attachedDeepCopy: assert(false, "cannot happen")
-    of attachedTrace, attachedDispose: discard
+    of attachedTrace: discard
+    of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x)
 
 proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   let xx = genBuiltin(c, mAccessEnv, "accessEnv", x)
@@ -830,7 +849,8 @@ proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   of attachedDestructor:
     body.add genIf(c, xx, actions)
   of attachedDeepCopy: assert(false, "cannot happen")
-  of attachedTrace, attachedDispose: discard
+  of attachedTrace: discard
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x)
 
 proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   case t.kind
@@ -946,7 +966,7 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp
 
   result.typ = newProcType(info, nextTypeId(idgen), owner)
   result.typ.addParam dest
-  if kind notin {attachedDestructor, attachedDispose}:
+  if kind notin {attachedDestructor, attachedWasMoved}:
     result.typ.addParam src
 
   if kind == attachedAsgn and g.config.selectedGC == gcOrc and
@@ -965,6 +985,12 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp
   incl result.flags, sfFromGeneric
   incl result.flags, sfGeneratedOp
 
+proc genTypeFieldCopy(c: var TLiftCtx; t: PType; body, x, y: PNode) =
+  let xx = genBuiltin(c, mAccessTypeField, "accessTypeField", x)
+  let yy = genBuiltin(c, mAccessTypeField, "accessTypeField", y)
+  xx.typ = getSysType(c.g, c.info, tyPointer)
+  yy.typ = xx.typ
+  body.add newAsgnStmt(xx, yy)
 
 proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
               info: TLineInfo; idgen: IdGenerator): PSym =
@@ -980,7 +1006,7 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
 
   let dest = result.typ.n[1].sym
   let d = newDeref(newSymNode(dest))
-  let src = if kind in {attachedDestructor, attachedDispose}: newNodeIT(nkSym, info, getSysType(g, info, tyPointer))
+  let src = if kind in {attachedDestructor, attachedWasMoved}: newNodeIT(nkSym, info, getSysType(g, info, tyPointer))
             else: newSymNode(result.typ.n[2].sym)
 
   # register this operation already:
@@ -1004,6 +1030,10 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
       fillStrOp(a, typ, result.ast[bodyPos], d, src)
     else:
       fillBody(a, typ, result.ast[bodyPos], d, src)
+      if tk == tyObject and a.kind in {attachedAsgn, attachedSink, attachedDeepCopy} and not lacksMTypeField(typ):
+        # bug #19205: Do not forget to also copy the hidden type field:
+        genTypeFieldCopy(a, typ, result.ast[bodyPos], d, src)
+
   if not a.canRaise: incl result.flags, sfNeverRaises
   completePartialOp(g, idgen.module, typ, kind, result)
 
@@ -1055,13 +1085,7 @@ proc inst(g: ModuleGraph; c: PContext; t: PType; kind: TTypeAttachedOp; idgen: I
   let op = getAttachedOp(g, t, kind)
   if op != nil and op.ast != nil and op.ast.isGenericRoutine:
     if t.typeInst != nil:
-      var a: TLiftCtx
-      a.info = info
-      a.g = g
-      a.kind = kind
-      a.c = c
-      a.idgen = idgen
-
+      var a = TLiftCtx(info: info, g: g, kind: kind, c: c, idgen: idgen)
       let opInst = instantiateGeneric(a, op, t, t.typeInst)
       if opInst.ast != nil:
         patchBody(g, c, opInst.ast, info, a.idgen)
@@ -1083,7 +1107,7 @@ proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInf
   let skipped = orig.skipTypes({tyGenericInst, tyAlias, tySink})
   if isEmptyContainer(skipped) or skipped.kind == tyStatic: return
 
-  let h = sighashes.hashType(skipped, {CoType, CoConsiderOwned, CoDistinct})
+  let h = sighashes.hashType(skipped, g.config, {CoType, CoConsiderOwned, CoDistinct})
   var canon = g.canonTypes.getOrDefault(h)
   if canon == nil:
     g.canonTypes[h] = skipped
@@ -1096,23 +1120,23 @@ proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInf
   # 4. We have a custom destructor.
   # 5. We have a (custom) generic destructor.
 
-  # we do not generate '=trace' nor '=dispose' procs if we
+  # we do not generate '=trace' procs if we
   # have the cycle detection disabled, saves code size.
-  let lastAttached = if g.config.selectedGC == gcOrc: attachedDispose
+  let lastAttached = if g.config.selectedGC == gcOrc: attachedTrace
                      else: attachedSink
 
   # bug #15122: We need to produce all prototypes before entering the
   # mind boggling recursion. Hacks like these imply we should rewrite
   # this module.
-  var generics: array[attachedDestructor..attachedDispose, bool]
-  for k in attachedDestructor..lastAttached:
+  var generics: array[attachedWasMoved..attachedTrace, bool]
+  for k in attachedWasMoved..lastAttached:
     generics[k] = getAttachedOp(g, canon, k) != nil
     if not generics[k]:
       setAttachedOp(g, idgen.module, canon, k,
           symPrototype(g, canon, canon.owner, k, info, idgen))
 
   # we generate the destructor first so that other operators can depend on it:
-  for k in attachedDestructor..lastAttached:
+  for k in attachedWasMoved..lastAttached:
     if not generics[k]:
       discard produceSym(g, c, canon, k, info, idgen)
     else:

compiler/lineinfos.nim

@@ -27,26 +27,39 @@ proc createDocLink*(urlSuffix: string): string =
 
 type
   TMsgKind* = enum
-    errUnknown, errInternal, errIllFormedAstX, errCannotOpenFile,
+    # fatal errors
+    errUnknown, errFatal, errInternal,
+    # non-fatal errors
+    errIllFormedAstX, errCannotOpenFile,
     errXExpected,
-    errGridTableNotImplemented,
-    errMarkdownIllformedTable,
-    errGeneralParseError,
-    errNewSectionExpected,
-    errInvalidDirectiveX,
-    errFootnoteMismatch,
+    errRstMissingClosing,
+    errRstGridTableNotImplemented,
+    errRstMarkdownIllformedTable,
+    errRstIllformedTable,
+    errRstNewSectionExpected,
+    errRstGeneralParseError,
+    errRstInvalidDirectiveX,
+    errRstInvalidField,
+    errRstFootnoteMismatch,
+    errRstSandboxedDirective,
     errProveInit, # deadcode
     errGenerated,
     errUser,
-
+    # warnings
     warnCannotOpenFile = "CannotOpenFile", warnOctalEscape = "OctalEscape",
     warnXIsNeverRead = "XIsNeverRead", warnXmightNotBeenInit = "XmightNotBeenInit",
     warnDeprecated = "Deprecated", warnConfigDeprecated = "ConfigDeprecated",
+    warnDotLikeOps = "DotLikeOps",
     warnSmallLshouldNotBeUsed = "SmallLshouldNotBeUsed", warnUnknownMagic = "UnknownMagic",
-    warnRedefinitionOfLabel = "RedefinitionOfLabel", warnUnknownSubstitutionX = "UnknownSubstitutionX",
-    warnLanguageXNotSupported = "LanguageXNotSupported",
-    warnFieldXNotSupported = "FieldXNotSupported",
-    warnRstStyle = "warnRstStyle", warnCommentXIgnored = "CommentXIgnored",
+    warnRstRedefinitionOfLabel = "RedefinitionOfLabel",
+    warnRstUnknownSubstitutionX = "UnknownSubstitutionX",
+    warnRstAmbiguousLink = "AmbiguousLink",
+    warnRstBrokenLink = "BrokenLink",
+    warnRstLanguageXNotSupported = "LanguageXNotSupported",
+    warnRstFieldXNotSupported = "FieldXNotSupported",
+    warnRstUnusedImportdoc = "UnusedImportdoc",
+    warnRstStyle = "warnRstStyle",
+    warnCommentXIgnored = "CommentXIgnored",
     warnTypelessParam = "TypelessParam",
     warnUseBase = "UseBase", warnWriteToForeignHeap = "WriteToForeignHeap",
     warnUnsafeCode = "UnsafeCode", warnUnusedImportX = "UnusedImport",
@@ -56,26 +69,40 @@ type
     warnUnreachableElse = "UnreachableElse", warnUnreachableCode = "UnreachableCode",
     warnStaticIndexCheck = "IndexCheck", warnGcUnsafe = "GcUnsafe", warnGcUnsafe2 = "GcUnsafe2",
     warnUninit = "Uninit", warnGcMem = "GcMem", warnDestructor = "Destructor",
-    warnLockLevel = "LockLevel", warnResultShadowed = "ResultShadowed",
+    warnLockLevel = "LockLevel", # deadcode
+    warnResultShadowed = "ResultShadowed",
     warnInconsistentSpacing = "Spacing",  warnCaseTransition = "CaseTransition",
     warnCycleCreated = "CycleCreated", warnObservableStores = "ObservableStores",
     warnStrictNotNil = "StrictNotNil",
     warnResultUsed = "ResultUsed",
     warnCannotOpen = "CannotOpen",
     warnFileChanged = "FileChanged",
+    warnSuspiciousEnumConv = "EnumConv",
+    warnAnyEnumConv = "AnyEnumConv",
+    warnHoleEnumConv = "HoleEnumConv",
+    warnCstringConv = "CStringConv",
+    warnPtrToCstringConv = "PtrToCstringConv",
+    warnEffect = "Effect",
+    warnCastSizes = "CastSizes"
+    warnImplicitTemplateRedefinition = "ImplicitTemplateRedefinition",
+    warnUnnamedBreak = "UnnamedBreak",
+    warnStmtListLambda = "StmtListLambda",
+    warnBareExcept = "BareExcept",
     warnUser = "User",
-
-    hintSuccess = "Success", hintSuccessX = "SuccessX", hintBuildMode = "BuildMode",
+    # hints
+    hintSuccess = "Success", hintSuccessX = "SuccessX",
     hintCC = "CC",
-    hintLineTooLong = "LineTooLong", hintXDeclaredButNotUsed = "XDeclaredButNotUsed",
+    hintLineTooLong = "LineTooLong",
+    hintXDeclaredButNotUsed = "XDeclaredButNotUsed", hintDuplicateModuleImport = "DuplicateModuleImport",
     hintXCannotRaiseY = "XCannotRaiseY", hintConvToBaseNotNeeded = "ConvToBaseNotNeeded",
     hintConvFromXtoItselfNotNeeded = "ConvFromXtoItselfNotNeeded", hintExprAlwaysX = "ExprAlwaysX",
-    hintQuitCalled = "QuitCalled", hintProcessing = "Processing", hintCodeBegin = "CodeBegin",
+    hintQuitCalled = "QuitCalled", hintProcessing = "Processing", hintProcessingStmt = "ProcessingStmt", hintCodeBegin = "CodeBegin",
     hintCodeEnd = "CodeEnd", hintConf = "Conf", hintPath = "Path",
     hintConditionAlwaysTrue = "CondTrue", hintConditionAlwaysFalse = "CondFalse", hintName = "Name",
     hintPattern = "Pattern", hintExecuting = "Exec", hintLinking = "Link", hintDependency = "Dependency",
     hintSource = "Source", hintPerformance = "Performance", hintStackTrace = "StackTrace",
     hintGCStats = "GCStats", hintGlobalVar = "GlobalVar", hintExpandMacro = "ExpandMacro",
+    hintAmbiguousEnum = "AmbiguousEnum",
     hintUser = "User", hintUserRaw = "UserRaw", hintExtendedContext = "ExtendedContext",
     hintMsgOrigin = "MsgOrigin", # since 1.3.5
     hintDeclaredLoc = "DeclaredLoc", # since 1.5.1
@@ -83,16 +110,21 @@ type
 const
   MsgKindToStr*: array[TMsgKind, string] = [
     errUnknown: "unknown error",
+    errFatal: "fatal error: $1",
     errInternal: "internal error: $1",
     errIllFormedAstX: "illformed AST: $1",
     errCannotOpenFile: "cannot open '$1'",
     errXExpected: "'$1' expected",
-    errGridTableNotImplemented: "grid table is not implemented",
-    errMarkdownIllformedTable: "illformed delimiter row of a markdown table",
-    errGeneralParseError: "general parse error",
-    errNewSectionExpected: "new section expected $1",
-    errInvalidDirectiveX: "invalid directive: '$1'",
-    errFootnoteMismatch: "number of footnotes and their references don't match: $1",
+    errRstMissingClosing: "$1",
+    errRstGridTableNotImplemented: "grid table is not implemented",
+    errRstMarkdownIllformedTable: "illformed delimiter row of a markdown table",
+    errRstIllformedTable: "Illformed table: $1",
+    errRstNewSectionExpected: "new section expected $1",
+    errRstGeneralParseError: "general parse error",
+    errRstInvalidDirectiveX: "invalid directive: '$1'",
+    errRstInvalidField: "invalid field: $1",
+    errRstFootnoteMismatch: "number of footnotes and their references don't match: $1",
+    errRstSandboxedDirective: "disabled directive: '$1'",
     errProveInit: "Cannot prove that '$1' is initialized.",  # deadcode
     errGenerated: "$1",
     errUser: "$1",
@@ -102,15 +134,19 @@ const
     warnXmightNotBeenInit: "'$1' might not have been initialized",
     warnDeprecated: "$1",
     warnConfigDeprecated: "config file '$1' is deprecated",
+    warnDotLikeOps: "$1",
     warnSmallLshouldNotBeUsed: "'l' should not be used as an identifier; may look like '1' (one)",
     warnUnknownMagic: "unknown magic '$1' might crash the compiler",
-    warnRedefinitionOfLabel: "redefinition of label '$1'",
-    warnUnknownSubstitutionX: "unknown substitution '$1'",
-    warnLanguageXNotSupported: "language '$1' not supported",
-    warnFieldXNotSupported: "field '$1' not supported",
+    warnRstRedefinitionOfLabel: "redefinition of label '$1'",
+    warnRstUnknownSubstitutionX: "unknown substitution '$1'",
+    warnRstAmbiguousLink: "ambiguous doc link $1",
+    warnRstBrokenLink: "broken link '$1'",
+    warnRstLanguageXNotSupported: "language '$1' not supported",
+    warnRstFieldXNotSupported: "field '$1' not supported",
+    warnRstUnusedImportdoc: "importdoc for '$1' is not used",
     warnRstStyle: "RST style: $1",
     warnCommentXIgnored: "comment '$1' ignored",
-    warnTypelessParam: "'$1' has no type. Typeless parameters are deprecated; only allowed for 'template'",
+    warnTypelessParam: "", # deadcode
     warnUseBase: "use {.base.} for base methods; baseless methods are deprecated",
     warnWriteToForeignHeap: "write to foreign heap",
     warnUnsafeCode: "unsafe code: '$1'",
@@ -132,7 +168,7 @@ const
     warnUninit: "use explicit initialization of '$1' for clarity",
     warnGcMem: "'$1' uses GC'ed memory",
     warnDestructor: "usage of a type with a destructor in a non destructible context. This will become a compile time error in the future.",
-    warnLockLevel: "$1",
+    warnLockLevel: "$1", # deadcode
     warnResultShadowed: "Special variable 'result' is shadowed.",
     warnInconsistentSpacing: "Number of spaces around '$#' is not consistent",
     warnCaseTransition: "Potential object case transition, instantiate new object instead",
@@ -142,20 +178,32 @@ const
     warnResultUsed: "used 'result' variable",
     warnCannotOpen: "cannot open: $1",
     warnFileChanged: "file changed: $1",
+    warnSuspiciousEnumConv: "$1",
+    warnAnyEnumConv: "$1",
+    warnHoleEnumConv: "$1",
+    warnCstringConv: "$1",
+    warnPtrToCstringConv: "unsafe conversion to 'cstring' from '$1'; this will become a compile time error in the future",
+    warnEffect: "$1",
+    warnCastSizes: "$1",
+    warnImplicitTemplateRedefinition: "template '$1' is implicitly redefined; this is deprecated, add an explicit .redefine pragma",
+    warnUnnamedBreak: "Using an unnamed break in a block is deprecated; Use a named block with a named break instead",
+    warnStmtListLambda: "statement list expression assumed to be anonymous proc; this is deprecated, use `do (): ...` or `proc () = ...` instead",
+    warnBareExcept: "$1",
     warnUser: "$1",
     hintSuccess: "operation successful: $#",
     # keep in sync with `testament.isSuccess`
-    hintSuccessX: "$loc lines; ${sec}s; $mem; proj: $project; out: $output",
-    hintBuildMode: "$1",
+    hintSuccessX: "$build\n$loc lines; ${sec}s; $mem; proj: $project; out: $output",
     hintCC: "CC: $1",
     hintLineTooLong: "line too long",
     hintXDeclaredButNotUsed: "'$1' is declared but not used",
+    hintDuplicateModuleImport: "$1",
     hintXCannotRaiseY: "$1",
     hintConvToBaseNotNeeded: "conversion to base object is not needed",
     hintConvFromXtoItselfNotNeeded: "conversion from $1 to itself is pointless",
     hintExprAlwaysX: "expression evaluates always to '$1'",
     hintQuitCalled: "quit() called",
     hintProcessing: "$1",
+    hintProcessingStmt: "$1",
     hintCodeBegin: "generated code listing:",
     hintCodeEnd: "end of listing",
     hintConf: "used config file '$1'",
@@ -173,6 +221,7 @@ const
     hintGCStats: "$1",
     hintGlobalVar: "global variable declared here",
     hintExpandMacro: "expanded macro: $1",
+    hintAmbiguousEnum: "$1",
     hintUser: "$1",
     hintUserRaw: "$1",
     hintExtendedContext: "$1",
@@ -181,26 +230,26 @@ const
   ]
 
 const
-  fatalMin* = errUnknown
-  fatalMax* = errInternal
+  fatalMsgs* = {errUnknown..errInternal}
   errMin* = errUnknown
   errMax* = errUser
   warnMin* = warnCannotOpenFile
   warnMax* = pred(hintSuccess)
   hintMin* = hintSuccess
   hintMax* = high(TMsgKind)
+  rstWarnings* = {warnRstRedefinitionOfLabel..warnRstStyle}
 
 type
   TNoteKind* = range[warnMin..hintMax] # "notes" are warnings or hints
   TNoteKinds* = set[TNoteKind]
 
 proc computeNotesVerbosity(): array[0..3, TNoteKinds] =
-  result[3] = {low(TNoteKind)..high(TNoteKind)} - {warnObservableStores, warnResultUsed}
-  result[2] = result[3] - {hintStackTrace, warnUninit, hintExtendedContext, hintDeclaredLoc}
+  result[3] = {low(TNoteKind)..high(TNoteKind)} - {warnObservableStores, warnResultUsed, warnAnyEnumConv}
+  result[2] = result[3] - {hintStackTrace, hintExtendedContext, hintDeclaredLoc, hintProcessingStmt}
   result[1] = result[2] - {warnProveField, warnProveIndex,
     warnGcUnsafe, hintPath, hintDependency, hintCodeBegin, hintCodeEnd,
     hintSource, hintGlobalVar, hintGCStats, hintMsgOrigin, hintPerformance}
-  result[0] = result[1] - {hintSuccessX, hintBuildMode, hintSuccess, hintConf,
+  result[0] = result[1] - {hintSuccessX, hintSuccess, hintConf,
     hintProcessing, hintPattern, hintExecuting, hintLinking, hintCC}
 
 const

compiler/linter.nim

@@ -12,7 +12,8 @@
 import std/strutils
 from std/sugar import dup
 
-import options, ast, msgs, idents, lineinfos, wordrecg, astmsgs
+import options, ast, msgs, idents, lineinfos, wordrecg, astmsgs, semdata, packages
+export packages
 
 const
   Letters* = {'a'..'z', 'A'..'Z', '0'..'9', '\x80'..'\xFF', '_'}
@@ -85,24 +86,33 @@ proc differ*(line: string, a, b: int, x: string): string =
       result = y
 
 proc nep1CheckDefImpl(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
-  # operators stay as they are:
-  if k in {skResult, skTemp} or s.name.s[0] notin Letters: return
-  if k in {skType, skGenericParam} and sfAnon in s.flags: return
-  if s.typ != nil and s.typ.kind == tyTypeDesc: return
-  if {sfImportc, sfExportc} * s.flags != {}: return
-  if optStyleCheck notin s.options: return
   let beau = beautifyName(s.name.s, k)
   if s.name.s != beau:
     lintReport(conf, info, beau, s.name.s)
 
-template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
-  if {optStyleHint, optStyleError} * conf.globalOptions != {} and optStyleUsages notin conf.globalOptions:
-    nep1CheckDefImpl(conf, info, s, k)
+template styleCheckDef*(ctx: PContext; info: TLineInfo; sym: PSym; k: TSymKind) =
+  ## Check symbol definitions adhere to NEP1 style rules.
+  if optStyleCheck in ctx.config.options and # ignore if styleChecks are off
+     {optStyleHint, optStyleError} * ctx.config.globalOptions != {} and # check only if hint/error is enabled
+     hintName in ctx.config.notes and # ignore if name checks are not requested
+     ctx.config.belongsToProjectPackage(ctx.module) and # ignore foreign packages
+     optStyleUsages notin ctx.config.globalOptions and # ignore if requested to only check name usage
+     sym.kind != skResult and # ignore `result`
+     sym.kind != skTemp and # ignore temporary variables created by the compiler
+     sym.name.s[0] in Letters and # ignore operators TODO: what about unicode symbols???
+     k notin {skType, skGenericParam} and # ignore types and generic params
+     (sym.typ == nil or sym.typ.kind != tyTypeDesc) and # ignore `typedesc`
+     {sfImportc, sfExportc} * sym.flags == {} and # ignore FFI
+     sfAnon notin sym.flags: # ignore if created by compiler
+    nep1CheckDefImpl(ctx.config, info, sym, k)
 
-template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym) =
-  styleCheckDef(conf, info, s, s.kind)
-template styleCheckDef*(conf: ConfigRef; s: PSym) =
-  styleCheckDef(conf, s.info, s, s.kind)
+template styleCheckDef*(ctx: PContext; info: TLineInfo; s: PSym) =
+  ## Check symbol definitions adhere to NEP1 style rules.
+  styleCheckDef(ctx, info, s, s.kind)
+
+template styleCheckDef*(ctx: PContext; s: PSym) =
+  ## Check symbol definitions adhere to NEP1 style rules.
+  styleCheckDef(ctx, s.info, s, s.kind)
 
 proc differs(conf: ConfigRef; info: TLineInfo; newName: string): string =
   let line = sourceLine(conf, info)
@@ -116,23 +126,27 @@ proc differs(conf: ConfigRef; info: TLineInfo; newName: string): string =
   let last = first+identLen(line, first)-1
   result = differ(line, first, last, newName)
 
-proc styleCheckUse*(conf: ConfigRef; info: TLineInfo; s: PSym) =
-  if info.fileIndex.int < 0: return
-  # we simply convert it to what it looks like in the definition
-  # for consistency
-
-  # operators stay as they are:
-  if s.kind == skTemp or s.name.s[0] notin Letters or sfAnon in s.flags:
-    return
-
+proc styleCheckUseImpl(conf: ConfigRef; info: TLineInfo; s: PSym) =
   let newName = s.name.s
   let badName = differs(conf, info, newName)
   if badName.len > 0:
-    # special rules for historical reasons
-    let forceHint = badName == "nnkArgList" and newName == "nnkArglist" or badName == "nnkArglist" and newName == "nnkArgList"
-    lintReport(conf, info, newName, badName, forceHint = forceHint, extraMsg = "".dup(addDeclaredLoc(conf, s)))
+    lintReport(conf, info, newName, badName, "".dup(addDeclaredLoc(conf, s)))
 
-proc checkPragmaUse*(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) =
+template styleCheckUse*(ctx: PContext; info: TLineInfo; sym: PSym) =
+  ## Check symbol uses match their definition's style.
+  if {optStyleHint, optStyleError} * ctx.config.globalOptions != {} and # ignore if styleChecks are off
+     hintName in ctx.config.notes and # ignore if name checks are not requested
+     ctx.config.belongsToProjectPackage(ctx.module) and # ignore foreign packages
+     sym.kind != skTemp and # ignore temporary variables created by the compiler
+     sym.name.s[0] in Letters and # ignore operators TODO: what about unicode symbols???
+     sfAnon notin sym.flags: # ignore temporary variables created by the compiler
+    styleCheckUseImpl(ctx.config, info, sym)
+
+proc checkPragmaUseImpl(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) =
   let wanted = $w
   if pragmaName != wanted:
     lintReport(conf, info, wanted, pragmaName)
+
+template checkPragmaUse*(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) =
+  if {optStyleHint, optStyleError} * conf.globalOptions != {}:
+    checkPragmaUseImpl(conf, info, w, pragmaName)

compiler/llstream.nim

@@ -12,6 +12,9 @@
 import
   pathutils
 
+when defined(nimPreviewSlimSystem):
+  import std/syncio
+
 # support `useGnuReadline`, `useLinenoise` for backwards compatibility
 const hasRstdin = (defined(nimUseLinenoise) or defined(useLinenoise) or defined(useGnuReadline)) and
   not defined(windows)

compiler/lookups.nim

@@ -8,11 +8,14 @@
 #
 
 # This module implements lookup helpers.
+import std/[algorithm, strutils]
+
+when defined(nimPreviewSlimSystem):
+  import std/assertions
 
 import
   intsets, ast, astalgo, idents, semdata, types, msgs, options,
-  renderer, nimfix/prettybase, lineinfos, strutils,
-  modulegraphs, astmsgs
+  renderer, nimfix/prettybase, lineinfos, modulegraphs, astmsgs, sets
 
 proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope)
 
@@ -45,6 +48,11 @@ proc considerQuotedIdent*(c: PContext; n: PNode, origin: PNode = nil): PIdent =
         case x.kind
         of nkIdent: id.add(x.ident.s)
         of nkSym: id.add(x.sym.name.s)
+        of nkSymChoices:
+          if x[0].kind == nkSym:
+            id.add(x[0].sym.name.s)
+          else:
+            handleError(n, origin)
         of nkLiterals - nkFloatLiterals: id.add(x.renderTree)
         else: handleError(n, origin)
       result = getIdent(c.cache, id)
@@ -162,6 +170,7 @@ iterator allSyms*(c: PContext): (PSym, int, bool) =
   # really iterate over all symbols in all the scopes. This is expensive
   # and only used by suggest.nim.
   var isLocal = true
+
   var scopeN = 0
   for scope in allScopes(c.currentScope):
     if scope == c.topLevelScope: isLocal = false
@@ -176,16 +185,29 @@ iterator allSyms*(c: PContext): (PSym, int, bool) =
       assert s != nil
       yield (s, scopeN, isLocal)
 
+iterator uniqueSyms*(c: PContext): (PSym, int, bool) =
+  ## Like [allSyms] except only returns unique symbols (Uniqueness determined by line + name)
+  # Track seen symbols so we don't duplicate them.
+  # The int is for the symbols name, and line info is
+  # to be able to tell apart symbols with same name but on different lines
+  var seen = initHashSet[(TLineInfo, int)]()
+  for res in allSyms(c):
+    if not seen.containsOrIncl((res[0].info, res[0].name.id)):
+      yield res
+
+
 proc someSymFromImportTable*(c: PContext; name: PIdent; ambiguous: var bool): PSym =
   var marked = initIntSet()
+  var symSet = OverloadableSyms
   result = nil
-  for im in c.imports.mitems:
-    for s in symbols(im, marked, name, c.graph):
-      if result == nil:
-        result = s
-      else:
-        if s.kind notin OverloadableSyms or result.kind notin OverloadableSyms:
+  block outer:
+    for im in c.imports.mitems:
+      for s in symbols(im, marked, name, c.graph):
+        if result == nil:
+          result = s
+        elif s.kind notin symSet or result.kind notin symSet:
           ambiguous = true
+          break outer
 
 proc searchInScopes*(c: PContext, s: PIdent; ambiguous: var bool): PSym =
   for scope in allScopes(c.currentScope):
@@ -206,15 +228,14 @@ proc debugScopes*(c: PContext; limit=0, max = int.high) {.deprecated.} =
     if i == limit: return
     inc i
 
-proc searchInScopesFilterBy*(c: PContext, s: PIdent, filter: TSymKinds): seq[PSym] =
+proc searchInScopesAllCandidatesFilterBy*(c: PContext, s: PIdent, filter: TSymKinds): seq[PSym] =
   result = @[]
   for scope in allScopes(c.currentScope):
     var ti: TIdentIter
     var candidate = initIdentIter(ti, scope.symbols, s)
     while candidate != nil:
       if candidate.kind in filter:
-        if result.len == 0:
-          result.add candidate
+        result.add candidate
       candidate = nextIdentIter(ti, scope.symbols)
 
   if result.len == 0:
@@ -224,6 +245,26 @@ proc searchInScopesFilterBy*(c: PContext, s: PIdent, filter: TSymKinds): seq[PSy
         if s.kind in filter:
           result.add s
 
+proc searchInScopesFilterBy*(c: PContext, s: PIdent, filter: TSymKinds): seq[PSym] =
+  result = @[]
+  block outer:
+    for scope in allScopes(c.currentScope):
+      var ti: TIdentIter
+      var candidate = initIdentIter(ti, scope.symbols, s)
+      while candidate != nil:
+        if candidate.kind in filter:
+          result.add candidate
+          # Break here, because further symbols encountered would be shadowed
+          break outer
+        candidate = nextIdentIter(ti, scope.symbols)
+
+  if result.len == 0:
+    var marked = initIntSet()
+    for im in c.imports.mitems:
+      for s in symbols(im, marked, s, c.graph):
+        if s.kind in filter:
+          result.add s
+
 proc errorSym*(c: PContext, n: PNode): PSym =
   ## creates an error symbol to avoid cascading errors (for IDE support)
   var m = n
@@ -268,6 +309,7 @@ proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope) =
   var it: TTabIter
   var s = initTabIter(it, scope.symbols)
   var missingImpls = 0
+  var unusedSyms: seq[tuple[sym: PSym, key: string]]
   while s != nil:
     if sfForward in s.flags and s.kind notin {skType, skModule}:
       # too many 'implementation of X' errors are annoying
@@ -282,39 +324,53 @@ proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope) =
         # maybe they can be made skGenericParam as well.
         if s.typ != nil and tfImplicitTypeParam notin s.typ.flags and
            s.typ.kind != tyGenericParam:
-          message(c.config, s.info, hintXDeclaredButNotUsed, s.name.s)
+          unusedSyms.add (s, toFileLineCol(c.config, s.info))
     s = nextIter(it, scope.symbols)
+  for (s, _) in sortedByIt(unusedSyms, it.key):
+    message(c.config, s.info, hintXDeclaredButNotUsed, s.name.s)
 
 proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string;
-                        conflictsWith: TLineInfo) =
+                        conflictsWith: TLineInfo, note = errGenerated) =
   ## Emit a redefinition error if in non-interactive mode
   if c.config.cmd != cmdInteractive:
-    localError(c.config, info,
+    localError(c.config, info, note,
       "redefinition of '$1'; previous declaration here: $2" %
       [s, c.config $ conflictsWith])
 
-proc addDecl*(c: PContext, sym: PSym, info: TLineInfo) =
-  let conflict = c.currentScope.addUniqueSym(sym)
+# xxx pending bootstrap >= 1.4, replace all those overloads with a single one:
+# proc addDecl*(c: PContext, sym: PSym, info = sym.info, scope = c.currentScope) {.inline.} =
+proc addDeclAt*(c: PContext; scope: PScope, sym: PSym, info: TLineInfo) =
+  if sym.name.s == "_": return
+  let conflict = scope.addUniqueSym(sym)
   if conflict != nil:
-    wrongRedefinition(c, info, sym.name.s, conflict.info)
+    if sym.kind == skModule and conflict.kind == skModule and
+                        sym.position == conflict.position:
+      # e.g.: import foo; import foo
+      # xxx we could refine this by issuing a different hint for the case
+      # where a duplicate import happens inside an include.
+      localError(c.config, info, hintDuplicateModuleImport,
+        "duplicate import of '$1'; previous import here: $2" %
+        [sym.name.s, c.config $ conflict.info])
+    else:
+      wrongRedefinition(c, info, sym.name.s, conflict.info, errGenerated)
 
-proc addDecl*(c: PContext, sym: PSym) =
-  let conflict = strTableInclReportConflict(c.currentScope.symbols, sym, true)
-  if conflict != nil:
-    wrongRedefinition(c, sym.info, sym.name.s, conflict.info)
+proc addDeclAt*(c: PContext; scope: PScope, sym: PSym) {.inline.} =
+  addDeclAt(c, scope, sym, sym.info)
+
+proc addDecl*(c: PContext, sym: PSym, info: TLineInfo) {.inline.} =
+  addDeclAt(c, c.currentScope, sym, info)
+
+proc addDecl*(c: PContext, sym: PSym) {.inline.} =
+  addDeclAt(c, c.currentScope, sym)
 
 proc addPrelimDecl*(c: PContext, sym: PSym) =
   discard c.currentScope.addUniqueSym(sym)
 
-proc addDeclAt*(c: PContext; scope: PScope, sym: PSym) =
-  let conflict = scope.addUniqueSym(sym)
-  if conflict != nil:
-    wrongRedefinition(c, sym.info, sym.name.s, conflict.info)
-
 from ic / ic import addHidden
 
-proc addInterfaceDeclAux*(c: PContext, sym: PSym, forceExport = false) =
-  if sfExported in sym.flags or forceExport:
+proc addInterfaceDeclAux(c: PContext, sym: PSym) =
+  ## adds symbol to the module for either private or public access.
+  if sfExported in sym.flags:
     # add to interface:
     if c.module != nil: exportSym(c, sym)
     else: internalError(c.config, sym.info, "addInterfaceDeclAux")
@@ -324,10 +380,19 @@ proc addInterfaceDeclAux*(c: PContext, sym: PSym, forceExport = false) =
       addHidden(c.encoder, c.packedRepr, sym)
 
 proc addInterfaceDeclAt*(c: PContext, scope: PScope, sym: PSym) =
+  ## adds a symbol on the scope and the interface if appropriate
   addDeclAt(c, scope, sym)
-  addInterfaceDeclAux(c, sym)
+  if not scope.isShadowScope:
+    # adding into a non-shadow scope, we need to handle exports, etc
+    addInterfaceDeclAux(c, sym)
+
+proc addInterfaceDecl*(c: PContext, sym: PSym) {.inline.} =
+  ## adds a decl and the interface if appropriate
+  addInterfaceDeclAt(c, c.currentScope, sym)
 
 proc addOverloadableSymAt*(c: PContext; scope: PScope, fn: PSym) =
+  ## adds an symbol to the given scope, will check for and raise errors if it's
+  ## a redefinition as opposed to an overload.
   if fn.kind notin OverloadableSyms:
     internalError(c.config, fn.info, "addOverloadableSymAt")
     return
@@ -337,25 +402,33 @@ proc addOverloadableSymAt*(c: PContext; scope: PScope, fn: PSym) =
   else:
     scope.addSym(fn)
 
-proc addInterfaceDecl*(c: PContext, sym: PSym) =
-  # it adds the symbol to the interface if appropriate
-  addDecl(c, sym)
-  addInterfaceDeclAux(c, sym)
-
 proc addInterfaceOverloadableSymAt*(c: PContext, scope: PScope, sym: PSym) =
-  # it adds the symbol to the interface if appropriate
+  ## adds an overloadable symbol on the scope and the interface if appropriate
   addOverloadableSymAt(c, scope, sym)
-  addInterfaceDeclAux(c, sym)
+  if not scope.isShadowScope:
+    # adding into a non-shadow scope, we need to handle exports, etc
+    addInterfaceDeclAux(c, sym)
 
 proc openShadowScope*(c: PContext) =
+  ## opens a shadow scope, just like any other scope except the depth is the
+  ## same as the parent -- see `isShadowScope`.
   c.currentScope = PScope(parent: c.currentScope,
                           symbols: newStrTable(),
                           depthLevel: c.scopeDepth)
 
 proc closeShadowScope*(c: PContext) =
-  c.closeScope
+  ## closes the shadow scope, but doesn't merge any of the symbols
+  ## Does not check for unused symbols or missing forward decls since a macro
+  ## or template consumes this AST
+  rawCloseScope(c)
 
 proc mergeShadowScope*(c: PContext) =
+  ## close the existing scope and merge in all defined symbols, this will also
+  ## trigger any export related code if this is into a non-shadow scope.
+  ##
+  ## Merges:
+  ## shadow -> shadow: add symbols to the parent but check for redefinitions etc
+  ## shadow -> non-shadow: the above, but also handle exports and all that
   let shadowScope = c.currentScope
   c.rawCloseScope
   for sym in shadowScope.symbols:
@@ -380,7 +453,7 @@ type SpellCandidate = object
   msg: string
   sym: PSym
 
-template toOrderTup(a: SpellCandidate): auto =
+template toOrderTup(a: SpellCandidate): (int, int, string) =
   # `dist` is first, to favor nearby matches
   # `depth` is next, to favor nearby enclosing scopes among ties
   # `sym.name.s` is last, to make the list ordered and deterministic among ties
@@ -403,12 +476,13 @@ proc fixSpelling(c: PContext, n: PNode, ident: PIdent, result: var string) =
     let dist = editDistance(name0, sym.name.s.nimIdentNormalize)
     var msg: string
     msg.add "\n ($1, $2): '$3'" % [$dist, $depth, sym.name.s]
-    addDeclaredLoc(msg, c.config, sym) # `msg` needed for deterministic ordering.
     list.push SpellCandidate(dist: dist, depth: depth, msg: msg, sym: sym)
 
   if list.len == 0: return
   let e0 = list[0]
-  var count = 0
+  var
+    count = 0
+    last: PIdent = nil
   while true:
     # pending https://github.com/timotheecour/Nim/issues/373 use more efficient `itemsSorted`.
     if list.len == 0: break
@@ -424,8 +498,10 @@ proc fixSpelling(c: PContext, n: PNode, ident: PIdent, result: var string) =
     elif count >= c.config.spellSuggestMax: break
     if count == 0:
       result.add "\ncandidates (edit distance, scope distance); see '--spellSuggest': "
-    result.add e.msg
-    count.inc
+    if e.sym.name != last:
+      result.add e.msg
+      count.inc
+      last = e.sym.name
 
 proc errorUseQualifier(c: PContext; info: TLineInfo; s: PSym; amb: var bool): PSym =
   var err = "ambiguous identifier: '" & s.name.s & "'"
@@ -451,17 +527,24 @@ proc errorUseQualifier*(c: PContext; info: TLineInfo; s: PSym) =
   var amb: bool
   discard errorUseQualifier(c, info, s, amb)
 
-proc errorUseQualifier(c: PContext; info: TLineInfo; candidates: seq[PSym]) =
+proc errorUseQualifier(c: PContext; info: TLineInfo; candidates: seq[PSym]; prefix = "use one of") =
   var err = "ambiguous identifier: '" & candidates[0].name.s & "'"
   var i = 0
   for candidate in candidates:
-    if i == 0: err.add " -- use one of the following:\n"
+    if i == 0: err.add " -- $1 the following:\n" % prefix
     else: err.add "\n"
     err.add "  " & candidate.owner.name.s & "." & candidate.name.s
     err.add ": " & typeToString(candidate.typ)
     inc i
   localError(c.config, info, errGenerated, err)
 
+proc errorUseQualifier*(c: PContext; info:TLineInfo; choices: PNode) =
+  var candidates = newSeq[PSym](choices.len)
+  let prefix = if choices[0].typ.kind != tyProc: "use one of" else: "you need a helper proc to disambiguate"
+  for i, n in choices:
+    candidates[i] = n.sym
+  errorUseQualifier(c, info, candidates, prefix)
+
 proc errorUndeclaredIdentifier*(c: PContext; info: TLineInfo; name: string, extra = "") =
   var err = "undeclared identifier: '" & name & "'" & extra
   if c.recursiveDep.len > 0:

compiler/lowerings.nim

@@ -15,6 +15,9 @@ const
 import ast, astalgo, types, idents, magicsys, msgs, options, modulegraphs,
   lineinfos
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 proc newDeref*(n: PNode): PNode {.inline.} =
   result = newNodeIT(nkHiddenDeref, n.info, n.typ[0])
   result.add n
@@ -71,14 +74,20 @@ proc lowerTupleUnpacking*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: P
   let value = n.lastSon
   result = newNodeI(nkStmtList, n.info)
 
-  var temp = newSym(skTemp, getIdent(g.cache, genPrefix), nextSymId(idgen),
-                    owner, value.info, g.config.options)
-  temp.typ = skipTypes(value.typ, abstractInst)
-  incl(temp.flags, sfFromGeneric)
+  var tempAsNode: PNode
+  let avoidTemp = value.kind == nkSym
+  if avoidTemp:
+    tempAsNode = value
+  else:
+    var temp = newSym(skTemp, getIdent(g.cache, genPrefix), nextSymId(idgen),
+                  owner, value.info, g.config.options)
+    temp.typ = skipTypes(value.typ, abstractInst)
+    incl(temp.flags, sfFromGeneric)
+    tempAsNode = newSymNode(temp)
 
   var v = newNodeI(nkVarSection, value.info)
-  let tempAsNode = newSymNode(temp)
-  v.addVar(tempAsNode, value)
+  if not avoidTemp:
+    v.addVar(tempAsNode, value)
   result.add(v)
 
   for i in 0..<n.len-2:
@@ -221,7 +230,7 @@ proc lookupInRecord(n: PNode, id: ItemId): PSym =
     if n.sym.itemId.module == id.module and n.sym.itemId.item == -abs(id.item): result = n.sym
   else: discard
 
-proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator) =
+proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator): PSym =
   # because of 'gensym' support, we have to mangle the name with its ID.
   # This is hacky but the clean solution is much more complex than it looks.
   var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len),
@@ -232,9 +241,11 @@ proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator) =
   assert t.kind != tyTyped
   propagateToOwner(obj, t)
   field.position = obj.n.len
-  field.flags = s.flags * {sfCursor}
+  # sfNoInit flag for skField is used in closureiterator codegen
+  field.flags = s.flags * {sfCursor, sfNoInit}
   obj.n.add newSymNode(field)
   fieldCheck()
+  result = field
 
 proc addUniqueField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator): PSym {.discardable.} =
   result = lookupInRecord(obj.n, s.itemId)

compiler/main.nim

@@ -13,23 +13,26 @@ when not defined(nimcore):
   {.error: "nimcore MUST be defined for Nim's core tooling".}
 
 import
-  llstream, strutils, os, ast, lexer, syntaxes, options, msgs,
-  condsyms, times,
-  sem, idents, passes, extccomp,
-  cgen, json, nversion,
-  platform, nimconf, passaux, depends, vm,
+  std/[strutils, os, times, tables, sha1, with, json],
+  llstream, ast, lexer, syntaxes, options, msgs,
+  condsyms,
+  idents, extccomp,
+  cgen, nversion,
+  platform, nimconf, depends,
   modules,
-  modulegraphs, tables, lineinfos, pathutils, vmprofiler, std/[sha1, with]
+  modulegraphs, lineinfos, pathutils, vmprofiler
+
+
+when defined(nimPreviewSlimSystem):
+  import std/[syncio, assertions]
 
 import ic / [cbackend, integrity, navigator]
 from ic / ic import rodViewer
 
-when not defined(leanCompiler):
-  import jsgen, docgen, docgen2
+import pipelines
 
-proc semanticPasses(g: ModuleGraph) =
-  registerPass g, verbosePass
-  registerPass g, semPass
+when not defined(leanCompiler):
+  import docgen
 
 proc writeDepsFile(g: ModuleGraph) =
   let fname = g.config.nimcacheDir / RelativeFile(g.config.projectName & ".deps")
@@ -42,13 +45,39 @@ proc writeDepsFile(g: ModuleGraph) =
       f.writeLine(toFullPath(g.config, k))
   f.close()
 
+proc writeCMakeDepsFile(conf: ConfigRef) =
+  ## write a list of C files for build systems like CMake.
+  ## only updated when the C file list changes.
+  let fname = getNimcacheDir(conf) / conf.outFile.changeFileExt("cdeps")
+  # generate output files list
+  var cfiles: seq[string] = @[]
+  for it in conf.toCompile: cfiles.add(it.cname.string)
+  let fileset = cfiles.toCountTable()
+  # read old cfiles list
+  var fl: File
+  var prevset = initCountTable[string]()
+  if open(fl, fname.string, fmRead):
+    for line in fl.lines: prevset.inc(line)
+    fl.close()
+  # write cfiles out
+  if fileset != prevset:
+    fl = open(fname.string, fmWrite)
+    for line in cfiles: fl.writeLine(line)
+    fl.close()
+
 proc commandGenDepend(graph: ModuleGraph) =
-  semanticPasses(graph)
-  registerPass(graph, gendependPass)
-  compileProject(graph)
+  setPipeLinePass(graph, GenDependPass)
+  compilePipelineProject(graph)
   let project = graph.config.projectFull
   writeDepsFile(graph)
   generateDot(graph, project)
+
+  # dot in graphivz tool kit is required
+  let graphvizDotPath = findExe("dot")
+  if graphvizDotPath.len == 0:
+    quit("gendepend: Graphviz's tool dot is required," &
+    "see https://graphviz.org/download for downloading")
+
   execExternalProgram(graph.config, "dot -Tpng -o" &
       changeFileExt(project, "png").string &
       ' ' & changeFileExt(project, "dot").string)
@@ -57,8 +86,13 @@ proc commandCheck(graph: ModuleGraph) =
   let conf = graph.config
   conf.setErrorMaxHighMaybe
   defineSymbol(conf.symbols, "nimcheck")
-  semanticPasses(graph)  # use an empty backend for semantic checking only
-  compileProject(graph)
+  if optWasNimscript in conf.globalOptions:
+    defineSymbol(conf.symbols, "nimscript")
+    defineSymbol(conf.symbols, "nimconfig")
+  elif conf.backend == backendJs:
+    setTarget(conf.target, osJS, cpuJS)
+  setPipeLinePass(graph, SemPass)
+  compilePipelineProject(graph)
 
   if conf.symbolFiles != disabledSf:
     case conf.ideCmd
@@ -72,22 +106,20 @@ when not defined(leanCompiler):
   proc commandDoc2(graph: ModuleGraph; ext: string) =
     handleDocOutputOptions graph.config
     graph.config.setErrorMaxHighMaybe
-    semanticPasses(graph)
     case ext:
-    of TexExt:  registerPass(graph, docgen2TexPass)
-    of JsonExt: registerPass(graph, docgen2JsonPass)
-    of HtmlExt: registerPass(graph, docgen2Pass)
+    of TexExt:
+      setPipeLinePass(graph, Docgen2TexPass)
+    of JsonExt:
+      setPipeLinePass(graph, Docgen2JsonPass)
+    of HtmlExt:
+      setPipeLinePass(graph, Docgen2Pass)
     else: doAssert false, $ext
-    compileProject(graph)
-    finishDoc2Pass(graph.config.projectName)
+    compilePipelineProject(graph)
 
 proc commandCompileToC(graph: ModuleGraph) =
   let conf = graph.config
   extccomp.initVars(conf)
-  semanticPasses(graph)
   if conf.symbolFiles == disabledSf:
-    registerPass(graph, cgenPass)
-
     if {optRun, optForceFullMake} * conf.globalOptions == {optRun} or isDefined(conf, "nimBetterRun"):
       if not changeDetectedViaJsonBuildInstructions(conf, conf.jsonBuildInstructionsFile):
         # nothing changed
@@ -97,7 +129,11 @@ proc commandCompileToC(graph: ModuleGraph) =
   if not extccomp.ccHasSaneOverflow(conf):
     conf.symbols.defineSymbol("nimEmulateOverflowChecks")
 
-  compileProject(graph)
+  if conf.symbolFiles == disabledSf:
+    setPipeLinePass(graph, CgenPass)
+  else:
+    setPipeLinePass(graph, SemPass)
+  compilePipelineProject(graph)
   if graph.config.errorCounter > 0:
     return # issue #9933
   if conf.symbolFiles == disabledSf:
@@ -116,6 +152,8 @@ proc commandCompileToC(graph: ModuleGraph) =
       extccomp.writeJsonBuildInstructions(conf)
     if optGenScript in graph.config.globalOptions:
       writeDepsFile(graph)
+    if optGenCDeps in graph.config.globalOptions:
+      writeCMakeDepsFile(conf)
 
 proc commandJsonScript(graph: ModuleGraph) =
   extccomp.runJsonBuildInstructions(graph.config, graph.config.jsonBuildInstructionsFile)
@@ -128,33 +166,27 @@ proc commandCompileToJS(graph: ModuleGraph) =
     conf.exc = excCpp
     setTarget(conf.target, osJS, cpuJS)
     defineSymbol(conf.symbols, "ecmascript") # For backward compatibility
-    semanticPasses(graph)
-    registerPass(graph, JSgenPass)
-    compileProject(graph)
+    setPipeLinePass(graph, JSgenPass)
+    compilePipelineProject(graph)
     if optGenScript in conf.globalOptions:
       writeDepsFile(graph)
 
-proc interactivePasses(graph: ModuleGraph) =
+proc commandInteractive(graph: ModuleGraph) =
+  graph.config.setErrorMaxHighMaybe
   initDefines(graph.config.symbols)
   defineSymbol(graph.config.symbols, "nimscript")
   # note: seems redundant with -d:nimHasLibFFI
   when hasFFI: defineSymbol(graph.config.symbols, "nimffi")
-  registerPass(graph, verbosePass)
-  registerPass(graph, semPass)
-  registerPass(graph, evalPass)
-
-proc commandInteractive(graph: ModuleGraph) =
-  graph.config.setErrorMaxHighMaybe
-  interactivePasses(graph)
-  compileSystemModule(graph)
+  setPipeLinePass(graph, InterpreterPass)
+  compilePipelineSystemModule(graph)
   if graph.config.commandArgs.len > 0:
-    discard graph.compileModule(fileInfoIdx(graph.config, graph.config.projectFull), {})
+    discard graph.compilePipelineModule(fileInfoIdx(graph.config, graph.config.projectFull), {})
   else:
     var m = graph.makeStdinModule()
     incl(m.flags, sfMainModule)
     var idgen = IdGenerator(module: m.itemId.module, symId: m.itemId.item, typeId: 0)
     let s = llStreamOpenStdIn(onPrompt = proc() = flushDot(graph.config))
-    processModule(graph, m, idgen, s)
+    discard processPipelineModule(graph, m, idgen, s)
 
 proc commandScan(cache: IdentCache, config: ConfigRef) =
   var f = addFileExt(AbsoluteFile mainCommandArg(config), NimExt)
@@ -209,8 +241,6 @@ proc mainCommand*(graph: ModuleGraph) =
   let conf = graph.config
   let cache = graph.cache
 
-  # In "nim serve" scenario, each command must reset the registered passes
-  clearPasses(graph)
   conf.lastCmdTime = epochTime()
   conf.searchPaths.add(conf.libpath)
 
@@ -266,7 +296,8 @@ proc mainCommand*(graph: ModuleGraph) =
     var ret = if optUseNimcache in conf.globalOptions: getNimcacheDir(conf)
               else: conf.projectPath
     doAssert ret.string.isAbsolute # `AbsoluteDir` is not a real guarantee
-    if conf.cmd in cmdDocLike + {cmdRst2html, cmdRst2tex}: ret = ret / htmldocsDir
+    if conf.cmd in cmdDocLike + {cmdRst2html, cmdRst2tex, cmdMd2html, cmdMd2tex}:
+      ret = ret / htmldocsDir
     conf.outDir = ret
 
   ## process all commands
@@ -281,10 +312,9 @@ proc mainCommand*(graph: ModuleGraph) =
     else:
       rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc")
   of cmdDoc0: docLikeCmd commandDoc(cache, conf)
-  of cmdDoc2:
+  of cmdDoc:
     docLikeCmd():
-      conf.setNoteDefaults(warnLockLevel, false) # issue #13218
-      conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218
+      conf.setNoteDefaults(warnRstRedefinitionOfLabel, false) # issue #13218
         # because currently generates lots of false positives due to conflation
         # of labels links in doc comments, e.g. for random.rand:
         #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
@@ -292,32 +322,33 @@ proc mainCommand*(graph: ModuleGraph) =
       commandDoc2(graph, HtmlExt)
       if optGenIndex in conf.globalOptions and optWholeProject in conf.globalOptions:
         commandBuildIndex(conf, $conf.outDir)
-  of cmdRst2html:
+  of cmdRst2html, cmdMd2html:
     # XXX: why are warnings disabled by default for rst2html and rst2tex?
-    for warn in [warnUnknownSubstitutionX, warnLanguageXNotSupported,
-                 warnFieldXNotSupported, warnRstStyle]:
+    for warn in rstWarnings:
       conf.setNoteDefaults(warn, true)
-    conf.setNoteDefaults(warnRedefinitionOfLabel, false) # similar to issue #13218
+    conf.setNoteDefaults(warnRstRedefinitionOfLabel, false) # similar to issue #13218
     when defined(leanCompiler):
       conf.quitOrRaise "compiler wasn't built with documentation generator"
     else:
       loadConfigs(DocConfig, cache, conf, graph.idgen)
-      commandRst2Html(cache, conf)
-  of cmdRst2tex, cmdDoc2tex:
-    for warn in [warnRedefinitionOfLabel, warnUnknownSubstitutionX,
-                 warnLanguageXNotSupported,
-                 warnFieldXNotSupported, warnRstStyle]:
+      commandRst2Html(cache, conf, preferMarkdown = (conf.cmd == cmdMd2html))
+  of cmdRst2tex, cmdMd2tex, cmdDoc2tex:
+    for warn in rstWarnings:
       conf.setNoteDefaults(warn, true)
     when defined(leanCompiler):
       conf.quitOrRaise "compiler wasn't built with documentation generator"
     else:
-      if conf.cmd == cmdRst2tex:
+      if conf.cmd in {cmdRst2tex, cmdMd2tex}:
         loadConfigs(DocTexConfig, cache, conf, graph.idgen)
-        commandRst2TeX(cache, conf)
+        commandRst2TeX(cache, conf, preferMarkdown = (conf.cmd == cmdMd2tex))
       else:
         docLikeCmd commandDoc2(graph, TexExt)
   of cmdJsondoc0: docLikeCmd commandJson(cache, conf)
-  of cmdJsondoc: docLikeCmd commandDoc2(graph, JsonExt)
+  of cmdJsondoc:
+    docLikeCmd():
+      commandDoc2(graph, JsonExt)
+      if optGenIndex in conf.globalOptions and optWholeProject in conf.globalOptions:
+        commandBuildIndexJson(conf, $conf.outDir)
   of cmdCtags: docLikeCmd commandTags(cache, conf)
   of cmdBuildindex: docLikeCmd commandBuildIndex(conf, $conf.projectFull, conf.outFile)
   of cmdGendepend: commandGenDepend(graph)
@@ -365,7 +396,8 @@ proc mainCommand*(graph: ModuleGraph) =
       msgWriteln(conf, "-- end of list --", {msgStdout, msgSkipHook})
 
       for it in conf.searchPaths: msgWriteln(conf, it.string)
-  of cmdCheck: commandCheck(graph)
+  of cmdCheck:
+    commandCheck(graph)
   of cmdParse:
     wantMainModule(conf)
     discard parseFile(conf.projectMainIdx, cache, conf)
@@ -387,37 +419,9 @@ proc mainCommand*(graph: ModuleGraph) =
     rawMessage(conf, errGenerated, "invalid command: " & conf.command)
 
   if conf.errorCounter == 0 and conf.cmd notin {cmdTcc, cmdDump, cmdNop}:
-    # D20210419T170230:here
-    let mem =
-      when declared(system.getMaxMem): formatSize(getMaxMem()) & " peakmem"
-      else: formatSize(getTotalMem()) & " totmem"
-    let loc = $conf.linesCompiled
-    let build = if isDefined(conf, "danger"): "Dangerous Release build"
-                elif isDefined(conf, "release"): "Release build"
-                else: "***SLOW, DEBUG BUILD***; -d:release makes code run faster."
-    let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3)
-    let project = if conf.filenameOption == foAbs: $conf.projectFull else: $conf.projectName
-      # xxx honor conf.filenameOption more accurately
-    var output: string
-    if optCompileOnly in conf.globalOptions and conf.cmd != cmdJsonscript:
-      output = $conf.jsonBuildFile
-    elif conf.outFile.isEmpty and conf.cmd notin {cmdJsonscript} + cmdDocLike + cmdBackends:
-      # for some cmd we expect a valid absOutFile
-      output = "unknownOutput"
-    else:
-      output = $conf.absOutFile
-    if conf.filenameOption != foAbs: output = output.AbsoluteFile.extractFilename
-      # xxx honor filenameOption more accurately
     if optProfileVM in conf.globalOptions:
       echo conf.dump(conf.vmProfileData)
-    rawMessage(conf, hintSuccessX, [
-      "loc", loc,
-      "sec", sec,
-      "mem", mem,
-      "project", project,
-      "output", output,
-      ])
-    rawMessage(conf, hintBuildMode, build)
+    genSuccessX(conf)
 
   when PrintRopeCacheStats:
     echo "rope cache stats: "

compiler/modulegraphs.nim

@@ -12,9 +12,12 @@
 ## or stored in a rod-file.
 
 import intsets, tables, hashes, md5
-import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils
+import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages
 import ic / [packed_ast, ic]
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 type
   SigHash* = distinct MD5Digest
 
@@ -50,6 +53,22 @@ type
     concreteTypes*: seq[FullId]
     inst*: PInstantiation
 
+  SymInfoPair* = object
+    sym*: PSym
+    info*: TLineInfo
+
+  PipelinePass* = enum
+    NonePass
+    SemPass
+    JSgenPass
+    CgenPass
+    EvalPass
+    InterpreterPass
+    GenDependPass
+    Docgen2TexPass
+    Docgen2JsonPass
+    Docgen2Pass
+
   ModuleGraph* {.acyclic.} = ref object
     ifaces*: seq[Iface]  ## indexed by int32 fileIdx
     packed*: PackedModuleGraph
@@ -57,14 +76,13 @@ type
 
     typeInstCache*: Table[ItemId, seq[LazyType]] # A symbol's ItemId.
     procInstCache*: Table[ItemId, seq[LazyInstantiation]] # A symbol's ItemId.
-    attachedOps*: array[TTypeAttachedOp, Table[ItemId, PSym]] # Type ID, destructors, etc.
+    attachedOps*: array[TTypeAttachedOp, Table[ItemId, LazySym]] # Type ID, destructors, etc.
     methodsPerType*: Table[ItemId, seq[(int, LazySym)]] # Type ID, attached methods
     enumToStringProcs*: Table[ItemId, LazySym]
     emittedTypeInfo*: Table[string, FileIndex]
 
     startupPackedConfig*: PackedConfig
     packageSyms*: TStrTable
-    modulesPerPackage*: Table[ItemId, TStrTable]
     deps*: IntSet # the dependency graph or potentially its transitive closure.
     importDeps*: Table[FileIndex, seq[FileIndex]] # explicit import module dependencies
     suggestMode*: bool # whether we are in nimsuggest mode or not.
@@ -81,6 +99,8 @@ type
     doStopCompile*: proc(): bool {.closure.}
     usageSym*: PSym # for nimsuggest
     owners*: seq[PSym]
+    suggestSymbols*: Table[FileIndex, seq[SymInfoPair]]
+    suggestErrors*: Table[FileIndex, seq[Suggest]]
     methods*: seq[tuple[methods: seq[PSym], dispatcher: PSym]] # needs serialization!
     systemModule*: PSym
     sysTypes*: array[TTypeKind, PType]
@@ -96,6 +116,7 @@ type
     cacheCounters*: Table[string, BiggestInt] # IC: implemented
     cacheTables*: Table[string, BTree[string, PNode]] # IC: implemented
     passes*: seq[TPass]
+    pipelinePass*: PipelinePass
     onDefinition*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
     onDefinitionResolveForward*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
     onUsage*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
@@ -277,6 +298,13 @@ proc resolveInst(g: ModuleGraph; t: var LazyInstantiation): PInstantiation =
     t.inst = result
   assert result != nil
 
+proc resolveAttachedOp(g: ModuleGraph; t: var LazySym): PSym =
+  result = t.sym
+  if result == nil:
+    result = loadSymFromId(g.config, g.cache, g.packed, t.id.module, t.id.packed)
+    t.sym = result
+  assert result != nil
+
 iterator typeInstCacheItems*(g: ModuleGraph; s: PSym): PType =
   if g.typeInstCache.contains(s.itemId):
     let x = addr(g.typeInstCache[s.itemId])
@@ -292,22 +320,25 @@ iterator procInstCacheItems*(g: ModuleGraph; s: PSym): PInstantiation =
 proc getAttachedOp*(g: ModuleGraph; t: PType; op: TTypeAttachedOp): PSym =
   ## returns the requested attached operation for type `t`. Can return nil
   ## if no such operation exists.
-  result = g.attachedOps[op].getOrDefault(t.itemId)
+  if g.attachedOps[op].contains(t.itemId):
+    result = resolveAttachedOp(g, g.attachedOps[op][t.itemId])
+  else:
+    result = nil
 
 proc setAttachedOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) =
   ## we also need to record this to the packed module.
-  g.attachedOps[op][t.itemId] = value
+  g.attachedOps[op][t.itemId] = LazySym(sym: value)
 
 proc setAttachedOpPartial*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) =
   ## we also need to record this to the packed module.
-  g.attachedOps[op][t.itemId] = value
-  # XXX Also add to the packed module!
+  g.attachedOps[op][t.itemId] = LazySym(sym: value)
 
 proc completePartialOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) =
   if g.config.symbolFiles != disabledSf:
     assert module < g.encoders.len
     assert isActive(g.encoders[module])
     toPackedGeneratedProcDef(value, g.encoders[module], g.packed[module].fromDisk)
+    #storeAttachedProcDef(t, op, value, g.encoders[module], g.packed[module].fromDisk)
 
 proc getToStringProc*(g: ModuleGraph; t: PType): PSym =
   result = resolveSym(g, g.enumToStringProcs[t.itemId])
@@ -368,20 +399,9 @@ template getPContext(): untyped =
   when c is PContext: c
   else: c.c
 
-when defined(nimfind):
-  template onUse*(info: TLineInfo; s: PSym) =
-    let c = getPContext()
-    if c.graph.onUsage != nil: c.graph.onUsage(c.graph, s, info)
-
-  template onDef*(info: TLineInfo; s: PSym) =
-    let c = getPContext()
-    if c.graph.onDefinition != nil: c.graph.onDefinition(c.graph, s, info)
-
-  template onDefResolveForward*(info: TLineInfo; s: PSym) =
-    let c = getPContext()
-    if c.graph.onDefinitionResolveForward != nil:
-      c.graph.onDefinitionResolveForward(c.graph, s, info)
-
+when defined(nimsuggest):
+  template onUse*(info: TLineInfo; s: PSym) = discard
+  template onDefResolveForward*(info: TLineInfo; s: PSym) = discard
 else:
   template onUse*(info: TLineInfo; s: PSym) = discard
   template onDef*(info: TLineInfo; s: PSym) = discard
@@ -432,8 +452,7 @@ proc initOperators*(g: ModuleGraph): Operators =
   result.opNot = createMagic(g, "not", mNot)
   result.opContains = createMagic(g, "contains", mInSet)
 
-proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
-  result = ModuleGraph()
+proc initModuleGraphFields(result: ModuleGraph) =
   # A module ID of -1 means that the symbol is not attached to a module at all,
   # but to the module graph:
   result.idgen = IdGenerator(module: -1'i32, symId: 0'i32, typeId: 0'i32)
@@ -443,9 +462,9 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result.ifaces = @[]
   result.importStack = @[]
   result.inclToMod = initTable[FileIndex, FileIndex]()
-  result.config = config
-  result.cache = cache
   result.owners = @[]
+  result.suggestSymbols = initTable[FileIndex, seq[SymInfoPair]]()
+  result.suggestErrors = initTable[FileIndex, seq[Suggest]]()
   result.methods = @[]
   initStrTable(result.compilerprocs)
   initStrTable(result.exposed)
@@ -459,6 +478,12 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result.operators = initOperators(result)
   result.emittedTypeInfo = initTable[string, FileIndex]()
 
+proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
+  result = ModuleGraph()
+  result.config = config
+  result.cache = cache
+  initModuleGraphFields(result)
+
 proc resetAllModules*(g: ModuleGraph) =
   initStrTable(g.packageSyms)
   g.deps = initIntSet()
@@ -470,6 +495,7 @@ proc resetAllModules*(g: ModuleGraph) =
   g.methods = @[]
   initStrTable(g.compilerprocs)
   initStrTable(g.exposed)
+  initModuleGraphFields(g)
 
 proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym =
   if fileIdx.int32 >= 0:
@@ -548,7 +574,19 @@ proc transitiveClosure(g: var IntSet; n: int) =
 
 proc markDirty*(g: ModuleGraph; fileIdx: FileIndex) =
   let m = g.getModule fileIdx
-  if m != nil: incl m.flags, sfDirty
+  if m != nil:
+    g.suggestSymbols.del(fileIdx)
+    g.suggestErrors.del(fileIdx)
+    incl m.flags, sfDirty
+
+proc unmarkAllDirty*(g: ModuleGraph) =
+  for i in 0i32..<g.ifaces.len.int32:
+    let m = g.ifaces[i].module
+    if m != nil:
+      m.flags.excl sfDirty
+
+proc isDirty*(g: ModuleGraph; m: PSym): bool =
+  result = g.suggestMode and sfDirty in m.flags
 
 proc markClientsDirty*(g: ModuleGraph; fileIdx: FileIndex) =
   # we need to mark its dependent modules D as dirty right away because after
@@ -560,12 +598,26 @@ proc markClientsDirty*(g: ModuleGraph; fileIdx: FileIndex) =
 
   # every module that *depends* on this file is also dirty:
   for i in 0i32..<g.ifaces.len.int32:
-    let m = g.ifaces[i].module
-    if m != nil and g.deps.contains(i.dependsOn(fileIdx.int)):
-      incl m.flags, sfDirty
+    if g.deps.contains(i.dependsOn(fileIdx.int)):
+      g.markDirty(FileIndex(i))
 
-proc isDirty*(g: ModuleGraph; m: PSym): bool =
-  result = g.suggestMode and sfDirty in m.flags
+proc needsCompilation*(g: ModuleGraph): bool =
+  # every module that *depends* on this file is also dirty:
+  for i in 0i32..<g.ifaces.len.int32:
+    let m = g.ifaces[i].module
+    if m != nil:
+      if sfDirty in m.flags:
+        return true
+
+proc needsCompilation*(g: ModuleGraph, fileIdx: FileIndex): bool =
+  let module = g.getModule(fileIdx)
+  if module != nil and g.isDirty(module):
+    return true
+
+  for i in 0i32..<g.ifaces.len.int32:
+    let m = g.ifaces[i].module
+    if m != nil and g.isDirty(m) and g.deps.contains(fileIdx.int32.dependsOn(i)):
+      return true
 
 proc getBody*(g: ModuleGraph; s: PSym): PNode {.inline.} =
   result = s.ast[bodyPos]
@@ -582,3 +634,46 @@ proc moduleFromRodFile*(g: ModuleGraph; fileIdx: FileIndex;
 
 proc configComplete*(g: ModuleGraph) =
   rememberStartupConfig(g.startupPackedConfig, g.config)
+
+from std/strutils import repeat, `%`
+
+proc onProcessing*(graph: ModuleGraph, fileIdx: FileIndex, moduleStatus: string, fromModule: PSym, ) =
+  let conf = graph.config
+  let isNimscript = conf.isDefined("nimscript")
+  if (not isNimscript) or hintProcessing in conf.cmdlineNotes:
+    let path = toFilenameOption(conf, fileIdx, conf.filenameOption)
+    let indent = ">".repeat(graph.importStack.len)
+    let fromModule2 = if fromModule != nil: $fromModule.name.s else: "(toplevel)"
+    let mode = if isNimscript: "(nims) " else: ""
+    rawMessage(conf, hintProcessing, "$#$# $#: $#: $#" % [mode, indent, fromModule2, moduleStatus, path])
+
+proc getPackage*(graph: ModuleGraph; fileIdx: FileIndex): PSym =
+  ## Returns a package symbol for yet to be defined module for fileIdx.
+  ## The package symbol is added to the graph if it doesn't exist.
+  let pkgSym = getPackage(graph.config, graph.cache, fileIdx)
+  # check if the package is already in the graph
+  result = graph.packageSyms.strTableGet(pkgSym.name)
+  if result == nil:
+     # the package isn't in the graph, so create and add it
+    result = pkgSym
+    graph.packageSyms.strTableAdd(pkgSym)
+
+func belongsToStdlib*(graph: ModuleGraph, sym: PSym): bool =
+  ## Check if symbol belongs to the 'stdlib' package.
+  sym.getPackageSymbol.getPackageId == graph.systemModule.getPackageId
+
+proc `==`*(a, b: SymInfoPair): bool =
+  result = a.sym == b.sym and a.info.exactEquals(b.info)
+
+proc fileSymbols*(graph: ModuleGraph, fileIdx: FileIndex): seq[SymInfoPair] =
+  result = graph.suggestSymbols.getOrDefault(fileIdx, @[])
+
+iterator suggestSymbolsIter*(g: ModuleGraph): SymInfoPair =
+  for xs in g.suggestSymbols.values:
+    for x in xs:
+      yield x
+
+iterator suggestErrorsIter*(g: ModuleGraph): Suggest =
+  for xs in g.suggestErrors.values:
+    for x in xs:
+      yield x

compiler/modulepaths.nim

@@ -10,100 +10,6 @@
 import ast, renderer, strutils, msgs, options, idents, os, lineinfos,
   pathutils
 
-when false:
-  const
-    considerParentDirs = not defined(noParentProjects)
-    considerNimbleDirs = not defined(noNimbleDirs)
-
-  proc findInNimbleDir(pkg, subdir, dir: string): string =
-    var best = ""
-    var bestv = ""
-    for k, p in os.walkDir(dir, relative=true):
-      if k == pcDir and p.len > pkg.len+1 and
-          p[pkg.len] == '-' and p.startsWith(pkg):
-        let (_, a) = getPathVersion(p)
-        if bestv.len == 0 or bestv < a:
-          bestv = a
-          best = dir / p
-
-    if best.len > 0:
-      var f: File
-      if open(f, best / changeFileExt(pkg, ".nimble-link")):
-        # the second line contains what we're interested in, see:
-        # https://github.com/nim-lang/nimble#nimble-link
-        var override = ""
-        discard readLine(f, override)
-        discard readLine(f, override)
-        close(f)
-        if not override.isAbsolute():
-          best = best / override
-        else:
-          best = override
-    let f = if subdir.len == 0: pkg else: subdir
-    let res = addFileExt(best / f, "nim")
-    if best.len > 0 and fileExists(res):
-      result = res
-
-when false:
-  proc resolveDollar(project, source, pkg, subdir: string; info: TLineInfo): string =
-    template attempt(a) =
-      let x = addFileExt(a, "nim")
-      if fileExists(x): return x
-
-    case pkg
-    of "stdlib":
-      if subdir.len == 0:
-        return options.libpath
-      else:
-        for candidate in stdlibDirs:
-          attempt(options.libpath / candidate / subdir)
-    of "root":
-      let root = project.splitFile.dir
-      if subdir.len == 0:
-        return root
-      else:
-        attempt(root / subdir)
-    else:
-      when considerParentDirs:
-        var p = parentDir(source.splitFile.dir)
-        # support 'import $karax':
-        let f = if subdir.len == 0: pkg else: subdir
-
-        while p.len > 0:
-          let dir = p / pkg
-          if dirExists(dir):
-            attempt(dir / f)
-            # 2nd attempt: try to use 'karax/karax'
-            attempt(dir / pkg / f)
-            # 3rd attempt: try to use 'karax/src/karax'
-            attempt(dir / "src" / f)
-            attempt(dir / "src" / pkg / f)
-          p = parentDir(p)
-
-      when considerNimbleDirs:
-        if not options.gNoNimblePath:
-          var nimbleDir = getEnv("NIMBLE_DIR")
-          if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble"
-          result = findInNimbleDir(pkg, subdir, nimbleDir / "pkgs")
-          if result.len > 0: return result
-          when not defined(windows):
-            result = findInNimbleDir(pkg, subdir, "/opt/nimble/pkgs")
-            if result.len > 0: return result
-
-  proc scriptableImport(pkg, sub: string; info: TLineInfo): string =
-    resolveDollar(gProjectFull, info.toFullPath(), pkg, sub, info)
-
-  proc lookupPackage(pkg, subdir: PNode): string =
-    let sub = if subdir != nil: renderTree(subdir, {renderNoComments}).replace(" ") else: ""
-    case pkg.kind
-    of nkStrLit, nkRStrLit, nkTripleStrLit:
-      result = scriptableImport(pkg.strVal, sub, pkg.info)
-    of nkIdent:
-      result = scriptableImport(pkg.ident.s, sub, pkg.info)
-    else:
-      localError(pkg.info, "package name must be an identifier or string literal")
-      result = ""
-
 proc getModuleName*(conf: ConfigRef; n: PNode): string =
   # This returns a short relative module name without the nim extension
   # e.g. like "system", "importer" or "somepath/module"
@@ -163,3 +69,18 @@ proc checkModuleName*(conf: ConfigRef; n: PNode; doLocalError=true): FileIndex =
     result = InvalidFileIdx
   else:
     result = fileInfoIdx(conf, fullPath)
+
+proc mangleModuleName*(conf: ConfigRef; path: AbsoluteFile): string =
+  ## Mangle a relative module path to avoid path and symbol collisions.
+  ##
+  ## Used by backends that need to generate intermediary files from Nim modules.
+  ## This is needed because the compiler uses a flat cache file hierarchy.
+  ##
+  ## Example:
+  ## `foo-#head/../bar` becomes `@foo-@hhead@s..@sbar`
+  "@m" & relativeTo(path, conf.projectPath).string.multiReplace(
+    {$os.DirSep: "@s", $os.AltSep: "@s", "#": "@h", "@": "@@", ":": "@c"})
+
+proc demangleModuleName*(path: string): string =
+  ## Demangle a relative module path.
+  result = path.multiReplace({"@@": "@", "@h": "#", "@s": "/", "@m": "", "@c": ":"})

compiler/modules.nim

@@ -10,11 +10,9 @@
 ## Implements the module handling, including the caching of modules.
 
 import
-  ast, astalgo, magicsys, msgs, options,
-  idents, lexer, passes, syntaxes, llstream, modulegraphs,
-  lineinfos, pathutils, tables
-
-import ic / replayer
+  ast, magicsys, msgs, options,
+  idents, lexer, syntaxes, modulegraphs,
+  lineinfos, pathutils
 
 proc resetSystemArtifacts*(g: ModuleGraph) =
   magicsys.resetSysTypes(g)
@@ -22,57 +20,12 @@ proc resetSystemArtifacts*(g: ModuleGraph) =
 template getModuleIdent(graph: ModuleGraph, filename: AbsoluteFile): PIdent =
   getIdent(graph.cache, splitFile(filename).name)
 
-template packageId(): untyped {.dirty.} = ItemId(module: PackageModuleId, item: int32(fileIdx))
-
-proc getPackage(graph: ModuleGraph; fileIdx: FileIndex): PSym =
-  ## returns package symbol (skPackage) for yet to be defined module for fileIdx
-  let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
-  let name = getModuleIdent(graph, filename)
-  let info = newLineInfo(fileIdx, 1, 1)
-  let
-    pck = getPackageName(graph.config, filename.string)
-    pck2 = if pck.len > 0: pck else: "unknown"
-    pack = getIdent(graph.cache, pck2)
-  result = graph.packageSyms.strTableGet(pack)
-  if result == nil:
-    result = newSym(skPackage, getIdent(graph.cache, pck2), packageId(), nil, info)
-    #initStrTable(packSym.tab)
-    graph.packageSyms.strTableAdd(result)
-  else:
-    let modules = graph.modulesPerPackage.getOrDefault(result.itemId)
-    let existing = if modules.data.len > 0: strTableGet(modules, name) else: nil
-    if existing != nil and existing.info.fileIndex != info.fileIndex:
-      when false:
-        # we used to produce an error:
-        localError(graph.config, info,
-          "module names need to be unique per Nimble package; module clashes with " &
-            toFullPath(graph.config, existing.info.fileIndex))
-      else:
-        # but starting with version 0.20 we now produce a fake Nimble package instead
-        # to resolve the conflicts:
-        let pck3 = fakePackageName(graph.config, filename)
-        # this makes the new `result`'s owner be the original `result`
-        result = newSym(skPackage, getIdent(graph.cache, pck3), packageId(), result, info)
-        #initStrTable(packSym.tab)
-        graph.packageSyms.strTableAdd(result)
-
-proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) =
+proc partialInitModule*(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) =
   let packSym = getPackage(graph, fileIdx)
   result.owner = packSym
   result.position = int fileIdx
 
-  #initStrTable(result.tab(graph))
-  when false:
-    strTableAdd(result.tab, result) # a module knows itself
-    # This is now implemented via
-    #   c.moduleScope.addSym(module) # a module knows itself
-    # in sem.nim, around line 527
-
-  if graph.modulesPerPackage.getOrDefault(packSym.itemId).data.len == 0:
-    graph.modulesPerPackage[packSym.itemId] = newStrTable()
-  graph.modulesPerPackage[packSym.itemId].strTableAdd(result)
-
-proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
+proc newModule*(graph: ModuleGraph; fileIdx: FileIndex): PSym =
   let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
   # We cannot call ``newSym`` here, because we have to circumvent the ID
   # mechanism, which we do in order to assign each module a persistent ID.
@@ -80,103 +33,21 @@ proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
                 name: getModuleIdent(graph, filename),
                 info: newLineInfo(fileIdx, 1, 1))
   if not isNimIdentifier(result.name.s):
-    rawMessage(graph.config, errGenerated, "invalid module name: " & result.name.s)
+    rawMessage(graph.config, errGenerated, "invalid module name: '" & result.name.s &
+              "'; a module name must be a valid Nim identifier.")
   partialInitModule(result, graph, fileIdx, filename)
   graph.registerModule(result)
 
-proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): PSym =
-  var flags = flags
-  if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule
-  result = graph.getModule(fileIdx)
-
-  template processModuleAux =
-    var s: PLLStream
-    if sfMainModule in flags:
-      if graph.config.projectIsStdin: s = stdin.llStreamOpen
-      elif graph.config.projectIsCmd: s = llStreamOpen(graph.config.cmdInput)
-    discard processModule(graph, result, idGeneratorFromModule(result), s)
-  if result == nil:
-    var cachedModules: seq[FileIndex]
-    result = moduleFromRodFile(graph, fileIdx, cachedModules)
-    let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
-    if result == nil:
-      result = newModule(graph, fileIdx)
-      result.flags.incl flags
-      registerModule(graph, result)
-      processModuleAux()
-    else:
-      if sfSystemModule in flags:
-        graph.systemModule = result
-      partialInitModule(result, graph, fileIdx, filename)
-    for m in cachedModules:
-      registerModuleById(graph, m)
-      replayStateChanges(graph.packed[m.int].module, graph)
-      replayGenericCacheInformation(graph, m.int)
-  elif graph.isDirty(result):
-    result.flags.excl sfDirty
-    # reset module fields:
-    initStrTables(graph, result)
-    result.ast = nil
-    processModuleAux()
-    graph.markClientsDirty(fileIdx)
-
-proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym =
-  # this is called by the semantic checking phase
-  assert graph.config != nil
-  result = compileModule(graph, fileIdx, {})
-  graph.addDep(s, fileIdx)
-  # keep track of import relationships
-  if graph.config.hcrOn:
-    graph.importDeps.mgetOrPut(FileIndex(s.position), @[]).add(fileIdx)
-  #if sfSystemModule in result.flags:
-  #  localError(result.info, errAttemptToRedefine, result.name.s)
-  # restore the notes for outer module:
-  graph.config.notes =
-    if s.getnimblePkgId == graph.config.mainPackageId or isDefined(graph.config, "booting"): graph.config.mainPackageNotes
-    else: graph.config.foreignPackageNotes
-
 proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode =
   result = syntaxes.parseFile(fileIdx, graph.cache, graph.config)
   graph.addDep(s, fileIdx)
   graph.addIncludeDep(s.position.FileIndex, fileIdx)
 
-proc connectCallbacks*(graph: ModuleGraph) =
-  graph.includeFileCallback = includeModule
-  graph.importModuleCallback = importModule
-
-proc compileSystemModule*(graph: ModuleGraph) =
-  if graph.systemModule == nil:
-    connectCallbacks(graph)
-    graph.config.m.systemFileIdx = fileInfoIdx(graph.config,
-        graph.config.libpath / RelativeFile"system.nim")
-    discard graph.compileModule(graph.config.m.systemFileIdx, {sfSystemModule})
-
 proc wantMainModule*(conf: ConfigRef) =
   if conf.projectFull.isEmpty:
-    fatal(conf, newLineInfo(conf, AbsoluteFile(commandLineDesc), 1, 1), errGenerated,
-        "command expects a filename")
+    fatal(conf, gCmdLineInfo, "command expects a filename")
   conf.projectMainIdx = fileInfoIdx(conf, addFileExt(conf.projectFull, NimExt))
 
-proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) =
-  connectCallbacks(graph)
-  let conf = graph.config
-  wantMainModule(conf)
-  configComplete(graph)
-
-  let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim")
-  let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx
-  conf.projectMainIdx2 = projectFile
-
-  let packSym = getPackage(graph, projectFile)
-  graph.config.mainPackageId = packSym.getnimblePkgId
-  graph.importStack.add projectFile
-
-  if projectFile == systemFileIdx:
-    discard graph.compileModule(projectFile, {sfMainModule, sfSystemModule})
-  else:
-    graph.compileSystemModule()
-    discard graph.compileModule(projectFile, {sfMainModule})
-
 proc makeModule*(graph: ModuleGraph; filename: AbsoluteFile): PSym =
   result = graph.newModule(fileInfoIdx(graph.config, filename))
   registerModule(graph, result)

compiler/msgs.nim

@@ -8,10 +8,15 @@
 #
 
 import
-  options, strutils, os, tables, ropes, terminal, macros,
-  lineinfos, pathutils
-import std/private/miscdollars
-import strutils2
+  std/[strutils, os, tables, terminal, macros, times],
+  std/private/miscdollars,
+  options, lineinfos, pathutils
+
+import ropes except `%`
+
+when defined(nimPreviewSlimSystem):
+  import std/[syncio, assertions]
+
 
 type InstantiationInfo* = typeof(instantiationInfo())
 template instLoc*(): InstantiationInfo = instantiationInfo(-2, fullPaths = true)
@@ -19,6 +24,10 @@ template instLoc*(): InstantiationInfo = instantiationInfo(-2, fullPaths = true)
 template toStdOrrKind(stdOrr): untyped =
   if stdOrr == stdout: stdOrrStdout else: stdOrrStderr
 
+proc toLowerAscii(a: var string) {.inline.} =
+  for c in mitems(a):
+    if isUpperAscii(c): c = char(uint8(c) xor 0b0010_0000'u8)
+
 proc flushDot*(conf: ConfigRef) =
   ## safe to call multiple times
   # xxx one edge case not yet handled is when `printf` is called at CT with `compiletimeFFI`.
@@ -28,7 +37,7 @@ proc flushDot*(conf: ConfigRef) =
     conf.lastMsgWasDot.excl stdOrrKind
     write(stdOrr, "\n")
 
-proc toCChar*(c: char; result: var string) =
+proc toCChar*(c: char; result: var string) {.inline.} =
   case c
   of '\0'..'\x1F', '\x7F'..'\xFF':
     result.add '\\'
@@ -40,18 +49,16 @@ proc toCChar*(c: char; result: var string) =
     result.add c
 
 proc makeCString*(s: string): Rope =
-  result = nil
-  var res = newStringOfCap(int(s.len.toFloat * 1.1) + 1)
-  res.add("\"")
+  result = newStringOfCap(int(s.len.toFloat * 1.1) + 1)
+  result.add("\"")
   for i in 0..<s.len:
     # line wrapping of string litterals in cgen'd code was a bad idea, e.g. causes: bug #16265
     # It also makes reading c sources or grepping harder, for zero benefit.
     # const MaxLineLength = 64
     # if (i + 1) mod MaxLineLength == 0:
     #   res.add("\"\L\"")
-    toCChar(s[i], res)
-  res.add('\"')
-  result.add(rope(res))
+    toCChar(s[i], result)
+  result.add('\"')
 
 proc newFileInfo(fullPath: AbsoluteFile, projPath: RelativeFile): TFileInfo =
   result.fullPath = fullPath
@@ -75,7 +82,7 @@ when defined(nimpretty):
   proc fileSection*(conf: ConfigRef; fid: FileIndex; a, b: int): string =
     substr(conf.m.fileInfos[fid.int].fullContent, a, b)
 
-proc canonicalCase(path: var string) =
+proc canonicalCase(path: var string) {.inline.} =
   ## the idea is to only use this for checking whether a path is already in
   ## the table but otherwise keep the original case
   when FileSystemCaseSensitive: discard
@@ -98,15 +105,13 @@ proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile; isKnownFile: var bool
 
   try:
     canon = canonicalizePath(conf, filename)
-    shallow(canon.string)
   except OSError:
     canon = filename
     # The compiler uses "filenames" such as `command line` or `stdin`
     # This flag indicates that we are working with such a path here
     pseudoPath = true
 
-  var canon2: string
-  forceCopy(canon2, canon.string) # because `canon` may be shallow
+  var canon2 = canon.string
   canon2.canonicalCase
 
   if conf.m.filenameToIndexTbl.hasKey(canon2):
@@ -115,7 +120,6 @@ proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile; isKnownFile: var bool
   else:
     isKnownFile = false
     result = conf.m.fileInfos.len.FileIndex
-    #echo "ID ", result.int, " ", canon2
     conf.m.fileInfos.add(newFileInfo(canon, if pseudoPath: RelativeFile filename
                                             else: relativeTo(canon, conf.projectPath)))
     conf.m.filenameToIndexTbl[canon2] = result
@@ -124,6 +128,13 @@ proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile): FileIndex =
   var dummy: bool
   result = fileInfoIdx(conf, filename, dummy)
 
+proc fileInfoIdx*(conf: ConfigRef; filename: RelativeFile; isKnownFile: var bool): FileIndex =
+  fileInfoIdx(conf, AbsoluteFile expandFilename(filename.string), isKnownFile)
+
+proc fileInfoIdx*(conf: ConfigRef; filename: RelativeFile): FileIndex =
+  var dummy: bool
+  fileInfoIdx(conf, AbsoluteFile expandFilename(filename.string), dummy)
+
 proc newLineInfo*(fileInfoIdx: FileIndex, line, col: int): TLineInfo =
   result.fileIndex = fileInfoIdx
   if line < int high(uint16):
@@ -215,11 +226,18 @@ proc setDirtyFile*(conf: ConfigRef; fileIdx: FileIndex; filename: AbsoluteFile)
 
 proc setHash*(conf: ConfigRef; fileIdx: FileIndex; hash: string) =
   assert fileIdx.int32 >= 0
-  shallowCopy(conf.m.fileInfos[fileIdx.int32].hash, hash)
+  when defined(gcArc) or defined(gcOrc):
+    conf.m.fileInfos[fileIdx.int32].hash = hash
+  else:
+    shallowCopy(conf.m.fileInfos[fileIdx.int32].hash, hash)
+
 
 proc getHash*(conf: ConfigRef; fileIdx: FileIndex): string =
   assert fileIdx.int32 >= 0
-  shallowCopy(result, conf.m.fileInfos[fileIdx.int32].hash)
+  when defined(gcArc) or defined(gcOrc):
+    result = conf.m.fileInfos[fileIdx.int32].hash
+  else:
+    shallowCopy(result, conf.m.fileInfos[fileIdx.int32].hash)
 
 proc toFullPathConsiderDirty*(conf: ConfigRef; fileIdx: FileIndex): AbsoluteFile =
   if fileIdx.int32 < 0:
@@ -289,10 +307,6 @@ proc `??`* (conf: ConfigRef; info: TLineInfo, filename: string): bool =
   # only for debugging purposes
   result = filename in toFilename(conf, info)
 
-const
-  UnitSep = "\31"
-    # this needs care to avoid issues similar to https://github.com/nim-lang/Nim/issues/17853
-
 type
   MsgFlag* = enum  ## flags altering msgWriteln behavior
     msgStdout,     ## force writing to stdout, even stderr is default
@@ -309,7 +323,7 @@ proc msgWriteln*(conf: ConfigRef; s: string, flags: MsgFlags = {}) =
   ## This is used for 'nim dump' etc. where we don't have nimsuggest
   ## support.
   #if conf.cmd == cmdIdeTools and optCDebug notin gGlobalOptions: return
-  let sep = if msgNoUnitSep notin flags: UnitSep else: ""
+  let sep = if msgNoUnitSep notin flags: conf.unitSep else: ""
   if not isNil(conf.writelnHook) and msgSkipHook notin flags:
     conf.writelnHook(s & sep)
   elif optStdout in conf.globalOptions or msgStdout in flags:
@@ -409,15 +423,15 @@ proc quit(conf: ConfigRef; msg: TMsgKind) {.gcsafe.} =
         styledMsgWriteln(fgRed, """
 No stack traceback available
 To create a stacktrace, rerun compilation with './koch temp $1 <file>', see $2 for details""" %
-          [conf.command, "intern.html#debugging-the-compiler".createDocLink], UnitSep)
+          [conf.command, "intern.html#debugging-the-compiler".createDocLink], conf.unitSep)
   quit 1
 
-proc handleError(conf: ConfigRef; msg: TMsgKind, eh: TErrorHandling, s: string) =
-  if msg >= fatalMin and msg <= fatalMax:
+proc handleError(conf: ConfigRef; msg: TMsgKind, eh: TErrorHandling, s: string, ignoreMsg: bool) =
+  if msg in fatalMsgs:
     if conf.cmd == cmdIdeTools: log(s)
     quit(conf, msg)
   if msg >= errMin and msg <= errMax or
-      (msg in warnMin..hintMax and msg in conf.warningAsErrors):
+      (msg in warnMin..hintMax and msg in conf.warningAsErrors and not ignoreMsg):
     inc(conf.errorCounter)
     conf.exitcode = 1'i8
     if conf.errorCounter >= conf.errorMax:
@@ -497,12 +511,18 @@ proc formatMsg*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string): s
   conf.toFileLineCol(info) & " " & title & getMessageStr(msg, arg)
 
 proc liMessage*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
-               eh: TErrorHandling, info2: InstantiationInfo, isRaw = false) {.noinline.} =
+               eh: TErrorHandling, info2: InstantiationInfo, isRaw = false) {.gcsafe, noinline.} =
   var
     title: string
     color: ForegroundColor
     ignoreMsg = false
     sev: Severity
+  let errorOutputsOld = conf.m.errorOutputs
+  if msg in fatalMsgs:
+    # don't gag, refs bug #7080, bug #18278; this can happen with `{.fatal.}`
+    # or inside a `tryConstExpr`.
+    conf.m.errorOutputs = {eStdOut, eStdErr}
+
   let kind = if msg in warnMin..hintMax and msg != hintUserRaw: $msg else: "" # xxx not sure why hintUserRaw is special
   case msg
   of errMin..errMax:
@@ -519,19 +539,18 @@ proc liMessage*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
   of warnMin..warnMax:
     sev = Severity.Warning
     ignoreMsg = not conf.hasWarn(msg)
-    if msg in conf.warningAsErrors:
-      ignoreMsg = false
+    if not ignoreMsg and msg in conf.warningAsErrors:
       title = ErrorTitle
+      color = ErrorColor
     else:
       title = WarningTitle
+      color = WarningColor
     if not ignoreMsg: writeContext(conf, info)
-    color = WarningColor
     inc(conf.warnCounter)
   of hintMin..hintMax:
     sev = Severity.Hint
     ignoreMsg = not conf.hasHint(msg)
-    if msg in conf.warningAsErrors:
-      ignoreMsg = false
+    if not ignoreMsg and msg in conf.warningAsErrors:
       title = ErrorTitle
     else:
       title = HintTitle
@@ -550,14 +569,17 @@ proc liMessage*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
         msgWrite(conf, ".")
       else:
         styledMsgWriteln(styleBright, loc, resetStyle, color, title, resetStyle, s, KindColor, kindmsg,
-                         resetStyle, conf.getSurroundingSrc(info), UnitSep)
+                         resetStyle, conf.getSurroundingSrc(info), conf.unitSep)
         if hintMsgOrigin in conf.mainPackageNotes:
           # xxx needs a bit of refactoring to honor `conf.filenameOption`
           styledMsgWriteln(styleBright, toFileLineCol(info2), resetStyle,
             " compiler msg initiated here", KindColor,
             KindFormat % $hintMsgOrigin,
-            resetStyle, UnitSep)
-  handleError(conf, msg, eh, s)
+            resetStyle, conf.unitSep)
+  handleError(conf, msg, eh, s, ignoreMsg)
+  if msg in fatalMsgs:
+    # most likely would have died here but just in case, we restore state
+    conf.m.errorOutputs = errorOutputsOld
 
 template rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) =
   let arg = msgKindToString(msg) % args
@@ -566,9 +588,7 @@ template rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) =
 template rawMessage*(conf: ConfigRef; msg: TMsgKind, arg: string) =
   liMessage(conf, unknownLineInfo, msg, arg, eh = doAbort, instLoc())
 
-template fatal*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
-  # this fixes bug #7080 so that it is at least obvious 'fatal' was executed.
-  conf.m.errorOutputs = {eStdOut, eStdErr}
+template fatal*(conf: ConfigRef; info: TLineInfo, arg = "", msg = errFatal) =
   liMessage(conf, info, msg, arg, doAbort, instLoc())
 
 template globalAssert*(conf: ConfigRef; cond: untyped, info: TLineInfo = unknownLineInfo, arg = "") =
@@ -616,9 +636,9 @@ template internalAssert*(conf: ConfigRef, e: bool) =
     let arg = info2.toFileLineCol
     internalErrorImpl(conf, unknownLineInfo, arg, info2)
 
-template lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string, forceHint = false, extraMsg = "") =
+template lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string, extraMsg = "") =
   let m = "'$1' should be: '$2'$3" % [got, beau, extraMsg]
-  let msg = if optStyleError in conf.globalOptions and not forceHint: errGenerated else: hintName
+  let msg = if optStyleError in conf.globalOptions: errGenerated else: hintName
   liMessage(conf, info, msg, m, doNothing, instLoc())
 
 proc quotedFilename*(conf: ConfigRef; i: TLineInfo): Rope =
@@ -660,3 +680,54 @@ proc uniqueModuleName*(conf: ConfigRef; fid: FileIndex): string =
       # We mangle upper letters and digits too so that there cannot
       # be clashes with our special meanings of 'Z' and 'O'
       result.addInt ord(c)
+
+proc genSuccessX*(conf: ConfigRef) =
+  let mem =
+    when declared(system.getMaxMem): formatSize(getMaxMem()) & " peakmem"
+    else: formatSize(getTotalMem()) & " totmem"
+  let loc = $conf.linesCompiled
+  var build = ""
+  var flags = ""
+  const debugModeHints = "none (DEBUG BUILD, `-d:release` generates faster code)"
+  if conf.cmd in cmdBackends:
+    if conf.backend != backendJs:
+      build.add "mm: $#; " % $conf.selectedGC
+      if optThreads in conf.globalOptions: build.add "threads: on; "
+      build.add "opt: "
+      if optOptimizeSpeed in conf.options: build.add "speed"
+      elif optOptimizeSize in conf.options: build.add "size"
+      else: build.add debugModeHints
+        # pending https://github.com/timotheecour/Nim/issues/752, point to optimization.html
+      if isDefined(conf, "danger"): flags.add " -d:danger"
+      elif isDefined(conf, "release"): flags.add " -d:release"
+    else:
+      build.add "opt: "
+      if isDefined(conf, "danger"):
+        build.add "speed"
+        flags.add " -d:danger"
+      elif isDefined(conf, "release"):
+        build.add "speed"
+        flags.add " -d:release"
+      else: build.add debugModeHints
+    if flags.len > 0: build.add "; options:" & flags
+  let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3)
+  let project = if conf.filenameOption == foAbs: $conf.projectFull else: $conf.projectName
+    # xxx honor conf.filenameOption more accurately
+  var output: string
+  if optCompileOnly in conf.globalOptions and conf.cmd != cmdJsonscript:
+    output = $conf.jsonBuildFile
+  elif conf.outFile.isEmpty and conf.cmd notin {cmdJsonscript} + cmdDocLike + cmdBackends:
+    # for some cmd we expect a valid absOutFile
+    output = "unknownOutput"
+  else:
+    output = $conf.absOutFile
+  if conf.filenameOption != foAbs: output = output.AbsoluteFile.extractFilename
+    # xxx honor filenameOption more accurately
+  rawMessage(conf, hintSuccessX, [
+    "build", build,
+    "loc", loc,
+    "sec", sec,
+    "mem", mem,
+    "project", project,
+    "output", output,
+    ])

compiler/ndi.nim

@@ -12,6 +12,9 @@
 
 import ast, msgs, ropes, options, pathutils
 
+when defined(nimPreviewSlimSystem):
+  import std/[syncio, assertions]
+
 type
   NdiFile* = object
     enabled: bool