fix some issues with --backend (#14363)

* fix some issues with --backend
* fix https://github.com/timotheecour/Nim/issues/175; improve upon #14306

changelog.md

@@ -160,8 +160,8 @@ proc mydiv(a, b): int {.raises: [].} =
 
 - Specific warnings can now be turned into errors via `--warningAsError[X]:on|off`.
 - The `define` and `undef` pragmas have been de-deprecated.
-- New command: `nim r main.nim [args...]` which compiles and runs main.nim, saving
+- New command: `nim r main.nim [args...]` which compiles and runs main.nim, and implies `--usenimcache`
-  the binary to $nimcache/main$exeExt, using the same logic as `nim c -r` to
+  so that output is saved to $nimcache/main$exeExt, using the same logic as `nim c -r` to
   avoid recompiling when sources don't change. This is now the preferred way to
   run tests, avoiding the usual pain of clobbering your repo with binaries or
   using tricky gitignore rules on posix. Example:
@@ -178,6 +178,12 @@ proc mydiv(a, b): int {.raises: [].} =
 - new hint: `--hint:msgOrigin` will show where a compiler msg (hint|warning|error) was generated; this
   helps in particular when it's non obvious where it came from either because multiple locations generate
   the same message, or because the message involves runtime formatting.
+- new flag `--backend:js|c|cpp|objc (or -b:js etc), to change backend; can be used with any command
+  (eg nim r, doc, check etc); safe to re-assign.
+- new flag `--doccmd:cmd` to pass additional flags for runnableExamples, eg: `--doccmd:-d:foo --threads`
+  use `--doccmd:skip` to skip runnableExamples and rst test snippets.
+- new flag `--usenimcache` to output to nimcache (whatever it resolves to after all commands are processed)
+  and avoids polluting both $pwd and $projectdir. It can be used with any command.
 
 ## Tool changes
 

compiler/cmdlinehelper.nim

@@ -75,14 +75,6 @@ proc loadConfigsAndRunMainCommand*(self: NimProg, cache: IdentCache; conf: Confi
   # now process command line arguments again, because some options in the
   # command line can overwrite the config file's settings
   extccomp.initVars(conf)
-  # XXX This is hacky. We need to find a better way.
-  case conf.command
-  of "cpp", "compiletocpp":
-    conf.backend = backendCpp
-    conf.cmd = cmdCompileToBackend
-  else:
-    discard
-
   self.processCmdLine(passCmd2, "", conf)
   if conf.command == "":
     rawMessage(conf, errGenerated, "command missing")

compiler/commands.nim

@@ -502,8 +502,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
         defineSymbol(conf.symbols, "gcmarkandsweep")
       of "destructors", "arc":
         conf.selectedGC = gcArc
-        if conf.backend != backendCpp:
-          conf.exc = excGoto
         defineSymbol(conf.symbols, "gcdestructors")
         defineSymbol(conf.symbols, "gcarc")
         incl conf.globalOptions, optSeqDestructors
@@ -513,8 +511,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
           defineSymbol(conf.symbols, "nimV2")
       of "orc":
         conf.selectedGC = gcOrc
-        if conf.backend != backendCpp:
-          conf.exc = excGoto
         defineSymbol(conf.symbols, "gcdestructors")
         defineSymbol(conf.symbols, "gcorc")
         incl conf.globalOptions, optSeqDestructors

compiler/docgen.nim

@@ -216,13 +216,20 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
       # Include the current file if we're parsing a nim file
       let importStmt = if d.isPureRst: "" else: "import \"$1\"\n" % [d.filename.replace("\\", "/")]
       writeFile(outp, importStmt & content)
-      let c = if cmd.startsWith("nim <backend> "): os.getAppFilename() & " " & $conf.backend & cmd.substr("nim <backend>".len)
+
-              elif cmd.startsWith("nim "): os.getAppFilename() & cmd.substr("nim".len)
+      proc interpSnippetCmd(cmd: string): string =
-              else: cmd
+        # backward compatibility hacks; interpolation commands should explicitly use `$`
-      let c2 = c % quoteShell(outp)
+        if cmd.startsWith "nim ": result = "$nim " & cmd[4..^1]
-      rawMessage(conf, hintExecuting, c2)
+        else: result = cmd
-      if execShellCmd(c2) != status:
+        result = result.replace("$1", "$options") % [
-        rawMessage(conf, errGenerated, "executing of external program failed: " & c2)
+          "nim", os.getAppFilename().quoteShell,
+          "backend", $d.conf.backend,
+          "options", outp.quoteShell,
+        ]
+      let cmd = cmd.interpSnippetCmd
+      rawMessage(conf, hintExecuting, cmd)
+      if execShellCmd(cmd) != status:
+        rawMessage(conf, errGenerated, "executing of external program failed: " & cmd)
   result.emitted = initIntSet()
   result.destFile = getOutFile2(conf, presentationPath(conf, filename),
                                 outExt, htmldocsDir, false)

compiler/extccomp.nim

@@ -306,13 +306,14 @@ proc isVSCompatible*(conf: ConfigRef): bool =
 proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string =
   # use ``cpu.os.cc`` for cross compilation, unless ``--compileOnly`` is given
   # for niminst support
-  let fullSuffix =
+  var fullSuffix = suffix
-    case conf.backend
+  case conf.backend
-    of backendCpp, backendJs, backendObjc: "." & $conf.backend & suffix
+  of backendCpp, backendJs, backendObjc: fullSuffix = "." & $conf.backend & suffix
-    of backendC: suffix
+  of backendC: discard
-    else:
+  of backendInvalid:
-      doAssert false
+    # during parsing of cfg files; we don't know the backend yet, no point in
-      ""
+    # guessing wrong thing
+    return ""
 
   if (conf.target.hostOS != conf.target.targetOS or conf.target.hostCPU != conf.target.targetCPU) and
       optCompileOnly notin conf.globalOptions:

compiler/main.nim

@@ -114,8 +114,10 @@ proc commandJsonScript(graph: ModuleGraph) =
   let proj = changeFileExt(graph.config.projectFull, "")
   extccomp.runJsonBuildInstructions(graph.config, proj)
 
-when not defined(leanCompiler):
+proc commandCompileToJS(graph: ModuleGraph) =
-  proc commandCompileToJS(graph: ModuleGraph) =
+  when defined(leanCompiler):
+    globalError(graph.config, unknownLineInfo, "compiler wasn't built with JS code generator")
+  else:
     let conf = graph.config
     conf.exc = excCpp
 
@@ -194,47 +196,65 @@ proc mainCommand*(graph: ModuleGraph) =
   when false: setOutDir(conf)
   if optUseNimcache in conf.globalOptions: setOutDir(conf)
 
-  template handleBackend(backend2: TBackend) =
+  proc customizeForBackend(backend: TBackend) =
-    conf.backend = backend2
+    ## Sets backend specific options but don't compile to backend yet in
-    conf.cmd = cmdCompileToBackend
+    ## case command doesn't require it. This must be called by all commands.
-    defineSymbol(graph.config.symbols, $backend2)
+    if conf.backend == backendInvalid:
-    case backend2
+      # only set if wasn't already set, to allow override via `nim c -b:cpp`
+      conf.backend = backend
+
+    defineSymbol(graph.config.symbols, $conf.backend)
+    case conf.backend
     of backendC:
       if conf.exc == excNone: conf.exc = excSetjmp
-      commandCompileToC(graph)
     of backendCpp:
       if conf.exc == excNone: conf.exc = excCpp
-      commandCompileToC(graph)
+    of backendObjc: discard
-    of backendObjc:
-      commandCompileToC(graph)
     of backendJs:
-      when defined(leanCompiler):
+      if conf.hcrOn:
-        globalError(conf, unknownLineInfo, "compiler wasn't built with JS code generator")
+        # XXX: At the moment, system.nim cannot be compiled in JS mode
-      else:
+        # with "-d:useNimRtl". The HCR option has been processed earlier
-        if conf.hcrOn:
+        # and it has added this define implictly, so we must undo that here.
-          # XXX: At the moment, system.nim cannot be compiled in JS mode
+        # A better solution might be to fix system.nim
-          # with "-d:useNimRtl". The HCR option has been processed earlier
+        undefSymbol(conf.symbols, "useNimRtl")
-          # and it has added this define implictly, so we must undo that here.
+    of backendInvalid: doAssert false
-          # A better solution might be to fix system.nim
+    if conf.selectedGC in {gcArc, gcOrc} and conf.backend != backendCpp:
-          undefSymbol(conf.symbols, "useNimRtl")
+      conf.exc = excGoto
-        commandCompileToJS(graph)
+
+  var commandAlreadyProcessed = false
+
+  proc compileToBackend(backend: TBackend, cmd = cmdCompileToBackend) =
+    commandAlreadyProcessed = true
+    conf.cmd = cmd
+    customizeForBackend(backend)
+    case conf.backend
+    of backendC: commandCompileToC(graph)
+    of backendCpp: commandCompileToC(graph)
+    of backendObjc: commandCompileToC(graph)
+    of backendJs: commandCompileToJS(graph)
     of backendInvalid: doAssert false
 
+  ## process all backend commands
   case conf.command.normalize
-  of "c", "cc", "compile", "compiletoc": handleBackend(backendC) # compile means compileToC currently
+  of "c", "cc", "compile", "compiletoc": compileToBackend(backendC) # compile means compileToC currently
-  of "cpp", "compiletocpp": handleBackend(backendCpp)
+  of "cpp", "compiletocpp": compileToBackend(backendCpp)
-  of "objc", "compiletooc": handleBackend(backendObjc)
+  of "objc", "compiletooc": compileToBackend(backendObjc)
-  of "js", "compiletojs": handleBackend(backendJs)
+  of "js", "compiletojs": compileToBackend(backendJs)
   of "r": # different from `"run"`!
     conf.globalOptions.incl {optRun, optUseNimcache}
-    handleBackend(conf.backend)
+    compileToBackend(backendC)
   of "run":
-    conf.cmd = cmdRun
     when hasTinyCBackend:
       extccomp.setCC(conf, "tcc", unknownLineInfo)
-      commandCompileToC(graph)
+      if conf.backend notin {backendC, backendInvalid}:
+        rawMessage(conf, errGenerated, "'run' requires c backend, got: '$1'" % $conf.backend)
+      compileToBackend(backendC, cmd = cmdRun)
     else:
       rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc")
+  else: customizeForBackend(backendC) # fallback for other commands
+
+  ## process all other commands
+  case conf.command.normalize
   of "doc0":
     when defined(leanCompiler):
       quit "compiler wasn't built with documentation generator"
@@ -383,6 +403,7 @@ proc mainCommand*(graph: ModuleGraph) =
   of "jsonscript":
     conf.cmd = cmdJsonScript
     commandJsonScript(graph)
+  elif commandAlreadyProcessed: discard # already handled
   else:
     rawMessage(conf, errGenerated, "invalid command: " & conf.command)
 

compiler/options.nim

@@ -223,7 +223,7 @@ type
                           ## fields marked with '*' are subject to
                           ## the incremental compilation mechanisms
                           ## (+) means "part of the dependency"
-    backend*: TBackend
+    backend*: TBackend # set via `nim x` or `nim --backend:x`
     target*: Target       # (+)
     linesCompiled*: int   # all lines that have been compiled
     options*: TOptions    # (+)
@@ -419,7 +419,7 @@ proc newConfigRef*(): ConfigRef =
     cIncludes: @[],   # directories to search for included files
     cLibs: @[],       # directories to search for lib files
     cLinkedLibs: @[],  # libraries to link
-    backend: backendC,
+    backend: backendInvalid,
     externalToLink: @[],
     linkOptionsCmd: "",
     compileOptionsCmd: @[],

lib/packages/docutils/rstgen.nim

@@ -865,7 +865,7 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) =
   of "test":
     params.testCmd = n.getFieldValue.strip
     if params.testCmd.len == 0:
-      params.testCmd = "nim <backend> -r $1" # The nim backend is auto-set in docgen.nim
+      params.testCmd = "$nim r --backend:$backend $options" # see `interpSnippetCmd`
     else:
       params.testCmd = unescape(params.testCmd)
   of "status", "exitcode":

tests/misc/mbackend.nim

@@ -0,0 +1,31 @@
+#[
+We can't merge this test inside a `when defined(cpp)` because some bug that was
+fixed would not trigger in that case.
+]#
+
+import std/compilesettings
+import std/unittest
+
+static:
+  ## bugfix 1: this used to CT error with: Error: unhandled exception: mimportcpp.nim(6, 18) `defined(cpp)`
+  doAssert defined(cpp)
+  doAssert querySetting(backend) == "cpp"
+
+  ## checks that `--backend:c` has no side effect (ie, can be overridden by subsequent commands)
+  doAssert not defined(c)
+  doAssert not defined(js)
+  doAssert not defined(js)
+
+type
+  std_exception {.importcpp: "std::exception", header: "<exception>".} = object
+proc what(s: std_exception): cstring {.importcpp: "((char *)#.what())".}
+
+var isThrown = false
+try:
+  ## bugfix 2: this used to CT error with: Error: only a 'ref object' can be raised
+  raise std_exception()
+except std_exception as ex:
+  doAssert ex.what().len > 0
+  isThrown = true
+
+doAssert isThrown

tests/nimdoc/m13129.nim

@@ -1,3 +1,5 @@
+# issue #13129
+
 when defined(cpp):
   {.push header: "<vector>".}
   type

tests/trunner.nim

@@ -123,6 +123,16 @@ else: # don't run twice the same test
       let cmd = fmt"{nim} r --backend:{mode} --hints:off --nimcache:{nimcache} {file}"
       check execCmdEx(cmd) == ("ok3\n", 0)
 
+  block: # further issues with `--backend`
+    let file = testsDir / "misc/mbackend.nim"
+    var cmd = fmt"{nim} doc -b:cpp --hints:off --nimcache:{nimcache} {file}"
+    check execCmdEx(cmd) == ("", 0)
+    cmd = fmt"{nim} check -b:c -b:cpp --hints:off --nimcache:{nimcache} {file}"
+    check execCmdEx(cmd) == ("", 0)
+    # issue https://github.com/timotheecour/Nim/issues/175
+    cmd = fmt"{nim} c -b:js -b:cpp --hints:off --nimcache:{nimcache} {file}"
+    check execCmdEx(cmd) == ("", 0)
+
   block: # some importc tests
     # issue #14314
     let file = testsDir / "misc/mimportc.nim"