--usenimcache (implied by nim r main) now caches some compile options to avoid recompiling when project was previously compiled with such options. (#17829)

* `--usenimcache` (implied by `nim r main`) now caches some compile options
to avoid recompiling when project was previously compiled with such options.

* works
* add test
* changelog
* use std/with

changelog.md

@@ -377,7 +377,16 @@
 
 - `--hint:CC` now goes to stderr (like all other hints) instead of stdout.
 
+- json build instructions are now generated in `$nimcache/outFileBasename.json`
+  instead of `$nimcache/projectName.json`. This allows avoiding recompiling a given project
+  compiled with different options if the output file differs.
 
+- `--usenimcache` (implied by `nim r main`) now generates an output file that includes a hash of
+  some of the compilation options, which allows caching generated binaries:
+  nim r main # recompiles
+  nim r -d:foo main # recompiles
+  nim r main # uses cached binary
+  nim r main arg1 arg2 # ditto (runtime arguments are irrelevant)
 
 ## Tool changes
 

compiler/extccomp.nim

@@ -934,9 +934,14 @@ proc callCCompiler*(conf: ConfigRef) =
     script.add("\n")
     generateScript(conf, script)
 
-
 template hashNimExe(): string = $secureHashFile(os.getAppFilename())
 
+proc jsonBuildInstructionsFile*(conf: ConfigRef): AbsoluteFile =
+  # `outFile` is better than `projectName`, as it allows having different json
+  # files for a given source file compiled with different options; it also
+  # works out of the box with `hashMainCompilationParams`.
+  result = getNimcacheDir(conf) / conf.outFile.changeFileExt("json")
+
 proc writeJsonBuildInstructions*(conf: ConfigRef) =
   template lit(x: string) = f.write x
   template str(x: string) =
@@ -993,8 +998,7 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) =
 
 
   var buf = newStringOfCap(50)
-
+  let jsonFile = conf.jsonBuildInstructionsFile
-  let jsonFile = conf.getNimcacheDir / RelativeFile(conf.projectName & ".json")
   conf.jsonBuildFile = jsonFile
   let output = conf.absOutFile
 
@@ -1038,8 +1042,7 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) =
     lit "\L}\L"
     close(f)
 
-proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile): bool =
+proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile): bool =
-  let jsonFile = toGeneratedFile(conf, projectfile, "json")
   if not fileExists(jsonFile): return true
   if not fileExists(conf.absOutFile): return true
   result = false
@@ -1090,11 +1093,9 @@ proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; projectfile: Absol
     echo "Warning: JSON processing failed: ", getCurrentExceptionMsg()
     result = true
 
-proc runJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) =
+proc runJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile) =
-  let jsonFile = toGeneratedFile(conf, projectfile, "json")
   try:
     let data = json.parseFile(jsonFile.string)
-
     let output = data["outputFile"].getStr
     createDir output.parentDir
     let outputCurrent = $conf.absOutFile

compiler/main.nim

@@ -19,7 +19,7 @@ import
   cgen, json, nversion,
   platform, nimconf, passaux, depends, vm,
   modules,
-  modulegraphs, tables, lineinfos, pathutils, vmprofiler
+  modulegraphs, tables, lineinfos, pathutils, vmprofiler, std/[sha1, with]
 
 import ic / [cbackend, integrity, navigator]
 from ic / ic import rodViewer
@@ -80,15 +80,13 @@ when not defined(leanCompiler):
 
 proc commandCompileToC(graph: ModuleGraph) =
   let conf = graph.config
-  setOutFile(conf)
   extccomp.initVars(conf)
   semanticPasses(graph)
   if conf.symbolFiles == disabledSf:
     registerPass(graph, cgenPass)
 
     if {optRun, optForceFullMake} * conf.globalOptions == {optRun} or isDefined(conf, "nimBetterRun"):
-      let proj = changeFileExt(conf.projectFull, "")
+      if not changeDetectedViaJsonBuildInstructions(conf, conf.jsonBuildInstructionsFile):
-      if not changeDetectedViaJsonBuildInstructions(conf, proj):
         # nothing changed
         graph.config.notes = graph.config.mainPackageNotes
         return
@@ -117,27 +115,20 @@ proc commandCompileToC(graph: ModuleGraph) =
       writeDepsFile(graph)
 
 proc commandJsonScript(graph: ModuleGraph) =
-  let proj = changeFileExt(graph.config.projectFull, "")
+  extccomp.runJsonBuildInstructions(graph.config, graph.config.jsonBuildInstructionsFile)
-  extccomp.runJsonBuildInstructions(graph.config, proj)
 
 proc commandCompileToJS(graph: ModuleGraph) =
-  when defined(leanCompiler):
-    globalError(graph.config, unknownLineInfo, "compiler wasn't built with JS code generator")
-  else:
   let conf = graph.config
+  when defined(leanCompiler):
+    globalError(conf, unknownLineInfo, "compiler wasn't built with JS code generator")
+  else:
     conf.exc = excCpp
-
+    setTarget(conf.target, osJS, cpuJS)
-    if conf.outFile.isEmpty:
+    defineSymbol(conf.symbols, "ecmascript") # For backward compatibility
-      conf.outFile = RelativeFile(conf.projectName & ".js")
-
-    #incl(gGlobalOptions, optSafeCode)
-    setTarget(graph.config.target, osJS, cpuJS)
-    #initDefines()
-    defineSymbol(graph.config.symbols, "ecmascript") # For backward compatibility
     semanticPasses(graph)
     registerPass(graph, JSgenPass)
     compileProject(graph)
-    if optGenScript in graph.config.globalOptions:
+    if optGenScript in conf.globalOptions:
       writeDepsFile(graph)
 
 proc interactivePasses(graph: ModuleGraph) =
@@ -186,6 +177,31 @@ proc commandView(graph: ModuleGraph) =
 const
   PrintRopeCacheStats = false
 
+proc hashMainCompilationParams*(conf: ConfigRef): string =
+  ## doesn't have to be complete; worst case is a cache hit and recompilation.
+  var state = newSha1State()
+  with state:
+    update os.getAppFilename() # nim compiler
+    update conf.commandLine # excludes `arguments`, as it should
+    update $conf.projectFull # so that running `nim r main` from 2 directories caches differently
+  result = $SecureHash(state.finalize())
+
+proc setOutFile*(conf: ConfigRef) =
+  proc libNameTmpl(conf: ConfigRef): string {.inline.} =
+    result = if conf.target.targetOS == osWindows: "$1.lib" else: "lib$1.a"
+
+  if conf.outFile.isEmpty:
+    var base = conf.projectName
+    if optUseNimcache in conf.globalOptions:
+      base.add "_" & hashMainCompilationParams(conf)
+    let targetName =
+      if conf.backend == backendJs: base & ".js"
+      elif optGenDynLib in conf.globalOptions:
+        platform.OS[conf.target.targetOS].dllFrmt % base
+      elif optGenStaticLib in conf.globalOptions: libNameTmpl(conf) % base
+      else: base & platform.OS[conf.target.targetOS].exeExt
+    conf.outFile = RelativeFile targetName
+
 proc mainCommand*(graph: ModuleGraph) =
   let conf = graph.config
   let cache = graph.cache
@@ -220,6 +236,7 @@ proc mainCommand*(graph: ModuleGraph) =
 
   proc compileToBackend() =
     customizeForBackend(conf.backend)
+    setOutFile(conf)
     case conf.backend
     of backendC: commandCompileToC(graph)
     of backendCpp: commandCompileToC(graph)

compiler/options.nim

@@ -970,18 +970,3 @@ proc floatInt64Align*(conf: ConfigRef): int16 =
       # to 4bytes (except with -malign-double)
       return 4
   return 8
-
-proc setOutFile*(conf: ConfigRef) =
-  proc libNameTmpl(conf: ConfigRef): string {.inline.} =
-    result = if conf.target.targetOS == osWindows: "$1.lib" else: "lib$1.a"
-
-  if conf.outFile.isEmpty:
-    let base = conf.projectName
-    let targetName =
-      if optGenDynLib in conf.globalOptions:
-        platform.OS[conf.target.targetOS].dllFrmt % base
-      elif optGenStaticLib in conf.globalOptions:
-        libNameTmpl(conf) % base
-      else:
-        base & platform.OS[conf.target.targetOS].exeExt
-    conf.outFile = RelativeFile targetName

tests/misc/mbetterrun.nim

@@ -0,0 +1,3 @@
+const mbetterrunVal {.strdefine.} = ""
+static: echo "compiling: " & mbetterrunVal
+echo "running: " & mbetterrunVal

tests/misc/trunner.nim

@@ -29,13 +29,18 @@ const
   nimcache = buildDir / "nimcacheTrunner"
     # instead of `querySetting(nimcacheDir)`, avoids stomping on other parallel tests
 
-proc runCmd(file, options = ""): auto =
+proc runNimCmd(file, options = "", rtarg = ""): auto =
   let fileabs = testsDir / file.unixToNativePath
   doAssert fileabs.fileExists, fileabs
-  let cmd = fmt"{nim} {mode} {options} --hints:off {fileabs}"
+  let cmd = fmt"{nim} {mode} {options} --hints:off {fileabs} {rtarg}"
   result = execCmdEx(cmd)
   when false:  echo result[0] & "\n" & result[1] # for debugging
 
+proc runNimCmdChk(file, options = "", rtarg = ""): string =
+  let (ret, status) = runNimCmd(file, options, rtarg = rtarg)
+  doAssert status == 0, $(file, options) & "\n" & ret
+  ret
+
 when defined(nimTrunnerFfi):
   block: # mevalffi
     when defined(openbsd):
@@ -53,8 +58,8 @@ when defined(nimTrunnerFfi):
 hello world stderr
 hi stderr
 """
-    let (output, exitCode) = runCmd("vm/mevalffi.nim", fmt"{opt} --experimental:compiletimeFFI")
+    let output = runNimCmdChk("vm/mevalffi.nim", fmt"{opt} --experimental:compiletimeFFI")
-    let expected = fmt"""
+    doAssert output == fmt"""
 {prefix}foo
 foo:100
 foo:101
@@ -62,12 +67,10 @@ foo:102:103
 foo:102:103:104
 foo:0.03:asdf:103:105
 ret=[s1:foobar s2:foobar age:25 pi:3.14]
-"""
+""", output
-    doAssert output == expected, output
-    doAssert exitCode == 0
 
 else: # don't run twice the same test
-  import std/[strutils]
+  import std/strutils
   template check2(msg) = doAssert msg in output, output
 
   block: # tests with various options `nim doc --project --index --docroot`
@@ -142,17 +145,16 @@ sub/mmain.idx""", context
       else: doAssert false
 
   block: # mstatic_assert
-    let (output, exitCode) = runCmd("ccgbugs/mstatic_assert.nim", "-d:caseBad")
+    let (output, exitCode) = runNimCmd("ccgbugs/mstatic_assert.nim", "-d:caseBad")
     check2 "sizeof(bool) == 2"
     check exitCode != 0
 
   block: # ABI checks
     let file = "misc/msizeof5.nim"
     block:
-      let (output, exitCode) = runCmd(file, "-d:checkAbi")
+      discard runNimCmdChk(file, "-d:checkAbi")
-      doAssert exitCode == 0, output
     block:
-      let (output, exitCode) = runCmd(file, "-d:checkAbi -d:caseBad")
+      let (output, exitCode) = runNimCmd(file, "-d:checkAbi -d:caseBad")
       # on platforms that support _StaticAssert natively, errors will show full context, e.g.:
       # error: static_assert failed due to requirement 'sizeof(unsigned char) == 8'
       # "backend & Nim disagree on size for: BadImportcType{int64} [declared in mabi_check.nim(1, 6)]"
@@ -293,3 +295,25 @@ tests/newconfig/bar/mfoo.nims""".splitLines
         let (outp, exitCode) = run "echo 1+2; quit(2)"
         check3 "3" in outp
         doAssert exitCode == 2
+
+  block: # nimBetterRun
+    let file = "misc/mbetterrun.nim"
+    const nimcache2 = buildDir / "D20210423T185116"
+    removeDir nimcache2
+    # related to `-d:nimBetterRun`
+    let opt = fmt"-r --usenimcache --nimcache:{nimcache2}"
+    var ret = ""
+    for a in @["v1", "v2", "v1", "v3"]:
+      ret.add runNimCmdChk(file, fmt"{opt} -d:mbetterrunVal:{a}")
+    ret.add runNimCmdChk(file, fmt"{opt} -d:mbetterrunVal:v2", rtarg = "arg1 arg2")
+      # rt arguments should not cause a recompilation
+    doAssert ret == """
+compiling: v1
+running: v1
+compiling: v2
+running: v2
+running: v1
+compiling: v3
+running: v3
+running: v2
+""", ret