better run [feature] (#11709)

* track the checksums of all involved Nim files for smarter 'nim c -r' recompiles
* don't recompile unless necessary for 'nim c -r' [feature]
* [feature] koch boot uses a two step process in order to free the RAM before the GCC/Clang invocations
* fixes a serious regression

changelog.md

@@ -32,6 +32,9 @@
 
 ### Tool changes
 
+- The Nim compiler now does not recompile the Nim project via ``nim c -r`` if
+  no dependent Nim file changed. This feature can be overridden by
+  the ``--forceBuild`` command line option.
 
 ### Compiler changes
 

compiler/commands.nim

@@ -784,7 +784,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if strutils.find(switch, '.') >= 0: options.setConfigVar(conf, switch, arg)
     else: invalidCmdLineOption(conf, pass, switch, info)
 
-template gCmdLineInfo*(): untyped = newLineInfo(config, AbsoluteFile"command line", 1, 1)
+template gCmdLineInfo*(): untyped = newLineInfo(commandLineIdx, 1, 1)
 
 proc processCommand*(switch: string, pass: TCmdLinePass; config: ConfigRef) =
   var cmd, arg: string

compiler/extccomp.nim

@@ -947,7 +947,7 @@ proc callCCompiler*(conf: ConfigRef) =
     generateScript(conf, script)
 
 #from json import escapeJson
-import json
+import json, std / sha1
 
 proc writeJsonBuildInstructions*(conf: ConfigRef) =
   template lit(x: untyped) = f.write x
@@ -960,17 +960,17 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) =
       f.write escapeJson(x)
 
   proc cfiles(conf: ConfigRef; f: File; buf: var string; clist: CfileList, isExternal: bool) =
-    var pastStart = false
+    var i = 0
     for it in clist:
       if CfileFlag.Cached in it.flags: continue
       let compileCmd = getCompileCFileCmd(conf, it)
-      if pastStart: lit "],\L"
+      if i > 0: lit ",\L"
       lit "["
       str it.cname.string
       lit ", "
       str compileCmd
-      pastStart = true
+      lit "]"
-    lit "]\L"
+      inc i
 
   proc linkfiles(conf: ConfigRef; f: File; buf, objfiles: var string; clist: CfileList;
                  llist: seq[string]) =
@@ -994,6 +994,19 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) =
       pastStart = true
     lit "\L"
 
+  proc nimfiles(conf: ConfigRef; f: File) =
+    var i = 0
+    for it in conf.m.fileInfos:
+      if isAbsolute(it.fullPath.string):
+        if i > 0: lit "],\L"
+        lit "["
+        str it.fullPath.string
+        lit ", "
+        str $secureHashFile(it.fullPath.string)
+        inc i
+    lit "]\L"
+
+
   var buf = newStringOfCap(50)
 
   let jsonFile = conf.getNimcacheDir / RelativeFile(conf.projectName & ".json")
@@ -1009,9 +1022,37 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) =
 
     lit "],\L\"linkcmd\": "
     str getLinkCmd(conf, conf.absOutFile, objfiles)
+
+    if optRun in conf.globalOptions:
+      lit ",\L\"nimfiles\":[\L"
+      nimfiles(conf, f)
+      lit "]\L"
+
     lit "\L}\L"
     close(f)
 
+proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile): bool =
+  let jsonFile = toGeneratedFile(conf, projectfile, "json")
+  if not fileExists(jsonFile): return true
+  if not fileExists(conf.absOutFile): return true
+  result = false
+  try:
+    let data = json.parseFile(jsonFile.string)
+    let nimfilesPairs = data["nimfiles"]
+    doAssert nimfilesPairs.kind == JArray
+    for p in nimfilesPairs:
+      doAssert p.kind == JArray
+      # >= 2 for forwards compatibility with potential later .json files:
+      doAssert p.len >= 2
+      let nimFilename = p[0].getStr
+      let oldHashValue = p[1].getStr
+      let newHashValue = $secureHashFile(nimFilename)
+      if oldHashValue != newHashValue:
+        result = true
+  except IOError, OSError, ValueError:
+    echo "Warning: JSON processing failed: ", getCurrentExceptionMsg()
+    result = true
+
 proc runJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) =
   let jsonFile = toGeneratedFile(conf, projectfile, "json")
   try:

compiler/lineinfos.nim

@@ -240,8 +240,10 @@ type
   Severity* {.pure.} = enum ## VS Code only supports these three
     Hint, Warning, Error
 
-const trackPosInvalidFileIdx* = FileIndex(-2) # special marker so that no suggestions
+const
+  trackPosInvalidFileIdx* = FileIndex(-2) # special marker so that no suggestions
                                           # are produced within comments and string literals
+  commandLineIdx* = FileIndex(-3)
 
 type
   MsgConfig* = object ## does not need to be stored in the incremental cache

compiler/main.nim

@@ -87,6 +87,15 @@ proc commandCompileToC(graph: ModuleGraph) =
   semanticPasses(graph)
   registerPass(graph, cgenPass)
 
+  if {optRun, optForceFullMake} * conf.globalOptions == {optRun}:
+    let proj = changeFileExt(conf.projectFull, "")
+    if not changeDetectedViaJsonBuildInstructions(conf, proj):
+      # nothing changed
+      # Little hack here in order to not lose our precious
+      # hintSuccessX message:
+      conf.notes.incl hintSuccessX
+      return
+
   compileProject(graph)
   if graph.config.errorCounter > 0:
     return # issue #9933

compiler/msgs.nim

@@ -160,19 +160,25 @@ proc getInfoContext*(conf: ConfigRef; index: int): TLineInfo =
   if i >=% L: result = unknownLineInfo()
   else: result = conf.m.msgContext[i].info
 
+const
+  commandLineDesc = "command line"
+
 template toFilename*(conf: ConfigRef; fileIdx: FileIndex): string =
   if fileIdx.int32 < 0 or conf == nil:
-    "???"
+    (if fileIdx == commandLineIdx: commandLineDesc else: "???")
   else:
     conf.m.fileInfos[fileIdx.int32].shortName
 
 proc toProjPath*(conf: ConfigRef; fileIdx: FileIndex): string =
-  if fileIdx.int32 < 0 or conf == nil: "???"
+  if fileIdx.int32 < 0 or conf == nil:
+    (if fileIdx == commandLineIdx: commandLineDesc else: "???")
   else: conf.m.fileInfos[fileIdx.int32].projPath.string
 
 proc toFullPath*(conf: ConfigRef; fileIdx: FileIndex): string =
-  if fileIdx.int32 < 0 or conf == nil: result = "???"
+  if fileIdx.int32 < 0 or conf == nil:
-  else: result = conf.m.fileInfos[fileIdx.int32].fullPath.string
+    result = (if fileIdx == commandLineIdx: commandLineDesc else: "???")
+  else:
+    result = conf.m.fileInfos[fileIdx.int32].fullPath.string
 
 proc setDirtyFile*(conf: ConfigRef; fileIdx: FileIndex; filename: AbsoluteFile) =
   assert fileIdx.int32 >= 0
@@ -189,7 +195,7 @@ proc getHash*(conf: ConfigRef; fileIdx: FileIndex): string =
 
 proc toFullPathConsiderDirty*(conf: ConfigRef; fileIdx: FileIndex): AbsoluteFile =
   if fileIdx.int32 < 0:
-    result = AbsoluteFile"???"
+    result = AbsoluteFile(if fileIdx == commandLineIdx: commandLineDesc else: "???")
   elif not conf.m.fileInfos[fileIdx.int32].dirtyFile.isEmpty:
     result = conf.m.fileInfos[fileIdx.int32].dirtyFile
   else:

compiler/passes.nim

@@ -97,7 +97,6 @@ proc resolveMod(conf: ConfigRef; module, relativeTo: string): FileIndex =
 proc processImplicits(graph: ModuleGraph; implicits: seq[string], nodeKind: TNodeKind,
                       a: var TPassContextArray; m: PSym) =
   # XXX fixme this should actually be relative to the config file!
-  let gCmdLineInfo = newLineInfo(FileIndex(0), 1, 1)
   let relativeTo = toFullPath(graph.config, m.info)
   for module in items(implicits):
     # implicit imports should not lead to a module importing itself

compiler/rodimpl.nim

@@ -427,12 +427,8 @@ proc storeRemaining*(g: ModuleGraph; module: PSym) =
       stillForwarded.add s
   swap w.forwardedSyms, stillForwarded
   transitiveClosure(g)
-  var nimid = 0
   for x in items(g.config.m.fileInfos):
-    # don't store the "command line" entry:
-    if nimid != 0:
     storeFilename(g, x.fullPath, FileIndex(nimid))
-    inc nimid
 
 # ---------------- decoder -----------------------------------
 

koch.nim

@@ -308,8 +308,14 @@ proc boot(args: string) =
         extraOption.add " -d:nimBoostrapCsources0_19_0"
         # remove this when csources get updated
 
-    exec "$# $# $# $# --nimcache:$# compiler" / "nim.nim" %
+    # in order to use less memory, we split the build into two steps:
+    # --compileOnly produces a $project.json file and does not run GCC/Clang.
+    # jsonbuild then uses the $project.json file to build the Nim binary.
+    exec "$# $# $# $# --nimcache:$# --compileOnly compiler" / "nim.nim" %
       [nimi, bootOptions, extraOption, args, smartNimcache]
+    exec "$# jsonscript --nimcache:$# compiler" / "nim.nim" %
+      [nimi, smartNimcache]
+
     if sameFileContent(output, i.thVersion):
       copyExe(output, finalDest)
       echo "executables are equal: SUCCESS!"

nimdoc/tester.nim

@@ -14,6 +14,10 @@ type
     doc: seq[string]
     buildIndex: seq[string]
 
+proc exec(cmd: string) =
+  if execShellCmd(cmd) != 0:
+    quit("FAILURE: " & cmd)
+
 proc testNimDoc(prjDir, docsDir: string; switches: NimSwitches; fixup = false) =
   let
     nimDocSwitches = switches.doc.join(" ")
@@ -22,12 +26,10 @@ proc testNimDoc(prjDir, docsDir: string; switches: NimSwitches; fixup = false) =
   putEnv("SOURCE_DATE_EPOCH", "100000")
 
   if nimDocSwitches != "":
-    if execShellCmd("nim doc $1" % [nimDocSwitches]) != 0:
+    exec("nim doc $1" % [nimDocSwitches])
-      quit("FAILURE: nim doc failed")
 
   if nimBuildIndexSwitches != "":
-    if execShellCmd("nim buildIndex $1" % [nimBuildIndexSwitches]) != 0:
+    exec("nim buildIndex $1" % [nimBuildIndexSwitches])
-      quit("FAILURE: nim buildIndex failed")
 
   for expected in walkDirRec(prjDir / "expected/"):
     let produced = expected.replace('\\', '/').replace("/expected/", "/$1/" % [docsDir])