fixes #19795; fixes #11852; fixes #19974; remove parsing pipeline, Nim now parses the whole module at one time (#21379)

* fixes #19795; remove parse pipeline

* isScript

* fixes nimscriptapi

* don't touch reorder

* check script

* fixes tests

* it seems implicit imports of system cause troubles

* access the first child of `nkStmtList`

* ignore comments

* minor messages

* perhaps increases hloLoopDetector

* the module is a stmtList, which changes the errors

* fixes nimdoc

* fixes tlinter

* fixes nim  secret tests

* fixes arc_misc

* fixes nim secret tests again

* safe; fixes one more test

* GlobalError is the root cause too

* fixes parsing errors

* put emit types to the cfsForwardTypes section

* fixes #11852; `{.push checks:off}` now works in procs

* disable navigator

* fixes nimdoc

* add tests for JS

* fixes nimsuggest

compiler/backendpragmas.nim

@@ -0,0 +1,25 @@
+import pragmas, options, ast, trees
+
+proc pushBackendOption(optionsStack: var seq[TOptions], options: var TOptions) =
+  optionsStack.add options
+
+proc popBackendOption(optionsStack: var seq[TOptions], options: var TOptions) =
+  options = optionsStack[^1]
+  optionsStack.setLen(optionsStack.len-1)
+
+proc processPushBackendOption*(optionsStack: var seq[TOptions], options: var TOptions,
+                           n: PNode, start: int) =
+  pushBackendOption(optionsStack, options)
+  for i in start..<n.len:
+    let it = n[i]
+    if it.kind in nkPragmaCallKinds and it.len == 2 and it[1].kind == nkIntLit:
+      let sw = whichPragma(it[0])
+      let opts = pragmaToOptions(sw)
+      if opts != {}:
+        if it[1].intVal != 0:
+          options.incl opts
+        else:
+          options.excl opts
+
+template processPopBackendOption*(optionsStack: var seq[TOptions], options: var TOptions) =
+  popBackendOption(optionsStack, options)

compiler/ccgstmts.nim

@@ -8,7 +8,6 @@
 #
 
 # included from cgen.nim
-
 const
   RangeExpandLimit = 256      # do not generate ranges
                               # over 'RangeExpandLimit' elements
@@ -1538,7 +1537,7 @@ proc determineSection(n: PNode): TCFileSection =
   result = cfsProcHeaders
   if n.len >= 1 and n[0].kind in {nkStrLit..nkTripleStrLit}:
     let sec = n[0].strVal
-    if sec.startsWith("/*TYPESECTION*/"): result = cfsTypes
+    if sec.startsWith("/*TYPESECTION*/"): result = cfsForwardTypes # TODO WORKAROUND
     elif sec.startsWith("/*VARSECTION*/"): result = cfsVars
     elif sec.startsWith("/*INCLUDESECTION*/"): result = cfsHeaders
 
@@ -1555,9 +1554,14 @@ proc genEmit(p: BProc, t: PNode) =
     line(p, cpsStmts, s)
 
 proc genPragma(p: BProc, n: PNode) =
-  for it in n.sons:
+  for i in 0..<n.len:
+    let it = n[i]
     case whichPragma(it)
     of wEmit: genEmit(p, it)
+    of wPush:
+      processPushBackendOption(p.optionsStack, p.options, n, i+1)
+    of wPop:
+      processPopBackendOption(p.optionsStack, p.options)
     else: discard
 
 

compiler/cgen.nim

@@ -15,7 +15,7 @@ import
   ccgutils, os, ropes, math, passes, wordrecg, treetab, cgmeth,
   rodutils, renderer, cgendata, aliases,
   lowerings, tables, sets, ndi, lineinfos, pathutils, transf,
-  injectdestructors, astmsgs, modulepaths
+  injectdestructors, astmsgs, modulepaths, backendpragmas
 
 when defined(nimPreviewSlimSystem):
   import std/assertions

compiler/cgendata.nim

@@ -86,6 +86,7 @@ type
     options*: TOptions        # options that should be used for code
                               # generation; this is the same as prc.options
                               # unless prc == nil
+    optionsStack*: seq[TOptions]
     module*: BModule          # used to prevent excessive parameter passing
     withinLoop*: int          # > 0 if we are within a loop
     splitDecls*: int          # > 0 if we are in some context for C++ that

compiler/jsgen.nim

@@ -33,7 +33,7 @@ import
   nversion, msgs, idents, types,
   ropes, passes, ccgutils, wordrecg, renderer,
   cgmeth, lowerings, sighashes, modulegraphs, lineinfos, rodutils,
-  transf, injectdestructors, sourcemap, astmsgs
+  transf, injectdestructors, sourcemap, astmsgs, backendpragmas
 
 import json, sets, math, tables, intsets
 import strutils except addf
@@ -98,6 +98,7 @@ type
     prc: PSym
     globals, locals, body: Rope
     options: TOptions
+    optionsStack: seq[TOptions]
     module: BModule
     g: PGlobals
     generatedParamCopies: IntSet
@@ -2557,9 +2558,14 @@ proc genStmt(p: PProc, n: PNode) =
   if r.res != "": lineF(p, "$#;$n", [r.res])
 
 proc genPragma(p: PProc, n: PNode) =
-  for it in n.sons:
+  for i in 0..<n.len:
+    let it = n[i]
     case whichPragma(it)
     of wEmit: genAsmOrEmitStmt(p, it[1])
+    of wPush:
+      processPushBackendOption(p.optionsStack, p.options, n, i+1)
+    of wPop:
+      processPopBackendOption(p.optionsStack, p.options)
     else: discard
 
 proc genCast(p: PProc, n: PNode, r: var TCompRes) =

compiler/passes.nim

@@ -155,11 +155,10 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
     checkFirstLineIndentation(p)
     while true:
       if graph.stopCompile(): break
-      var n = parseTopLevelStmt(p)
+      var n = parseTopLevelStmt(p) # todo merge it
       if n.kind == nkEmpty: break
-      if (sfSystemModule notin module.flags and
-          ({sfNoForward, sfReorder} * module.flags != {} or
-          codeReordering in graph.config.features)):
+
+      if true:
         # read everything, no streaming possible
         var sl = newNodeI(nkStmtList, n.info)
         sl.add n

compiler/pragmas.nim

@@ -364,7 +364,7 @@ proc processNote(c: PContext, n: PNode) =
     else: invalidPragma(c, n)
   else: invalidPragma(c, n)
 
-proc pragmaToOptions(w: TSpecialWord): TOptions {.inline.} =
+proc pragmaToOptions*(w: TSpecialWord): TOptions {.inline.} =
   case w
   of wChecks: ChecksOptions
   of wObjChecks: {optObjCheck}

compiler/sem.nim

@@ -701,6 +701,12 @@ proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext
 
 proc isImportSystemStmt(g: ModuleGraph; n: PNode): bool =
   if g.systemModule == nil: return false
+  var n = n
+  if n.kind == nkStmtList:
+    for i in 0..<n.len-1:
+      if n[i].kind notin {nkCommentStmt, nkEmpty}:
+        n = n[i]
+        break
   case n.kind
   of nkImportStmt:
     for x in n: