symbol files: more progress

compiler/modules.nim

@@ -169,12 +169,14 @@ proc compileModule*(graph: ModuleGraph; fileIdx: int32; cache: IdentCache, flags
     if sfMainModule in result.flags:
       gMainPackageId = result.owner.id
 
+    when false:
       if gCmd in {cmdCompileToC, cmdCompileToCpp, cmdCheck, cmdIdeTools}:
         rd = handleSymbolFile(result, cache)
         if result.id < 0:
           internalError("handleSymbolFile should have set the module's ID")
           return
       else:
+        discard
     result.id = getModuleId(toFullPath(fileIdx))
     discard processModule(graph, result,
       if sfMainModule in flags and gProjectIsStdin: stdin.llStreamOpen else: nil,

compiler/passes.nim

@@ -7,13 +7,13 @@
 #    distribution, for details about the copyright.
 #
 
-# This module implements the passes functionality. A pass must implement the
+## This module implements the passes functionality. A pass must implement the
-# `TPass` interface.
+## `TPass` interface.
 
 import
   strutils, options, ast, astalgo, llstream, msgs, platform, os,
   condsyms, idents, renderer, types, extccomp, math, magicsys, nversion,
-  nimsets, syntaxes, times, rodread, idgen, modulegraphs, reorder
+  nimsets, syntaxes, times, rodread, idgen, modulegraphs, reorder, rod
 
 
 type
@@ -29,7 +29,8 @@ type
   TPassProcess* = proc (p: PPassContext, topLevelStmt: PNode): PNode {.nimcall.}
 
   TPass* = tuple[open: TPassOpen, openCached: TPassOpenCached,
-                 process: TPassProcess, close: TPassClose]
+                 process: TPassProcess, close: TPassClose,
+                 isFrontend: bool]
 
   TPassData* = tuple[input: PNode, closeOutput: PNode]
   TPasses* = openArray[TPass]
@@ -41,11 +42,13 @@ type
 proc makePass*(open: TPassOpen = nil,
                openCached: TPassOpenCached = nil,
                process: TPassProcess = nil,
-               close: TPassClose = nil): TPass =
+               close: TPassClose = nil,
+               isFrontend = false): TPass =
   result.open = open
   result.openCached = openCached
   result.close = close
   result.process = process
+  result.isFrontend = isFrontend
 
 # the semantic checker needs these:
 var
@@ -178,7 +181,34 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream,
     a: TPassContextArray
     s: PLLStream
     fileIdx = module.fileIdx
-  if rd == nil:
+  if module.id < 0:
+    # new module caching mechanism:
+    for i in 0..<gPassesLen:
+      if not isNil(gPasses[i].open) and not gPasses[i].isFrontend:
+        a[i] = gPasses[i].open(graph, module, cache)
+      else:
+        a[i] = nil
+
+    var stmtIndex = 0
+    var doContinue = true
+    while doContinue:
+      let n = loadNode(module, stmtIndex)
+      if n == nil or graph.stopCompile(): break
+      inc stmtIndex
+      var m = n
+      for i in 0..<gPassesLen:
+        if not isNil(gPasses[i].process) and not gPasses[i].isFrontend:
+          m = gPasses[i].process(a[i], m)
+          if isNil(m):
+            doContinue = false
+            break
+
+    var m: PNode = nil
+    for i in 0..<gPassesLen:
+      if not isNil(gPasses[i].close) and not gPasses[i].isFrontend:
+        m = gPasses[i].close(graph, a[i], m)
+      a[i] = nil
+  elif rd == nil:
     openPasses(graph, a, module, cache)
     if stream == nil:
       let filename = fileIdx.toFullPathConsiderDirty

compiler/rod.nim

@@ -14,7 +14,7 @@ import ast, idgen
 when not defined(nimSymbolfiles):
   template setupModuleCache* = discard
   template storeNode*(module: PSym; n: PNode) = discard
-  template loadNode*(module: PSym; index: var int): PNode = discard
+  template loadNode*(module: PSym; index: var int): PNode = PNode(nil)
 
   template getModuleId*(fullpath: string): int = getID()
 

compiler/rodimpl.nim

@@ -7,10 +7,10 @@
 #    distribution, for details about the copyright.
 #
 
-## This module implements the canonalization for the various caching mechanisms.
+## This module implements the new compilation cache.
 
 import strutils, os, intsets, tables, ropes, db_sqlite, msgs, options, types,
-  renderer, rodutils, std / sha1, idents
+  renderer, rodutils, std / sha1, idents, astalgo, magicsys
 
 var db: DbConn
 
@@ -20,7 +20,7 @@ proc getModuleId*(fullpath: string): int =
     sql"select id, fullHash from modules where fullpath = ?", fullpath)
   let currentFullhash = $secureHashFile(fullpath)
   if module[0].len == 0:
-    result = int db.insertID(sql"insert into modules(fullpath, interfHash, fullHash) values (?, ?)",
+    result = int db.insertID(sql"insert into modules(fullpath, interfHash, fullHash) values (?, ?, ?)",
       fullpath, "", currentFullhash)
   else:
     result = parseInt(module[0])
@@ -74,6 +74,19 @@ proc pushSym(w: PRodWriter, s: PSym) =
   if not containsOrIncl(w.smarks, s.id):
     w.sstack.add(s)
 
+proc toDbFileId(fullpath: string): int =
+  let id = db.getValue(sql"select id from filenames where fullpath = ?",
+    fullpath)
+  if id.len == 0:
+    result = int db.insertID(sql"insert into filenames(fullpath) values (?)", fullpath)
+  else:
+    result = parseInt(id)
+
+proc fromDbFileId(dbId: int): int32 =
+  let fullpath = db.getValue(sql"select fullpath from filenames where id = ?", dbId)
+  doAssert fullpath.len > 0, "cannot find file name for DB ID " & $dbId
+  result = fileInfoIdx(fullpath)
+
 proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode,
                 result: var string) =
   if n == nil:
@@ -91,7 +104,7 @@ proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode,
     result.add(',')
     encodeVInt(n.info.line, result)
     result.add(',')
-    encodeVInt(n.info.fileIndex, result)
+    encodeVInt(toDbFileId(n.info.toFullPath), result)
   elif fInfo.line != n.info.line:
     result.add('?')
     encodeVInt(n.info.col, result)
@@ -269,7 +282,7 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
   result.add(',')
   if s.info.line != -1'i16: encodeVInt(s.info.line, result)
   result.add(',')
-  encodeVInt(s.info.fileIndex, result)
+  encodeVInt(toDbFileId(s.info.toFullPath), result)
   if s.owner != nil:
     result.add('*')
     encodeVInt(s.owner.id, result)
@@ -331,8 +344,8 @@ proc storeSym(w: PRodWriter; s: PSym) =
   encodeSym(w, s, buf)
   # XXX only store the name for exported symbols in order to speed up lookup
   # times once we enable the skStub logic.
-  db.exec(sql"insert into syms(nimid, module, name, data) values (?, ?, ?, ?)",
+  db.exec(sql"insert into syms(nimid, module, name, data, exported) values (?, ?, ?, ?, ?)",
-    s.id, abs(w.module.id), s.name.s, buf)
+    s.id, abs(w.module.id), s.name.s, buf, ord(sfExported in s.flags))
 
 proc storeType(w: PRodWriter; t: PType) =
   var buf = newStringOfCap(160)
@@ -410,7 +423,7 @@ proc decodeLineInfo(r; b; info: var TLineInfo) =
       else: info.line = int16(decodeVInt(b.s, b.pos))
       if b.s[b.pos] == ',':
         inc(b.pos)
-        info.fileIndex = int32(decodeVInt(b.s, b.pos))
+        info.fileIndex = fromDbFileId(decodeVInt(b.s, b.pos))
 
 proc skipNode(b) =
   assert b.s[b.pos] == '('
@@ -642,25 +655,21 @@ proc decodeInstantiations(r; b; info: TLineInfo;
       ii.compilesId = decodeVInt(b.s, b.pos)
     s.safeAdd ii
 
-proc loadSym(r; id: int; info: TLineInfo): PSym =
+proc loadSymFromBlob(r; b; info: TLineInfo): PSym =
-  var
-    id: int
-    ident: PIdent
-  result = r.syms.getOrDefault(id)
-  if result != nil: return result
-  var b = loadBlob(sql"select data from syms where nimid = ?", id)
   if b.s[b.pos] == '{':
     inc(b.pos)
     if b.s[b.pos] == '}':
       inc(b.pos)
       return                  # nil sym
   var k = TSymKind(decodeVInt(b.s, b.pos))
+  var id: int
   if b.s[b.pos] == '+':
     inc(b.pos)
     id = decodeVInt(b.s, b.pos)
     setId(id)
   else:
     internalError(info, "decodeSym: no id")
+  var ident: PIdent
   if b.s[b.pos] == '&':
     inc(b.pos)
     ident = r.cache.getIdent(decodeStr(b.s, b.pos))
@@ -729,9 +738,34 @@ proc loadSym(r; id: int; info: TLineInfo): PSym =
     #  result.ast = decodeNodeLazyBody(b, result.info, result)
     #else:
     result.ast = decodeNode(r, b, result.info)
+  if sfCompilerProc in result.flags:
+    registerCompilerProc(result)
+
+proc loadSym(r; id: int; info: TLineInfo): PSym =
+  result = r.syms.getOrDefault(id)
+  if result != nil: return result
+  var b = loadBlob(sql"select data from syms where nimid = ?", id)
+  result = loadSymFromBlob(r, b, info)
+  doAssert id == result.id, "symbol ID is not consistent!"
+
+proc loadModuleSymTab(r; module: PSym) =
+  ## goal: fill  module.tab
+  gr.syms[module.id] = module
+  for row in db.fastRows(sql"select nimid, data from syms where module = ? and exported = 1", abs(module.id)):
+    let id = parseInt(row[0])
+    var s = r.syms.getOrDefault(id)
+    if s == nil:
+      var b = BlobReader(pos: 0)
+      shallowCopy(b.s, row[1])
+      s = loadSymFromBlob(r, b, module.info)
+    assert s != nil
+    strTableAdd(module.tab, s)
+  if sfSystemModule in module.flags:
+    magicsys.systemModule = module
 
 proc loadNode*(module: PSym; index: var int): PNode =
   if index == 0:
+    loadModuleSymTab(gr, module)
     index = parseInt db.getValue(
       sql"select min(id) from toplevelstmts where module = ?", abs module.id)
   var b = BlobReader(pos: 0)
@@ -749,6 +783,14 @@ proc createDb() =
     );
   """)
 
+  db.exec(sql"""
+    create table if not exists filenames(
+      id integer primary key,
+      fullpath varchar(8000) not null
+    );
+  """)
+  db.exec sql"create index if not exists FilenameIx on filenames(fullpath);"
+
   db.exec(sql"""
     create table if not exists modules(
       id integer primary key,
@@ -779,6 +821,7 @@ proc createDb() =
       module integer not null,
       name varchar(256) not null,
       data blob not null,
+      exported int not null,
       foreign key (module) references module(id)
     );
   """)
@@ -798,7 +841,6 @@ proc createDb() =
   """)
   db.exec sql"create index TopLevelStmtByModuleIdx on toplevelstmts(module);"
 
-
   db.exec(sql"""
     create table if not exists statics(
       id integer primary key,

compiler/sem.nim

@@ -621,4 +621,5 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode =
   popProcCon(c)
   if c.runnableExamples != nil: testExamples(c)
 
-const semPass* = makePass(myOpen, myOpenCached, myProcess, myClose)
+const semPass* = makePass(myOpen, myOpenCached, myProcess, myClose,
+                          isFrontend = true)